Initial Dever 0.0.1
This commit is contained in:
Codex committed
commit
81233d098c
1479 files changed
+163782
No files matched your search
@@ -0,0 +1,53 @@
|
||||
# 外部 Worker SDK
|
||||
|
||||
通过 Port 和外部 Adapter，把 Python、JavaScript、Go 或 C/C++ 库接入 Dever。Worker 在独立进程中运行，Dever 负责通信、取消和回收。
|
||||
|
||||
应用接入先看 [完整示例](../skills/references/external-lib.md)，无需手写协议或操作注册表。
|
||||
|
||||
## 处理函数
|
||||
|
||||
| 语言 | 接口 |
|
||||
| --- | --- |
|
||||
| Python | 异步函数，使用 `Worker.from_manifest(manifest, handlers)` |
|
||||
| JavaScript | 接收 AbortSignal，使用 `Worker.fromManifest(manifest, handlers)` |
|
||||
| Go | 接收 context.Context，使用 `NewFromManifest` 和 `RegisterHandlers` |
|
||||
| C/C++ | 同步 `dever_ffi_handler` 回调，使用 `dever_ffi_serve` |
|
||||
|
||||
函数名对应 Port 操作名，输入输出及业务错误由编译器生成的接口定义检查。结果按命名字段返回 JSON；业务拒绝使用已声明的 BusinessError。stdout 只用于协议，日志写 stderr。
|
||||
|
||||
Python/JavaScript/Go 处理函数须响应取消信号。一秒内不退出会使 Worker 停止接收新调用；已取消调用的结果不再交付。自行调用 Go Serve 时，还须解除自己提供的 Reader 阻塞。
|
||||
|
||||
Worker 调用没有统一的固定倒计时，启动握手、写入和停止清理各自有限时。Worker 不是不可信代码沙箱，文件、网络和设备权限由运行环境管理。
|
||||
|
||||
## 接口与运行资源
|
||||
|
||||
编译器通过 `ExternalWorkerContract::sdk_manifest()` 生成接口定义，SDK 在 hello 时核对身份并读取配置。通信为 4 字节大端长度加 UTF-8 JSON；示例中的 fixture-schema-v1 仅用于测试。
|
||||
|
||||
运行时使用锁定的目标解释器、依赖和 Worker 资源，不借用宿主 PATH。Python 使用 `-I -S -B` 启动，再加载打包的 SDK、Adapter 和依赖。
|
||||
|
||||
`dever lib install <project>` 恢复已有锁；当前 CLI 不提供 Lib add/update。源码构建及目标运行资源由发行方准备，不能用本机 Python/Node/Go 代替缺失资源。
|
||||
|
||||
## C/C++
|
||||
|
||||
使用 `external exec "worker/native" {}`，shim 包含 `c/dever_ffi.h` 并与 `c/dever_ffi.c` 编译为目标平台程序。
|
||||
|
||||
- ABI 为 DEVER_FFI_ABI_V1。open 初始化一次，invoke 串行处理操作，close 释放状态。
|
||||
- operation、setting 和 payload 是当前回调借用的字节或 JSON，不保证 NUL 结尾，不得保留指针。
|
||||
- 输出缓冲由 SDK 分配。只写容量以内的数据并设置 size，不替换或释放缓冲。
|
||||
- DEVER_FFI_OK 返回结果 JSON；DEVER_FFI_ERROR 返回声明错误及 payload；其他状态作为运行故障。
|
||||
- C++ 异常不得跨 C ABI。同步原生调用不保证协作取消，宽限结束后由进程监管终止。
|
||||
|
||||
原生库须与目标系统 ABI 匹配。它们运行在 Worker 中，不直接加载进 Dever 主进程。
|
||||
|
||||
## 验证
|
||||
|
||||
从仓库根执行，指定已准备好的工具绝对路径：
|
||||
|
||||
```sh
|
||||
python3 -B test/external-sdk/check.py --python /path/to/python3 --node /path/to/node --go /path/to/go
|
||||
python3 -B test/external-sdk/test_c_ffi.py --cc /path/to/cc
|
||||
```
|
||||
|
||||
未提供的语言会报告 unavailable。SDK 测试验证真实通信和类型，普通应用测试使用 Port fake；后者不证明第三方库可以实际运行。
|
||||
|
||||
`sdk/python-runtime.py` 可从锁定摘要的 Python Build Standalone 归档准备解释器输入，参数见 `--help`；它不下载资源。脚本验证目标 ELF、glibc、模块及移动后的解释器，并保留许可证，默认 glibc 上限为 2.17。
|
||||
@@ -0,0 +1,310 @@
|
||||
#include "dever_ffi.h"
|
||||
#include <ctype.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#define MAX_BYTES (16U * 1024U * 1024U)
|
||||
#define MAX_TOKENS 65536U
|
||||
#define MAX_DEPTH 64U
|
||||
|
||||
typedef struct {
|
||||
size_t start, end, after, count;
|
||||
char kind;
|
||||
} token;
|
||||
|
||||
typedef struct {
|
||||
const char *data;
|
||||
size_t length, cursor, used;
|
||||
token *tokens;
|
||||
} document;
|
||||
|
||||
static void whitespace(document *doc) {
|
||||
while (doc->cursor < doc->length && strchr(" \t\r\n", doc->data[doc->cursor]) != NULL)
|
||||
doc->cursor++;
|
||||
}
|
||||
|
||||
static int string_end(document *doc) {
|
||||
if (doc->data[doc->cursor++] != '"') return 0;
|
||||
while (doc->cursor < doc->length) {
|
||||
unsigned char byte = (unsigned char)doc->data[doc->cursor++];
|
||||
if (byte == '"') return 1;
|
||||
if (byte < 32) return 0;
|
||||
if (byte == '\\') {
|
||||
if (doc->cursor == doc->length) return 0;
|
||||
byte = (unsigned char)doc->data[doc->cursor++];
|
||||
if (byte == 'u') {
|
||||
for (unsigned digit = 0; digit < 4; digit++)
|
||||
if (doc->cursor == doc->length || !isxdigit((unsigned char)doc->data[doc->cursor++])) return 0;
|
||||
} else if (strchr("\"\\/bfnrt", byte) == NULL) return 0;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int number_end(document *doc) {
|
||||
if (doc->data[doc->cursor] == '-') doc->cursor++;
|
||||
if (doc->cursor == doc->length) return 0;
|
||||
if (doc->data[doc->cursor] == '0') doc->cursor++;
|
||||
else {
|
||||
if (doc->data[doc->cursor] < '1' || doc->data[doc->cursor] > '9') return 0;
|
||||
while (doc->cursor < doc->length && isdigit((unsigned char)doc->data[doc->cursor])) doc->cursor++;
|
||||
}
|
||||
if (doc->cursor < doc->length && doc->data[doc->cursor] == '.') {
|
||||
size_t start = ++doc->cursor;
|
||||
while (doc->cursor < doc->length && isdigit((unsigned char)doc->data[doc->cursor])) doc->cursor++;
|
||||
if (doc->cursor == start) return 0;
|
||||
}
|
||||
if (doc->cursor < doc->length && strchr("eE", doc->data[doc->cursor]) != NULL) {
|
||||
doc->cursor++;
|
||||
if (doc->cursor < doc->length && strchr("+-", doc->data[doc->cursor]) != NULL) doc->cursor++;
|
||||
size_t start = doc->cursor;
|
||||
while (doc->cursor < doc->length && isdigit((unsigned char)doc->data[doc->cursor])) doc->cursor++;
|
||||
if (doc->cursor == start) return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int value(document *doc, unsigned depth) {
|
||||
whitespace(doc);
|
||||
if (depth > MAX_DEPTH || doc->used == MAX_TOKENS || doc->cursor == doc->length) return 0;
|
||||
token *item = &doc->tokens[doc->used++];
|
||||
item->start = doc->cursor;
|
||||
item->kind = doc->data[doc->cursor];
|
||||
if (item->kind == '"') {
|
||||
if (!string_end(doc)) return 0;
|
||||
} else if (item->kind == '{' || item->kind == '[') {
|
||||
char end = item->kind == '{' ? '}' : ']';
|
||||
doc->cursor++;
|
||||
whitespace(doc);
|
||||
if (doc->cursor == doc->length) return 0;
|
||||
if (doc->data[doc->cursor] != end) {
|
||||
for (;;) {
|
||||
whitespace(doc);
|
||||
if (item->kind == '{') {
|
||||
if (doc->cursor == doc->length || doc->data[doc->cursor] != '"' || !value(doc, depth + 1)) return 0;
|
||||
whitespace(doc);
|
||||
if (doc->cursor == doc->length || doc->data[doc->cursor++] != ':') return 0;
|
||||
}
|
||||
if (!value(doc, depth + 1)) return 0;
|
||||
item->count++;
|
||||
whitespace(doc);
|
||||
if (doc->cursor == doc->length) return 0;
|
||||
if (doc->data[doc->cursor] == end) break;
|
||||
if (doc->data[doc->cursor++] != ',') return 0;
|
||||
}
|
||||
}
|
||||
doc->cursor++;
|
||||
} else {
|
||||
const char *literal = item->kind == 't' ? "true" : item->kind == 'f' ? "false" : item->kind == 'n' ? "null" : NULL;
|
||||
if (literal != NULL) {
|
||||
size_t size = strlen(literal);
|
||||
if (size > doc->length - doc->cursor || memcmp(doc->data + doc->cursor, literal, size) != 0) return 0;
|
||||
doc->cursor += size;
|
||||
} else if (!number_end(doc)) return 0;
|
||||
}
|
||||
item->end = doc->cursor;
|
||||
item->after = doc->used;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int parse(document *doc, const char *data, size_t size) {
|
||||
*doc = (document){.data = data, .length = size};
|
||||
if (size == 0 || size > MAX_BYTES || memchr(data, 0, size) != NULL) return 0;
|
||||
doc->tokens = calloc(MAX_TOKENS, sizeof(token));
|
||||
if (doc->tokens == NULL) return 0;
|
||||
if (value(doc, 0)) {
|
||||
whitespace(doc);
|
||||
if (doc->cursor == size) return 1;
|
||||
}
|
||||
free(doc->tokens);
|
||||
doc->tokens = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static dever_ffi_slice slice(const document *doc, size_t index) {
|
||||
const token *item = &doc->tokens[index];
|
||||
return (dever_ffi_slice){doc->data + item->start, item->end - item->start};
|
||||
}
|
||||
|
||||
static int text_is(const document *doc, size_t index, const char *text) {
|
||||
if (index >= doc->used || doc->tokens[index].kind != '"') return 0;
|
||||
dever_ffi_slice part = slice(doc, index);
|
||||
return part.size == strlen(text) + 2 && memcmp(part.data + 1, text, (size_t)part.size - 2) == 0;
|
||||
}
|
||||
|
||||
static size_t field(const document *doc, const char *name) {
|
||||
size_t found = MAX_TOKENS;
|
||||
if (doc->tokens[0].kind != '{') return found;
|
||||
for (size_t index = 1; index < doc->tokens[0].after;) {
|
||||
size_t entry = doc->tokens[index].after;
|
||||
if (text_is(doc, index, name)) {
|
||||
if (found != MAX_TOKENS) return MAX_TOKENS;
|
||||
found = entry;
|
||||
}
|
||||
index = doc->tokens[entry].after;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
static int same(const document *left, size_t left_index, const document *right, size_t right_index) {
|
||||
if (left_index >= left->used || right_index >= right->used) return 0;
|
||||
const token *left_item = &left->tokens[left_index], *right_item = &right->tokens[right_index];
|
||||
if (left_item->kind != right_item->kind || left_item->count != right_item->count) return 0;
|
||||
if (left_item->kind == '[') {
|
||||
for (size_t offset = 1; offset <= left_item->count; offset++)
|
||||
if (!same(left, left_index + offset, right, right_index + offset)) return 0;
|
||||
return 1;
|
||||
}
|
||||
dever_ffi_slice left_slice = slice(left, left_index), right_slice = slice(right, right_index);
|
||||
return left_slice.size == right_slice.size && memcmp(left_slice.data, right_slice.data, (size_t)left_slice.size) == 0;
|
||||
}
|
||||
|
||||
static int append(dever_ffi_output *output, dever_ffi_slice input) {
|
||||
if (input.size > output->capacity - output->size) return 0;
|
||||
memcpy(output->data + output->size, input.data, (size_t)input.size);
|
||||
output->size += input.size;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int literal(dever_ffi_output *output, const char *text) {
|
||||
return append(output, (dever_ffi_slice){text, strlen(text)});
|
||||
}
|
||||
|
||||
static int send(dever_ffi_slice data) {
|
||||
if (data.size == 0 || data.size > MAX_BYTES) return 0;
|
||||
unsigned char size[4] = {(unsigned char)(data.size >> 24), (unsigned char)(data.size >> 16), (unsigned char)(data.size >> 8), (unsigned char)data.size};
|
||||
return fwrite(size, 1, 4, stdout) == 4 && fwrite(data.data, 1, (size_t)data.size, stdout) == data.size && fflush(stdout) == 0;
|
||||
}
|
||||
|
||||
static char *receive(size_t *length) {
|
||||
unsigned char size[4];
|
||||
if (fread(size, 1, 4, stdin) != 4) return NULL;
|
||||
*length = ((uint32_t)size[0] << 24) | ((uint32_t)size[1] << 16) | ((uint32_t)size[2] << 8) | size[3];
|
||||
if (*length == 0 || *length > MAX_BYTES) return NULL;
|
||||
char *data = malloc(*length);
|
||||
if (data == NULL) return NULL;
|
||||
if (fread(data, 1, *length, stdin) != *length) { free(data); return NULL; }
|
||||
return data;
|
||||
}
|
||||
|
||||
static int identifier(const document *doc, size_t index, uint64_t *result) {
|
||||
if (index >= doc->used) return 0;
|
||||
dever_ffi_slice input = slice(doc, index);
|
||||
uint64_t number = 0;
|
||||
for (uint64_t offset = 0; offset < input.size; offset++) {
|
||||
unsigned digit = (unsigned char)input.data[offset] - '0';
|
||||
if (digit > 9 || number > ((uint64_t)INT64_MAX - digit) / 10) return 0;
|
||||
number = number * 10 + digit;
|
||||
}
|
||||
*result = number;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int serve(const dever_ffi_handler *handler, const document *manifest, int *opened) {
|
||||
size_t length = 0;
|
||||
char *input = receive(&length);
|
||||
document hello = {0};
|
||||
char *buffer = malloc(MAX_BYTES), *returned = malloc(MAX_BYTES);
|
||||
int okay = input != NULL && buffer != NULL && returned != NULL && parse(&hello, input, length);
|
||||
const char *fields[] = {"port", "schema", "adapter", "capabilities", "operations"};
|
||||
if (okay) okay = hello.tokens[0].count == 8 && text_is(&hello, field(&hello, "kind"), "hello") && text_is(&hello, field(&hello, "version"), "dever-component-1");
|
||||
for (size_t index = 0; okay && index < 5; index++) okay = same(&hello, field(&hello, fields[index]), manifest, field(manifest, fields[index]));
|
||||
size_t setting = okay ? field(&hello, "setting") : MAX_TOKENS;
|
||||
if (okay) okay = setting < hello.used;
|
||||
if (okay && handler->open != NULL) okay = handler->open(handler->context, slice(&hello, setting)) == DEVER_FFI_OK;
|
||||
dever_ffi_output output = {buffer, MAX_BYTES, 0};
|
||||
if (okay) {
|
||||
*opened = 1;
|
||||
okay = literal(&output, "{\"kind\":\"ready\",\"version\":\"dever-component-1\"");
|
||||
for (size_t index = 0; okay && index < 5; index++)
|
||||
okay = literal(&output, ",\"") && literal(&output, fields[index]) && literal(&output, "\":") && append(&output, slice(manifest, field(manifest, fields[index])));
|
||||
okay = okay && literal(&output, "}") && send((dever_ffi_slice){output.data, output.size});
|
||||
}
|
||||
free(hello.tokens);
|
||||
free(input);
|
||||
uint64_t next_id = 1;
|
||||
int cancelled = 0, stopped = 0;
|
||||
while (okay && !stopped) {
|
||||
input = receive(&length);
|
||||
document message = {0};
|
||||
okay = input != NULL && parse(&message, input, length);
|
||||
if (!okay) { free(input); break; }
|
||||
size_t kind = field(&message, "kind"), id_index = field(&message, "id");
|
||||
uint64_t request_id = 0;
|
||||
output.size = 0;
|
||||
if (text_is(&message, kind, "shutdown")) {
|
||||
okay = message.tokens[0].count == 1 && send((dever_ffi_slice){"{\"kind\":\"shutdown\"}", 19});
|
||||
stopped = 1;
|
||||
} else if (text_is(&message, kind, "health")) {
|
||||
okay = message.tokens[0].count == 2 && identifier(&message, id_index, &request_id) && request_id == 0 && send((dever_ffi_slice){input, length});
|
||||
} else if (text_is(&message, kind, "cancel")) {
|
||||
okay = message.tokens[0].count == 2 && identifier(&message, id_index, &request_id) && request_id > 0 && request_id == next_id - 1 && !cancelled;
|
||||
cancelled = 1;
|
||||
} else if (text_is(&message, kind, "call")) {
|
||||
size_t operation = field(&message, "operation"), payload = field(&message, "payload");
|
||||
okay = message.tokens[0].count == 4 && identifier(&message, id_index, &request_id) && request_id == next_id && payload < message.used && operation < message.used && message.tokens[operation].kind == '"';
|
||||
size_t operations = field(manifest, "operations");
|
||||
int found = 0;
|
||||
for (size_t index = operations + 1; okay && index < manifest->tokens[operations].after; index = manifest->tokens[index].after)
|
||||
if (same(&message, operation, manifest, index)) found = 1;
|
||||
okay = okay && found;
|
||||
if (okay) {
|
||||
next_id++;
|
||||
cancelled = 0;
|
||||
dever_ffi_slice operation_name = slice(&message, operation);
|
||||
operation_name.data++;
|
||||
operation_name.size -= 2;
|
||||
dever_ffi_output result = {returned, MAX_BYTES - 256, 0};
|
||||
int status = handler->invoke(handler->context, operation_name, slice(&message, payload), &result);
|
||||
document result_doc = {0};
|
||||
okay = result.data == returned && result.capacity == MAX_BYTES - 256 && result.size <= result.capacity && parse(&result_doc, returned, (size_t)result.size);
|
||||
if (okay && status == DEVER_FFI_OK) {
|
||||
okay = literal(&output, "{\"kind\":\"result\",\"id\":") && append(&output, slice(&message, id_index)) && literal(&output, ",\"payload\":") && append(&output, (dever_ffi_slice){returned, result.size});
|
||||
} else if (okay && status == DEVER_FFI_ERROR) {
|
||||
size_t error = field(&result_doc, "error"), error_payload = field(&result_doc, "payload");
|
||||
okay = result_doc.tokens[0].count == 2 && error < result_doc.used && result_doc.tokens[error].kind == '"' && error_payload < result_doc.used;
|
||||
okay = okay && literal(&output, "{\"kind\":\"error\",\"id\":") && append(&output, slice(&message, id_index)) && literal(&output, ",\"error\":") && append(&output, slice(&result_doc, error)) && literal(&output, ",\"payload\":") && append(&output, slice(&result_doc, error_payload));
|
||||
} else okay = 0;
|
||||
okay = okay && literal(&output, "}") && send((dever_ffi_slice){output.data, output.size});
|
||||
free(result_doc.tokens);
|
||||
}
|
||||
} else okay = 0;
|
||||
free(message.tokens);
|
||||
free(input);
|
||||
}
|
||||
free(buffer);
|
||||
free(returned);
|
||||
return okay && stopped;
|
||||
}
|
||||
|
||||
int dever_ffi_serve(const dever_ffi_handler *handler) {
|
||||
if (handler == NULL || handler->abi != DEVER_FFI_ABI_V1 || handler->invoke == NULL) return 1;
|
||||
FILE *file = fopen("contract.json", "rb");
|
||||
if (file == NULL) return 1;
|
||||
char *data = malloc(MAX_BYTES + 1);
|
||||
if (data == NULL) { fclose(file); return 1; }
|
||||
size_t size = fread(data, 1, MAX_BYTES + 1, file);
|
||||
int okay = !ferror(file) && size <= MAX_BYTES;
|
||||
fclose(file);
|
||||
document manifest = {0};
|
||||
okay = okay && parse(&manifest, data, size);
|
||||
if (okay) {
|
||||
size_t operations = field(&manifest, "operations");
|
||||
okay = operations < manifest.used && manifest.tokens[operations].kind == '[' && manifest.tokens[operations].count > 0;
|
||||
const char *names[] = {"port", "schema", "adapter"};
|
||||
for (size_t index = 0; okay && index < 3; index++) {
|
||||
size_t entry = field(&manifest, names[index]);
|
||||
okay = entry < manifest.used && manifest.tokens[entry].kind == '"';
|
||||
}
|
||||
size_t capabilities = field(&manifest, "capabilities");
|
||||
okay = okay && capabilities < manifest.used && manifest.tokens[capabilities].kind == '[';
|
||||
}
|
||||
int opened = 0;
|
||||
okay = okay && serve(handler, &manifest, &opened);
|
||||
if (opened && handler->close != NULL) handler->close(handler->context);
|
||||
free(manifest.tokens);
|
||||
free(data);
|
||||
if (!okay) fputs("native FFI Worker failed\n", stderr);
|
||||
return okay ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
#ifndef DEVER_FFI_H
|
||||
#define DEVER_FFI_H
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define DEVER_FFI_ABI_V1 1
|
||||
#define DEVER_FFI_OK 0
|
||||
#define DEVER_FFI_ERROR 1
|
||||
#define DEVER_FFI_FAULT 2
|
||||
|
||||
typedef struct {
|
||||
const char *data;
|
||||
uint64_t size;
|
||||
} dever_ffi_slice;
|
||||
|
||||
typedef struct {
|
||||
char *data;
|
||||
uint64_t capacity;
|
||||
uint64_t size;
|
||||
} dever_ffi_output;
|
||||
|
||||
typedef struct {
|
||||
uint32_t abi;
|
||||
void *context;
|
||||
int (*open)(void *context, dever_ffi_slice setting);
|
||||
int (*invoke)(void *context, dever_ffi_slice operation, dever_ffi_slice payload, dever_ffi_output *output);
|
||||
void (*close)(void *context);
|
||||
} dever_ffi_handler;
|
||||
|
||||
int dever_ffi_serve(const dever_ffi_handler *handler);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,699 @@
|
||||
// Package component implements the Dever external Worker process protocol.
|
||||
// The compiler owns schema and typed Port codecs; callers bind its identity here.
|
||||
package component
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
Version = "dever-component-1"
|
||||
maxBytes = 16 * 1024 * 1024
|
||||
maxDepth = 64
|
||||
maxElements = 65536
|
||||
cancelGrace = time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
decimalPattern = regexp.MustCompile(`^[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$`)
|
||||
uuidPattern = regexp.MustCompile(`(?i)^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$`)
|
||||
timePattern = regexp.MustCompile(`^[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{3})?$`)
|
||||
datetimePattern = regexp.MustCompile(`^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,3})?(?:Z|[+-][0-9]{2}:[0-9]{2})$`)
|
||||
)
|
||||
|
||||
type Contract struct {
|
||||
Port, Schema, Adapter string
|
||||
Operations, Capabilities, Errors []string
|
||||
}
|
||||
|
||||
type TypeSchema struct {
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Value *TypeSchema `json:"value,omitempty"`
|
||||
Fields map[string]TypeSchema `json:"fields,omitempty"`
|
||||
}
|
||||
|
||||
type Manifest struct {
|
||||
Port string `json:"port"`
|
||||
Schema string `json:"schema"`
|
||||
Adapter string `json:"adapter"`
|
||||
Operations []string `json:"operations"`
|
||||
Capabilities []string `json:"capabilities"`
|
||||
Setting *TypeSchema `json:"setting"`
|
||||
Inputs map[string]TypeSchema `json:"inputs"`
|
||||
Outputs map[string]TypeSchema `json:"outputs"`
|
||||
Errors map[string]TypeSchema `json:"errors"`
|
||||
}
|
||||
|
||||
type BusinessError struct {
|
||||
Identity string
|
||||
Payload any
|
||||
}
|
||||
|
||||
func (e BusinessError) Error() string { return e.Identity }
|
||||
|
||||
type handler[S any] func(context.Context, any, S) (any, error)
|
||||
|
||||
type Worker[S any] struct {
|
||||
contract Contract
|
||||
decodeSetting func(any) (S, error)
|
||||
handlers map[string]handler[S]
|
||||
inputTypes map[string]TypeSchema
|
||||
outputTypes map[string]TypeSchema
|
||||
errorTypes map[string]TypeSchema
|
||||
}
|
||||
|
||||
func NewFromManifest(manifest Manifest) (*Worker[any], error) {
|
||||
if len(manifest.Inputs) != len(manifest.Operations) || len(manifest.Outputs) != len(manifest.Operations) {
|
||||
return nil, errors.New("incomplete typed Worker contract")
|
||||
}
|
||||
for _, operation := range manifest.Operations {
|
||||
if _, ok := manifest.Inputs[operation]; !ok {
|
||||
return nil, errors.New("missing operation input contract")
|
||||
}
|
||||
if _, ok := manifest.Outputs[operation]; !ok {
|
||||
return nil, errors.New("missing operation output contract")
|
||||
}
|
||||
}
|
||||
identities := make([]string, 0, len(manifest.Errors))
|
||||
for identity := range manifest.Errors {
|
||||
identities = append(identities, identity)
|
||||
}
|
||||
sort.Strings(identities)
|
||||
worker, err := New(Contract{
|
||||
Port: manifest.Port, Schema: manifest.Schema, Adapter: manifest.Adapter,
|
||||
Operations: manifest.Operations, Capabilities: manifest.Capabilities, Errors: identities,
|
||||
}, func(value any) (any, error) {
|
||||
if manifest.Setting == nil {
|
||||
if value != nil {
|
||||
return nil, errors.New("unexpected setting")
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
return decodeTyped(value, *manifest.Setting)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
worker.inputTypes, worker.outputTypes, worker.errorTypes = manifest.Inputs, manifest.Outputs, manifest.Errors
|
||||
return worker, nil
|
||||
}
|
||||
|
||||
// ServeManifest binds the checked contract to compile-time selected handlers.
|
||||
func ServeManifest(manifestBytes []byte, handlers map[string]func(context.Context, any, any) (any, error), input io.Reader, output io.Writer) error {
|
||||
var manifest Manifest
|
||||
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
worker, err := NewFromManifest(manifest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := worker.RegisterHandlers(handlers); err != nil {
|
||||
return err
|
||||
}
|
||||
return worker.Serve(input, output)
|
||||
}
|
||||
|
||||
func (w *Worker[S]) RegisterHandler(operation string, run func(context.Context, any, S) (any, error)) error {
|
||||
schema, ok := w.inputTypes[operation]
|
||||
if !ok || run == nil {
|
||||
return errors.New("operation is not in a typed Worker contract")
|
||||
}
|
||||
if w.handlers[operation] != nil {
|
||||
return errors.New("duplicate component operation")
|
||||
}
|
||||
w.handlers[operation] = func(ctx context.Context, raw any, setting S) (any, error) {
|
||||
payload, err := decodeTyped(raw, schema)
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid component payload")
|
||||
}
|
||||
result, err := run(ctx, payload, setting)
|
||||
if err != nil {
|
||||
var business BusinessError
|
||||
if errors.As(err, &business) {
|
||||
typeSchema, declared := w.errorTypes[business.Identity]
|
||||
if !declared {
|
||||
return nil, errors.New("undeclared business error identity")
|
||||
}
|
||||
business.Payload, err = decodeTyped(business.Payload, typeSchema)
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid business error payload")
|
||||
}
|
||||
return nil, business
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return decodeTyped(result, w.outputTypes[operation])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker[S]) RegisterHandlers(handlers map[string]func(context.Context, any, S) (any, error)) error {
|
||||
if len(handlers) != len(w.contract.Operations) {
|
||||
return errors.New("component operations are not fully registered")
|
||||
}
|
||||
for _, operation := range w.contract.Operations {
|
||||
if err := w.RegisterHandler(operation, handlers[operation]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func decodeTyped(value any, schema TypeSchema) (any, error) {
|
||||
switch schema.Type {
|
||||
case "nullable":
|
||||
if value == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if schema.Value == nil {
|
||||
return nil, errors.New("invalid nullable contract")
|
||||
}
|
||||
return decodeTyped(value, *schema.Value)
|
||||
case "list":
|
||||
values, ok := value.([]any)
|
||||
if !ok || schema.Value == nil {
|
||||
return nil, errors.New("expected list")
|
||||
}
|
||||
result := make([]any, len(values))
|
||||
for index, item := range values {
|
||||
decoded, err := decodeTyped(item, *schema.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result[index] = decoded
|
||||
}
|
||||
return result, nil
|
||||
case "record":
|
||||
object, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
return nil, errors.New("expected record")
|
||||
}
|
||||
for name := range object {
|
||||
if _, ok := schema.Fields[name]; !ok {
|
||||
return nil, errors.New("unknown record field")
|
||||
}
|
||||
}
|
||||
result := make(map[string]any, len(schema.Fields))
|
||||
for name, field := range schema.Fields {
|
||||
entry, exists := object[name]
|
||||
if !exists && field.Type != "nullable" {
|
||||
return nil, errors.New("missing record field")
|
||||
}
|
||||
decoded, err := decodeTyped(entry, field)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result[name] = decoded
|
||||
}
|
||||
return result, nil
|
||||
case "int64", "duration", "model_id":
|
||||
switch number := value.(type) {
|
||||
case json.Number:
|
||||
if _, err := number.Int64(); err == nil {
|
||||
return number, nil
|
||||
}
|
||||
case int64:
|
||||
return number, nil
|
||||
case int:
|
||||
return int64(number), nil
|
||||
}
|
||||
case "float64":
|
||||
switch number := value.(type) {
|
||||
case json.Number:
|
||||
parsed, err := number.Float64()
|
||||
if err == nil && !math.IsInf(parsed, 0) && !math.IsNaN(parsed) {
|
||||
return number, nil
|
||||
}
|
||||
case float64:
|
||||
if !math.IsInf(number, 0) && !math.IsNaN(number) {
|
||||
return number, nil
|
||||
}
|
||||
}
|
||||
case "bool":
|
||||
if boolean, ok := value.(bool); ok {
|
||||
return boolean, nil
|
||||
}
|
||||
case "json":
|
||||
return value, nil
|
||||
case "text", "id", "secret", "decimal", "uuid", "datetime", "date", "time":
|
||||
if text, ok := value.(string); ok {
|
||||
switch schema.Type {
|
||||
case "decimal":
|
||||
if !decimalPattern.MatchString(text) {
|
||||
return nil, errors.New("invalid decimal")
|
||||
}
|
||||
case "uuid":
|
||||
if !uuidPattern.MatchString(text) {
|
||||
return nil, errors.New("invalid UUID")
|
||||
}
|
||||
case "date":
|
||||
if _, err := time.Parse("2006-01-02", text); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case "time":
|
||||
if !timePattern.MatchString(text) {
|
||||
return nil, errors.New("invalid Time")
|
||||
}
|
||||
layout := "15:04:05"
|
||||
if len(text) == 12 {
|
||||
layout = "15:04:05.000"
|
||||
}
|
||||
if _, err := time.Parse(layout, text); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case "datetime":
|
||||
if !datetimePattern.MatchString(text) {
|
||||
return nil, errors.New("invalid DateTime")
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339Nano, text)
|
||||
if err != nil || parsed.Nanosecond()%1_000_000 != 0 {
|
||||
return nil, errors.New("invalid DateTime")
|
||||
}
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
}
|
||||
return nil, errors.New("invalid typed Worker value")
|
||||
}
|
||||
|
||||
func New[S any](contract Contract, decodeSetting func(any) (S, error)) (*Worker[S], error) {
|
||||
if contract.Port == "" || contract.Schema == "" || contract.Adapter == "" || len(contract.Operations) == 0 || decodeSetting == nil || !unique(contract.Operations) || !unique(contract.Capabilities) || !unique(contract.Errors) {
|
||||
return nil, errors.New("incomplete component contract")
|
||||
}
|
||||
if contract.Capabilities == nil {
|
||||
contract.Capabilities = []string{}
|
||||
}
|
||||
if contract.Errors == nil {
|
||||
contract.Errors = []string{}
|
||||
}
|
||||
contract.Operations = append([]string(nil), contract.Operations...)
|
||||
contract.Capabilities = append([]string{}, contract.Capabilities...)
|
||||
contract.Errors = append([]string{}, contract.Errors...)
|
||||
return &Worker[S]{contract: contract, decodeSetting: decodeSetting, handlers: make(map[string]handler[S])}, nil
|
||||
}
|
||||
|
||||
func unique(values []string) bool {
|
||||
seen := make(map[string]bool, len(values))
|
||||
for _, value := range values {
|
||||
if value == "" || seen[value] {
|
||||
return false
|
||||
}
|
||||
seen[value] = true
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// RegisterTyped binds a compiler-declared operation to its application codec.
|
||||
func RegisterTyped[S, P any](worker *Worker[S], operation string, decode func(any) (P, error), run func(context.Context, P, S) (any, error)) error {
|
||||
if !contains(worker.contract.Operations, operation) || worker.handlers[operation] != nil || decode == nil || run == nil {
|
||||
return errors.New("unknown or duplicate component operation")
|
||||
}
|
||||
worker.handlers[operation] = func(ctx context.Context, raw any, setting S) (any, error) {
|
||||
payload, err := decode(raw)
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid component payload")
|
||||
}
|
||||
return run(ctx, payload, setting)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func contains(values []string, target string) bool {
|
||||
for _, value := range values {
|
||||
if value == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type event struct {
|
||||
message map[string]any
|
||||
err error
|
||||
}
|
||||
|
||||
type result struct {
|
||||
id int64
|
||||
payload any
|
||||
err error
|
||||
}
|
||||
|
||||
func (w *Worker[S]) Serve(input io.Reader, output io.Writer) error {
|
||||
if len(w.handlers) != len(w.contract.Operations) {
|
||||
return errors.New("component operations are not fully registered")
|
||||
}
|
||||
messages := make(chan event, 1)
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
go func() {
|
||||
for {
|
||||
message, err := readFrame(input)
|
||||
select {
|
||||
case messages <- event{message, err}:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
first := <-messages
|
||||
if first.err != nil {
|
||||
return first.err
|
||||
}
|
||||
setting, err := w.handshake(first.message, output)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
results := make(chan result, 1)
|
||||
var cancel context.CancelFunc
|
||||
var active int64
|
||||
var activeDone chan struct{}
|
||||
defer func() {
|
||||
if active != 0 {
|
||||
_ = stopHandler(cancel, activeDone)
|
||||
}
|
||||
}()
|
||||
nextID := int64(1)
|
||||
lastCancelled := false
|
||||
for {
|
||||
select {
|
||||
case received := <-messages:
|
||||
if received.err != nil {
|
||||
return received.err
|
||||
}
|
||||
message := received.message
|
||||
kind, ok := message["kind"].(string)
|
||||
if !ok {
|
||||
return errors.New("invalid component kind")
|
||||
}
|
||||
switch kind {
|
||||
case "health":
|
||||
if !exact(message, "kind", "id") || requestID(message["id"]) != 0 {
|
||||
return errors.New("invalid component health")
|
||||
}
|
||||
if err := writeFrame(output, message); err != nil {
|
||||
return err
|
||||
}
|
||||
case "call":
|
||||
if !exact(message, "kind", "id", "operation", "payload") {
|
||||
return errors.New("invalid component call")
|
||||
}
|
||||
id := requestID(message["id"])
|
||||
operation, ok := message["operation"].(string)
|
||||
run := w.handlers[operation]
|
||||
if id != nextID || active != 0 || !ok || run == nil {
|
||||
return errors.New("invalid component call")
|
||||
}
|
||||
nextID++
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
cancel, active = stop, id
|
||||
activeDone = make(chan struct{})
|
||||
payload := message["payload"]
|
||||
go func(finished chan struct{}) {
|
||||
defer close(finished)
|
||||
value, err := run(ctx, payload, setting)
|
||||
select {
|
||||
case results <- result{id, value, err}:
|
||||
case <-ctx.Done():
|
||||
case <-done:
|
||||
}
|
||||
}(activeDone)
|
||||
case "cancel":
|
||||
if !exact(message, "kind", "id") {
|
||||
return errors.New("invalid component cancel")
|
||||
}
|
||||
id := requestID(message["id"])
|
||||
if id <= 0 {
|
||||
return errors.New("invalid component cancel")
|
||||
}
|
||||
if id == active {
|
||||
if err := stopHandler(cancel, activeDone); err != nil {
|
||||
active = 0
|
||||
activeDone = nil
|
||||
return err
|
||||
}
|
||||
active = 0
|
||||
activeDone = nil
|
||||
lastCancelled = true
|
||||
if err := writeFrame(output, map[string]any{"kind": "error", "id": id, "error": "dever.cancelled", "payload": nil}); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if id != nextID-1 || lastCancelled {
|
||||
return errors.New("unknown component cancel id")
|
||||
}
|
||||
case "shutdown":
|
||||
if !exact(message, "kind") {
|
||||
return errors.New("invalid component shutdown")
|
||||
}
|
||||
if active != 0 {
|
||||
if err := stopHandler(cancel, activeDone); err != nil {
|
||||
active = 0
|
||||
activeDone = nil
|
||||
return err
|
||||
}
|
||||
active = 0
|
||||
activeDone = nil
|
||||
}
|
||||
return writeFrame(output, map[string]any{"kind": "shutdown"})
|
||||
default:
|
||||
return errors.New("unknown component message")
|
||||
}
|
||||
case completed := <-results:
|
||||
if completed.id != active {
|
||||
continue
|
||||
}
|
||||
cancel()
|
||||
active = 0
|
||||
activeDone = nil
|
||||
lastCancelled = false
|
||||
if completed.err == nil {
|
||||
if err := writeFrame(output, map[string]any{"kind": "result", "id": completed.id, "payload": completed.payload}); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
var business BusinessError
|
||||
if !errors.As(completed.err, &business) || !contains(w.contract.Errors, business.Identity) {
|
||||
return errors.New("component handler failed")
|
||||
}
|
||||
if err := writeFrame(output, map[string]any{"kind": "error", "id": completed.id, "error": business.Identity, "payload": business.Payload}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stopHandler(cancel context.CancelFunc, finished <-chan struct{}) error {
|
||||
cancel()
|
||||
timer := time.NewTimer(cancelGrace)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-finished:
|
||||
return nil
|
||||
case <-timer.C:
|
||||
return errors.New("component handler did not stop after cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker[S]) handshake(hello map[string]any, output io.Writer) (S, error) {
|
||||
var zero S
|
||||
if !exact(hello, "kind", "version", "port", "schema", "adapter", "capabilities", "operations", "setting") || hello["kind"] != "hello" || hello["version"] != Version || hello["port"] != w.contract.Port || hello["schema"] != w.contract.Schema || hello["adapter"] != w.contract.Adapter || !sameNames(hello["capabilities"], w.contract.Capabilities) || !sameNames(hello["operations"], w.contract.Operations) {
|
||||
return zero, errors.New("component handshake does not match registered contract")
|
||||
}
|
||||
setting, err := w.decodeSetting(hello["setting"])
|
||||
if err != nil {
|
||||
return zero, errors.New("invalid component setting")
|
||||
}
|
||||
if err := writeFrame(output, map[string]any{"kind": "ready", "version": Version, "port": w.contract.Port, "schema": w.contract.Schema, "adapter": w.contract.Adapter, "capabilities": w.contract.Capabilities, "operations": w.contract.Operations}); err != nil {
|
||||
return zero, err
|
||||
}
|
||||
return setting, nil
|
||||
}
|
||||
|
||||
func sameNames(raw any, expected []string) bool {
|
||||
values, ok := raw.([]any)
|
||||
if !ok || len(values) != len(expected) {
|
||||
return false
|
||||
}
|
||||
for i, value := range values {
|
||||
if value != expected[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func exact(message map[string]any, names ...string) bool {
|
||||
if len(message) != len(names) {
|
||||
return false
|
||||
}
|
||||
for _, name := range names {
|
||||
if _, ok := message[name]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func requestID(value any) int64 {
|
||||
number, ok := value.(json.Number)
|
||||
if !ok {
|
||||
return -1
|
||||
}
|
||||
id, err := number.Int64()
|
||||
if err != nil || id < 0 {
|
||||
return -1
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func readFrame(input io.Reader) (map[string]any, error) {
|
||||
var header [4]byte
|
||||
if _, err := io.ReadFull(input, header[:]); err != nil {
|
||||
return nil, errors.New("incomplete component frame")
|
||||
}
|
||||
size := binary.BigEndian.Uint32(header[:])
|
||||
if size == 0 || size > maxBytes {
|
||||
return nil, errors.New("component frame exceeds byte limit")
|
||||
}
|
||||
body := make([]byte, size)
|
||||
if _, err := io.ReadFull(input, body); err != nil {
|
||||
return nil, errors.New("incomplete component frame")
|
||||
}
|
||||
return parseJSON(body)
|
||||
}
|
||||
|
||||
func writeFrame(output io.Writer, message map[string]any) error {
|
||||
body, err := json.Marshal(message)
|
||||
if err != nil {
|
||||
return errors.New("invalid handler response")
|
||||
}
|
||||
if len(body) == 0 || len(body) > maxBytes {
|
||||
return errors.New("component frame exceeds byte limit")
|
||||
}
|
||||
if _, err := parseJSON(body); err != nil {
|
||||
return err
|
||||
}
|
||||
var header [4]byte
|
||||
binary.BigEndian.PutUint32(header[:], uint32(len(body)))
|
||||
if err := writeAll(output, header[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeAll(output, body)
|
||||
}
|
||||
|
||||
func writeAll(output io.Writer, data []byte) error {
|
||||
for len(data) > 0 {
|
||||
count, err := output.Write(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if count == 0 {
|
||||
return io.ErrShortWrite
|
||||
}
|
||||
data = data[count:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseJSON(body []byte) (map[string]any, error) {
|
||||
if !utf8.Valid(body) {
|
||||
return nil, errors.New("component frame is not UTF-8")
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(body))
|
||||
decoder.UseNumber()
|
||||
remaining := maxElements
|
||||
value, err := parseValue(decoder, 0, &remaining)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := decoder.Token(); err != io.EOF {
|
||||
return nil, errors.New("invalid component JSON")
|
||||
}
|
||||
object, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
return nil, errors.New("component message must be an object")
|
||||
}
|
||||
return object, nil
|
||||
}
|
||||
|
||||
func parseValue(decoder *json.Decoder, depth int, remaining *int) (any, error) {
|
||||
if depth > maxDepth || *remaining <= 0 {
|
||||
return nil, errors.New("wire JSON exceeds limit")
|
||||
}
|
||||
*remaining--
|
||||
token, err := decoder.Token()
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid component JSON")
|
||||
}
|
||||
delim, ok := token.(json.Delim)
|
||||
if !ok {
|
||||
if number, isNumber := token.(json.Number); isNumber && strings.ContainsAny(string(number), ".eE") {
|
||||
value, err := strconv.ParseFloat(string(number), 64)
|
||||
if err != nil || math.IsInf(value, 0) || math.IsNaN(value) {
|
||||
return nil, errors.New("non-finite wire Float")
|
||||
}
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
if depth == maxDepth {
|
||||
return nil, errors.New("wire JSON exceeds depth limit")
|
||||
}
|
||||
switch delim {
|
||||
case '{':
|
||||
object := make(map[string]any)
|
||||
for decoder.More() {
|
||||
key, err := decoder.Token()
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid component JSON")
|
||||
}
|
||||
name, ok := key.(string)
|
||||
if !ok {
|
||||
return nil, errors.New("invalid component JSON")
|
||||
}
|
||||
if _, exists := object[name]; exists {
|
||||
return nil, errors.New("duplicate JSON field")
|
||||
}
|
||||
value, err := parseValue(decoder, depth+1, remaining)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
object[name] = value
|
||||
}
|
||||
if end, err := decoder.Token(); err != nil || end != json.Delim('}') {
|
||||
return nil, errors.New("invalid component JSON")
|
||||
}
|
||||
return object, nil
|
||||
case '[':
|
||||
array := make([]any, 0)
|
||||
for decoder.More() {
|
||||
value, err := parseValue(decoder, depth+1, remaining)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
array = append(array, value)
|
||||
}
|
||||
if end, err := decoder.Token(); err != nil || end != json.Delim(']') {
|
||||
return nil, errors.New("invalid component JSON")
|
||||
}
|
||||
return array, nil
|
||||
default:
|
||||
return nil, errors.New("invalid component JSON")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
component "dever-component"
|
||||
)
|
||||
|
||||
type payload struct {
|
||||
Text string
|
||||
DelayMS int64
|
||||
}
|
||||
|
||||
func setting(raw any) (string, error) {
|
||||
object, ok := raw.(map[string]any)
|
||||
if !ok || len(object) != 1 {
|
||||
return "", errors.New("invalid setting")
|
||||
}
|
||||
value, ok := object["prefix"].(string)
|
||||
if !ok {
|
||||
return "", errors.New("invalid setting")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func decode(raw any) (payload, error) {
|
||||
object, ok := raw.(map[string]any)
|
||||
if !ok || len(object) != 2 {
|
||||
return payload{}, errors.New("invalid payload")
|
||||
}
|
||||
text, ok := object["text"].(string)
|
||||
if !ok {
|
||||
return payload{}, errors.New("invalid payload")
|
||||
}
|
||||
delay, ok := object["delay_ms"].(interface{ Int64() (int64, error) })
|
||||
if !ok {
|
||||
return payload{}, errors.New("invalid payload")
|
||||
}
|
||||
milliseconds, err := delay.Int64()
|
||||
if err != nil || milliseconds < 0 || milliseconds > 10000 {
|
||||
return payload{}, errors.New("invalid payload")
|
||||
}
|
||||
return payload{Text: text, DelayMS: milliseconds}, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
worker, err := component.New(component.Contract{
|
||||
Port: "example.Text", Schema: "fixture-schema-v1", Adapter: "example.TextAdapter",
|
||||
Operations: []string{"text.render", "text.fail", "number.echo", "text.stubborn"}, Errors: []string{"example.rejected"},
|
||||
}, setting)
|
||||
if err == nil {
|
||||
err = component.RegisterTyped(worker, "text.render", decode, func(ctx context.Context, value payload, prefix string) (any, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(time.Duration(value.DelayMS) * time.Millisecond):
|
||||
return map[string]any{"value": prefix + value.Text}, nil
|
||||
}
|
||||
})
|
||||
}
|
||||
if err == nil {
|
||||
err = component.RegisterTyped(worker, "text.fail", decode, func(context.Context, payload, string) (any, error) {
|
||||
return nil, component.BusinessError{Identity: "example.rejected", Payload: map[string]any{"reason": "rejected"}}
|
||||
})
|
||||
}
|
||||
if err == nil {
|
||||
err = component.RegisterTyped(worker, "number.echo", func(raw any) (json.Number, error) {
|
||||
object, ok := raw.(map[string]any)
|
||||
if !ok || len(object) != 1 {
|
||||
return "", errors.New("invalid number")
|
||||
}
|
||||
number, ok := object["number"].(json.Number)
|
||||
if !ok {
|
||||
return "", errors.New("invalid number")
|
||||
}
|
||||
if _, err := number.Int64(); err != nil {
|
||||
return "", errors.New("invalid number")
|
||||
}
|
||||
return number, nil
|
||||
}, func(_ context.Context, value json.Number, _ string) (any, error) {
|
||||
return map[string]any{"number": value}, nil
|
||||
})
|
||||
}
|
||||
if err == nil {
|
||||
err = component.RegisterTyped(worker, "text.stubborn", decode, func(_ context.Context, value payload, prefix string) (any, error) {
|
||||
time.Sleep(5 * time.Second)
|
||||
return map[string]any{"value": prefix + value.Text}, nil
|
||||
})
|
||||
}
|
||||
if err == nil {
|
||||
err = worker.Serve(os.Stdin, os.Stdout)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "component worker:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module dever-component
|
||||
|
||||
go 1.21
|
||||
@@ -0,0 +1,368 @@
|
||||
const VERSION = 'dever-component-1';
|
||||
const MAX_BYTES = 16 * 1024 * 1024;
|
||||
const MAX_DEPTH = 64;
|
||||
const MAX_ELEMENTS = 65536;
|
||||
const CANCEL_GRACE_MS = 1000;
|
||||
const decoder = new TextDecoder('utf-8', { fatal: true });
|
||||
|
||||
export class ProtocolError extends Error {}
|
||||
|
||||
export class BusinessError extends Error {
|
||||
constructor(identity, payload = null) {
|
||||
super(identity);
|
||||
this.identity = identity;
|
||||
this.payload = payload;
|
||||
}
|
||||
}
|
||||
|
||||
// JSON.parse accepts duplicate object keys; this parser enforces the runtime's
|
||||
// duplicate, depth and value budgets before returning ordinary JS values.
|
||||
function parseJSON(text) {
|
||||
let offset = 0;
|
||||
let remaining = MAX_ELEMENTS;
|
||||
const fail = () => { throw new ProtocolError('invalid component JSON'); };
|
||||
const space = () => { while (offset < text.length && /[ \t\r\n]/.test(text[offset])) offset++; };
|
||||
function string() {
|
||||
const start = offset++;
|
||||
while (offset < text.length) {
|
||||
const char = text[offset++];
|
||||
if (char === '"') return JSON.parse(text.slice(start, offset));
|
||||
if (char === '\\') offset++;
|
||||
else if (char.charCodeAt(0) < 32) fail();
|
||||
}
|
||||
fail();
|
||||
}
|
||||
function value(depth) {
|
||||
if (depth > MAX_DEPTH || --remaining < 0) throw new ProtocolError('wire JSON exceeds limit');
|
||||
space();
|
||||
const char = text[offset];
|
||||
if (char === '"') return string();
|
||||
if (char === '[' || char === '{') {
|
||||
if (depth === MAX_DEPTH) throw new ProtocolError('wire JSON exceeds depth limit');
|
||||
offset++;
|
||||
const array = char === '[';
|
||||
const result = array ? [] : {};
|
||||
const seen = new Set();
|
||||
space();
|
||||
if (text[offset] === (array ? ']' : '}')) { offset++; return result; }
|
||||
for (;;) {
|
||||
if (array) result.push(value(depth + 1));
|
||||
else {
|
||||
if (text[offset] !== '"') fail();
|
||||
const key = string();
|
||||
if (seen.has(key)) throw new ProtocolError('duplicate JSON field');
|
||||
seen.add(key);
|
||||
space();
|
||||
if (text[offset++] !== ':') fail();
|
||||
Object.defineProperty(result, key, { value: value(depth + 1), enumerable: true, writable: true, configurable: true });
|
||||
}
|
||||
space();
|
||||
const end = array ? ']' : '}';
|
||||
if (text[offset] === end) { offset++; return result; }
|
||||
if (text[offset++] !== ',') fail();
|
||||
space();
|
||||
}
|
||||
}
|
||||
const match = /^(?:true|false|null|-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?)/.exec(text.slice(offset));
|
||||
if (!match) fail();
|
||||
offset += match[0].length;
|
||||
if (match[0] === 'true' || match[0] === 'false' || match[0] === 'null') return JSON.parse(match[0]);
|
||||
if (match[0] === '-0') return -0;
|
||||
if (/^-?(?:0|[1-9]\d*)$/.test(match[0])) {
|
||||
const integer = BigInt(match[0]);
|
||||
return integer <= BigInt(Number.MAX_SAFE_INTEGER) && integer >= BigInt(Number.MIN_SAFE_INTEGER) ? Number(integer) : integer;
|
||||
}
|
||||
const number = JSON.parse(match[0]);
|
||||
if (!Number.isFinite(number)) throw new ProtocolError('invalid JSON number');
|
||||
return number;
|
||||
}
|
||||
const result = value(0);
|
||||
space();
|
||||
if (offset !== text.length || result === null || Array.isArray(result) || typeof result !== 'object') fail();
|
||||
return result;
|
||||
}
|
||||
|
||||
function encodeJSON(value) {
|
||||
let remaining = MAX_ELEMENTS;
|
||||
function encode(current, depth) {
|
||||
if (depth > MAX_DEPTH || --remaining < 0) throw new ProtocolError('wire JSON exceeds limit');
|
||||
if (current === null || typeof current === 'boolean') return String(current);
|
||||
if (typeof current === 'string') return JSON.stringify(current);
|
||||
if (typeof current === 'bigint') return current.toString();
|
||||
if (typeof current === 'number') {
|
||||
if (!Number.isFinite(current)) throw new ProtocolError('invalid JSON number');
|
||||
return Object.is(current, -0) ? '-0' : String(current);
|
||||
}
|
||||
if (depth === MAX_DEPTH) throw new ProtocolError('wire JSON exceeds depth limit');
|
||||
if (Array.isArray(current)) return `[${current.map(item => encode(item, depth + 1)).join(',')}]`;
|
||||
if (current && typeof current === 'object') {
|
||||
return `{${Object.keys(current).map(key => `${JSON.stringify(key)}:${encode(current[key], depth + 1)}`).join(',')}}`;
|
||||
}
|
||||
throw new ProtocolError('invalid handler response');
|
||||
}
|
||||
return encode(value, 0);
|
||||
}
|
||||
|
||||
function exact(message, names) {
|
||||
const actual = Object.keys(message);
|
||||
if (actual.length !== names.length || names.some(name => !Object.hasOwn(message, name))) {
|
||||
throw new ProtocolError('component message has missing or extra fields');
|
||||
}
|
||||
}
|
||||
|
||||
function requestID(value) {
|
||||
if (!Number.isSafeInteger(value) || value < 0) throw new ProtocolError('invalid request id');
|
||||
return value;
|
||||
}
|
||||
|
||||
function names(values) {
|
||||
return Array.isArray(values) && values.every(value => typeof value === 'string' && value.length) && new Set(values).size === values.length;
|
||||
}
|
||||
|
||||
function typed(value, schema) {
|
||||
switch (schema.type) {
|
||||
case 'nullable': return value === null ? null : typed(value, schema.value);
|
||||
case 'list':
|
||||
if (!Array.isArray(value)) throw new ProtocolError('expected list');
|
||||
return value.map(item => typed(item, schema.value));
|
||||
case 'record': {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).some(name => !Object.hasOwn(schema.fields, name))) throw new ProtocolError('invalid record fields');
|
||||
const result = {};
|
||||
for (const [name, field] of Object.entries(schema.fields)) {
|
||||
if (!Object.hasOwn(value, name) && field.type !== 'nullable') throw new ProtocolError('missing required record field');
|
||||
Object.defineProperty(result, name, { value: Object.hasOwn(value, name) ? typed(value[name], field) : null, enumerable: true });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
case 'int64': case 'duration': case 'model_id': {
|
||||
if ((typeof value !== 'bigint' && !Number.isSafeInteger(value)) || BigInt(value) < -(1n << 63n) || BigInt(value) >= (1n << 63n)) throw new ProtocolError('expected signed 64-bit integer');
|
||||
return value;
|
||||
}
|
||||
case 'float64':
|
||||
if (typeof value !== 'number' || !Number.isFinite(value)) throw new ProtocolError('expected finite float');
|
||||
return value;
|
||||
case 'bool':
|
||||
if (typeof value !== 'boolean') throw new ProtocolError('expected boolean');
|
||||
return value;
|
||||
case 'json': return value;
|
||||
case 'text': case 'id': case 'secret': case 'decimal': case 'uuid': case 'datetime': case 'date': case 'time':
|
||||
if (typeof value !== 'string') throw new ProtocolError('expected text');
|
||||
if (schema.type === 'decimal' && !/^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/.test(value)) throw new ProtocolError('invalid decimal');
|
||||
if (schema.type === 'uuid' && !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)) throw new ProtocolError('invalid UUID');
|
||||
if (schema.type === 'date' && (!/^\d{4}-\d{2}-\d{2}$/.test(value) || Number.isNaN(Date.parse(`${value}T00:00:00Z`)))) throw new ProtocolError('invalid Date');
|
||||
if (schema.type === 'time' && !/^\d{2}:\d{2}:\d{2}(?:\.\d{3})?$/.test(value)) throw new ProtocolError('invalid Time');
|
||||
if (schema.type === 'datetime' && (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|[+-]\d{2}:\d{2})$/.test(value) || Number.isNaN(Date.parse(value)))) throw new ProtocolError('invalid DateTime');
|
||||
return value;
|
||||
default: throw new ProtocolError('unknown worker type');
|
||||
}
|
||||
}
|
||||
|
||||
async function stopHandler(current) {
|
||||
current.controller.abort();
|
||||
let timeout;
|
||||
try {
|
||||
await Promise.race([
|
||||
current.settled,
|
||||
new Promise((_, reject) => {
|
||||
timeout = setTimeout(() => reject(new ProtocolError('component handler did not stop after cancellation')), CANCEL_GRACE_MS);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
class Frames {
|
||||
constructor() {
|
||||
this.buffer = Buffer.alloc(0);
|
||||
this.queue = [];
|
||||
this.queuedBytes = 0;
|
||||
this.waiter = null;
|
||||
this.failed = null;
|
||||
process.stdin.on('data', chunk => this.receive(chunk));
|
||||
process.stdin.on('end', () => this.fail(new ProtocolError('incomplete component frame')));
|
||||
process.stdin.on('error', error => this.fail(error));
|
||||
}
|
||||
|
||||
fail(error) {
|
||||
this.failed = error;
|
||||
if (this.waiter) { this.waiter.reject(error); this.waiter = null; }
|
||||
}
|
||||
|
||||
receive(chunk) {
|
||||
if (this.failed) return;
|
||||
this.buffer = Buffer.concat([this.buffer, chunk]);
|
||||
try {
|
||||
while (this.buffer.length >= 4) {
|
||||
const size = this.buffer.readUInt32BE(0);
|
||||
if (!size || size > MAX_BYTES) throw new ProtocolError('component frame exceeds byte limit');
|
||||
if (this.buffer.length < size + 4) break;
|
||||
const message = parseJSON(decoder.decode(this.buffer.subarray(4, size + 4)));
|
||||
this.buffer = this.buffer.subarray(size + 4);
|
||||
if (this.waiter) { this.waiter.resolve(message); this.waiter = null; }
|
||||
else {
|
||||
if (this.queue.length === 4 || this.queuedBytes + size > MAX_BYTES) throw new ProtocolError('component input queue exceeds limit');
|
||||
this.queue.push({ message, size });
|
||||
this.queuedBytes += size;
|
||||
}
|
||||
}
|
||||
} catch (error) { this.fail(error); }
|
||||
}
|
||||
|
||||
next() {
|
||||
if (this.failed) return Promise.reject(this.failed);
|
||||
if (this.queue.length) {
|
||||
const next = this.queue.shift();
|
||||
this.queuedBytes -= next.size;
|
||||
return Promise.resolve(next.message);
|
||||
}
|
||||
return new Promise((resolve, reject) => { this.waiter = { resolve, reject }; });
|
||||
}
|
||||
|
||||
async send(message) {
|
||||
const body = Buffer.from(encodeJSON(message));
|
||||
if (!body.length || body.length > MAX_BYTES) throw new ProtocolError('component frame exceeds byte limit');
|
||||
parseJSON(decoder.decode(body));
|
||||
const frame = Buffer.allocUnsafe(body.length + 4);
|
||||
frame.writeUInt32BE(body.length);
|
||||
body.copy(frame, 4);
|
||||
await new Promise((resolve, reject) => process.stdout.write(frame, error => error ? reject(error) : resolve()));
|
||||
}
|
||||
}
|
||||
|
||||
export class Worker {
|
||||
constructor(contract, decodeSetting) {
|
||||
if (!contract || ![contract.port, contract.schema, contract.adapter].every(value => typeof value === 'string' && value.length) || !names(contract.operations) || !contract.operations.length || !names(contract.capabilities ?? []) || !names(contract.errors ?? []) || typeof decodeSetting !== 'function') {
|
||||
throw new Error('incomplete component contract');
|
||||
}
|
||||
this.contract = Object.freeze({
|
||||
port: contract.port, schema: contract.schema, adapter: contract.adapter,
|
||||
operations: Object.freeze([...contract.operations]),
|
||||
capabilities: Object.freeze([...(contract.capabilities ?? [])]),
|
||||
errors: Object.freeze([...(contract.errors ?? [])]),
|
||||
});
|
||||
this.decodeSetting = decodeSetting;
|
||||
this.handlers = new Map();
|
||||
this.outputTypes = null;
|
||||
this.errorTypes = null;
|
||||
}
|
||||
|
||||
static fromManifest(manifest, handlers = null) {
|
||||
const operations = manifest.operations;
|
||||
if (!names(operations) || Object.keys(manifest.inputs).sort().join() !== [...operations].sort().join() || Object.keys(manifest.outputs).sort().join() !== [...operations].sort().join()) throw new Error('incomplete typed Worker contract');
|
||||
const worker = new Worker({
|
||||
port: manifest.port, schema: manifest.schema, adapter: manifest.adapter,
|
||||
operations, capabilities: manifest.capabilities, errors: Object.keys(manifest.errors),
|
||||
}, value => {
|
||||
if (manifest.setting !== null) return typed(value, manifest.setting);
|
||||
if (value !== null) throw new ProtocolError('unexpected setting');
|
||||
return null;
|
||||
});
|
||||
worker.inputTypes = manifest.inputs;
|
||||
worker.outputTypes = manifest.outputs;
|
||||
worker.errorTypes = manifest.errors;
|
||||
if (handlers !== null) {
|
||||
if (Object.keys(handlers).sort().join() !== [...operations].sort().join()) throw new Error('component operations are not fully registered');
|
||||
for (const [operation, handler] of Object.entries(handlers)) worker.registerHandler(operation, handler);
|
||||
}
|
||||
return worker;
|
||||
}
|
||||
|
||||
registerHandler(operation, handler) {
|
||||
if (!this.inputTypes || !Object.hasOwn(this.inputTypes, operation)) throw new Error('operation is not in a typed Worker contract');
|
||||
this.register(operation, value => typed(value, this.inputTypes[operation]), handler);
|
||||
}
|
||||
|
||||
register(operation, decodePayload, handler) {
|
||||
if (!this.contract.operations.includes(operation) || this.handlers.has(operation) || typeof decodePayload !== 'function' || typeof handler !== 'function') throw new Error('unknown or duplicate operation');
|
||||
this.handlers.set(operation, { decodePayload, handler });
|
||||
}
|
||||
|
||||
async serve() {
|
||||
const contract = this.contract;
|
||||
if (this.handlers.size !== contract.operations.length) throw new Error('component operations are not fully registered');
|
||||
const frames = new Frames();
|
||||
const hello = await frames.next();
|
||||
exact(hello, ['kind', 'version', 'port', 'schema', 'adapter', 'capabilities', 'operations', 'setting']);
|
||||
if (hello.kind !== 'hello' || hello.version !== VERSION || hello.port !== contract.port || hello.schema !== contract.schema || hello.adapter !== contract.adapter || JSON.stringify(hello.capabilities) !== JSON.stringify(contract.capabilities) || JSON.stringify(hello.operations) !== JSON.stringify(contract.operations)) {
|
||||
throw new ProtocolError('component handshake does not match registered contract');
|
||||
}
|
||||
let setting;
|
||||
try { setting = this.decodeSetting(hello.setting); }
|
||||
catch { throw new ProtocolError('invalid component setting'); }
|
||||
await frames.send({ kind: 'ready', version: VERSION, port: contract.port, schema: contract.schema, adapter: contract.adapter, capabilities: contract.capabilities, operations: contract.operations });
|
||||
|
||||
let active = null;
|
||||
let nextID = 1;
|
||||
let lastCancelled = false;
|
||||
for (;;) {
|
||||
const message = await frames.next();
|
||||
switch (message.kind) {
|
||||
case 'health':
|
||||
exact(message, ['kind', 'id']);
|
||||
if (requestID(message.id) !== 0) throw new ProtocolError('invalid health id');
|
||||
await frames.send(message);
|
||||
break;
|
||||
case 'call': {
|
||||
exact(message, ['kind', 'id', 'operation', 'payload']);
|
||||
const id = requestID(message.id);
|
||||
const operation = this.handlers.get(message.operation);
|
||||
if (id !== nextID++ || active || !operation) throw new ProtocolError('invalid component call');
|
||||
let payload;
|
||||
try { payload = operation.decodePayload(message.payload); }
|
||||
catch { throw new ProtocolError('invalid component payload'); }
|
||||
const current = { id, controller: new AbortController() };
|
||||
active = current;
|
||||
current.settled = Promise.resolve().then(() => operation.handler(payload, setting, current.controller.signal)).then(
|
||||
async result => {
|
||||
if (active !== current) return;
|
||||
active = null;
|
||||
lastCancelled = false;
|
||||
if (this.outputTypes) result = typed(result, this.outputTypes[message.operation]);
|
||||
await frames.send({ kind: 'result', id, payload: result });
|
||||
},
|
||||
async error => {
|
||||
if (active !== current) return;
|
||||
active = null;
|
||||
lastCancelled = false;
|
||||
if (!(error instanceof BusinessError) || !contract.errors.includes(error.identity)) throw new ProtocolError('component handler failed');
|
||||
if (this.errorTypes) error.payload = typed(error.payload, this.errorTypes[error.identity]);
|
||||
await frames.send({ kind: 'error', id, error: error.identity, payload: error.payload });
|
||||
},
|
||||
).catch(error => { frames.fail(error); });
|
||||
break;
|
||||
}
|
||||
case 'cancel':
|
||||
exact(message, ['kind', 'id']);
|
||||
requestID(message.id);
|
||||
if (active?.id === message.id) {
|
||||
const current = active;
|
||||
active = null;
|
||||
await stopHandler(current);
|
||||
lastCancelled = true;
|
||||
await frames.send({ kind: 'error', id: message.id, error: 'dever.cancelled', payload: null });
|
||||
} else if (message.id !== nextID - 1 || lastCancelled) throw new ProtocolError('unknown component cancel id');
|
||||
break;
|
||||
case 'shutdown':
|
||||
exact(message, ['kind']);
|
||||
if (active) {
|
||||
const current = active;
|
||||
active = null;
|
||||
await stopHandler(current);
|
||||
}
|
||||
await frames.send({ kind: 'shutdown' });
|
||||
return;
|
||||
default:
|
||||
throw new ProtocolError('unknown component message');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(worker) {
|
||||
try { await worker.serve(); }
|
||||
catch (error) {
|
||||
console.error(`component worker: ${error instanceof ProtocolError ? error.message : 'internal failure'}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { BusinessError, Worker, main } from './dever_component.mjs';
|
||||
|
||||
function setting(value) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).join() !== 'prefix' || typeof value.prefix !== 'string') throw new Error('invalid setting');
|
||||
return value.prefix;
|
||||
}
|
||||
|
||||
function payload(value) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).sort().join() !== 'delay_ms,text' || typeof value.text !== 'string' || !Number.isInteger(value.delay_ms) || value.delay_ms < 0 || value.delay_ms > 10000) throw new Error('invalid payload');
|
||||
return value;
|
||||
}
|
||||
|
||||
function number(value) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).join() !== 'number' || (typeof value.number !== 'bigint' && !Number.isSafeInteger(value.number))) throw new Error('invalid number');
|
||||
return value.number;
|
||||
}
|
||||
|
||||
const worker = new Worker({ port: 'example.Text', schema: 'fixture-schema-v1', adapter: 'example.TextAdapter', operations: ['text.render', 'text.fail', 'number.echo', 'text.stubborn'], errors: ['example.rejected'] }, setting);
|
||||
worker.register('text.render', payload, async (value, prefix, signal) => {
|
||||
await new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(resolve, value.delay_ms);
|
||||
signal.addEventListener('abort', () => { clearTimeout(timer); reject(new Error('cancelled')); }, { once: true });
|
||||
});
|
||||
return { value: prefix + value.text };
|
||||
});
|
||||
worker.register('text.fail', payload, async () => { throw new BusinessError('example.rejected', { reason: 'rejected' }); });
|
||||
worker.register('number.echo', number, async value => ({ number: value }));
|
||||
worker.register('text.stubborn', payload, async () => new Promise(() => {}));
|
||||
await main(worker);
|
||||
@@ -0,0 +1,182 @@
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
|
||||
|
||||
MAX_BYTES = 256 * 1024 * 1024
|
||||
TARGETS = {
|
||||
"linux-x86_64": ("x86_64", "Advanced Micro Devices X86-64"),
|
||||
"linux-aarch64": ("aarch64", "AArch64"),
|
||||
}
|
||||
OMIT = {"__pycache__", "site-packages", "test", "tests", "tkinter", "turtledemo", "idlelib", "ensurepip"}
|
||||
PROBE = """import ctypes, hashlib, importlib.machinery, json, platform, sqlite3, ssl, sys, zlib
|
||||
print(json.dumps({"version": platform.python_version(), "machine": platform.machine(),
|
||||
"executable": sys.executable, "prefix": sys.prefix,
|
||||
"suffixes": importlib.machinery.EXTENSION_SUFFIXES}))
|
||||
"""
|
||||
|
||||
|
||||
def checksum(path):
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def selected(name, minor):
|
||||
path = PurePosixPath(name)
|
||||
if path == PurePosixPath(f"python/bin/python{minor}"):
|
||||
return True
|
||||
if len(path.parts) == 3 and path.parts[:2] == ("python", "lib"):
|
||||
return bool(re.fullmatch(r"libpython[\d.]+\.so(?:\.\d+)*", path.name))
|
||||
prefix = ("python", "lib", f"python{minor}")
|
||||
return (path.parts[:3] == prefix and not OMIT.intersection(path.parts[3:])
|
||||
and not any(part.startswith("config-") for part in path.parts[3:])
|
||||
and not path.name.startswith("_test") and path.suffix not in (".pyc", ".pyo", ".a"))
|
||||
|
||||
|
||||
def extract(archive, output, minor):
|
||||
total = 0
|
||||
seen = set()
|
||||
with tarfile.open(archive, "r:gz") as source:
|
||||
for member in source:
|
||||
name = member.name.rstrip("/")
|
||||
parts = name.split("/")
|
||||
if (parts[0] != "python" or any(part in ("", ".", "..") for part in parts)
|
||||
or "\\" in name or name in seen or len(seen) >= 65536):
|
||||
raise ValueError("invalid, repeated or excessive archive paths")
|
||||
seen.add(name)
|
||||
total += member.size
|
||||
if total > MAX_BYTES:
|
||||
raise ValueError("Python source exceeds the runtime byte budget")
|
||||
if not member.isfile() or not selected(name, minor):
|
||||
continue
|
||||
destination = output / name
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with source.extractfile(member) as incoming, destination.open("xb") as outgoing:
|
||||
shutil.copyfileobj(incoming, outgoing)
|
||||
destination.chmod(0o755 if name == f"python/bin/python{minor}" else 0o644)
|
||||
|
||||
|
||||
def check_elf(path, readelf, target, baseline):
|
||||
with path.open("rb") as stream:
|
||||
if stream.read(4) != b"\x7fELF":
|
||||
return
|
||||
result = subprocess.run([str(readelf), "--wide", "--file-header", "--version-info", str(path)],
|
||||
check=True, capture_output=True, text=True, timeout=15)
|
||||
if len(result.stdout) > 256 * 1024 or TARGETS[target][1] not in result.stdout:
|
||||
raise ValueError("runtime ELF has the wrong target or excessive metadata")
|
||||
if "Name: GLIBC_PRIVATE" in result.stdout:
|
||||
raise ValueError("runtime ELF depends on private glibc interfaces")
|
||||
versions = [tuple(map(int, value.split(".")))
|
||||
for value in re.findall(r"Name: GLIBC_(\d+(?:\.\d+)+)", result.stdout)]
|
||||
if versions and max(versions) > baseline:
|
||||
raise ValueError(f"{path.name} requires GLIBC_{'.'.join(map(str, max(versions)))}")
|
||||
|
||||
|
||||
def wheel_tags(minor, target, baseline):
|
||||
architecture = TARGETS[target][0]
|
||||
oldest = 5 if architecture == "x86_64" else 17
|
||||
if baseline[0] != 2 or baseline[1] < oldest:
|
||||
raise ValueError("unsupported target glibc baseline")
|
||||
aliases = {17: "manylinux2014", 12: "manylinux2010", 5: "manylinux1"}
|
||||
platforms = []
|
||||
for glibc_minor in range(baseline[1], oldest - 1, -1):
|
||||
platforms.append(f"manylinux_2_{glibc_minor}_{architecture}")
|
||||
if glibc_minor in aliases:
|
||||
platforms.append(f"{aliases[glibc_minor]}_{architecture}")
|
||||
platforms.append(f"linux_{architecture}")
|
||||
python_minor = int(minor.split(".")[1])
|
||||
abi = "cp" + minor.replace(".", "")
|
||||
tags = [f"{abi}-{abi}-{platform}" for platform in platforms]
|
||||
tags += [f"{abi}-abi3-{platform}" for platform in platforms]
|
||||
tags += [f"{abi}-none-{platform}" for platform in platforms]
|
||||
tags += [f"cp3{version}-abi3-{platform}"
|
||||
for version in range(python_minor - 1, 1, -1) for platform in platforms]
|
||||
interpreters = [f"py3{python_minor}", "py3"]
|
||||
interpreters += [f"py3{version}" for version in range(python_minor - 1, -1, -1)]
|
||||
tags += [f"{interpreter}-none-{platform}" for interpreter in interpreters for platform in platforms]
|
||||
tags += [f"{abi}-none-any"]
|
||||
tags += [f"{interpreter}-none-any" for interpreter in interpreters]
|
||||
return tags
|
||||
|
||||
|
||||
def prepare(archive, expected, version, target, baseline, readelf, output, runner=None, sysroot=None):
|
||||
if (not re.fullmatch(r"[0-9a-f]{64}", expected) or archive.stat().st_size > MAX_BYTES
|
||||
or checksum(archive) != expected):
|
||||
raise ValueError("Python archive SHA-256 or size mismatch")
|
||||
if not re.fullmatch(r"3\.\d+\.\d+", version) or target not in TARGETS:
|
||||
raise ValueError("invalid Python version or target")
|
||||
if not readelf.is_absolute() or not readelf.is_file():
|
||||
raise ValueError("readelf must be an explicit absolute executable")
|
||||
if ((runner is None) != (sysroot is None)
|
||||
or runner is not None and (not runner.is_absolute() or not runner.is_file()
|
||||
or not sysroot.is_absolute() or not sysroot.is_dir())):
|
||||
raise ValueError("cross probing requires an explicit QEMU runner and target sysroot")
|
||||
minor = ".".join(version.split(".")[:2])
|
||||
output.mkdir(mode=0o700)
|
||||
try:
|
||||
extract(archive, output, minor)
|
||||
root = output / "python"
|
||||
binary = root / "bin" / f"python{minor}"
|
||||
if not binary.is_file() or not (root / "lib" / f"python{minor}" / "LICENSE.txt").is_file():
|
||||
raise ValueError("Python archive lacks its interpreter or license")
|
||||
for path in root.rglob("*"):
|
||||
if path.is_file():
|
||||
check_elf(path, readelf, target, baseline)
|
||||
environment = {"HOME": str(root), "TMPDIR": str(output), "PATH": "/usr/bin:/bin"}
|
||||
prefix = [str(runner), "-L", str(sysroot)] if runner else []
|
||||
result = subprocess.run(prefix + [str(binary), "-I", "-S", "-B", "-c", PROBE],
|
||||
check=True, capture_output=True, text=True, env=environment, timeout=30)
|
||||
probe = json.loads(result.stdout)
|
||||
if (probe["version"] != version or probe["machine"] != TARGETS[target][0]
|
||||
or Path(probe["prefix"]) != root or Path(probe["executable"]) != binary):
|
||||
raise ValueError("Python runtime is not relocatable or has the wrong identity")
|
||||
tags = wheel_tags(minor, target, baseline)
|
||||
markers = {"implementation_name": "cpython", "implementation_version": version,
|
||||
"os_name": "posix", "platform_machine": TARGETS[target][0],
|
||||
"platform_python_implementation": "CPython", "platform_release": "",
|
||||
"platform_system": "Linux", "platform_version": "", "python_full_version": version,
|
||||
"python_version": minor, "sys_platform": "linux"}
|
||||
descriptor = {"format": "dever-worker-runtime-v1", "ecosystem": "pip", "target": target,
|
||||
"executable": f"bin/python{minor}", "arguments": ["-I", "-S", "-B"],
|
||||
"python_markers": markers, "python_wheel_tags": tags,
|
||||
"python_extension_suffixes": probe["suffixes"]}
|
||||
(root / "dever-runtime.json").write_text(json.dumps(descriptor, indent=2) + "\n")
|
||||
files = [{"source": str(path.relative_to(output)), "path": str(path.relative_to(root)),
|
||||
"sha256": checksum(path)} for path in sorted(root.rglob("*")) if path.is_file()]
|
||||
author = {"name": "cpython", "version": version, "python_markers": markers,
|
||||
"python_wheel_tags": tags, "files": files}
|
||||
(output / "python-runtime.json").write_text(json.dumps(author, indent=2) + "\n")
|
||||
return {"version": version, "target": target, "glibc_max": ".".join(map(str, baseline)),
|
||||
"source_sha256": expected, "files": len(files)}
|
||||
except BaseException:
|
||||
shutil.rmtree(output)
|
||||
raise
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Prepare verified portable Python inputs for native-release")
|
||||
parser.add_argument("--archive", type=Path, required=True)
|
||||
parser.add_argument("--sha256", required=True)
|
||||
parser.add_argument("--version", required=True)
|
||||
parser.add_argument("--target", choices=TARGETS, required=True)
|
||||
parser.add_argument("--glibc-max", default="2.17")
|
||||
parser.add_argument("--readelf", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--runner", type=Path, help="explicit QEMU user-mode executable for cross probing")
|
||||
parser.add_argument("--sysroot", type=Path, help="target system libraries for the explicit QEMU runner")
|
||||
arguments = parser.parse_args()
|
||||
if not re.fullmatch(r"2\.\d+", arguments.glibc_max):
|
||||
parser.error("glibc-max must be a 2.x baseline")
|
||||
print(json.dumps(prepare(arguments.archive.resolve(), arguments.sha256, arguments.version,
|
||||
arguments.target, tuple(map(int, arguments.glibc_max.split("."))),
|
||||
arguments.readelf, arguments.output.absolute(), arguments.runner,
|
||||
arguments.sysroot), indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,404 @@
|
||||
"""Python Worker for the Dever Component Protocol.
|
||||
|
||||
The compiler owns the Port schema. Pass its exact identity to Contract; this
|
||||
module only transports and decodes values supplied by the application.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
import uuid
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
VERSION = "dever-component-1"
|
||||
MAX_BYTES = 16 * 1024 * 1024
|
||||
MAX_DEPTH = 64
|
||||
MAX_ELEMENTS = 65_536
|
||||
CANCEL_GRACE = 1
|
||||
|
||||
|
||||
class ProtocolError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class BusinessError(Exception):
|
||||
def __init__(self, identity: str, payload: Any = None):
|
||||
super().__init__(identity)
|
||||
self.identity = identity
|
||||
self.payload = payload
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Contract:
|
||||
port: str
|
||||
schema: str
|
||||
adapter: str
|
||||
operations: tuple[str, ...]
|
||||
capabilities: tuple[str, ...] = ()
|
||||
errors: tuple[str, ...] = ()
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name in ("operations", "capabilities", "errors"):
|
||||
values = getattr(self, name)
|
||||
if not isinstance(values, (list, tuple)):
|
||||
raise ValueError("component contract entries must be lists")
|
||||
object.__setattr__(self, name, tuple(values))
|
||||
if not all(isinstance(value, str) and value for value in (self.port, self.schema, self.adapter)) or not self.operations:
|
||||
raise ValueError("incomplete component contract")
|
||||
for names in (self.operations, self.capabilities, self.errors):
|
||||
if any(not isinstance(name, str) or not name for name in names) or len(set(names)) != len(names):
|
||||
raise ValueError("invalid or duplicate component contract entry")
|
||||
|
||||
|
||||
def _typed(value: Any, schema: dict[str, Any]) -> Any:
|
||||
kind = schema["type"]
|
||||
if kind == "nullable":
|
||||
return None if value is None else _typed(value, schema["value"])
|
||||
if kind == "list":
|
||||
if not isinstance(value, list):
|
||||
raise ValueError("expected list")
|
||||
return [_typed(item, schema["value"]) for item in value]
|
||||
if kind == "record":
|
||||
fields = schema["fields"]
|
||||
if not isinstance(value, dict) or not value.keys() <= fields.keys():
|
||||
raise ValueError("invalid record fields")
|
||||
return {name: _typed(value[name], field) if name in value else
|
||||
(None if field["type"] == "nullable" else _missing_field())
|
||||
for name, field in fields.items()}
|
||||
if kind in ("int64", "duration", "model_id"):
|
||||
if type(value) is not int or not -(2**63) <= value < 2**63:
|
||||
raise ValueError("expected signed 64-bit integer")
|
||||
elif kind == "float64":
|
||||
if type(value) not in (int, float) or not math.isfinite(value):
|
||||
raise ValueError("expected finite float")
|
||||
elif kind == "bool":
|
||||
if type(value) is not bool:
|
||||
raise ValueError("expected boolean")
|
||||
elif kind == "json":
|
||||
_bounded(value, 0, [MAX_ELEMENTS])
|
||||
elif kind in ("text", "id", "secret", "decimal", "uuid", "datetime", "date", "time"):
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("expected text")
|
||||
if kind == "decimal" and not re.fullmatch(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?", value):
|
||||
raise ValueError("invalid decimal")
|
||||
if kind == "uuid":
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value):
|
||||
raise ValueError("invalid UUID")
|
||||
uuid.UUID(value)
|
||||
if kind == "date":
|
||||
import datetime
|
||||
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
||||
raise ValueError("invalid Date")
|
||||
datetime.date.fromisoformat(value)
|
||||
if kind == "time":
|
||||
import datetime
|
||||
if not re.fullmatch(r"\d{2}:\d{2}:\d{2}(?:\.\d{3})?", value):
|
||||
raise ValueError("invalid Time")
|
||||
datetime.time.fromisoformat(value)
|
||||
if kind == "datetime":
|
||||
import datetime
|
||||
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|[+-]\d{2}:\d{2})", value):
|
||||
raise ValueError("invalid DateTime")
|
||||
datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
else:
|
||||
raise ValueError("unknown worker type")
|
||||
return value
|
||||
|
||||
|
||||
def _missing_field() -> Any:
|
||||
raise ValueError("missing required record field")
|
||||
|
||||
|
||||
def _pairs(values: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result = {}
|
||||
for key, value in values:
|
||||
if key in result:
|
||||
raise ProtocolError("duplicate JSON field")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _reject_constant(_: str) -> None:
|
||||
raise ProtocolError("invalid JSON number")
|
||||
|
||||
|
||||
def _bounded(value: Any, depth: int, budget: list[int]) -> None:
|
||||
if depth > MAX_DEPTH or (depth == MAX_DEPTH and isinstance(value, (dict, list))):
|
||||
raise ProtocolError("wire JSON exceeds depth limit")
|
||||
budget[0] -= 1
|
||||
if budget[0] < 0:
|
||||
raise ProtocolError("wire JSON exceeds element limit")
|
||||
if isinstance(value, dict):
|
||||
for child in value.values():
|
||||
_bounded(child, depth + 1, budget)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
_bounded(child, depth + 1, budget)
|
||||
elif isinstance(value, float) and not math.isfinite(value):
|
||||
raise ProtocolError("non-finite wire Float")
|
||||
|
||||
|
||||
def _decode(body: bytes) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(body.decode("utf-8"), object_pairs_hook=_pairs,
|
||||
parse_constant=_reject_constant)
|
||||
except (UnicodeError, ValueError, RecursionError) as error:
|
||||
raise ProtocolError("invalid component JSON") from error
|
||||
_bounded(value, 0, [MAX_ELEMENTS])
|
||||
if not isinstance(value, dict):
|
||||
raise ProtocolError("component message must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _fields(message: dict[str, Any], names: set[str]) -> None:
|
||||
if message.keys() != names:
|
||||
raise ProtocolError("component message has missing or extra fields")
|
||||
|
||||
|
||||
def _id(value: Any) -> int:
|
||||
if type(value) is not int or not 0 <= value <= 2**63 - 1:
|
||||
raise ProtocolError("invalid request id")
|
||||
return value
|
||||
|
||||
|
||||
async def _read_frame(reader: asyncio.StreamReader) -> dict[str, Any]:
|
||||
try:
|
||||
size = struct.unpack(">I", await reader.readexactly(4))[0]
|
||||
if not 0 < size <= MAX_BYTES:
|
||||
raise ProtocolError("component frame exceeds byte limit")
|
||||
return _decode(await reader.readexactly(size))
|
||||
except asyncio.IncompleteReadError as error:
|
||||
raise ProtocolError("incomplete component frame") from error
|
||||
|
||||
|
||||
def _write_frame(message: dict[str, Any]) -> None:
|
||||
try:
|
||||
body = json.dumps(message, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode("utf-8")
|
||||
except (TypeError, ValueError) as error:
|
||||
raise ProtocolError("invalid handler response") from error
|
||||
_bounded(_decode(body), 0, [MAX_ELEMENTS])
|
||||
if not 0 < len(body) <= MAX_BYTES:
|
||||
raise ProtocolError("component frame exceeds byte limit")
|
||||
sys.__stdout__.buffer.write(struct.pack(">I", len(body)) + body)
|
||||
sys.__stdout__.buffer.flush()
|
||||
|
||||
|
||||
async def _stop_handler(task: asyncio.Task) -> None:
|
||||
task.cancel()
|
||||
completed, _ = await asyncio.wait({task}, timeout=CANCEL_GRACE)
|
||||
if not completed:
|
||||
raise ProtocolError("component handler did not stop after cancellation")
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self, contract: Contract, decode_setting: Callable[[Any], Any]):
|
||||
if not callable(decode_setting):
|
||||
raise ValueError("component setting decoder is required")
|
||||
self.contract = contract
|
||||
self.decode_setting = decode_setting
|
||||
self.handlers: dict[str, tuple[Callable[[Any], Any], Callable[..., Any]]] = {}
|
||||
self.output_types: dict[str, Any] = {}
|
||||
self.error_types: dict[str, Any] = {}
|
||||
|
||||
@classmethod
|
||||
def from_manifest(cls, manifest: dict[str, Any], handlers: dict[str, Callable[..., Any]] | None = None) -> "Worker":
|
||||
operations = tuple(manifest["operations"])
|
||||
inputs = manifest["inputs"]
|
||||
outputs = manifest["outputs"]
|
||||
errors = manifest["errors"]
|
||||
if set(inputs) != set(operations) or set(outputs) != set(operations):
|
||||
raise ValueError("incomplete typed Worker contract")
|
||||
contract = Contract(manifest["port"], manifest["schema"], manifest["adapter"],
|
||||
operations, tuple(manifest["capabilities"]), tuple(errors))
|
||||
setting = manifest["setting"]
|
||||
worker = cls(contract, lambda value: _typed(value, setting) if setting is not None else
|
||||
(None if value is None else _missing_field()))
|
||||
worker.input_types = inputs
|
||||
worker.output_types = outputs
|
||||
worker.error_types = errors
|
||||
if handlers is not None:
|
||||
if set(handlers) != set(operations):
|
||||
raise ValueError("component operations are not fully registered")
|
||||
for operation, handler in handlers.items():
|
||||
worker.register_handler(operation, handler)
|
||||
return worker
|
||||
|
||||
def register_handler(self, operation: str, handler: Callable[..., Any]) -> None:
|
||||
if not hasattr(self, "input_types") or operation not in self.input_types:
|
||||
raise ValueError("operation is not in a typed Worker contract")
|
||||
self.register(operation, lambda value: _typed(value, self.input_types[operation]), handler)
|
||||
|
||||
def register(self, operation: str, decode_payload: Callable[[Any], Any], handler: Callable[..., Any]) -> None:
|
||||
if operation not in self.contract.operations or operation in self.handlers:
|
||||
raise ValueError("unknown or duplicate operation")
|
||||
if not callable(decode_payload):
|
||||
raise ValueError("component payload decoder is required")
|
||||
if not asyncio.iscoroutinefunction(handler):
|
||||
raise ValueError("component handler must be async")
|
||||
self.handlers[operation] = (decode_payload, handler)
|
||||
|
||||
async def serve(self) -> None:
|
||||
if set(self.handlers) != set(self.contract.operations):
|
||||
raise ValueError("component operations are not fully registered")
|
||||
loop = asyncio.get_running_loop()
|
||||
reader = asyncio.StreamReader()
|
||||
transport, _ = await loop.connect_read_pipe(
|
||||
lambda: asyncio.StreamReaderProtocol(reader), sys.stdin.buffer
|
||||
)
|
||||
try:
|
||||
await self._serve(reader)
|
||||
finally:
|
||||
transport.close()
|
||||
|
||||
async def _serve(self, reader: asyncio.StreamReader) -> None:
|
||||
hello = await _read_frame(reader)
|
||||
_fields(hello, {"kind", "version", "port", "schema", "adapter", "capabilities", "operations", "setting"})
|
||||
expected = self.contract
|
||||
if (hello["kind"], hello["version"], hello["port"], hello["schema"], hello["adapter"], hello["capabilities"], hello["operations"]) != ("hello", VERSION, expected.port, expected.schema, expected.adapter, list(expected.capabilities), list(expected.operations)):
|
||||
raise ProtocolError("component handshake does not match registered contract")
|
||||
try:
|
||||
setting = self.decode_setting(hello["setting"])
|
||||
except Exception as error:
|
||||
raise ProtocolError("invalid component setting") from error
|
||||
_write_frame({"kind": "ready", "version": VERSION, "port": expected.port, "schema": expected.schema, "adapter": expected.adapter, "capabilities": list(expected.capabilities), "operations": list(expected.operations)})
|
||||
|
||||
messages: asyncio.Queue = asyncio.Queue(maxsize=4)
|
||||
|
||||
async def read_messages() -> None:
|
||||
while True:
|
||||
try:
|
||||
message = await _read_frame(reader)
|
||||
except (OSError, ProtocolError) as error:
|
||||
await messages.put(("fault", error))
|
||||
return
|
||||
await messages.put(("frame", message))
|
||||
|
||||
receiver = asyncio.create_task(read_messages())
|
||||
try:
|
||||
await self._dispatch(messages, setting)
|
||||
finally:
|
||||
# Cancel and join the pipe reader before Python finalizes stdio.
|
||||
receiver.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await receiver
|
||||
|
||||
async def _dispatch(self, messages: asyncio.Queue, setting: Any) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
active_id: int | None = None
|
||||
active_task: asyncio.Task | None = None
|
||||
active_operation: str | None = None
|
||||
next_id = 1
|
||||
last_cancelled = False
|
||||
completions: set[asyncio.Task] = set()
|
||||
|
||||
def on_done(task: asyncio.Task) -> None:
|
||||
try:
|
||||
task.exception()
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
notification = loop.create_task(messages.put(("done", task)))
|
||||
completions.add(notification)
|
||||
notification.add_done_callback(completions.discard)
|
||||
|
||||
try:
|
||||
while True:
|
||||
event, value = await messages.get()
|
||||
if event == "fault":
|
||||
raise value
|
||||
if event == "done":
|
||||
task = value
|
||||
if task is active_task:
|
||||
try:
|
||||
payload = task.result()
|
||||
if self.output_types:
|
||||
payload = _typed(payload, self.output_types[active_operation])
|
||||
_write_frame({"kind": "result", "id": active_id, "payload": payload})
|
||||
except BusinessError as error:
|
||||
if error.identity not in self.contract.errors:
|
||||
raise ProtocolError("undeclared business error identity") from error
|
||||
if self.error_types:
|
||||
error.payload = _typed(error.payload, self.error_types[error.identity])
|
||||
_write_frame({"kind": "error", "id": active_id, "error": error.identity, "payload": error.payload})
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as error:
|
||||
raise ProtocolError("component handler failed") from error
|
||||
active_id, active_task, active_operation = None, None, None
|
||||
last_cancelled = False
|
||||
continue
|
||||
message = value
|
||||
kind = message.get("kind")
|
||||
if kind == "health":
|
||||
_fields(message, {"kind", "id"})
|
||||
if _id(message["id"]) != 0:
|
||||
raise ProtocolError("invalid health id")
|
||||
_write_frame(message)
|
||||
elif kind == "call":
|
||||
_fields(message, {"kind", "id", "operation", "payload"})
|
||||
request_id = _id(message["id"])
|
||||
operation = message["operation"]
|
||||
if request_id != next_id or active_task is not None or not isinstance(operation, str) or operation not in self.handlers:
|
||||
raise ProtocolError("invalid component call")
|
||||
next_id += 1
|
||||
decoder, handler = self.handlers[operation]
|
||||
try:
|
||||
payload = decoder(message["payload"])
|
||||
except Exception as error:
|
||||
raise ProtocolError("invalid component payload") from error
|
||||
async def invoke() -> Any:
|
||||
return await handler(payload, setting)
|
||||
active_id = request_id
|
||||
active_operation = operation
|
||||
active_task = loop.create_task(invoke())
|
||||
active_task.add_done_callback(on_done)
|
||||
elif kind == "cancel":
|
||||
_fields(message, {"kind", "id"})
|
||||
request_id = _id(message["id"])
|
||||
if request_id == 0:
|
||||
raise ProtocolError("invalid cancel id")
|
||||
if active_id == request_id and active_task is not None:
|
||||
stopped_task = active_task
|
||||
active_id, active_task, active_operation = None, None, None
|
||||
await _stop_handler(stopped_task)
|
||||
last_cancelled = True
|
||||
_write_frame({"kind": "error", "id": request_id, "error": "dever.cancelled", "payload": None})
|
||||
elif request_id != next_id - 1 or last_cancelled:
|
||||
raise ProtocolError("unknown component cancel id")
|
||||
elif kind == "shutdown":
|
||||
_fields(message, {"kind"})
|
||||
if active_task is not None:
|
||||
stopped_task = active_task
|
||||
active_id, active_task = None, None
|
||||
await _stop_handler(stopped_task)
|
||||
_write_frame({"kind": "shutdown"})
|
||||
return
|
||||
else:
|
||||
raise ProtocolError("unknown component message")
|
||||
|
||||
finally:
|
||||
if active_task is not None:
|
||||
active_task.remove_done_callback(on_done)
|
||||
for notification in completions:
|
||||
notification.cancel()
|
||||
await asyncio.gather(*completions, return_exceptions=True)
|
||||
|
||||
|
||||
def main(worker: Worker) -> None:
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
asyncio.set_event_loop(loop)
|
||||
loop.run_until_complete(worker.serve())
|
||||
except (ProtocolError, OSError, ValueError) as error:
|
||||
print(f"component worker: {error}", file=sys.stderr)
|
||||
raise SystemExit(1) from None
|
||||
finally:
|
||||
for task in asyncio.all_tasks(loop):
|
||||
task.cancel()
|
||||
loop.close()
|
||||
asyncio.set_event_loop(None)
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Runnable protocol example with a fixed compiler-owned fixture schema."""
|
||||
|
||||
import asyncio
|
||||
from dever_component import BusinessError, Contract, Worker, main
|
||||
|
||||
|
||||
def setting(value):
|
||||
if not isinstance(value, dict) or set(value) != {"prefix"} or not isinstance(value["prefix"], str):
|
||||
raise ValueError("invalid setting")
|
||||
return value["prefix"]
|
||||
|
||||
|
||||
def payload(value):
|
||||
if not isinstance(value, dict) or set(value) != {"text", "delay_ms"}:
|
||||
raise ValueError("invalid payload")
|
||||
if not isinstance(value["text"], str) or type(value["delay_ms"]) is not int or not 0 <= value["delay_ms"] <= 10_000:
|
||||
raise ValueError("invalid payload")
|
||||
return value
|
||||
|
||||
|
||||
async def render(value, prefix):
|
||||
await asyncio.sleep(value["delay_ms"] / 1000)
|
||||
return {"value": prefix + value["text"]}
|
||||
|
||||
|
||||
async def fail(value, prefix):
|
||||
raise BusinessError("example.rejected", {"reason": "rejected"})
|
||||
|
||||
|
||||
def number(value):
|
||||
if not isinstance(value, dict) or set(value) != {"number"} or type(value["number"]) is not int:
|
||||
raise ValueError("invalid number")
|
||||
return value["number"]
|
||||
|
||||
|
||||
async def echo_number(value, prefix):
|
||||
return {"number": value}
|
||||
|
||||
|
||||
async def stubborn(value, prefix):
|
||||
try:
|
||||
await asyncio.sleep(5)
|
||||
except asyncio.CancelledError:
|
||||
await asyncio.sleep(5)
|
||||
return {"value": prefix + value["text"]}
|
||||
|
||||
|
||||
worker = Worker(Contract("example.Text", "fixture-schema-v1", "example.TextAdapter", ("text.render", "text.fail", "number.echo", "text.stubborn"), errors=("example.rejected",)), setting)
|
||||
worker.register("text.render", payload, render)
|
||||
worker.register("text.fail", payload, fail)
|
||||
worker.register("number.echo", number, echo_number)
|
||||
worker.register("text.stubborn", payload, stubborn)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(worker)
|
||||
Reference in new issue
Block a user