Initial Dever 0.0.1
This commit is contained in:
Codex committed
commit
81233d098c
1479 files changed
+163782
No files matched your search
@@ -0,0 +1,296 @@
|
||||
//! 阶段计时使用预借出的单连接；独立 Rust 仅共享表结构和返回字段，不调用 Dever 运行库。
|
||||
//! 原有生成程序基准另外覆盖连接借还、业务事务和完整分派，不能用这里替代它。
|
||||
|
||||
mod native;
|
||||
mod sql;
|
||||
use super::*;
|
||||
use deadpool_sqlite::rusqlite::Connection;
|
||||
use dever_runtime::{database::Executor, job::Claim, task::Scope};
|
||||
use std::{
|
||||
hint::black_box,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum Phase {
|
||||
Enqueue,
|
||||
Duplicate,
|
||||
Claim,
|
||||
Complete,
|
||||
Cycle,
|
||||
}
|
||||
|
||||
async fn clear(executor: &mut Executor, scope: &Scope<'_>) {
|
||||
executor
|
||||
.execute(scope, "DELETE FROM _dever_job", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn measure_dever(
|
||||
executor: &mut Executor,
|
||||
scope: &Scope<'_>,
|
||||
phase: Phase,
|
||||
keys: &[String],
|
||||
payload: &Json,
|
||||
) -> (u128, u64) {
|
||||
clear(executor, scope).await;
|
||||
if matches!(phase, Phase::Duplicate | Phase::Claim | Phase::Complete) {
|
||||
for key in keys {
|
||||
let mut value = submission(key);
|
||||
value.payload = Some(payload);
|
||||
job::enqueue(executor, scope, value, 1000).await.unwrap();
|
||||
}
|
||||
}
|
||||
let mut claims = Vec::new();
|
||||
if matches!(phase, Phase::Complete) {
|
||||
for _ in keys {
|
||||
claims.push(
|
||||
job::claim(executor, scope, 1000, 60000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
}
|
||||
let start = Instant::now();
|
||||
let mut checksum = 0;
|
||||
for (index, key) in keys.iter().enumerate() {
|
||||
match phase {
|
||||
Phase::Enqueue | Phase::Duplicate => {
|
||||
let mut value = submission(key);
|
||||
value.payload = Some(payload);
|
||||
checksum += black_box(
|
||||
job::enqueue(executor, scope, value, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_text(),
|
||||
)
|
||||
.len() as u64;
|
||||
}
|
||||
Phase::Claim => {
|
||||
let claim = job::claim(executor, scope, 1000, 60000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
checksum += black_box(claim).id.len() as u64;
|
||||
}
|
||||
Phase::Complete => {
|
||||
job::complete(executor, scope, &claims[index], 1000)
|
||||
.await
|
||||
.unwrap();
|
||||
checksum += 1;
|
||||
}
|
||||
Phase::Cycle => {
|
||||
let mut value = submission(key);
|
||||
value.payload = Some(payload);
|
||||
let id = job::enqueue(executor, scope, value, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_text();
|
||||
let claim = job::claim(executor, scope, 1000, 60000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(id, claim.id);
|
||||
job::complete(executor, scope, &claim, 1000).await.unwrap();
|
||||
checksum += black_box(claim).id.len() as u64;
|
||||
}
|
||||
}
|
||||
}
|
||||
let elapsed = start.elapsed().as_nanos();
|
||||
let state = state(phase);
|
||||
let count = executor
|
||||
.query(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state=?1",
|
||||
vec![Parameter::Text(state.into())],
|
||||
1,
|
||||
|row| row.get::<_, i64>(0),
|
||||
)
|
||||
.await
|
||||
.unwrap()[0];
|
||||
assert_eq!(count, keys.len() as i64);
|
||||
(elapsed, checksum)
|
||||
}
|
||||
|
||||
fn state(phase: Phase) -> &'static str {
|
||||
match phase {
|
||||
Phase::Enqueue | Phase::Duplicate => "pending",
|
||||
Phase::Claim => "running",
|
||||
Phase::Complete | Phase::Cycle => "completed",
|
||||
}
|
||||
}
|
||||
|
||||
async fn measure_native(
|
||||
native: &mut native::Native,
|
||||
phase: Phase,
|
||||
keys: &[String],
|
||||
payload: &str,
|
||||
) -> (u128, u64) {
|
||||
native
|
||||
.call(|connection, _| connection.execute("DELETE FROM _dever_job", []))
|
||||
.await
|
||||
.unwrap();
|
||||
if matches!(phase, Phase::Duplicate | Phase::Claim | Phase::Complete) {
|
||||
for key in keys {
|
||||
native.enqueue(key, payload).await;
|
||||
}
|
||||
}
|
||||
let mut claims: Vec<Claim> = Vec::new();
|
||||
if matches!(phase, Phase::Complete) {
|
||||
for _ in keys {
|
||||
claims.push(native.claim().await.unwrap());
|
||||
}
|
||||
}
|
||||
let start = Instant::now();
|
||||
let mut checksum = 0;
|
||||
for (index, key) in keys.iter().enumerate() {
|
||||
match phase {
|
||||
Phase::Enqueue | Phase::Duplicate => {
|
||||
checksum += black_box(native.enqueue(key, payload).await).len() as u64
|
||||
}
|
||||
Phase::Claim => checksum += black_box(native.claim().await.unwrap()).id.len() as u64,
|
||||
Phase::Complete => {
|
||||
native.complete(&claims[index]).await;
|
||||
checksum += 1;
|
||||
}
|
||||
Phase::Cycle => {
|
||||
let id = native.enqueue(key, payload).await;
|
||||
let claim = native.claim().await.unwrap();
|
||||
assert_eq!(id, claim.id);
|
||||
native.complete(&claim).await;
|
||||
checksum += black_box(claim).id.len() as u64;
|
||||
}
|
||||
}
|
||||
}
|
||||
let elapsed = start.elapsed().as_nanos();
|
||||
let count: i64 = native
|
||||
.call(move |connection, _| {
|
||||
connection.query_row(
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state=?1",
|
||||
[state(phase)],
|
||||
|row| row.get(0),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(count, keys.len() as i64);
|
||||
(elapsed, checksum)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "release-only owned SQLite phase timing; invoke explicitly"]
|
||||
fn independent_sqlite_phases() {
|
||||
let directory = Directory::new();
|
||||
let native_runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.worker_threads(1)
|
||||
.max_blocking_threads(1)
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap();
|
||||
let connection = Connection::open(directory.0.join("native.db")).unwrap();
|
||||
connection.busy_timeout(Duration::from_secs(10)).unwrap();
|
||||
connection
|
||||
.execute_batch(
|
||||
"PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON; PRAGMA synchronous=NORMAL;",
|
||||
)
|
||||
.unwrap();
|
||||
connection.execute_batch(job::schema::SQL.sqlite).unwrap();
|
||||
let mut native = native::Native::new(connection);
|
||||
let runtime = Runtime::new(Limits {
|
||||
threads: 1,
|
||||
workers: 1,
|
||||
tasks: 16,
|
||||
})
|
||||
.unwrap();
|
||||
let mut options = options();
|
||||
options.max_connections = 1;
|
||||
options.io_timeout_ms = 10000;
|
||||
let database = runtime
|
||||
.run(async |scope| Database::open(&directory.0, options, scope).await)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let mut executor = runtime
|
||||
.run(async |scope| {
|
||||
let mut executor = database.executor().await.unwrap();
|
||||
executor
|
||||
.schema(scope, job::schema::TABLE, job::schema::SQL)
|
||||
.await
|
||||
.unwrap();
|
||||
executor
|
||||
})
|
||||
.unwrap();
|
||||
println!("case,sample,implementation,iterations,nanoseconds,checksum");
|
||||
for (name, phase, count, payload) in [
|
||||
(
|
||||
"enqueue",
|
||||
Phase::Enqueue,
|
||||
1000,
|
||||
Json::parse("{\"number\":1}").unwrap(),
|
||||
),
|
||||
(
|
||||
"enqueue_64k",
|
||||
Phase::Enqueue,
|
||||
64,
|
||||
Json::parse(&format!("\"{}\"", "x".repeat(65536))).unwrap(),
|
||||
),
|
||||
(
|
||||
"duplicate",
|
||||
Phase::Duplicate,
|
||||
1000,
|
||||
Json::parse("{\"number\":1}").unwrap(),
|
||||
),
|
||||
(
|
||||
"claim",
|
||||
Phase::Claim,
|
||||
256,
|
||||
Json::parse("{\"number\":1}").unwrap(),
|
||||
),
|
||||
(
|
||||
"complete",
|
||||
Phase::Complete,
|
||||
256,
|
||||
Json::parse("{\"number\":1}").unwrap(),
|
||||
),
|
||||
(
|
||||
"cycle",
|
||||
Phase::Cycle,
|
||||
256,
|
||||
Json::parse("{\"number\":1}").unwrap(),
|
||||
),
|
||||
] {
|
||||
let keys: Vec<_> = (0..count).map(|index| format!("key-{index:04}")).collect();
|
||||
for sample in 0..16 {
|
||||
let mut order = [("dever", true), ("rust", false), ("control", true)];
|
||||
order.rotate_left(sample % 3);
|
||||
let mut expected = None;
|
||||
for (label, generated) in order {
|
||||
let (elapsed, checksum) = if generated {
|
||||
runtime
|
||||
.run(async |scope| {
|
||||
measure_dever(&mut executor, scope, phase, &keys, &payload).await
|
||||
})
|
||||
.unwrap()
|
||||
} else {
|
||||
native_runtime.block_on(measure_native(
|
||||
&mut native,
|
||||
phase,
|
||||
&keys,
|
||||
payload.as_str(),
|
||||
))
|
||||
};
|
||||
if let Some(expected) = expected {
|
||||
assert_eq!(checksum, expected);
|
||||
}
|
||||
expected = Some(checksum);
|
||||
if sample > 0 {
|
||||
println!("{name},{},{label},{count},{elapsed},{checksum}", sample - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
native_runtime.block_on(native.close());
|
||||
drop(executor);
|
||||
database.close();
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
//! 独立 Rust 对照仅使用 Tokio/rusqlite；连接移交、取消、期限和退出等待不调用 Dever。
|
||||
|
||||
use deadpool_sqlite::rusqlite::{Connection, Result};
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio_util::task::TaskTracker;
|
||||
|
||||
const TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
struct Cancel(Arc<AtomicBool>);
|
||||
|
||||
impl Drop for Cancel {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct Native {
|
||||
connection: Option<Connection>,
|
||||
workers: Arc<Semaphore>,
|
||||
tracker: TaskTracker,
|
||||
}
|
||||
|
||||
impl Native {
|
||||
pub(super) fn new(connection: Connection) -> Self {
|
||||
Self {
|
||||
connection: Some(connection),
|
||||
workers: Arc::new(Semaphore::new(1)),
|
||||
tracker: TaskTracker::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn call<Value: Send + 'static>(
|
||||
&mut self,
|
||||
operation: impl FnOnce(&mut Connection, &dyn Fn() -> Result<()>) -> Result<Value>
|
||||
+ Send
|
||||
+ 'static,
|
||||
) -> Result<Value> {
|
||||
let mut connection = self.connection.take().unwrap();
|
||||
let cancelled = Cancel(Arc::new(AtomicBool::new(false)));
|
||||
let signal = cancelled.0.clone();
|
||||
let deadline = Instant::now() + TIMEOUT;
|
||||
let workers = self.workers.clone();
|
||||
let tracker = self.tracker.clone();
|
||||
let work = async move {
|
||||
let worker = workers.acquire_owned().await.unwrap();
|
||||
let tracking = tracker.token();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _worker = worker;
|
||||
let check = || {
|
||||
if signal.load(Ordering::Acquire) || Instant::now() >= deadline {
|
||||
Err(deadpool_sqlite::rusqlite::Error::InvalidQuery)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
let outcome = (|| {
|
||||
check()?;
|
||||
let progress = signal.clone();
|
||||
connection.progress_handler(
|
||||
1000,
|
||||
Some(move || {
|
||||
progress.load(Ordering::Acquire) || Instant::now() >= deadline
|
||||
}),
|
||||
)?;
|
||||
let result = operation(&mut connection, &check);
|
||||
connection.progress_handler(0, None::<fn() -> bool>)?;
|
||||
check()?;
|
||||
result
|
||||
})();
|
||||
(connection, outcome, tracking)
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
};
|
||||
let (connection, result, _tracking) = tokio::time::timeout(TIMEOUT, work).await.unwrap();
|
||||
self.connection = Some(connection);
|
||||
result
|
||||
}
|
||||
|
||||
pub(super) async fn transaction<Value: Send + 'static>(
|
||||
&mut self,
|
||||
operation: impl FnOnce(&Connection) -> Result<Value> + Send + 'static,
|
||||
) -> Result<Value> {
|
||||
self.call(move |connection, check| {
|
||||
let transaction = connection.transaction_with_behavior(
|
||||
deadpool_sqlite::rusqlite::TransactionBehavior::Immediate,
|
||||
)?;
|
||||
check()?;
|
||||
let result = operation(&transaction)?;
|
||||
check()?;
|
||||
transaction.commit()?;
|
||||
Ok(result)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn close(self) {
|
||||
self.tracker.close();
|
||||
self.tracker.wait().await;
|
||||
drop(self.connection);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
//! 同一表合同的独立 SQL 实现；不复用生产队列函数，也不新增生产解析或调度层。
|
||||
|
||||
use super::native::Native;
|
||||
use deadpool_sqlite::rusqlite::{OptionalExtension, params};
|
||||
use dever_runtime::job::{Claim, Policy};
|
||||
use std::{fmt::Write, time::Instant};
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct Execution {
|
||||
kind: &'static str,
|
||||
tenant: Option<i64>,
|
||||
}
|
||||
|
||||
pub(super) fn identifier() -> String {
|
||||
let mut bytes = [0u8; 16];
|
||||
getrandom::getrandom(&mut bytes).unwrap();
|
||||
let mut value = String::with_capacity(32);
|
||||
write!(value, "{:032x}", u128::from_be_bytes(bytes)).unwrap();
|
||||
value
|
||||
}
|
||||
|
||||
impl Native {
|
||||
pub(super) async fn enqueue(&mut self, key: &str, payload: &str) -> String {
|
||||
assert!(!key.is_empty() && key.len() <= 1024);
|
||||
let id = identifier();
|
||||
let (key, payload, target, schema) = (
|
||||
key.to_owned(),
|
||||
payload.to_owned(),
|
||||
"demo.account.publish".to_owned(),
|
||||
"schema-1".to_owned(),
|
||||
);
|
||||
let execution = serde_json::to_string(&Execution {
|
||||
kind: "System",
|
||||
tenant: None,
|
||||
})
|
||||
.unwrap();
|
||||
self.transaction(move |connection| {
|
||||
let changed = connection.prepare_cached("INSERT INTO _dever_job(id,target,payload,schema,execution,task_key,state,retries,timeout_ms,available_at,created_at,execution_tenant) VALUES (?1,?2,?3,?4,?5,?6,'pending',?7,?8,?9,?10,?11) ON CONFLICT(execution_tenant,target,task_key) WHERE state IN ('pending','running') DO NOTHING")?.execute(params![id,target,payload,schema,execution,key,2,50,1000,1000,0])?;
|
||||
if changed == 1 {
|
||||
Ok(id)
|
||||
} else {
|
||||
connection.prepare_cached("SELECT id FROM _dever_job WHERE execution_tenant=?1 AND target=?2 AND task_key=?3 AND state IN ('pending','running')")?.query_row(params![0,target,key], |row| row.get(0))
|
||||
}
|
||||
}).await.unwrap()
|
||||
}
|
||||
|
||||
pub(super) async fn claim(&mut self) -> Option<Claim> {
|
||||
let started = Instant::now();
|
||||
let token = identifier();
|
||||
self.transaction(move |connection| {
|
||||
let now = 1000i64.checked_add(started.elapsed().as_millis().try_into().unwrap()).unwrap();
|
||||
let lease_until = now.checked_add(60000).unwrap();
|
||||
connection.prepare_cached("UPDATE _dever_job SET state=CASE WHEN attempt >= retries THEN 'dead' ELSE 'pending' END, token=NULL,lease_until=NULL,error='lease expired' WHERE id IN (SELECT id FROM _dever_job WHERE state='running' AND lease_until <= ?1 ORDER BY lease_until,id LIMIT 256)")?.execute([now])?;
|
||||
let value = connection.prepare_cached(
|
||||
"SELECT id,target,substr(payload,1,16777217),schema,substr(execution,1,16777217),retries,timeout_ms,attempt,execution_tenant FROM _dever_job
|
||||
WHERE id=(SELECT id FROM (
|
||||
SELECT * FROM (SELECT id,available_at FROM _dever_job WHERE state='pending' AND attempt < retries AND available_at <= ?1 ORDER BY available_at,id LIMIT 1)
|
||||
UNION ALL
|
||||
SELECT * FROM (SELECT id,available_at FROM _dever_job WHERE state='running' AND attempt < retries AND lease_until <= ?1 AND available_at <= ?1 ORDER BY available_at,id LIMIT 1)
|
||||
) ORDER BY available_at,id LIMIT 1)"
|
||||
)?.query_row([now], |row| Ok(Claim {
|
||||
id: row.get(0)?, target: row.get(1)?, payload: row.get(2)?, schema: row.get(3)?, execution: row.get(4)?,
|
||||
policy: Policy { retries: row.get(5)?, timeout_ms: row.get(6)? }, attempt: row.get::<_,i64>(7)? + 1,
|
||||
token, lease_until, execution_tenant: row.get(8)?,
|
||||
})).optional()?;
|
||||
if let Some(value) = &value {
|
||||
connection.prepare_cached("UPDATE _dever_job SET state='running',attempt=?1,token=?2,lease_until=?3,error=CASE WHEN state='running' THEN 'lease expired' ELSE error END WHERE id=?4")?.execute(params![value.attempt,value.token,lease_until,value.id])?;
|
||||
}
|
||||
Ok(value)
|
||||
}).await.unwrap()
|
||||
}
|
||||
|
||||
pub(super) async fn complete(&mut self, claim: &Claim) {
|
||||
let (id, token) = (claim.id.clone(), claim.token.clone());
|
||||
let started = Instant::now();
|
||||
self.transaction(move |connection| {
|
||||
let now = 1000i64.checked_add(started.elapsed().as_millis().try_into().unwrap()).unwrap();
|
||||
let changed = connection.prepare_cached("UPDATE _dever_job SET state=?1,available_at=?2,error=?3,token=NULL,lease_until=NULL WHERE id=?4 AND state='running' AND token=?5 AND lease_until > ?6")?.execute(params!["completed",now,None::<&str>,id,token,now])?;
|
||||
assert_eq!(changed, 1);
|
||||
Ok(())
|
||||
}).await.unwrap();
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user