316 lines
9.7 KiB
Rust
316 lines
9.7 KiB
Rust
use dever_runtime::{
|
|
database::{Database, Options, Parameter},
|
|
job::{self, Execution, Policy, Submission},
|
|
task::{Limits, Runtime},
|
|
wire::json::Json,
|
|
};
|
|
mod cleanup;
|
|
mod identity;
|
|
mod lease;
|
|
mod performance;
|
|
mod schedule;
|
|
use std::{
|
|
path::PathBuf,
|
|
sync::{
|
|
LazyLock,
|
|
atomic::{AtomicU64, Ordering},
|
|
},
|
|
};
|
|
|
|
static NEXT: AtomicU64 = AtomicU64::new(0);
|
|
struct Directory(PathBuf);
|
|
impl Directory {
|
|
fn new() -> Self {
|
|
let path = std::env::temp_dir().join(format!(
|
|
"dever-job-{}-{}",
|
|
std::process::id(),
|
|
NEXT.fetch_add(1, Ordering::Relaxed)
|
|
));
|
|
std::fs::create_dir(&path).unwrap();
|
|
Self(path)
|
|
}
|
|
}
|
|
impl Drop for Directory {
|
|
fn drop(&mut self) {
|
|
std::fs::remove_dir_all(&self.0).unwrap();
|
|
}
|
|
}
|
|
|
|
fn options() -> Options {
|
|
Options {
|
|
driver: "sqlite".into(),
|
|
path: "queue.db".into(),
|
|
url: None,
|
|
tls: None,
|
|
tenant_directory: None,
|
|
tenant_database_prefix: None,
|
|
min_connections: 1,
|
|
max_connections: 4,
|
|
max_page_size: 100,
|
|
wait_timeout_ms: 1000,
|
|
io_timeout_ms: 1000,
|
|
}
|
|
}
|
|
|
|
fn submission(key: &str) -> Submission<'_> {
|
|
static PAYLOAD: LazyLock<Json> = LazyLock::new(|| Json::parse(r#"{"number":1}"#).unwrap());
|
|
Submission {
|
|
target: "demo.account.publish",
|
|
payload: Some(&PAYLOAD),
|
|
schema: "schema-1",
|
|
execution: &Execution::System { tenant: None },
|
|
key,
|
|
policy: Policy {
|
|
retries: 2,
|
|
timeout_ms: 50,
|
|
},
|
|
at: 1000,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn durable_queue_keeps_idempotency_transactions_and_rejects_stale_leases() {
|
|
let directory = Directory::new();
|
|
let runtime = Runtime::new(Limits::default()).unwrap();
|
|
runtime
|
|
.run(async |scope| {
|
|
let database = Database::open(&directory.0, options(), scope)
|
|
.await
|
|
.unwrap();
|
|
let mut executor = database.executor().await.unwrap();
|
|
executor
|
|
.schema(scope, job::schema::TABLE, job::schema::SQL)
|
|
.await
|
|
.unwrap();
|
|
let id = job::enqueue(&mut executor, scope, submission("one"), 1000)
|
|
.await
|
|
.unwrap()
|
|
.into_text();
|
|
assert_eq!(
|
|
job::enqueue(&mut executor, scope, submission("one"), 1000)
|
|
.await
|
|
.unwrap()
|
|
.into_text(),
|
|
id
|
|
);
|
|
assert!(
|
|
job::claim(&mut executor, scope, 950, 100)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
let first = job::claim(&mut executor, scope, 1000, 100)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(first.id, id);
|
|
assert_eq!(first.attempt, 1);
|
|
assert!(
|
|
job::claim(&mut executor, scope, first.lease_until - 50, 100)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
let second = job::claim(&mut executor, scope, first.lease_until, 100)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(second.id, id);
|
|
assert_eq!(second.attempt, 2);
|
|
assert_ne!(first.token, second.token);
|
|
assert!(
|
|
job::complete(&mut executor, scope, &first, 1101)
|
|
.await
|
|
.is_err()
|
|
);
|
|
job::complete(&mut executor, scope, &second, 1101)
|
|
.await
|
|
.unwrap();
|
|
assert!(
|
|
job::complete(&mut executor, scope, &second, 1102)
|
|
.await
|
|
.is_err()
|
|
);
|
|
let next = job::enqueue(&mut executor, scope, submission("one"), 1102)
|
|
.await
|
|
.unwrap();
|
|
assert_ne!(next.into_text(), id);
|
|
let owner = executor.begin(scope).await.unwrap();
|
|
job::enqueue(&mut executor, scope, submission("rolled back"), 1102)
|
|
.await
|
|
.unwrap();
|
|
executor.finish(scope, owner, false).await.unwrap();
|
|
assert_eq!(
|
|
executor
|
|
.query(
|
|
scope,
|
|
"SELECT COUNT(*) FROM _dever_job WHERE task_key=?1",
|
|
vec![Parameter::Text("rolled back".into())],
|
|
1,
|
|
|row| row.get::<_, i64>(0)
|
|
)
|
|
.await
|
|
.unwrap(),
|
|
vec![0]
|
|
);
|
|
drop(executor);
|
|
database.close();
|
|
let database = Database::open(&directory.0, options(), scope)
|
|
.await
|
|
.unwrap();
|
|
let mut executor = database.executor().await.unwrap();
|
|
let persisted = job::claim(&mut executor, scope, 1103, 100)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_ne!(persisted.id, id);
|
|
assert!(matches!(
|
|
dever_runtime::wire::decode::<Execution>(&persisted.execution).unwrap(),
|
|
Execution::System { tenant: None }
|
|
));
|
|
job::block(&mut executor, scope, &persisted, 1104)
|
|
.await
|
|
.unwrap();
|
|
assert!(
|
|
job::claim(&mut executor, scope, 1204, 100)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
})
|
|
.unwrap();
|
|
}
|
|
|
|
#[test]
|
|
fn retry_backoff_and_exhaustion_survive_a_lost_worker() {
|
|
let directory = Directory::new();
|
|
let runtime = Runtime::new(Limits::default()).unwrap();
|
|
runtime
|
|
.run(async |scope| {
|
|
let database = Database::open(&directory.0, options(), scope)
|
|
.await
|
|
.unwrap();
|
|
let mut executor = database.executor().await.unwrap();
|
|
executor
|
|
.schema(scope, job::schema::TABLE, job::schema::SQL)
|
|
.await
|
|
.unwrap();
|
|
job::enqueue(&mut executor, scope, submission("retry"), 1000)
|
|
.await
|
|
.unwrap();
|
|
let first = job::claim(&mut executor, scope, 1000, 100)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
let config = job::Config {
|
|
retry_base_ms: 10,
|
|
retry_max_ms: 40,
|
|
..Default::default()
|
|
};
|
|
job::fail(&mut executor, scope, &first, 1001, &config)
|
|
.await
|
|
.unwrap();
|
|
let available = executor
|
|
.query(
|
|
scope,
|
|
"SELECT available_at FROM _dever_job",
|
|
vec![],
|
|
1,
|
|
|row| row.get::<_, i64>(0),
|
|
)
|
|
.await
|
|
.unwrap()[0];
|
|
assert!(
|
|
job::claim(&mut executor, scope, available - 50, 100)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
let last = job::claim(&mut executor, scope, available, 100)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(last.attempt, 2);
|
|
assert!(
|
|
job::claim(&mut executor, scope, last.lease_until, 100)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
assert_eq!(
|
|
executor
|
|
.query(scope, "SELECT state FROM _dever_job", vec![], 1, |row| {
|
|
row.get::<_, String>(0)
|
|
})
|
|
.await
|
|
.unwrap(),
|
|
vec!["dead"]
|
|
);
|
|
assert!(
|
|
job::complete(&mut executor, scope, &last, 1112)
|
|
.await
|
|
.is_err()
|
|
);
|
|
for (attempt, delay) in [(1, 10), (2, 20), (3, 40), (100, 40)] {
|
|
assert_eq!(config.delay(attempt), delay);
|
|
}
|
|
assert!(
|
|
config
|
|
.validate(
|
|
&[Policy {
|
|
retries: 2,
|
|
timeout_ms: 60000
|
|
}],
|
|
4,
|
|
true,
|
|
)
|
|
.is_err()
|
|
);
|
|
assert!(
|
|
config
|
|
.validate(
|
|
&[Policy {
|
|
retries: 2,
|
|
timeout_ms: 50
|
|
}],
|
|
2,
|
|
true,
|
|
)
|
|
.is_err()
|
|
);
|
|
config
|
|
.validate(
|
|
&[Policy {
|
|
retries: 2,
|
|
timeout_ms: 50,
|
|
}],
|
|
4,
|
|
true,
|
|
)
|
|
.unwrap();
|
|
config
|
|
.validate(
|
|
&[Policy {
|
|
retries: 2,
|
|
timeout_ms: 60000,
|
|
}],
|
|
4,
|
|
false,
|
|
)
|
|
.unwrap();
|
|
assert!(
|
|
config
|
|
.validate(
|
|
&[Policy {
|
|
retries: 0,
|
|
timeout_ms: 60000
|
|
}],
|
|
4,
|
|
false
|
|
)
|
|
.is_err()
|
|
);
|
|
})
|
|
.unwrap();
|
|
}
|