use dever_runtime::{ database::{Database, Options, Parameter}, job::{self, Execution, Policy, Submission}, task::{Limits, Runtime}, wire::json::Json, }; mod cleanup; mod identity; mod lease; mod performance; mod schedule; use std::{ path::PathBuf, sync::{ LazyLock, atomic::{AtomicU64, Ordering}, }, }; static NEXT: AtomicU64 = AtomicU64::new(0); struct Directory(PathBuf); impl Directory { fn new() -> Self { let path = std::env::temp_dir().join(format!( "dever-job-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed) )); std::fs::create_dir(&path).unwrap(); Self(path) } } impl Drop for Directory { fn drop(&mut self) { std::fs::remove_dir_all(&self.0).unwrap(); } } fn options() -> Options { Options { driver: "sqlite".into(), path: "queue.db".into(), url: None, tls: None, tenant_directory: None, tenant_database_prefix: None, min_connections: 1, max_connections: 4, max_page_size: 100, wait_timeout_ms: 1000, io_timeout_ms: 1000, } } fn submission(key: &str) -> Submission<'_> { static PAYLOAD: LazyLock = LazyLock::new(|| Json::parse(r#"{"number":1}"#).unwrap()); Submission { target: "demo.account.publish", payload: Some(&PAYLOAD), schema: "schema-1", execution: &Execution::System { tenant: None }, key, policy: Policy { retries: 2, timeout_ms: 50, }, at: 1000, } } #[test] fn durable_queue_keeps_idempotency_transactions_and_rejects_stale_leases() { let directory = Directory::new(); let runtime = Runtime::new(Limits::default()).unwrap(); runtime .run(async |scope| { let database = Database::open(&directory.0, options(), scope) .await .unwrap(); let mut executor = database.executor().await.unwrap(); executor .schema(scope, job::schema::TABLE, job::schema::SQL) .await .unwrap(); let id = job::enqueue(&mut executor, scope, submission("one"), 1000) .await .unwrap() .into_text(); assert_eq!( job::enqueue(&mut executor, scope, submission("one"), 1000) .await .unwrap() .into_text(), id ); assert!( job::claim(&mut executor, scope, 950, 100) .await .unwrap() .is_none() ); let first = job::claim(&mut executor, scope, 1000, 100) .await .unwrap() .unwrap(); assert_eq!(first.id, id); assert_eq!(first.attempt, 1); assert!( job::claim(&mut executor, scope, first.lease_until - 50, 100) .await .unwrap() .is_none() ); let second = job::claim(&mut executor, scope, first.lease_until, 100) .await .unwrap() .unwrap(); assert_eq!(second.id, id); assert_eq!(second.attempt, 2); assert_ne!(first.token, second.token); assert!( job::complete(&mut executor, scope, &first, 1101) .await .is_err() ); job::complete(&mut executor, scope, &second, 1101) .await .unwrap(); assert!( job::complete(&mut executor, scope, &second, 1102) .await .is_err() ); let next = job::enqueue(&mut executor, scope, submission("one"), 1102) .await .unwrap(); assert_ne!(next.into_text(), id); let owner = executor.begin(scope).await.unwrap(); job::enqueue(&mut executor, scope, submission("rolled back"), 1102) .await .unwrap(); executor.finish(scope, owner, false).await.unwrap(); assert_eq!( executor .query( scope, "SELECT COUNT(*) FROM _dever_job WHERE task_key=?1", vec![Parameter::Text("rolled back".into())], 1, |row| row.get::<_, i64>(0) ) .await .unwrap(), vec![0] ); drop(executor); database.close(); let database = Database::open(&directory.0, options(), scope) .await .unwrap(); let mut executor = database.executor().await.unwrap(); let persisted = job::claim(&mut executor, scope, 1103, 100) .await .unwrap() .unwrap(); assert_ne!(persisted.id, id); assert!(matches!( dever_runtime::wire::decode::(&persisted.execution).unwrap(), Execution::System { tenant: None } )); job::block(&mut executor, scope, &persisted, 1104) .await .unwrap(); assert!( job::claim(&mut executor, scope, 1204, 100) .await .unwrap() .is_none() ); }) .unwrap(); } #[test] fn retry_backoff_and_exhaustion_survive_a_lost_worker() { let directory = Directory::new(); let runtime = Runtime::new(Limits::default()).unwrap(); runtime .run(async |scope| { let database = Database::open(&directory.0, options(), scope) .await .unwrap(); let mut executor = database.executor().await.unwrap(); executor .schema(scope, job::schema::TABLE, job::schema::SQL) .await .unwrap(); job::enqueue(&mut executor, scope, submission("retry"), 1000) .await .unwrap(); let first = job::claim(&mut executor, scope, 1000, 100) .await .unwrap() .unwrap(); let config = job::Config { retry_base_ms: 10, retry_max_ms: 40, ..Default::default() }; job::fail(&mut executor, scope, &first, 1001, &config) .await .unwrap(); let available = executor .query( scope, "SELECT available_at FROM _dever_job", vec![], 1, |row| row.get::<_, i64>(0), ) .await .unwrap()[0]; assert!( job::claim(&mut executor, scope, available - 50, 100) .await .unwrap() .is_none() ); let last = job::claim(&mut executor, scope, available, 100) .await .unwrap() .unwrap(); assert_eq!(last.attempt, 2); assert!( job::claim(&mut executor, scope, last.lease_until, 100) .await .unwrap() .is_none() ); assert_eq!( executor .query(scope, "SELECT state FROM _dever_job", vec![], 1, |row| { row.get::<_, String>(0) }) .await .unwrap(), vec!["dead"] ); assert!( job::complete(&mut executor, scope, &last, 1112) .await .is_err() ); for (attempt, delay) in [(1, 10), (2, 20), (3, 40), (100, 40)] { assert_eq!(config.delay(attempt), delay); } assert!( config .validate( &[Policy { retries: 2, timeout_ms: 60000 }], 4, true, ) .is_err() ); assert!( config .validate( &[Policy { retries: 2, timeout_ms: 50 }], 2, true, ) .is_err() ); config .validate( &[Policy { retries: 2, timeout_ms: 50, }], 4, true, ) .unwrap(); config .validate( &[Policy { retries: 2, timeout_ms: 60000, }], 4, false, ) .unwrap(); assert!( config .validate( &[Policy { retries: 0, timeout_ms: 60000 }], 4, false ) .is_err() ); }) .unwrap(); }