use super::*; use crate::job::{Failure, heartbeat, queue}; use std::{ future::pending, sync::atomic::{AtomicBool, Ordering}, }; struct Finished(Arc); impl Drop for Finished { fn drop(&mut self) { self.0.store(true, Ordering::SeqCst); } } #[test] #[ignore = "requires explicit isolated database.postgres_test"] fn committed_handler_wins_renewal_race_while_scope_drains_children() { fixture(async |case, scope| { let registry = case.registry(scope).await; case.executor(scope) .await .initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)") .await .unwrap(); let claim = Arc::new(claimed(case, scope, "commit-race", 300).await); let value = claim.clone(); let owned = case.clone(); let finished = Arc::new(AtomicBool::new(false)); let observed = finished.clone(); let handler = scope .spawn(move |child| async move { let mut executor = owned.executor(&child).await; let owner = executor.begin(&child).await.unwrap(); executor .execute(&child, "INSERT INTO business VALUES (1)", vec![]) .await .unwrap(); job::complete( &mut executor, &child, &value, crate::database::now().unwrap(), ) .await .unwrap(); let (entered, ready) = tokio::sync::oneshot::channel(); let descendant = child .spawn(move |grandchild| async move { grandchild .blocking(move || { let _finished = Finished(observed); entered.send(()).unwrap(); std::thread::sleep(Duration::from_millis(450)); }) .await .unwrap(); }) .unwrap(); ready.await.unwrap(); tokio::time::sleep(Duration::from_millis(150)).await; executor.finish(&child, owner, true).await.unwrap(); drop(descendant); Ok(()) }) .unwrap(); let config = job::Config { lease_ms: 300, ..Default::default() }; heartbeat::run(scope, registry, (0, None), &config, &claim, handler) .await .unwrap(); assert!(finished.load(Ordering::SeqCst)); assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1); assert_eq!( case.count( scope, "SELECT COUNT(*) FROM _dever_job WHERE state='completed' AND attempt=1" ) .await, 1 ); let mut executor = case.executor(scope).await; assert!( job::claim( &mut executor, scope, crate::database::now().unwrap() + 1000, 300 ) .await .unwrap() .is_none() ); }); } async fn claimed(case: &Case, scope: &Scope<'_>, key: &str, lease: i64) -> Claim { let mut executor = case.executor(scope).await; let now = crate::database::now().unwrap(); let mut request = submission(key); request.policy.timeout_ms = 3000; job::enqueue(&mut executor, scope, request, now) .await .unwrap(); job::claim(&mut executor, scope, now, lease) .await .unwrap() .unwrap() } #[test] #[ignore = "requires explicit isolated database.postgres_test"] fn heartbeat_keeps_business_transaction_alive_across_multiple_leases() { fixture(async |case, scope| { let registry = case.registry(scope).await; let mut executor = case.executor(scope).await; executor .initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)") .await .unwrap(); let claim = Arc::new(claimed(case, scope, "long-transaction", 200).await); let value = claim.clone(); let owned = case.clone(); let handler = scope .spawn(move |child| async move { let mut executor = owned.executor(&child).await; let owner = executor.begin(&child).await.unwrap(); executor .execute(&child, "INSERT INTO business VALUES (1)", vec![]) .await .unwrap(); executor .postgres_operation(async |client| { client .query_one("SELECT pg_sleep(0.7)", &[]) .await .map_err(Error::from) }) .await .unwrap(); job::complete( &mut executor, &child, &value, crate::database::now().unwrap(), ) .await .unwrap(); executor.finish(&child, owner, true).await.unwrap(); Ok(()) }) .unwrap(); let config = job::Config { lease_ms: 200, ..Default::default() }; let (result, ()) = tokio::join!( heartbeat::run(scope, registry, (0, None), &config, &claim, handler), async { tokio::time::sleep(Duration::from_millis(450)).await; assert!( job::claim(&mut executor, scope, crate::database::now().unwrap(), 200) .await .unwrap() .is_none() ); }, ); result.unwrap(); assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1); assert_eq!(case.count(scope, "SELECT attempt FROM _dever_job").await, 1); assert_eq!( case.count( scope, "SELECT COUNT(*) FROM _dever_job WHERE state='completed'" ) .await, 1 ); }); } #[test] #[ignore = "requires explicit isolated database.postgres_test"] fn lost_fence_cancels_long_handler_and_joins_children_before_returning() { fixture(async |case, scope| { let registry = case.registry(scope).await; let mut executor = case.executor(scope).await; executor .initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)") .await .unwrap(); let claim = Arc::new(claimed(case, scope, "lost-fence", 300).await); let finished = Arc::new(AtomicBool::new(false)); let observed = finished.clone(); let (entered, ready) = tokio::sync::oneshot::channel(); let owned = case.clone(); let handler = scope .spawn(move |child| async move { let mut executor = owned.executor(&child).await; executor .execute(&child, "INSERT INTO business VALUES (1)", vec![]) .await .unwrap(); let owner = executor.begin(&child).await.unwrap(); executor .execute(&child, "INSERT INTO business VALUES (2)", vec![]) .await .unwrap(); let descendant = child .spawn(move |_| async move { let _finished = Finished(observed); entered.send(()).unwrap(); pending::<()>().await; }) .unwrap(); descendant.wait().await.map_err(|_| Failure::Handler)?; executor.finish(&child, owner, true).await.unwrap(); Ok(()) }) .unwrap(); let config = job::Config { lease_ms: 300, ..Default::default() }; let (result, ()) = tokio::join!( heartbeat::run(scope, registry, (0, None), &config, &claim, handler), async { ready.await.unwrap(); executor .execute(scope, "UPDATE _dever_job SET token='other-worker'", vec![]) .await .unwrap(); }, ); assert!(matches!(result, Err(Failure::LeaseLost))); assert!(finished.load(Ordering::SeqCst)); assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1); assert_eq!(case.count(scope, "SELECT id FROM business").await, 1); assert!( job::complete( &mut executor, scope, &claim, crate::database::now().unwrap() ) .await .is_err() ); assert_eq!(case.count(scope, "SELECT attempt FROM _dever_job").await, 1); assert_eq!( case.count( scope, "SELECT COUNT(*) FROM _dever_job WHERE token='other-worker' AND state='running'" ) .await, 1 ); }); } #[test] #[ignore = "requires explicit isolated database.postgres_test"] fn renewal_checks_expiry_after_waiting_for_row_lock() { fixture(async |case, scope| { let claim = claimed(case, scope, "row-wait", 100).await; let mut owner = case.executor(scope).await; let mut waiting = case.executor(scope).await; let transaction = owner.begin(scope).await.unwrap(); owner .query( scope, "SELECT id FROM _dever_job WHERE id=$1 FOR UPDATE", vec![Parameter::Text(claim.id.clone())], 1, |row| row.get::<_, String>(0), ) .await .unwrap(); let before = crate::database::now().unwrap(); let (result, ()) = tokio::join!( queue::renew(&mut waiting, scope, &claim, before, 1000), async { tokio::time::sleep(Duration::from_millis(200)).await; owner.finish(scope, transaction, true).await.unwrap(); }, ); assert!(result.is_err()); assert_eq!( case.count(scope, "SELECT lease_until FROM _dever_job") .await, claim.lease_until ); let replacement = job::claim(&mut waiting, scope, crate::database::now().unwrap(), 1000) .await .unwrap() .unwrap(); assert_ne!(replacement.token, claim.token); assert!( queue::renew( &mut waiting, scope, &claim, crate::database::now().unwrap(), 1000 ) .await .is_err() ); job::complete( &mut waiting, scope, &replacement, crate::database::now().unwrap(), ) .await .unwrap(); }); }