use super::*; #[test] #[ignore = "requires explicit isolated database.postgres_test"] fn identity_idempotency_retry_and_terminal_keys() { fixture(async |case, scope| { let mut executor = case.executor(scope).await; let identity = Execution::User { provider: "session".into(), subject: "user:7".into(), session: "test-session".into(), site: "admin".into(), tenant: None, tenant_id: None, permission: "demo.account.admin.publish".into(), }; let mut first = submission("same"); first.execution = &identity; let id = job::enqueue(&mut executor, scope, first, NOW) .await .unwrap() .into_text(); let mut duplicate = submission("same"); let payload = Json::parse(r#"{"number":999}"#).unwrap(); duplicate.payload = Some(&payload); assert_eq!( job::enqueue(&mut executor, scope, duplicate, NOW) .await .unwrap() .into_text(), id ); assert!( job::claim(&mut executor, scope, NOW - 1000, 1000) .await .unwrap() .is_none() ); let claim = job::claim(&mut executor, scope, NOW, 1000) .await .unwrap() .unwrap(); assert_eq!(claim.id, id); assert_eq!(claim.payload, r#"{"number":1}"#); assert_eq!(claim.schema, "schema-1"); assert_eq!( serde_json::from_str::(&claim.execution).unwrap(), serde_json::to_value(&identity).unwrap() ); let config = job::Config::default(); job::fail(&mut executor, scope, &claim, NOW + 1, &config) .await .unwrap(); assert!( job::claim(&mut executor, scope, NOW + 2, 1000) .await .unwrap() .is_none() ); let retry = job::claim(&mut executor, scope, NOW + 1 + config.delay(1) + 1000, 1000) .await .unwrap() .unwrap(); assert_eq!(retry.id, id); assert_eq!(retry.attempt, 2); assert_ne!(retry.token, claim.token); job::fail( &mut executor, scope, &retry, retry.lease_until - 500, &config, ) .await .unwrap(); assert!( job::claim(&mut executor, scope, NOW + 1_000_000, 1000) .await .unwrap() .is_none() ); assert_eq!( case.count(scope, "SELECT COUNT(*) FROM _dever_job WHERE state='dead'") .await, 1 ); let replacement = job::enqueue(&mut executor, scope, submission("same"), NOW) .await .unwrap() .into_text(); assert_ne!(replacement, id); let owner = executor.begin(scope).await.unwrap(); job::enqueue(&mut executor, scope, submission("rollback"), NOW) .await .unwrap(); executor.finish(scope, owner, false).await.unwrap(); assert_eq!( case.count( scope, "SELECT COUNT(*) FROM _dever_job WHERE task_key='rollback'" ) .await, 0 ); assert!(Json::parse("{").is_err()); assert_eq!( case.count( scope, "SELECT COUNT(*) FROM _dever_job WHERE task_key='invalid'" ) .await, 0 ); }); } #[test] #[ignore = "requires explicit isolated database.postgres_test"] fn concurrent_enqueue_claim_and_skip_locked_keep_single_ownership() { fixture(async |case, scope| { let mut left = case.executor(scope).await; let mut right = case.executor(scope).await; let (left_id, right_id) = tokio::join!( job::enqueue(&mut left, scope, submission("same"), NOW), job::enqueue(&mut right, scope, submission("same"), NOW), ); assert_eq!(left_id.unwrap().into_text(), right_id.unwrap().into_text()); let (first, second) = tokio::join!( job::claim(&mut left, scope, NOW, 10_000), job::claim(&mut right, scope, NOW, 10_000), ); assert_eq!( usize::from(first.unwrap().is_some()) + usize::from(second.unwrap().is_some()), 1 ); let locked = job::enqueue(&mut left, scope, submission("locked"), NOW) .await .unwrap() .into_text(); let free = job::enqueue(&mut right, scope, submission("free"), NOW) .await .unwrap() .into_text(); let owner = left.begin(scope).await.unwrap(); left.query( scope, "SELECT id FROM _dever_job WHERE id=$1 FOR UPDATE", vec![Parameter::Text(locked)], 1, |row| row.get::<_, String>(0), ) .await .unwrap(); let claim = tokio::time::timeout( Duration::from_millis(500), job::claim(&mut right, scope, NOW, 10_000), ) .await .unwrap() .unwrap() .unwrap(); assert_eq!(claim.id, free); left.finish(scope, owner, true).await.unwrap(); job::block(&mut right, scope, &claim, NOW + 1) .await .unwrap(); assert_eq!( case.count( scope, "SELECT COUNT(*) FROM _dever_job WHERE state='blocked'" ) .await, 1 ); }); } #[test] #[ignore = "requires explicit isolated database.postgres_test"] fn expired_claims_recover_or_exhaust_and_keep_token_fencing() { fixture(async |case, scope| { let mut executor = case.executor(scope).await; for key in ["first", "second", "third"] { job::enqueue(&mut executor, scope, submission(key), NOW) .await .unwrap(); } let mut originals = Vec::new(); for _ in 0..3 { originals.push( job::claim(&mut executor, scope, NOW, 100) .await .unwrap() .unwrap(), ); } let recovered = job::claim(&mut executor, scope, NOW + 1000, 1000) .await .unwrap() .unwrap(); assert_eq!(recovered.attempt, 2); assert_eq!( case.count( scope, "SELECT COUNT(*) FROM _dever_job WHERE state='pending'" ) .await, 2 ); for original in &originals { assert!( job::complete(&mut executor, scope, original, NOW + 1001) .await .is_err() ); } job::complete(&mut executor, scope, &recovered, NOW + 1001) .await .unwrap(); let exhausted = job::claim(&mut executor, scope, NOW + 1001, 100) .await .unwrap() .unwrap(); let next = job::claim(&mut executor, scope, NOW + 2000, 1000) .await .unwrap() .unwrap(); assert_ne!(next.id, exhausted.id); assert_eq!( case.count(scope, "SELECT COUNT(*) FROM _dever_job WHERE state='dead'") .await, 1 ); job::complete(&mut executor, scope, &next, NOW + 2001) .await .unwrap(); assert!( job::claim(&mut executor, scope, NOW + 10_000, 1000) .await .unwrap() .is_none() ); }); } #[test] #[ignore = "requires explicit isolated database.postgres_test"] fn expired_cleanup_is_bounded_and_workers_skip_each_others_locks() { fixture(async |case, scope| { let mut left = case.executor(scope).await; left.initialize(scope, "INSERT INTO _dever_job(id,target,payload,schema,execution,task_key,state,attempt,retries,timeout_ms,available_at,created_at,token,lease_until,execution_tenant) SELECT lpad(number::text,6,'0'),'demo.account.publish','null','zero','{\"kind\":\"System\",\"tenant\":null}',number::text,'running',1,2,50,0,0,'old',0,0 FROM generate_series(0,599) AS number").await.unwrap(); let mut right = case.executor(scope).await; let owner = left.begin(scope).await.unwrap(); let first = job::claim(&mut left, scope, NOW, 10_000) .await .unwrap() .unwrap(); let second = tokio::time::timeout( Duration::from_millis(500), job::claim(&mut right, scope, NOW, 10_000), ) .await .unwrap() .unwrap() .unwrap(); assert_eq!(first.id, "000000"); assert_eq!(second.id, "000257"); left.finish(scope, owner, true).await.unwrap(); assert_eq!( case.count( scope, "SELECT COUNT(*) FROM _dever_job WHERE state='pending'" ) .await, 512 ); assert_eq!( case.count( scope, "SELECT COUNT(*) FROM _dever_job WHERE state='running' AND token='old'" ) .await, 86 ); assert_eq!( case.count( scope, "SELECT COUNT(*) FROM _dever_job WHERE state='running' AND token<>'old'" ) .await, 2 ); }); } #[test] #[ignore = "requires explicit isolated database.postgres_test"] fn exhausted_cleanup_does_not_hide_eligible_expired_work() { fixture(async |case, scope| { let mut executor = case.executor(scope).await; executor.initialize(scope, "INSERT INTO _dever_job(id,target,payload,schema,execution,task_key,state,attempt,retries,timeout_ms,available_at,created_at,token,lease_until,execution_tenant) SELECT lpad(number::text,6,'0'),'demo.account.publish','null','zero','{\"kind\":\"System\",\"tenant\":null}',number::text,'running',1,CASE WHEN number=300 THEN 2 ELSE 1 END,50,0,0,'old',0,0 FROM generate_series(0,300) AS number").await.unwrap(); let claim = job::claim(&mut executor, scope, NOW, 1000) .await .unwrap() .unwrap(); assert_eq!(claim.id, "000300"); assert_eq!(claim.attempt, 2); assert_eq!( case.count(scope, "SELECT COUNT(*) FROM _dever_job WHERE state='dead'") .await, 256 ); assert_eq!( case.count( scope, "SELECT COUNT(*) FROM _dever_job WHERE state='running' AND token='old'" ) .await, 44 ); }); }