321 lines
11 KiB
Rust
321 lines
11 KiB
Rust
use super::*;
|
|
use crate::job::{Failure, heartbeat, queue};
|
|
use std::{
|
|
future::pending,
|
|
sync::atomic::{AtomicBool, Ordering},
|
|
};
|
|
|
|
struct Finished(Arc<AtomicBool>);
|
|
impl Drop for Finished {
|
|
fn drop(&mut self) {
|
|
self.0.store(true, Ordering::SeqCst);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires explicit isolated database.postgres_test"]
|
|
fn committed_handler_wins_renewal_race_while_scope_drains_children() {
|
|
fixture(async |case, scope| {
|
|
let registry = case.registry(scope).await;
|
|
case.executor(scope)
|
|
.await
|
|
.initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)")
|
|
.await
|
|
.unwrap();
|
|
let claim = Arc::new(claimed(case, scope, "commit-race", 300).await);
|
|
let value = claim.clone();
|
|
let owned = case.clone();
|
|
let finished = Arc::new(AtomicBool::new(false));
|
|
let observed = finished.clone();
|
|
let handler = scope
|
|
.spawn(move |child| async move {
|
|
let mut executor = owned.executor(&child).await;
|
|
let owner = executor.begin(&child).await.unwrap();
|
|
executor
|
|
.execute(&child, "INSERT INTO business VALUES (1)", vec![])
|
|
.await
|
|
.unwrap();
|
|
job::complete(
|
|
&mut executor,
|
|
&child,
|
|
&value,
|
|
crate::database::now().unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let (entered, ready) = tokio::sync::oneshot::channel();
|
|
let descendant = child
|
|
.spawn(move |grandchild| async move {
|
|
grandchild
|
|
.blocking(move || {
|
|
let _finished = Finished(observed);
|
|
entered.send(()).unwrap();
|
|
std::thread::sleep(Duration::from_millis(450));
|
|
})
|
|
.await
|
|
.unwrap();
|
|
})
|
|
.unwrap();
|
|
ready.await.unwrap();
|
|
tokio::time::sleep(Duration::from_millis(150)).await;
|
|
executor.finish(&child, owner, true).await.unwrap();
|
|
drop(descendant);
|
|
Ok(())
|
|
})
|
|
.unwrap();
|
|
let config = job::Config {
|
|
lease_ms: 300,
|
|
..Default::default()
|
|
};
|
|
heartbeat::run(scope, registry, (0, None), &config, &claim, handler)
|
|
.await
|
|
.unwrap();
|
|
assert!(finished.load(Ordering::SeqCst));
|
|
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1);
|
|
assert_eq!(
|
|
case.count(
|
|
scope,
|
|
"SELECT COUNT(*) FROM _dever_job WHERE state='completed' AND attempt=1"
|
|
)
|
|
.await,
|
|
1
|
|
);
|
|
let mut executor = case.executor(scope).await;
|
|
assert!(
|
|
job::claim(
|
|
&mut executor,
|
|
scope,
|
|
crate::database::now().unwrap() + 1000,
|
|
300
|
|
)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
});
|
|
}
|
|
|
|
async fn claimed(case: &Case, scope: &Scope<'_>, key: &str, lease: i64) -> Claim {
|
|
let mut executor = case.executor(scope).await;
|
|
let now = crate::database::now().unwrap();
|
|
let mut request = submission(key);
|
|
request.policy.timeout_ms = 3000;
|
|
job::enqueue(&mut executor, scope, request, now)
|
|
.await
|
|
.unwrap();
|
|
job::claim(&mut executor, scope, now, lease)
|
|
.await
|
|
.unwrap()
|
|
.unwrap()
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires explicit isolated database.postgres_test"]
|
|
fn heartbeat_keeps_business_transaction_alive_across_multiple_leases() {
|
|
fixture(async |case, scope| {
|
|
let registry = case.registry(scope).await;
|
|
let mut executor = case.executor(scope).await;
|
|
executor
|
|
.initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)")
|
|
.await
|
|
.unwrap();
|
|
let claim = Arc::new(claimed(case, scope, "long-transaction", 200).await);
|
|
let value = claim.clone();
|
|
let owned = case.clone();
|
|
let handler = scope
|
|
.spawn(move |child| async move {
|
|
let mut executor = owned.executor(&child).await;
|
|
let owner = executor.begin(&child).await.unwrap();
|
|
executor
|
|
.execute(&child, "INSERT INTO business VALUES (1)", vec![])
|
|
.await
|
|
.unwrap();
|
|
executor
|
|
.postgres_operation(async |client| {
|
|
client
|
|
.query_one("SELECT pg_sleep(0.7)", &[])
|
|
.await
|
|
.map_err(Error::from)
|
|
})
|
|
.await
|
|
.unwrap();
|
|
job::complete(
|
|
&mut executor,
|
|
&child,
|
|
&value,
|
|
crate::database::now().unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
executor.finish(&child, owner, true).await.unwrap();
|
|
Ok(())
|
|
})
|
|
.unwrap();
|
|
let config = job::Config {
|
|
lease_ms: 200,
|
|
..Default::default()
|
|
};
|
|
let (result, ()) = tokio::join!(
|
|
heartbeat::run(scope, registry, (0, None), &config, &claim, handler),
|
|
async {
|
|
tokio::time::sleep(Duration::from_millis(450)).await;
|
|
assert!(
|
|
job::claim(&mut executor, scope, crate::database::now().unwrap(), 200)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
},
|
|
);
|
|
result.unwrap();
|
|
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1);
|
|
assert_eq!(case.count(scope, "SELECT attempt FROM _dever_job").await, 1);
|
|
assert_eq!(
|
|
case.count(
|
|
scope,
|
|
"SELECT COUNT(*) FROM _dever_job WHERE state='completed'"
|
|
)
|
|
.await,
|
|
1
|
|
);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires explicit isolated database.postgres_test"]
|
|
fn lost_fence_cancels_long_handler_and_joins_children_before_returning() {
|
|
fixture(async |case, scope| {
|
|
let registry = case.registry(scope).await;
|
|
let mut executor = case.executor(scope).await;
|
|
executor
|
|
.initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)")
|
|
.await
|
|
.unwrap();
|
|
let claim = Arc::new(claimed(case, scope, "lost-fence", 300).await);
|
|
let finished = Arc::new(AtomicBool::new(false));
|
|
let observed = finished.clone();
|
|
let (entered, ready) = tokio::sync::oneshot::channel();
|
|
let owned = case.clone();
|
|
let handler = scope
|
|
.spawn(move |child| async move {
|
|
let mut executor = owned.executor(&child).await;
|
|
executor
|
|
.execute(&child, "INSERT INTO business VALUES (1)", vec![])
|
|
.await
|
|
.unwrap();
|
|
let owner = executor.begin(&child).await.unwrap();
|
|
executor
|
|
.execute(&child, "INSERT INTO business VALUES (2)", vec![])
|
|
.await
|
|
.unwrap();
|
|
let descendant = child
|
|
.spawn(move |_| async move {
|
|
let _finished = Finished(observed);
|
|
entered.send(()).unwrap();
|
|
pending::<()>().await;
|
|
})
|
|
.unwrap();
|
|
descendant.wait().await.map_err(|_| Failure::Handler)?;
|
|
executor.finish(&child, owner, true).await.unwrap();
|
|
Ok(())
|
|
})
|
|
.unwrap();
|
|
let config = job::Config {
|
|
lease_ms: 300,
|
|
..Default::default()
|
|
};
|
|
let (result, ()) = tokio::join!(
|
|
heartbeat::run(scope, registry, (0, None), &config, &claim, handler),
|
|
async {
|
|
ready.await.unwrap();
|
|
executor
|
|
.execute(scope, "UPDATE _dever_job SET token='other-worker'", vec![])
|
|
.await
|
|
.unwrap();
|
|
},
|
|
);
|
|
assert!(matches!(result, Err(Failure::LeaseLost)));
|
|
assert!(finished.load(Ordering::SeqCst));
|
|
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1);
|
|
assert_eq!(case.count(scope, "SELECT id FROM business").await, 1);
|
|
assert!(
|
|
job::complete(
|
|
&mut executor,
|
|
scope,
|
|
&claim,
|
|
crate::database::now().unwrap()
|
|
)
|
|
.await
|
|
.is_err()
|
|
);
|
|
assert_eq!(case.count(scope, "SELECT attempt FROM _dever_job").await, 1);
|
|
assert_eq!(
|
|
case.count(
|
|
scope,
|
|
"SELECT COUNT(*) FROM _dever_job WHERE token='other-worker' AND state='running'"
|
|
)
|
|
.await,
|
|
1
|
|
);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires explicit isolated database.postgres_test"]
|
|
fn renewal_checks_expiry_after_waiting_for_row_lock() {
|
|
fixture(async |case, scope| {
|
|
let claim = claimed(case, scope, "row-wait", 100).await;
|
|
let mut owner = case.executor(scope).await;
|
|
let mut waiting = case.executor(scope).await;
|
|
let transaction = owner.begin(scope).await.unwrap();
|
|
owner
|
|
.query(
|
|
scope,
|
|
"SELECT id FROM _dever_job WHERE id=$1 FOR UPDATE",
|
|
vec![Parameter::Text(claim.id.clone())],
|
|
1,
|
|
|row| row.get::<_, String>(0),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let before = crate::database::now().unwrap();
|
|
let (result, ()) = tokio::join!(
|
|
queue::renew(&mut waiting, scope, &claim, before, 1000),
|
|
async {
|
|
tokio::time::sleep(Duration::from_millis(200)).await;
|
|
owner.finish(scope, transaction, true).await.unwrap();
|
|
},
|
|
);
|
|
assert!(result.is_err());
|
|
assert_eq!(
|
|
case.count(scope, "SELECT lease_until FROM _dever_job")
|
|
.await,
|
|
claim.lease_until
|
|
);
|
|
let replacement = job::claim(&mut waiting, scope, crate::database::now().unwrap(), 1000)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_ne!(replacement.token, claim.token);
|
|
assert!(
|
|
queue::renew(
|
|
&mut waiting,
|
|
scope,
|
|
&claim,
|
|
crate::database::now().unwrap(),
|
|
1000
|
|
)
|
|
.await
|
|
.is_err()
|
|
);
|
|
job::complete(
|
|
&mut waiting,
|
|
scope,
|
|
&replacement,
|
|
crate::database::now().unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
});
|
|
}
|