Initial Dever 0.0.1
This commit is contained in:
Codex committed
commit
81233d098c
1479 files changed
+163782
No files matched your search
@@ -0,0 +1,320 @@
|
||||
use super::*;
|
||||
use crate::job::{Failure, heartbeat, queue};
|
||||
use std::{
|
||||
future::pending,
|
||||
sync::atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
struct Finished(Arc<AtomicBool>);
|
||||
impl Drop for Finished {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn committed_handler_wins_renewal_race_while_scope_drains_children() {
|
||||
fixture(async |case, scope| {
|
||||
let registry = case.registry(scope).await;
|
||||
case.executor(scope)
|
||||
.await
|
||||
.initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)")
|
||||
.await
|
||||
.unwrap();
|
||||
let claim = Arc::new(claimed(case, scope, "commit-race", 300).await);
|
||||
let value = claim.clone();
|
||||
let owned = case.clone();
|
||||
let finished = Arc::new(AtomicBool::new(false));
|
||||
let observed = finished.clone();
|
||||
let handler = scope
|
||||
.spawn(move |child| async move {
|
||||
let mut executor = owned.executor(&child).await;
|
||||
let owner = executor.begin(&child).await.unwrap();
|
||||
executor
|
||||
.execute(&child, "INSERT INTO business VALUES (1)", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
job::complete(
|
||||
&mut executor,
|
||||
&child,
|
||||
&value,
|
||||
crate::database::now().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (entered, ready) = tokio::sync::oneshot::channel();
|
||||
let descendant = child
|
||||
.spawn(move |grandchild| async move {
|
||||
grandchild
|
||||
.blocking(move || {
|
||||
let _finished = Finished(observed);
|
||||
entered.send(()).unwrap();
|
||||
std::thread::sleep(Duration::from_millis(450));
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
.unwrap();
|
||||
ready.await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
executor.finish(&child, owner, true).await.unwrap();
|
||||
drop(descendant);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let config = job::Config {
|
||||
lease_ms: 300,
|
||||
..Default::default()
|
||||
};
|
||||
heartbeat::run(scope, registry, (0, None), &config, &claim, handler)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(finished.load(Ordering::SeqCst));
|
||||
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1);
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state='completed' AND attempt=1"
|
||||
)
|
||||
.await,
|
||||
1
|
||||
);
|
||||
let mut executor = case.executor(scope).await;
|
||||
assert!(
|
||||
job::claim(
|
||||
&mut executor,
|
||||
scope,
|
||||
crate::database::now().unwrap() + 1000,
|
||||
300
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async fn claimed(case: &Case, scope: &Scope<'_>, key: &str, lease: i64) -> Claim {
|
||||
let mut executor = case.executor(scope).await;
|
||||
let now = crate::database::now().unwrap();
|
||||
let mut request = submission(key);
|
||||
request.policy.timeout_ms = 3000;
|
||||
job::enqueue(&mut executor, scope, request, now)
|
||||
.await
|
||||
.unwrap();
|
||||
job::claim(&mut executor, scope, now, lease)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn heartbeat_keeps_business_transaction_alive_across_multiple_leases() {
|
||||
fixture(async |case, scope| {
|
||||
let registry = case.registry(scope).await;
|
||||
let mut executor = case.executor(scope).await;
|
||||
executor
|
||||
.initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)")
|
||||
.await
|
||||
.unwrap();
|
||||
let claim = Arc::new(claimed(case, scope, "long-transaction", 200).await);
|
||||
let value = claim.clone();
|
||||
let owned = case.clone();
|
||||
let handler = scope
|
||||
.spawn(move |child| async move {
|
||||
let mut executor = owned.executor(&child).await;
|
||||
let owner = executor.begin(&child).await.unwrap();
|
||||
executor
|
||||
.execute(&child, "INSERT INTO business VALUES (1)", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
executor
|
||||
.postgres_operation(async |client| {
|
||||
client
|
||||
.query_one("SELECT pg_sleep(0.7)", &[])
|
||||
.await
|
||||
.map_err(Error::from)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
job::complete(
|
||||
&mut executor,
|
||||
&child,
|
||||
&value,
|
||||
crate::database::now().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
executor.finish(&child, owner, true).await.unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let config = job::Config {
|
||||
lease_ms: 200,
|
||||
..Default::default()
|
||||
};
|
||||
let (result, ()) = tokio::join!(
|
||||
heartbeat::run(scope, registry, (0, None), &config, &claim, handler),
|
||||
async {
|
||||
tokio::time::sleep(Duration::from_millis(450)).await;
|
||||
assert!(
|
||||
job::claim(&mut executor, scope, crate::database::now().unwrap(), 200)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
},
|
||||
);
|
||||
result.unwrap();
|
||||
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1);
|
||||
assert_eq!(case.count(scope, "SELECT attempt FROM _dever_job").await, 1);
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state='completed'"
|
||||
)
|
||||
.await,
|
||||
1
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn lost_fence_cancels_long_handler_and_joins_children_before_returning() {
|
||||
fixture(async |case, scope| {
|
||||
let registry = case.registry(scope).await;
|
||||
let mut executor = case.executor(scope).await;
|
||||
executor
|
||||
.initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)")
|
||||
.await
|
||||
.unwrap();
|
||||
let claim = Arc::new(claimed(case, scope, "lost-fence", 300).await);
|
||||
let finished = Arc::new(AtomicBool::new(false));
|
||||
let observed = finished.clone();
|
||||
let (entered, ready) = tokio::sync::oneshot::channel();
|
||||
let owned = case.clone();
|
||||
let handler = scope
|
||||
.spawn(move |child| async move {
|
||||
let mut executor = owned.executor(&child).await;
|
||||
executor
|
||||
.execute(&child, "INSERT INTO business VALUES (1)", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
let owner = executor.begin(&child).await.unwrap();
|
||||
executor
|
||||
.execute(&child, "INSERT INTO business VALUES (2)", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
let descendant = child
|
||||
.spawn(move |_| async move {
|
||||
let _finished = Finished(observed);
|
||||
entered.send(()).unwrap();
|
||||
pending::<()>().await;
|
||||
})
|
||||
.unwrap();
|
||||
descendant.wait().await.map_err(|_| Failure::Handler)?;
|
||||
executor.finish(&child, owner, true).await.unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let config = job::Config {
|
||||
lease_ms: 300,
|
||||
..Default::default()
|
||||
};
|
||||
let (result, ()) = tokio::join!(
|
||||
heartbeat::run(scope, registry, (0, None), &config, &claim, handler),
|
||||
async {
|
||||
ready.await.unwrap();
|
||||
executor
|
||||
.execute(scope, "UPDATE _dever_job SET token='other-worker'", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
},
|
||||
);
|
||||
assert!(matches!(result, Err(Failure::LeaseLost)));
|
||||
assert!(finished.load(Ordering::SeqCst));
|
||||
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1);
|
||||
assert_eq!(case.count(scope, "SELECT id FROM business").await, 1);
|
||||
assert!(
|
||||
job::complete(
|
||||
&mut executor,
|
||||
scope,
|
||||
&claim,
|
||||
crate::database::now().unwrap()
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(case.count(scope, "SELECT attempt FROM _dever_job").await, 1);
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE token='other-worker' AND state='running'"
|
||||
)
|
||||
.await,
|
||||
1
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn renewal_checks_expiry_after_waiting_for_row_lock() {
|
||||
fixture(async |case, scope| {
|
||||
let claim = claimed(case, scope, "row-wait", 100).await;
|
||||
let mut owner = case.executor(scope).await;
|
||||
let mut waiting = case.executor(scope).await;
|
||||
let transaction = owner.begin(scope).await.unwrap();
|
||||
owner
|
||||
.query(
|
||||
scope,
|
||||
"SELECT id FROM _dever_job WHERE id=$1 FOR UPDATE",
|
||||
vec![Parameter::Text(claim.id.clone())],
|
||||
1,
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let before = crate::database::now().unwrap();
|
||||
let (result, ()) = tokio::join!(
|
||||
queue::renew(&mut waiting, scope, &claim, before, 1000),
|
||||
async {
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
owner.finish(scope, transaction, true).await.unwrap();
|
||||
},
|
||||
);
|
||||
assert!(result.is_err());
|
||||
assert_eq!(
|
||||
case.count(scope, "SELECT lease_until FROM _dever_job")
|
||||
.await,
|
||||
claim.lease_until
|
||||
);
|
||||
let replacement = job::claim(&mut waiting, scope, crate::database::now().unwrap(), 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_ne!(replacement.token, claim.token);
|
||||
assert!(
|
||||
queue::renew(
|
||||
&mut waiting,
|
||||
scope,
|
||||
&claim,
|
||||
crate::database::now().unwrap(),
|
||||
1000
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
job::complete(
|
||||
&mut waiting,
|
||||
scope,
|
||||
&replacement,
|
||||
crate::database::now().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user