323 lines
11 KiB
Rust
323 lines
11 KiB
Rust
use super::*;
|
|
|
|
#[test]
|
|
#[ignore = "requires explicit isolated database.postgres_test"]
|
|
fn identity_idempotency_retry_and_terminal_keys() {
|
|
fixture(async |case, scope| {
|
|
let mut executor = case.executor(scope).await;
|
|
let identity = Execution::User {
|
|
provider: "session".into(),
|
|
subject: "user:7".into(),
|
|
session: "test-session".into(),
|
|
site: "admin".into(),
|
|
tenant: None,
|
|
tenant_id: None,
|
|
permission: "demo.account.admin.publish".into(),
|
|
};
|
|
let mut first = submission("same");
|
|
first.execution = &identity;
|
|
let id = job::enqueue(&mut executor, scope, first, NOW)
|
|
.await
|
|
.unwrap()
|
|
.into_text();
|
|
let mut duplicate = submission("same");
|
|
let payload = Json::parse(r#"{"number":999}"#).unwrap();
|
|
duplicate.payload = Some(&payload);
|
|
assert_eq!(
|
|
job::enqueue(&mut executor, scope, duplicate, NOW)
|
|
.await
|
|
.unwrap()
|
|
.into_text(),
|
|
id
|
|
);
|
|
assert!(
|
|
job::claim(&mut executor, scope, NOW - 1000, 1000)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
let claim = job::claim(&mut executor, scope, NOW, 1000)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(claim.id, id);
|
|
assert_eq!(claim.payload, r#"{"number":1}"#);
|
|
assert_eq!(claim.schema, "schema-1");
|
|
assert_eq!(
|
|
serde_json::from_str::<serde_json::Value>(&claim.execution).unwrap(),
|
|
serde_json::to_value(&identity).unwrap()
|
|
);
|
|
let config = job::Config::default();
|
|
job::fail(&mut executor, scope, &claim, NOW + 1, &config)
|
|
.await
|
|
.unwrap();
|
|
assert!(
|
|
job::claim(&mut executor, scope, NOW + 2, 1000)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
let retry = job::claim(&mut executor, scope, NOW + 1 + config.delay(1) + 1000, 1000)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(retry.id, id);
|
|
assert_eq!(retry.attempt, 2);
|
|
assert_ne!(retry.token, claim.token);
|
|
job::fail(
|
|
&mut executor,
|
|
scope,
|
|
&retry,
|
|
retry.lease_until - 500,
|
|
&config,
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert!(
|
|
job::claim(&mut executor, scope, NOW + 1_000_000, 1000)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
assert_eq!(
|
|
case.count(scope, "SELECT COUNT(*) FROM _dever_job WHERE state='dead'")
|
|
.await,
|
|
1
|
|
);
|
|
let replacement = job::enqueue(&mut executor, scope, submission("same"), NOW)
|
|
.await
|
|
.unwrap()
|
|
.into_text();
|
|
assert_ne!(replacement, id);
|
|
let owner = executor.begin(scope).await.unwrap();
|
|
job::enqueue(&mut executor, scope, submission("rollback"), NOW)
|
|
.await
|
|
.unwrap();
|
|
executor.finish(scope, owner, false).await.unwrap();
|
|
assert_eq!(
|
|
case.count(
|
|
scope,
|
|
"SELECT COUNT(*) FROM _dever_job WHERE task_key='rollback'"
|
|
)
|
|
.await,
|
|
0
|
|
);
|
|
assert!(Json::parse("{").is_err());
|
|
assert_eq!(
|
|
case.count(
|
|
scope,
|
|
"SELECT COUNT(*) FROM _dever_job WHERE task_key='invalid'"
|
|
)
|
|
.await,
|
|
0
|
|
);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires explicit isolated database.postgres_test"]
|
|
fn concurrent_enqueue_claim_and_skip_locked_keep_single_ownership() {
|
|
fixture(async |case, scope| {
|
|
let mut left = case.executor(scope).await;
|
|
let mut right = case.executor(scope).await;
|
|
let (left_id, right_id) = tokio::join!(
|
|
job::enqueue(&mut left, scope, submission("same"), NOW),
|
|
job::enqueue(&mut right, scope, submission("same"), NOW),
|
|
);
|
|
assert_eq!(left_id.unwrap().into_text(), right_id.unwrap().into_text());
|
|
let (first, second) = tokio::join!(
|
|
job::claim(&mut left, scope, NOW, 10_000),
|
|
job::claim(&mut right, scope, NOW, 10_000),
|
|
);
|
|
assert_eq!(
|
|
usize::from(first.unwrap().is_some()) + usize::from(second.unwrap().is_some()),
|
|
1
|
|
);
|
|
let locked = job::enqueue(&mut left, scope, submission("locked"), NOW)
|
|
.await
|
|
.unwrap()
|
|
.into_text();
|
|
let free = job::enqueue(&mut right, scope, submission("free"), NOW)
|
|
.await
|
|
.unwrap()
|
|
.into_text();
|
|
let owner = left.begin(scope).await.unwrap();
|
|
left.query(
|
|
scope,
|
|
"SELECT id FROM _dever_job WHERE id=$1 FOR UPDATE",
|
|
vec![Parameter::Text(locked)],
|
|
1,
|
|
|row| row.get::<_, String>(0),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let claim = tokio::time::timeout(
|
|
Duration::from_millis(500),
|
|
job::claim(&mut right, scope, NOW, 10_000),
|
|
)
|
|
.await
|
|
.unwrap()
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(claim.id, free);
|
|
left.finish(scope, owner, true).await.unwrap();
|
|
job::block(&mut right, scope, &claim, NOW + 1)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(
|
|
case.count(
|
|
scope,
|
|
"SELECT COUNT(*) FROM _dever_job WHERE state='blocked'"
|
|
)
|
|
.await,
|
|
1
|
|
);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires explicit isolated database.postgres_test"]
|
|
fn expired_claims_recover_or_exhaust_and_keep_token_fencing() {
|
|
fixture(async |case, scope| {
|
|
let mut executor = case.executor(scope).await;
|
|
for key in ["first", "second", "third"] {
|
|
job::enqueue(&mut executor, scope, submission(key), NOW)
|
|
.await
|
|
.unwrap();
|
|
}
|
|
let mut originals = Vec::new();
|
|
for _ in 0..3 {
|
|
originals.push(
|
|
job::claim(&mut executor, scope, NOW, 100)
|
|
.await
|
|
.unwrap()
|
|
.unwrap(),
|
|
);
|
|
}
|
|
let recovered = job::claim(&mut executor, scope, NOW + 1000, 1000)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(recovered.attempt, 2);
|
|
assert_eq!(
|
|
case.count(
|
|
scope,
|
|
"SELECT COUNT(*) FROM _dever_job WHERE state='pending'"
|
|
)
|
|
.await,
|
|
2
|
|
);
|
|
for original in &originals {
|
|
assert!(
|
|
job::complete(&mut executor, scope, original, NOW + 1001)
|
|
.await
|
|
.is_err()
|
|
);
|
|
}
|
|
job::complete(&mut executor, scope, &recovered, NOW + 1001)
|
|
.await
|
|
.unwrap();
|
|
let exhausted = job::claim(&mut executor, scope, NOW + 1001, 100)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
let next = job::claim(&mut executor, scope, NOW + 2000, 1000)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_ne!(next.id, exhausted.id);
|
|
assert_eq!(
|
|
case.count(scope, "SELECT COUNT(*) FROM _dever_job WHERE state='dead'")
|
|
.await,
|
|
1
|
|
);
|
|
job::complete(&mut executor, scope, &next, NOW + 2001)
|
|
.await
|
|
.unwrap();
|
|
assert!(
|
|
job::claim(&mut executor, scope, NOW + 10_000, 1000)
|
|
.await
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires explicit isolated database.postgres_test"]
|
|
fn expired_cleanup_is_bounded_and_workers_skip_each_others_locks() {
|
|
fixture(async |case, scope| {
|
|
let mut left = case.executor(scope).await;
|
|
left.initialize(scope, "INSERT INTO _dever_job(id,target,payload,schema,execution,task_key,state,attempt,retries,timeout_ms,available_at,created_at,token,lease_until,execution_tenant) SELECT lpad(number::text,6,'0'),'demo.account.publish','null','zero','{\"kind\":\"System\",\"tenant\":null}',number::text,'running',1,2,50,0,0,'old',0,0 FROM generate_series(0,599) AS number").await.unwrap();
|
|
let mut right = case.executor(scope).await;
|
|
let owner = left.begin(scope).await.unwrap();
|
|
let first = job::claim(&mut left, scope, NOW, 10_000)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
let second = tokio::time::timeout(
|
|
Duration::from_millis(500),
|
|
job::claim(&mut right, scope, NOW, 10_000),
|
|
)
|
|
.await
|
|
.unwrap()
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(first.id, "000000");
|
|
assert_eq!(second.id, "000257");
|
|
left.finish(scope, owner, true).await.unwrap();
|
|
assert_eq!(
|
|
case.count(
|
|
scope,
|
|
"SELECT COUNT(*) FROM _dever_job WHERE state='pending'"
|
|
)
|
|
.await,
|
|
512
|
|
);
|
|
assert_eq!(
|
|
case.count(
|
|
scope,
|
|
"SELECT COUNT(*) FROM _dever_job WHERE state='running' AND token='old'"
|
|
)
|
|
.await,
|
|
86
|
|
);
|
|
assert_eq!(
|
|
case.count(
|
|
scope,
|
|
"SELECT COUNT(*) FROM _dever_job WHERE state='running' AND token<>'old'"
|
|
)
|
|
.await,
|
|
2
|
|
);
|
|
});
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires explicit isolated database.postgres_test"]
|
|
fn exhausted_cleanup_does_not_hide_eligible_expired_work() {
|
|
fixture(async |case, scope| {
|
|
let mut executor = case.executor(scope).await;
|
|
executor.initialize(scope, "INSERT INTO _dever_job(id,target,payload,schema,execution,task_key,state,attempt,retries,timeout_ms,available_at,created_at,token,lease_until,execution_tenant) SELECT lpad(number::text,6,'0'),'demo.account.publish','null','zero','{\"kind\":\"System\",\"tenant\":null}',number::text,'running',1,CASE WHEN number=300 THEN 2 ELSE 1 END,50,0,0,'old',0,0 FROM generate_series(0,300) AS number").await.unwrap();
|
|
let claim = job::claim(&mut executor, scope, NOW, 1000)
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(claim.id, "000300");
|
|
assert_eq!(claim.attempt, 2);
|
|
assert_eq!(
|
|
case.count(scope, "SELECT COUNT(*) FROM _dever_job WHERE state='dead'")
|
|
.await,
|
|
256
|
|
);
|
|
assert_eq!(
|
|
case.count(
|
|
scope,
|
|
"SELECT COUNT(*) FROM _dever_job WHERE state='running' AND token='old'"
|
|
)
|
|
.await,
|
|
44
|
|
);
|
|
});
|
|
}
|