Initial Dever 0.0.1
This commit is contained in:
Codex committed
commit
81233d098c
1479 files changed
+163782
No files matched your search
@@ -0,0 +1,320 @@
|
||||
use super::*;
|
||||
use crate::job::{Failure, heartbeat, queue};
|
||||
use std::{
|
||||
future::pending,
|
||||
sync::atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
struct Finished(Arc<AtomicBool>);
|
||||
impl Drop for Finished {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn committed_handler_wins_renewal_race_while_scope_drains_children() {
|
||||
fixture(async |case, scope| {
|
||||
let registry = case.registry(scope).await;
|
||||
case.executor(scope)
|
||||
.await
|
||||
.initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)")
|
||||
.await
|
||||
.unwrap();
|
||||
let claim = Arc::new(claimed(case, scope, "commit-race", 300).await);
|
||||
let value = claim.clone();
|
||||
let owned = case.clone();
|
||||
let finished = Arc::new(AtomicBool::new(false));
|
||||
let observed = finished.clone();
|
||||
let handler = scope
|
||||
.spawn(move |child| async move {
|
||||
let mut executor = owned.executor(&child).await;
|
||||
let owner = executor.begin(&child).await.unwrap();
|
||||
executor
|
||||
.execute(&child, "INSERT INTO business VALUES (1)", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
job::complete(
|
||||
&mut executor,
|
||||
&child,
|
||||
&value,
|
||||
crate::database::now().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (entered, ready) = tokio::sync::oneshot::channel();
|
||||
let descendant = child
|
||||
.spawn(move |grandchild| async move {
|
||||
grandchild
|
||||
.blocking(move || {
|
||||
let _finished = Finished(observed);
|
||||
entered.send(()).unwrap();
|
||||
std::thread::sleep(Duration::from_millis(450));
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
.unwrap();
|
||||
ready.await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
executor.finish(&child, owner, true).await.unwrap();
|
||||
drop(descendant);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let config = job::Config {
|
||||
lease_ms: 300,
|
||||
..Default::default()
|
||||
};
|
||||
heartbeat::run(scope, registry, (0, None), &config, &claim, handler)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(finished.load(Ordering::SeqCst));
|
||||
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1);
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state='completed' AND attempt=1"
|
||||
)
|
||||
.await,
|
||||
1
|
||||
);
|
||||
let mut executor = case.executor(scope).await;
|
||||
assert!(
|
||||
job::claim(
|
||||
&mut executor,
|
||||
scope,
|
||||
crate::database::now().unwrap() + 1000,
|
||||
300
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async fn claimed(case: &Case, scope: &Scope<'_>, key: &str, lease: i64) -> Claim {
|
||||
let mut executor = case.executor(scope).await;
|
||||
let now = crate::database::now().unwrap();
|
||||
let mut request = submission(key);
|
||||
request.policy.timeout_ms = 3000;
|
||||
job::enqueue(&mut executor, scope, request, now)
|
||||
.await
|
||||
.unwrap();
|
||||
job::claim(&mut executor, scope, now, lease)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn heartbeat_keeps_business_transaction_alive_across_multiple_leases() {
|
||||
fixture(async |case, scope| {
|
||||
let registry = case.registry(scope).await;
|
||||
let mut executor = case.executor(scope).await;
|
||||
executor
|
||||
.initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)")
|
||||
.await
|
||||
.unwrap();
|
||||
let claim = Arc::new(claimed(case, scope, "long-transaction", 200).await);
|
||||
let value = claim.clone();
|
||||
let owned = case.clone();
|
||||
let handler = scope
|
||||
.spawn(move |child| async move {
|
||||
let mut executor = owned.executor(&child).await;
|
||||
let owner = executor.begin(&child).await.unwrap();
|
||||
executor
|
||||
.execute(&child, "INSERT INTO business VALUES (1)", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
executor
|
||||
.postgres_operation(async |client| {
|
||||
client
|
||||
.query_one("SELECT pg_sleep(0.7)", &[])
|
||||
.await
|
||||
.map_err(Error::from)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
job::complete(
|
||||
&mut executor,
|
||||
&child,
|
||||
&value,
|
||||
crate::database::now().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
executor.finish(&child, owner, true).await.unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let config = job::Config {
|
||||
lease_ms: 200,
|
||||
..Default::default()
|
||||
};
|
||||
let (result, ()) = tokio::join!(
|
||||
heartbeat::run(scope, registry, (0, None), &config, &claim, handler),
|
||||
async {
|
||||
tokio::time::sleep(Duration::from_millis(450)).await;
|
||||
assert!(
|
||||
job::claim(&mut executor, scope, crate::database::now().unwrap(), 200)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
},
|
||||
);
|
||||
result.unwrap();
|
||||
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1);
|
||||
assert_eq!(case.count(scope, "SELECT attempt FROM _dever_job").await, 1);
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state='completed'"
|
||||
)
|
||||
.await,
|
||||
1
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn lost_fence_cancels_long_handler_and_joins_children_before_returning() {
|
||||
fixture(async |case, scope| {
|
||||
let registry = case.registry(scope).await;
|
||||
let mut executor = case.executor(scope).await;
|
||||
executor
|
||||
.initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)")
|
||||
.await
|
||||
.unwrap();
|
||||
let claim = Arc::new(claimed(case, scope, "lost-fence", 300).await);
|
||||
let finished = Arc::new(AtomicBool::new(false));
|
||||
let observed = finished.clone();
|
||||
let (entered, ready) = tokio::sync::oneshot::channel();
|
||||
let owned = case.clone();
|
||||
let handler = scope
|
||||
.spawn(move |child| async move {
|
||||
let mut executor = owned.executor(&child).await;
|
||||
executor
|
||||
.execute(&child, "INSERT INTO business VALUES (1)", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
let owner = executor.begin(&child).await.unwrap();
|
||||
executor
|
||||
.execute(&child, "INSERT INTO business VALUES (2)", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
let descendant = child
|
||||
.spawn(move |_| async move {
|
||||
let _finished = Finished(observed);
|
||||
entered.send(()).unwrap();
|
||||
pending::<()>().await;
|
||||
})
|
||||
.unwrap();
|
||||
descendant.wait().await.map_err(|_| Failure::Handler)?;
|
||||
executor.finish(&child, owner, true).await.unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let config = job::Config {
|
||||
lease_ms: 300,
|
||||
..Default::default()
|
||||
};
|
||||
let (result, ()) = tokio::join!(
|
||||
heartbeat::run(scope, registry, (0, None), &config, &claim, handler),
|
||||
async {
|
||||
ready.await.unwrap();
|
||||
executor
|
||||
.execute(scope, "UPDATE _dever_job SET token='other-worker'", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
},
|
||||
);
|
||||
assert!(matches!(result, Err(Failure::LeaseLost)));
|
||||
assert!(finished.load(Ordering::SeqCst));
|
||||
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1);
|
||||
assert_eq!(case.count(scope, "SELECT id FROM business").await, 1);
|
||||
assert!(
|
||||
job::complete(
|
||||
&mut executor,
|
||||
scope,
|
||||
&claim,
|
||||
crate::database::now().unwrap()
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(case.count(scope, "SELECT attempt FROM _dever_job").await, 1);
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE token='other-worker' AND state='running'"
|
||||
)
|
||||
.await,
|
||||
1
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn renewal_checks_expiry_after_waiting_for_row_lock() {
|
||||
fixture(async |case, scope| {
|
||||
let claim = claimed(case, scope, "row-wait", 100).await;
|
||||
let mut owner = case.executor(scope).await;
|
||||
let mut waiting = case.executor(scope).await;
|
||||
let transaction = owner.begin(scope).await.unwrap();
|
||||
owner
|
||||
.query(
|
||||
scope,
|
||||
"SELECT id FROM _dever_job WHERE id=$1 FOR UPDATE",
|
||||
vec![Parameter::Text(claim.id.clone())],
|
||||
1,
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let before = crate::database::now().unwrap();
|
||||
let (result, ()) = tokio::join!(
|
||||
queue::renew(&mut waiting, scope, &claim, before, 1000),
|
||||
async {
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
owner.finish(scope, transaction, true).await.unwrap();
|
||||
},
|
||||
);
|
||||
assert!(result.is_err());
|
||||
assert_eq!(
|
||||
case.count(scope, "SELECT lease_until FROM _dever_job")
|
||||
.await,
|
||||
claim.lease_until
|
||||
);
|
||||
let replacement = job::claim(&mut waiting, scope, crate::database::now().unwrap(), 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_ne!(replacement.token, claim.token);
|
||||
assert!(
|
||||
queue::renew(
|
||||
&mut waiting,
|
||||
scope,
|
||||
&claim,
|
||||
crate::database::now().unwrap(),
|
||||
1000
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
job::complete(
|
||||
&mut waiting,
|
||||
scope,
|
||||
&replacement,
|
||||
crate::database::now().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
|
||||
async fn locked_claim(case: &Case, scope: &Scope<'_>) -> (Executor, Executor, Claim, bool) {
|
||||
let mut owner = case.executor(scope).await;
|
||||
let waiting = case.executor(scope).await;
|
||||
job::enqueue(&mut owner, scope, submission("locked"), NOW)
|
||||
.await
|
||||
.unwrap();
|
||||
let claim = job::claim(&mut owner, scope, NOW, 100)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let transaction = owner.begin(scope).await.unwrap();
|
||||
owner
|
||||
.query(
|
||||
scope,
|
||||
"SELECT id FROM _dever_job WHERE id=$1 FOR UPDATE",
|
||||
vec![Parameter::Text(claim.id.clone())],
|
||||
1,
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
(owner, waiting, claim, transaction)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn lease_expiration_after_row_lock_wait_rolls_back_business_writes() {
|
||||
fixture(async |case, scope| {
|
||||
let (mut owner, mut waiting, claim, transaction) = locked_claim(case, scope).await;
|
||||
waiting
|
||||
.initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)")
|
||||
.await
|
||||
.unwrap();
|
||||
let business = waiting.begin(scope).await.unwrap();
|
||||
waiting
|
||||
.execute(scope, "INSERT INTO business VALUES (1)", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
let (result, ()) =
|
||||
tokio::join!(job::complete(&mut waiting, scope, &claim, NOW + 1), async {
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
owner.finish(scope, transaction, true).await.unwrap();
|
||||
},);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"lease checked before the row lock was acquired"
|
||||
);
|
||||
assert!(waiting.finish(scope, business, true).await.is_err());
|
||||
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 0);
|
||||
let retry = job::claim(&mut waiting, scope, NOW + 1000, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_ne!(retry.token, claim.token);
|
||||
assert!(
|
||||
job::complete(&mut waiting, scope, &claim, NOW + 1001)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
job::complete(&mut waiting, scope, &retry, NOW + 1001)
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn claim_clock_includes_table_lock_wait() {
|
||||
fixture(async |case, scope| {
|
||||
let mut owner = case.executor(scope).await;
|
||||
let mut waiting = case.executor(scope).await;
|
||||
job::enqueue(&mut owner, scope, submission("wait"), NOW)
|
||||
.await
|
||||
.unwrap();
|
||||
let transaction = owner.begin(scope).await.unwrap();
|
||||
owner
|
||||
.initialize(scope, "LOCK TABLE _dever_job IN ACCESS EXCLUSIVE MODE")
|
||||
.await
|
||||
.unwrap();
|
||||
let (result, ()) = tokio::join!(job::claim(&mut waiting, scope, NOW, 100), async {
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
owner.finish(scope, transaction, true).await.unwrap();
|
||||
},);
|
||||
assert!(result.unwrap().unwrap().lease_until >= NOW + 300);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn cancellation_discards_waiting_transaction_and_preserves_recovery() {
|
||||
fixture(async |case, scope| {
|
||||
let (mut owner, mut waiting, claim, transaction) = locked_claim(case, scope).await;
|
||||
let claim = Arc::new(claim);
|
||||
let copy = claim.clone();
|
||||
let (started, ready) = tokio::sync::oneshot::channel();
|
||||
let task = scope
|
||||
.spawn(move |child| async move {
|
||||
started.send(()).unwrap();
|
||||
job::complete(&mut waiting, &child, ©, NOW + 1).await
|
||||
})
|
||||
.unwrap();
|
||||
ready.await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert!(task.stop().await.unwrap().is_none());
|
||||
owner.finish(scope, transaction, true).await.unwrap();
|
||||
let mut replacement = case.executor(scope).await;
|
||||
let next = job::claim(&mut replacement, scope, NOW + 1000, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(next.id, claim.id);
|
||||
assert_ne!(next.token, claim.token);
|
||||
job::complete(&mut replacement, scope, &next, NOW + 1001)
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
use crate::{
|
||||
adapter::Document,
|
||||
database::{Binding, Database, Error, Executor, Options, Parameter, Registry},
|
||||
job::{self, Claim, Execution, Policy, Submission},
|
||||
task::{Runtime, Scope},
|
||||
wire::json::Json,
|
||||
};
|
||||
use futures_util::FutureExt;
|
||||
use std::{
|
||||
panic::AssertUnwindSafe,
|
||||
path::PathBuf,
|
||||
sync::{Arc, LazyLock},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
mod heartbeat;
|
||||
mod lease;
|
||||
mod queue;
|
||||
mod schedule;
|
||||
mod workers;
|
||||
|
||||
const NOW: i64 = 1_700_000_000_000;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Case {
|
||||
database: Database,
|
||||
schema: String,
|
||||
directory: PathBuf,
|
||||
}
|
||||
|
||||
impl Case {
|
||||
async fn registry(&self, scope: &Scope<'_>) -> Arc<Registry> {
|
||||
Registry::load(
|
||||
&self.directory.join("setting.json"),
|
||||
&[Binding {
|
||||
storage: crate::database::Storage::Global,
|
||||
component: "demo",
|
||||
database: Some("default"),
|
||||
table: job::schema::TABLE,
|
||||
schema: job::schema::SQL,
|
||||
plan: None,
|
||||
references: &[],
|
||||
}],
|
||||
scope,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn executor(&self, scope: &Scope<'_>) -> Executor {
|
||||
let mut executor = self.database.executor().await.unwrap();
|
||||
executor
|
||||
.query(
|
||||
scope,
|
||||
"SELECT set_config('search_path',$1,false)",
|
||||
vec![Parameter::Text(self.schema.clone())],
|
||||
1,
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
executor
|
||||
}
|
||||
|
||||
async fn count(&self, scope: &Scope<'_>, sql: &'static str) -> i64 {
|
||||
self.executor(scope)
|
||||
.await
|
||||
.query(scope, sql, vec![], 1, |row| row.get::<_, i64>(0))
|
||||
.await
|
||||
.unwrap()[0]
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture(check: impl AsyncFnOnce(&Case, &Scope<'_>)) {
|
||||
let path =
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../config/setting.json");
|
||||
let document = Document::load(&path).expect("cannot read PostgreSQL test settings");
|
||||
let root: serde_json::Value = document.decode().expect("invalid PostgreSQL test settings");
|
||||
let value = root
|
||||
.get("database")
|
||||
.and_then(|database| database.get("postgres_test"))
|
||||
.expect("database.postgres_test must select an isolated test database");
|
||||
let mut options: Options = serde_json::from_value(value.clone())
|
||||
.unwrap_or_else(|_| panic!("invalid PostgreSQL test options"));
|
||||
assert_eq!(options.driver, "postgres");
|
||||
options.max_connections = 8;
|
||||
options.wait_timeout_ms = 1000;
|
||||
options.io_timeout_ms = 5000;
|
||||
Runtime::new(Default::default())
|
||||
.unwrap()
|
||||
.run(async |scope| {
|
||||
let database = Database::open(std::path::Path::new("."), options, scope)
|
||||
.await
|
||||
.unwrap();
|
||||
let schema = format!("dever_job_test_{}", job::identifier().unwrap());
|
||||
let directory = std::env::temp_dir().join(&schema);
|
||||
std::fs::create_dir(&directory).unwrap();
|
||||
let case = Case {
|
||||
database,
|
||||
schema,
|
||||
directory,
|
||||
};
|
||||
let mut owner = case.database.executor().await.unwrap();
|
||||
owner
|
||||
.postgres_operation(async |client| {
|
||||
client
|
||||
.batch_execute(&format!("CREATE SCHEMA {}", case.schema))
|
||||
.await
|
||||
.map_err(Error::from)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let outcome = AssertUnwindSafe(async {
|
||||
use std::{
|
||||
io::Write,
|
||||
os::unix::fs::{OpenOptionsExt, PermissionsExt},
|
||||
};
|
||||
std::fs::set_permissions(&case.directory, std::fs::Permissions::from_mode(0o700))
|
||||
.unwrap();
|
||||
let mut connection = value.clone();
|
||||
connection["max_connections"] = 8.into();
|
||||
let url = connection["url"].as_str().unwrap();
|
||||
assert!(
|
||||
url.parse::<tokio_postgres::Config>()
|
||||
.unwrap()
|
||||
.get_options()
|
||||
.is_none()
|
||||
);
|
||||
let separator = if url.contains('?') { '&' } else { '?' };
|
||||
connection["url"] =
|
||||
format!("{url}{separator}options=-csearch_path%3D{}", case.schema).into();
|
||||
let mut settings = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.open(case.directory.join("setting.json"))
|
||||
.unwrap();
|
||||
settings
|
||||
.write_all(
|
||||
&serde_json::to_vec(
|
||||
&serde_json::json!({"database":{"default":connection}}),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let mut executor = case.executor(scope).await;
|
||||
executor
|
||||
.schema(scope, job::schema::TABLE, job::schema::SQL)
|
||||
.await
|
||||
.unwrap();
|
||||
executor
|
||||
.schema(scope, job::schema::TABLE, job::schema::SQL)
|
||||
.await
|
||||
.unwrap();
|
||||
drop(executor);
|
||||
tokio::time::timeout(Duration::from_secs(30), check(&case, scope))
|
||||
.await
|
||||
.expect("PostgreSQL Job test deadline exceeded");
|
||||
})
|
||||
.catch_unwind()
|
||||
.await;
|
||||
let cleanup = owner
|
||||
.postgres_operation(async |client| {
|
||||
client
|
||||
.batch_execute(&format!("DROP SCHEMA {} CASCADE", case.schema))
|
||||
.await
|
||||
.map_err(Error::from)
|
||||
})
|
||||
.await;
|
||||
case.database.close();
|
||||
std::fs::remove_dir_all(&case.directory).unwrap();
|
||||
cleanup.expect("PostgreSQL Job test schema cleanup failed");
|
||||
if let Err(panic) = outcome {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn submission(key: &str) -> Submission<'_> {
|
||||
static PAYLOAD: LazyLock<Json> = LazyLock::new(|| Json::parse(r#"{"number":1}"#).unwrap());
|
||||
Submission {
|
||||
target: "demo.account.publish",
|
||||
payload: Some(&PAYLOAD),
|
||||
schema: "schema-1",
|
||||
execution: &Execution::System { tenant: None },
|
||||
key,
|
||||
policy: Policy {
|
||||
retries: 2,
|
||||
timeout_ms: 50,
|
||||
},
|
||||
at: NOW,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn identity_idempotency_retry_and_terminal_keys() {
|
||||
fixture(async |case, scope| {
|
||||
let mut executor = case.executor(scope).await;
|
||||
let identity = Execution::User {
|
||||
provider: "session".into(),
|
||||
subject: "user:7".into(),
|
||||
session: "test-session".into(),
|
||||
site: "admin".into(),
|
||||
tenant: None,
|
||||
tenant_id: None,
|
||||
permission: "demo.account.admin.publish".into(),
|
||||
};
|
||||
let mut first = submission("same");
|
||||
first.execution = &identity;
|
||||
let id = job::enqueue(&mut executor, scope, first, NOW)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_text();
|
||||
let mut duplicate = submission("same");
|
||||
let payload = Json::parse(r#"{"number":999}"#).unwrap();
|
||||
duplicate.payload = Some(&payload);
|
||||
assert_eq!(
|
||||
job::enqueue(&mut executor, scope, duplicate, NOW)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_text(),
|
||||
id
|
||||
);
|
||||
assert!(
|
||||
job::claim(&mut executor, scope, NOW - 1000, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let claim = job::claim(&mut executor, scope, NOW, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(claim.id, id);
|
||||
assert_eq!(claim.payload, r#"{"number":1}"#);
|
||||
assert_eq!(claim.schema, "schema-1");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<serde_json::Value>(&claim.execution).unwrap(),
|
||||
serde_json::to_value(&identity).unwrap()
|
||||
);
|
||||
let config = job::Config::default();
|
||||
job::fail(&mut executor, scope, &claim, NOW + 1, &config)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
job::claim(&mut executor, scope, NOW + 2, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
let retry = job::claim(&mut executor, scope, NOW + 1 + config.delay(1) + 1000, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(retry.id, id);
|
||||
assert_eq!(retry.attempt, 2);
|
||||
assert_ne!(retry.token, claim.token);
|
||||
job::fail(
|
||||
&mut executor,
|
||||
scope,
|
||||
&retry,
|
||||
retry.lease_until - 500,
|
||||
&config,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
job::claim(&mut executor, scope, NOW + 1_000_000, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
case.count(scope, "SELECT COUNT(*) FROM _dever_job WHERE state='dead'")
|
||||
.await,
|
||||
1
|
||||
);
|
||||
let replacement = job::enqueue(&mut executor, scope, submission("same"), NOW)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_text();
|
||||
assert_ne!(replacement, id);
|
||||
let owner = executor.begin(scope).await.unwrap();
|
||||
job::enqueue(&mut executor, scope, submission("rollback"), NOW)
|
||||
.await
|
||||
.unwrap();
|
||||
executor.finish(scope, owner, false).await.unwrap();
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE task_key='rollback'"
|
||||
)
|
||||
.await,
|
||||
0
|
||||
);
|
||||
assert!(Json::parse("{").is_err());
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE task_key='invalid'"
|
||||
)
|
||||
.await,
|
||||
0
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn concurrent_enqueue_claim_and_skip_locked_keep_single_ownership() {
|
||||
fixture(async |case, scope| {
|
||||
let mut left = case.executor(scope).await;
|
||||
let mut right = case.executor(scope).await;
|
||||
let (left_id, right_id) = tokio::join!(
|
||||
job::enqueue(&mut left, scope, submission("same"), NOW),
|
||||
job::enqueue(&mut right, scope, submission("same"), NOW),
|
||||
);
|
||||
assert_eq!(left_id.unwrap().into_text(), right_id.unwrap().into_text());
|
||||
let (first, second) = tokio::join!(
|
||||
job::claim(&mut left, scope, NOW, 10_000),
|
||||
job::claim(&mut right, scope, NOW, 10_000),
|
||||
);
|
||||
assert_eq!(
|
||||
usize::from(first.unwrap().is_some()) + usize::from(second.unwrap().is_some()),
|
||||
1
|
||||
);
|
||||
let locked = job::enqueue(&mut left, scope, submission("locked"), NOW)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_text();
|
||||
let free = job::enqueue(&mut right, scope, submission("free"), NOW)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_text();
|
||||
let owner = left.begin(scope).await.unwrap();
|
||||
left.query(
|
||||
scope,
|
||||
"SELECT id FROM _dever_job WHERE id=$1 FOR UPDATE",
|
||||
vec![Parameter::Text(locked)],
|
||||
1,
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let claim = tokio::time::timeout(
|
||||
Duration::from_millis(500),
|
||||
job::claim(&mut right, scope, NOW, 10_000),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(claim.id, free);
|
||||
left.finish(scope, owner, true).await.unwrap();
|
||||
job::block(&mut right, scope, &claim, NOW + 1)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state='blocked'"
|
||||
)
|
||||
.await,
|
||||
1
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn expired_claims_recover_or_exhaust_and_keep_token_fencing() {
|
||||
fixture(async |case, scope| {
|
||||
let mut executor = case.executor(scope).await;
|
||||
for key in ["first", "second", "third"] {
|
||||
job::enqueue(&mut executor, scope, submission(key), NOW)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let mut originals = Vec::new();
|
||||
for _ in 0..3 {
|
||||
originals.push(
|
||||
job::claim(&mut executor, scope, NOW, 100)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
let recovered = job::claim(&mut executor, scope, NOW + 1000, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(recovered.attempt, 2);
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state='pending'"
|
||||
)
|
||||
.await,
|
||||
2
|
||||
);
|
||||
for original in &originals {
|
||||
assert!(
|
||||
job::complete(&mut executor, scope, original, NOW + 1001)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
job::complete(&mut executor, scope, &recovered, NOW + 1001)
|
||||
.await
|
||||
.unwrap();
|
||||
let exhausted = job::claim(&mut executor, scope, NOW + 1001, 100)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let next = job::claim(&mut executor, scope, NOW + 2000, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_ne!(next.id, exhausted.id);
|
||||
assert_eq!(
|
||||
case.count(scope, "SELECT COUNT(*) FROM _dever_job WHERE state='dead'")
|
||||
.await,
|
||||
1
|
||||
);
|
||||
job::complete(&mut executor, scope, &next, NOW + 2001)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
job::claim(&mut executor, scope, NOW + 10_000, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn expired_cleanup_is_bounded_and_workers_skip_each_others_locks() {
|
||||
fixture(async |case, scope| {
|
||||
let mut left = case.executor(scope).await;
|
||||
left.initialize(scope, "INSERT INTO _dever_job(id,target,payload,schema,execution,task_key,state,attempt,retries,timeout_ms,available_at,created_at,token,lease_until,execution_tenant) SELECT lpad(number::text,6,'0'),'demo.account.publish','null','zero','{\"kind\":\"System\",\"tenant\":null}',number::text,'running',1,2,50,0,0,'old',0,0 FROM generate_series(0,599) AS number").await.unwrap();
|
||||
let mut right = case.executor(scope).await;
|
||||
let owner = left.begin(scope).await.unwrap();
|
||||
let first = job::claim(&mut left, scope, NOW, 10_000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let second = tokio::time::timeout(
|
||||
Duration::from_millis(500),
|
||||
job::claim(&mut right, scope, NOW, 10_000),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(first.id, "000000");
|
||||
assert_eq!(second.id, "000257");
|
||||
left.finish(scope, owner, true).await.unwrap();
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state='pending'"
|
||||
)
|
||||
.await,
|
||||
512
|
||||
);
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state='running' AND token='old'"
|
||||
)
|
||||
.await,
|
||||
86
|
||||
);
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state='running' AND token<>'old'"
|
||||
)
|
||||
.await,
|
||||
2
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn exhausted_cleanup_does_not_hide_eligible_expired_work() {
|
||||
fixture(async |case, scope| {
|
||||
let mut executor = case.executor(scope).await;
|
||||
executor.initialize(scope, "INSERT INTO _dever_job(id,target,payload,schema,execution,task_key,state,attempt,retries,timeout_ms,available_at,created_at,token,lease_until,execution_tenant) SELECT lpad(number::text,6,'0'),'demo.account.publish','null','zero','{\"kind\":\"System\",\"tenant\":null}',number::text,'running',1,CASE WHEN number=300 THEN 2 ELSE 1 END,50,0,0,'old',0,0 FROM generate_series(0,300) AS number").await.unwrap();
|
||||
let claim = job::claim(&mut executor, scope, NOW, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(claim.id, "000300");
|
||||
assert_eq!(claim.attempt, 2);
|
||||
assert_eq!(
|
||||
case.count(scope, "SELECT COUNT(*) FROM _dever_job WHERE state='dead'")
|
||||
.await,
|
||||
256
|
||||
);
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state='running' AND token='old'"
|
||||
)
|
||||
.await,
|
||||
44
|
||||
);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn concurrent_cron_deduplicates_and_bounds_catchup() {
|
||||
fixture(async |case, scope| {
|
||||
let mut left = case.executor(scope).await;
|
||||
let mut right = case.executor(scope).await;
|
||||
let entry = job::Schedule {
|
||||
binding: 0,
|
||||
target: "demo.account.hourly",
|
||||
schema: "zero",
|
||||
cron: "* * * * *",
|
||||
policy: Policy {
|
||||
retries: 1,
|
||||
timeout_ms: 50,
|
||||
},
|
||||
};
|
||||
let (first, second) = tokio::join!(
|
||||
job::tick(&mut left, scope, &entry, None, NOW),
|
||||
job::tick(&mut right, scope, &entry, None, NOW)
|
||||
);
|
||||
assert_eq!(first.unwrap() + second.unwrap(), 1);
|
||||
let claim = job::claim(&mut left, scope, NOW, 1000)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(claim.payload, "null");
|
||||
assert!(matches!(
|
||||
serde_json::from_str::<Execution>(&claim.execution).unwrap(),
|
||||
Execution::System { tenant: None }
|
||||
));
|
||||
job::complete(&mut left, scope, &claim, NOW + 1)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
job::tick(&mut right, scope, &entry, None, NOW)
|
||||
.await
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
job::tick(&mut left, scope, &entry, None, NOW + 60000 * 300)
|
||||
.await
|
||||
.unwrap(),
|
||||
256
|
||||
);
|
||||
assert_eq!(
|
||||
job::tick(&mut right, scope, &entry, None, NOW + 60000 * 300)
|
||||
.await
|
||||
.unwrap(),
|
||||
44
|
||||
);
|
||||
assert_eq!(
|
||||
job::tick(&mut left, scope, &entry, None, 0).await.unwrap(),
|
||||
0
|
||||
);
|
||||
let changed = job::Schedule {
|
||||
cron: "*/2 * * * *",
|
||||
..entry
|
||||
};
|
||||
assert!(
|
||||
job::tick(&mut right, scope, &changed, None, NOW)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
case.count(scope, "SELECT COUNT(*) FROM _dever_job_occurrence")
|
||||
.await,
|
||||
301
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn failed_scheduled_enqueue_rolls_back_cursor_and_occurrence() {
|
||||
fixture(async |case, scope| {
|
||||
let mut executor = case.executor(scope).await;
|
||||
executor.initialize(scope, "CREATE FUNCTION reject_scheduled_job() RETURNS trigger LANGUAGE plpgsql AS 'BEGIN RAISE EXCEPTION ''test-only failure''; END'; CREATE TRIGGER reject_scheduled_job BEFORE INSERT ON _dever_job FOR EACH ROW EXECUTE FUNCTION reject_scheduled_job();").await.unwrap();
|
||||
let entry = job::Schedule {
|
||||
binding: 0,
|
||||
target: "demo.account.hourly",
|
||||
schema: "zero",
|
||||
cron: "* * * * *",
|
||||
policy: Policy {
|
||||
retries: 1,
|
||||
timeout_ms: 50,
|
||||
},
|
||||
};
|
||||
assert!(
|
||||
job::tick(&mut executor, scope, &entry, None, NOW)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
case.count(scope, "SELECT COUNT(*) FROM _dever_job_schedule")
|
||||
.await,
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
case.count(scope, "SELECT COUNT(*) FROM _dever_job_occurrence")
|
||||
.await,
|
||||
0
|
||||
);
|
||||
executor
|
||||
.initialize(scope, "DROP TRIGGER reject_scheduled_job ON _dever_job")
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
job::tick(&mut executor, scope, &entry, None, NOW)
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
use super::*;
|
||||
use std::{
|
||||
future::pending,
|
||||
sync::atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
struct Finished(Arc<AtomicBool>);
|
||||
impl Drop for Finished {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
fn config() -> job::Config {
|
||||
job::Config {
|
||||
workers: 2,
|
||||
poll_ms: 10,
|
||||
lease_ms: 300,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn enqueue(case: &Case, scope: &Scope<'_>) -> Policy {
|
||||
let mut executor = case.executor(scope).await;
|
||||
executor
|
||||
.initialize(scope, "CREATE TABLE business(id BIGINT PRIMARY KEY)")
|
||||
.await
|
||||
.unwrap();
|
||||
let mut value = submission("worker");
|
||||
value.policy.timeout_ms = 3000;
|
||||
let policy = value.policy;
|
||||
job::enqueue(&mut executor, scope, value, crate::database::now().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
policy
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn short_lease_workers_renew_and_complete_once_without_replaying_handler() {
|
||||
fixture(async |case, scope| {
|
||||
let registry = case.registry(scope).await;
|
||||
let policy = enqueue(case, scope).await;
|
||||
let completed = Arc::new(tokio::sync::Notify::new());
|
||||
let signal = completed.clone();
|
||||
let owned = case.clone();
|
||||
let workers = job::start(
|
||||
scope,
|
||||
registry,
|
||||
config(),
|
||||
&[job::Entry { binding: 0, policy }],
|
||||
vec![],
|
||||
move |_, _, claim, child| {
|
||||
let (case, signal) = (owned.clone(), signal.clone());
|
||||
async move {
|
||||
let mut executor = case.executor(&child).await;
|
||||
let owner = executor.begin(&child).await.unwrap();
|
||||
executor
|
||||
.execute(&child, "INSERT INTO business VALUES (1)", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
executor
|
||||
.postgres_operation(async |client| {
|
||||
client
|
||||
.query_one("SELECT pg_sleep(0.8)", &[])
|
||||
.await
|
||||
.map_err(Error::from)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
job::complete(
|
||||
&mut executor,
|
||||
&child,
|
||||
&claim,
|
||||
crate::database::now().unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
executor.finish(&child, owner, true).await.unwrap();
|
||||
signal.notify_one();
|
||||
Ok(())
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(3), completed.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
workers.stop.cancel();
|
||||
workers.task.wait().await.unwrap().unwrap();
|
||||
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 1);
|
||||
assert_eq!(case.count(scope, "SELECT attempt FROM _dever_job").await, 1);
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state='completed'"
|
||||
)
|
||||
.await,
|
||||
1
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn failed_renewal_cancels_transaction_and_preserves_claim_for_expiry() {
|
||||
fixture(async |case, scope| {
|
||||
let registry = case.registry(scope).await;
|
||||
let policy = enqueue(case, scope).await;
|
||||
case.executor(scope).await.initialize(scope, "CREATE FUNCTION reject_renewal() RETURNS trigger LANGUAGE plpgsql AS 'BEGIN IF OLD.state=''running'' AND NEW.state=''running'' AND OLD.token=NEW.token AND OLD.lease_until<>NEW.lease_until THEN RAISE EXCEPTION ''test renewal failure''; END IF; RETURN NEW; END'; CREATE TRIGGER reject_renewal BEFORE UPDATE ON _dever_job FOR EACH ROW EXECUTE FUNCTION reject_renewal()").await.unwrap();
|
||||
let finished = Arc::new(AtomicBool::new(false));
|
||||
let observed = finished.clone();
|
||||
let owned = case.clone();
|
||||
let workers = job::start(
|
||||
scope,
|
||||
registry,
|
||||
config(),
|
||||
&[job::Entry { binding: 0, policy }],
|
||||
vec![],
|
||||
move |_, _, _, child| {
|
||||
let (case, observed) = (owned.clone(), observed.clone());
|
||||
async move {
|
||||
let _finished = Finished(observed);
|
||||
let mut executor = case.executor(&child).await;
|
||||
let owner = executor.begin(&child).await.unwrap();
|
||||
executor
|
||||
.execute(&child, "INSERT INTO business VALUES (1)", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
pending::<()>().await;
|
||||
executor.finish(&child, owner, true).await.unwrap();
|
||||
Ok(())
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while !finished.load(Ordering::SeqCst) {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
workers.stop.cancel();
|
||||
workers.task.wait().await.unwrap().unwrap();
|
||||
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 0);
|
||||
assert_eq!(case.count(scope, "SELECT attempt FROM _dever_job").await, 1);
|
||||
assert_eq!(
|
||||
case.count(
|
||||
scope,
|
||||
"SELECT COUNT(*) FROM _dever_job WHERE state='running' AND error IS NULL"
|
||||
)
|
||||
.await,
|
||||
1
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires explicit isolated database.postgres_test"]
|
||||
fn cancelling_worker_drains_handler_and_pending_renewal_before_reclaim() {
|
||||
fixture(async |case, scope| {
|
||||
let registry = case.registry(scope).await;
|
||||
let policy = enqueue(case, scope).await;
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let ready = entered.clone();
|
||||
let finished = Arc::new(AtomicBool::new(false));
|
||||
let observed = finished.clone();
|
||||
let owned = case.clone();
|
||||
let workers = job::start(
|
||||
scope,
|
||||
registry,
|
||||
config(),
|
||||
&[job::Entry { binding: 0, policy }],
|
||||
vec![],
|
||||
move |_, _, _, child| {
|
||||
let (case, observed, entered) = (owned.clone(), observed.clone(), entered.clone());
|
||||
async move {
|
||||
let _finished = Finished(observed);
|
||||
let mut executor = case.executor(&child).await;
|
||||
let owner = executor.begin(&child).await.unwrap();
|
||||
executor
|
||||
.execute(&child, "INSERT INTO business VALUES (1)", vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
entered.notify_one();
|
||||
pending::<()>().await;
|
||||
executor.finish(&child, owner, true).await.unwrap();
|
||||
Ok(())
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
ready.notified().await;
|
||||
let mut owner = case.executor(scope).await;
|
||||
let transaction = owner.begin(scope).await.unwrap();
|
||||
let token = owner
|
||||
.query(
|
||||
scope,
|
||||
"SELECT token FROM _dever_job FOR UPDATE",
|
||||
vec![],
|
||||
1,
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.remove(0);
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
assert!(!finished.load(Ordering::SeqCst));
|
||||
assert!(!matches!(workers.task.stop().await.unwrap(), Some(Ok(()))));
|
||||
assert!(finished.load(Ordering::SeqCst));
|
||||
owner.finish(scope, transaction, true).await.unwrap();
|
||||
assert_eq!(case.count(scope, "SELECT COUNT(*) FROM business").await, 0);
|
||||
let next = job::claim(
|
||||
&mut owner,
|
||||
scope,
|
||||
crate::database::now().unwrap() + 1000,
|
||||
300,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_ne!(next.token, token);
|
||||
assert_eq!(next.attempt, 2);
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user