feat: publish Dever core with project scaffolding and versioned AI skills

This commit is contained in:
Antigravity committed 2026-10-04 13:37:30 +00:00
1 parent 70fa73cdc2
commit ac458ffa7f
960 files changed
+221319 -1163

No files matched your search

@@ -0,0 +1,227 @@
use std::time::Duration;
use crate::settings::Settings;
pub const HEADER_BYTES: usize = 8_192;
pub const BODY_BYTES: usize = 65_536;
pub const HTTP2_FRAME_BYTES: u32 = 16_384;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum HttpVersion {
H1,
H2,
}
impl HttpVersion {
pub fn parse(value: &str) -> Result<Self, String> {
match value {
"h1" => Ok(Self::H1),
"h2" => Ok(Self::H2),
_ => Err("HTTP_VERSION must be 'h1' or 'h2'".into()),
}
}
pub fn as_str(self) -> &'static str {
match self {
Self::H1 => "h1",
Self::H2 => "h2",
}
}
pub fn alpn(self) -> &'static [u8] {
match self {
Self::H1 => b"http/1.1",
Self::H2 => b"h2",
}
}
}
#[derive(Clone, Copy, Debug)]
pub struct ConnectionShape {
pub physical_connections: usize,
pub streams_per_connection: usize,
pub request_slots: usize,
}
impl ConnectionShape {
pub fn new(
http_version: HttpVersion,
physical_connections: usize,
streams_per_connection: usize,
) -> Result<Self, String> {
if !(1..=65_536).contains(&physical_connections) {
return Err("PHYSICAL_CONNECTIONS must be between 1 and 65536".into());
}
match http_version {
HttpVersion::H1 if streams_per_connection != 1 => {
return Err("HTTP/1 requires STREAMS_PER_CONNECTION to be 1".into());
}
HttpVersion::H2 if !(1..=65_536).contains(&streams_per_connection) => {
return Err(
"HTTP/2 requires STREAMS_PER_CONNECTION to be between 1 and 65536".into(),
);
}
_ => {}
}
let request_slots = physical_connections
.checked_mul(streams_per_connection)
.ok_or("connection shape is too large")?;
if request_slots > 65_536 {
return Err(
"PHYSICAL_CONNECTIONS * STREAMS_PER_CONNECTION must not exceed 65536".into(),
);
}
Ok(Self {
physical_connections,
streams_per_connection,
request_slots,
})
}
}
#[derive(Clone, Copy, Debug)]
pub struct Http2Config {
pub streams: usize,
pub stream_window_bytes: u32,
pub connection_window_bytes: u32,
}
impl Http2Config {
pub fn from_settings(settings: &Settings, streams: usize) -> Result<Self, String> {
Ok(Self {
streams,
stream_window_bytes: settings.integer(
"http2_stream_window_bytes",
65_535,
1,
i32::MAX as usize,
)? as u32,
connection_window_bytes: settings.integer(
"http2_connection_window_bytes",
262_144,
65_535,
i32::MAX as usize,
)? as u32,
})
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ServiceKind {
RuntimeHttp,
RuntimeHttps,
RuntimeHttp2,
RuntimeHttps2,
HyperHttp,
HyperHttps,
HyperHttp2,
HyperHttps2,
}
impl ServiceKind {
pub fn parse(value: &str) -> Result<Self, String> {
match value {
"runtime-http" => Ok(Self::RuntimeHttp),
"runtime-https" => Ok(Self::RuntimeHttps),
"runtime-http2" => Ok(Self::RuntimeHttp2),
"runtime-https2" => Ok(Self::RuntimeHttps2),
"hyper-http" => Ok(Self::HyperHttp),
"hyper-https" => Ok(Self::HyperHttps),
"hyper-http2" => Ok(Self::HyperHttp2),
"hyper-https2" => Ok(Self::HyperHttps2),
_ => Err(format!("unknown service mode '{value}'")),
}
}
pub fn uses_runtime(self) -> bool {
matches!(
self,
Self::RuntimeHttp | Self::RuntimeHttps | Self::RuntimeHttp2 | Self::RuntimeHttps2
)
}
pub fn uses_tls(self) -> bool {
matches!(
self,
Self::RuntimeHttps | Self::RuntimeHttps2 | Self::HyperHttps | Self::HyperHttps2
)
}
pub fn http_version(self) -> HttpVersion {
match self {
Self::RuntimeHttp | Self::RuntimeHttps | Self::HyperHttp | Self::HyperHttps => {
HttpVersion::H1
}
Self::RuntimeHttp2 | Self::RuntimeHttps2 | Self::HyperHttp2 | Self::HyperHttps2 => {
HttpVersion::H2
}
}
}
}
#[derive(Clone, Debug)]
pub struct ServiceConfig {
pub workers: usize,
pub connections: usize,
pub task_capacity: usize,
pub timeout: Duration,
pub lifetime: Duration,
pub cert_path: Option<String>,
pub key_path: Option<String>,
pub http2: Option<Http2Config>,
}
impl ServiceConfig {
pub fn from_settings(settings: &Settings, kind: ServiceKind) -> Result<Self, String> {
let workers = settings.workers()?;
let connections = settings.integer("connections", 32, 1, 65_536)?;
let default_tasks = connections
.saturating_mul(2)
.saturating_add(16)
.clamp(272, 65_536);
let task_capacity = settings.integer("task_capacity", default_tasks, 1, 65_536)?;
let timeout_ms = settings.integer("timeout_ms", 2_000, 1, 300_000)?;
let lifetime_ms = settings.integer("lifetime_ms", 10_000, 1, 86_400_000)?;
let (cert_path, key_path) = if kind.uses_tls() {
(
Some(required_text(settings, "cert_path")?),
Some(required_text(settings, "key_path")?),
)
} else {
(None, None)
};
let http2 = if kind.http_version() == HttpVersion::H2 {
let streams = settings.integer("http2_streams", 16, 1, 65_536)?;
let request_slots = connections
.checked_mul(streams)
.ok_or("benchmark.connections * benchmark.http2_streams is too large")?;
if request_slots > 65_536 {
return Err(
"benchmark.connections * benchmark.http2_streams must not exceed 65536".into(),
);
}
Some(Http2Config::from_settings(settings, streams)?)
} else {
None
};
Ok(Self {
workers,
connections,
task_capacity,
timeout: Duration::from_millis(timeout_ms as u64),
lifetime: Duration::from_millis(lifetime_ms as u64),
cert_path,
key_path,
http2,
})
}
}
fn required_text(settings: &Settings, name: &str) -> Result<String, String> {
settings
.value(name)
.and_then(serde_json::Value::as_str)
.filter(|value| !value.is_empty())
.map(str::to_owned)
.ok_or_else(|| format!("benchmark.{name} must be nonempty Text"))
}
@@ -0,0 +1,767 @@
use std::collections::VecDeque;
use std::sync::Arc;
use std::time::Duration;
use bytes::Bytes;
use http_body_util::{BodyExt, Empty};
use hyper::body::Incoming;
use hyper::client::conn::{http1, http2};
use hyper::header::{CONTENT_TYPE, HOST, HeaderValue};
use hyper::{Request, StatusCode, Uri, Version};
use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer};
use tokio::runtime::Builder;
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use tokio::time::Instant;
use crate::config::{ConnectionShape, HEADER_BYTES, HTTP2_FRAME_BYTES, Http2Config, HttpVersion};
use crate::metrics::Histogram;
use crate::{response, tls};
const MAX_RATE: u64 = 1_000_000;
const MAX_SECONDS: f64 = 3_600.0;
#[derive(Clone, Debug)]
pub struct Arguments {
http_version: HttpVersion,
target: Target,
path: String,
expected: response::ExpectedResponse,
rate: u64,
seconds: Duration,
shape: ConnectionShape,
timeout: Duration,
reuse: bool,
ca_path: String,
}
impl Arguments {
pub fn parse(values: Vec<String>) -> Result<Self, String> {
let [
http_version,
url,
path,
rate,
seconds,
physical_connections,
streams_per_connection,
timeout_ms,
reuse,
ca,
] = values.as_slice()
else {
return Err(crate::usage());
};
let http_version = HttpVersion::parse(http_version)?;
let target = Target::parse(url)?;
let orm_count = if path == response::ORM_COUNT_PATH {
let settings = crate::settings::Settings::load()?;
if settings.value("orm_count").is_none() {
return Err("benchmark.orm_count is required for ORM count requests".into());
}
Some(settings.integer("orm_count", 0, 1, usize::MAX)?)
} else {
None
};
let expected = response::ExpectedResponse::for_path(path, orm_count)?;
let rate = number(rate, "RATE", 1, MAX_RATE)?;
let seconds = decimal_duration(seconds, "SECONDS", MAX_SECONDS)?;
let physical_connections =
number(physical_connections, "PHYSICAL_CONNECTIONS", 1, 65_536)? as usize;
let streams_per_connection =
number(streams_per_connection, "STREAMS_PER_CONNECTION", 1, 65_536)? as usize;
let shape =
ConnectionShape::new(http_version, physical_connections, streams_per_connection)?;
let timeout_ms = number(timeout_ms, "TIMEOUT_MS", 1, 300_000)?;
let reuse = match reuse.as_str() {
"true" => true,
"false" => false,
_ => return Err("REUSE must be 'true' or 'false'".into()),
};
if http_version == HttpVersion::H2 && !reuse {
return Err("HTTP/2 requires REUSE to be 'true'".into());
}
if target.tls && ca == "-" {
return Err("CA must name a root PEM file for HTTPS".into());
}
Ok(Self {
http_version,
target,
path: path.clone(),
expected,
rate,
seconds,
shape,
timeout: Duration::from_millis(timeout_ms),
reuse,
ca_path: ca.clone(),
})
}
}
#[derive(Clone, Debug)]
struct Target {
tls: bool,
port: u16,
authority: String,
}
impl Target {
fn parse(value: &str) -> Result<Self, String> {
let uri = value
.parse::<Uri>()
.map_err(|error| format!("URL is invalid: {error}"))?;
let scheme = uri
.scheme_str()
.ok_or("URL must include http:// or https://")?;
let host = uri.host().ok_or("URL must include a host")?;
let tls = match scheme {
"http" => false,
"https" => true,
_ => return Err("URL scheme must be http or https".into()),
};
if !matches!(host, "127.0.0.1" | "localhost" | "::1") {
return Err("benchmark URL must use a loopback host".into());
}
if tls && host != "localhost" {
return Err(
"HTTPS benchmark URL must use localhost for certificate verification".into(),
);
}
if uri.path() != "/" || uri.query().is_some() {
return Err(
"URL must contain only the scheme, loopback host and port; pass PATH separately"
.into(),
);
}
let port = uri.port_u16().ok_or("URL must include an explicit port")?;
let authority = uri
.authority()
.ok_or("URL must include an authority")?
.to_string();
Ok(Self {
tls,
port,
authority,
})
}
}
pub fn run(arguments: Arguments) -> Result<(), String> {
let settings = crate::settings::Settings::load()?;
let workers = settings.workers()?;
let runtime = Builder::new_multi_thread()
.worker_threads(workers)
.max_blocking_threads(1)
.enable_io()
.enable_time()
.build()
.map_err(|error| format!("cannot start load runtime: {error}"))?;
runtime.block_on(execute(arguments, &settings))
}
async fn execute(arguments: Arguments, settings: &crate::settings::Settings) -> Result<(), String> {
let tls = arguments
.target
.tls
.then(|| tls::client(&arguments.ca_path, arguments.http_version))
.transpose()?;
let http2 = (arguments.http_version == HttpVersion::H2)
.then(|| Http2Config::from_settings(settings, arguments.shape.streams_per_connection))
.transpose()?;
let client = Arc::new(ClientConfig {
target: arguments.target.clone(),
path: arguments.path.clone(),
expected: arguments.expected,
timeout: arguments.timeout,
reuse: arguments.reuse,
tls,
http2,
});
let mut http2_connections = if arguments.http_version == HttpVersion::H2 {
Http2Connections::connect(&client, arguments.shape).await?
} else {
Http2Connections::default()
};
let event_capacity = arguments.shape.request_slots.saturating_mul(2).max(1);
let (events, mut event_receiver) = mpsc::channel(event_capacity);
let mut commands = Vec::with_capacity(arguments.shape.request_slots);
let mut workers = Vec::with_capacity(arguments.shape.request_slots);
for id in 0..arguments.shape.request_slots {
let (sender, receiver) = mpsc::channel(1);
commands.push(sender);
let connection = match arguments.http_version {
HttpVersion::H1 => WorkerConnection::Http1(None),
HttpVersion::H2 => {
let connection = id / arguments.shape.streams_per_connection;
WorkerConnection::Http2(http2_connections.sender(connection))
}
};
workers.push(tokio::spawn(worker(
id,
receiver,
events.clone(),
client.clone(),
connection,
)));
}
drop(events);
let mut ready = VecDeque::with_capacity(arguments.shape.request_slots);
while ready.len() < arguments.shape.request_slots {
let event = tokio::time::timeout(arguments.timeout, event_receiver.recv())
.await
.map_err(|_| "load workers did not become ready before TIMEOUT_MS".to_owned())?
.ok_or("load workers stopped before becoming ready")?;
match event {
Event::Ready(id) => ready.push_back(id),
Event::Complete(_) => {
return Err("load worker completed before receiving a request".into());
}
}
}
let scheduled_float = arguments.rate as f64 * arguments.seconds.as_secs_f64();
if !scheduled_float.is_finite() || scheduled_float > u64::MAX as f64 {
return Err("RATE * SECONDS is too large".into());
}
let scheduled = scheduled_float.ceil() as u64;
let start = Instant::now();
let end = start + arguments.seconds;
let mut result = Results::new(
scheduled,
arguments.http_version,
arguments.shape,
http2_connections.successful_handshakes(),
);
for sequence in 0..scheduled {
let planned = start + Duration::from_secs_f64(sequence as f64 / arguments.rate as f64);
loop {
tokio::select! {
event = event_receiver.recv() => {
let event = event.ok_or("all load workers stopped unexpectedly")?;
result.observe(event, &mut ready);
}
_ = tokio::time::sleep_until(planned) => break,
}
}
while let Ok(event) = event_receiver.try_recv() {
result.observe(event, &mut ready);
}
if Instant::now() >= end {
result.dropped += 1;
result.dispatch_expired += 1;
continue;
}
let Some(worker) = ready.pop_front() else {
result.dropped += 1;
result.dropped_capacity += 1;
continue;
};
let job = Job { planned };
if commands[worker].try_send(job).is_ok() {
result.sent += 1;
} else {
result.dropped += 1;
result.dropped_capacity += 1;
}
}
while Instant::now() < end {
tokio::select! {
event = event_receiver.recv() => {
let event = event.ok_or("all load workers stopped unexpectedly")?;
result.observe(event, &mut ready);
}
_ = tokio::time::sleep_until(end) => break,
}
}
let drain_deadline = end + arguments.timeout;
while result.completed < result.sent {
match tokio::time::timeout_at(drain_deadline, event_receiver.recv()).await {
Ok(Some(event)) => result.observe(event, &mut ready),
_ => break,
}
}
drop(commands);
let pending = result.sent - result.completed;
result.completed += pending;
result.errors += pending;
for worker in &workers {
if !worker.is_finished() {
worker.abort();
}
}
for worker in workers {
let _ = worker.await;
}
http2_connections.close().await;
result.elapsed = start.elapsed();
println!("{}", result.json());
Ok(())
}
struct Job {
planned: Instant,
}
struct Completion {
succeeded: bool,
successful_handshakes: u64,
latency_us: u64,
dispatch_lag_us: u64,
request_latency_us: u64,
}
enum Event {
Ready(usize),
Complete(Completion),
}
async fn worker(
id: usize,
mut jobs: mpsc::Receiver<Job>,
events: mpsc::Sender<Event>,
config: Arc<ClientConfig>,
mut connection: WorkerConnection,
) {
if events.send(Event::Ready(id)).await.is_err() {
return;
}
while let Some(job) = jobs.recv().await {
let dispatched = Instant::now();
let outcome = request(&config, &mut connection).await;
let completed = Instant::now();
let completion = Completion {
succeeded: outcome.result.is_ok(),
successful_handshakes: outcome.successful_handshakes,
latency_us: micros(completed.saturating_duration_since(job.planned)),
dispatch_lag_us: micros(dispatched.saturating_duration_since(job.planned)),
request_latency_us: micros(completed.saturating_duration_since(dispatched)),
};
if events.send(Event::Complete(completion)).await.is_err()
|| events.send(Event::Ready(id)).await.is_err()
{
break;
}
}
if let WorkerConnection::Http1(Some(connection)) = connection {
connection.close().await;
}
}
struct RequestOutcome {
result: Result<(), String>,
successful_handshakes: u64,
}
async fn request(config: &ClientConfig, connection: &mut WorkerConnection) -> RequestOutcome {
let mut successful_handshakes = 0;
let result = match tokio::time::timeout(config.timeout, async {
match connection {
WorkerConnection::Http1(connection) => {
if connection.is_none() {
*connection = Some(Http1Connection::connect(config).await?);
successful_handshakes = 1;
}
connection
.as_mut()
.expect("connection established")
.request(config)
.await
}
WorkerConnection::Http2(sender) => http2_request(sender, config).await,
}
})
.await
{
Ok(result) => result,
Err(_) => Err("request timed out".to_owned()),
};
if let WorkerConnection::Http1(connection) = connection
&& (result.is_err() || !config.reuse)
&& let Some(connection) = connection.take()
{
connection.close().await;
}
RequestOutcome {
result,
successful_handshakes,
}
}
struct ClientConfig {
target: Target,
path: String,
expected: response::ExpectedResponse,
timeout: Duration,
reuse: bool,
tls: Option<Arc<rustls::ClientConfig>>,
http2: Option<Http2Config>,
}
enum WorkerConnection {
Http1(Option<Http1Connection>),
Http2(http2::SendRequest<Empty<Bytes>>),
}
struct Http1Connection {
sender: http1::SendRequest<Empty<Bytes>>,
driver: Option<JoinHandle<Result<(), hyper::Error>>>,
}
impl Http1Connection {
async fn connect(config: &ClientConfig) -> Result<Self, String> {
let stream = tls::connect(
config.target.port,
config.timeout,
config.tls.as_ref(),
HttpVersion::H1,
)
.await?;
let (sender, connection) =
tokio::time::timeout(config.timeout, http1::handshake(TokioIo::new(stream)))
.await
.map_err(|_| "HTTP handshake timed out".to_owned())?
.map_err(|error| error.to_string())?;
Ok(Self {
sender,
driver: Some(tokio::spawn(connection)),
})
}
async fn request(&mut self, config: &ClientConfig) -> Result<(), String> {
self.sender
.ready()
.await
.map_err(|error| error.to_string())?;
let request = Request::builder()
.method("GET")
.uri(config.path.as_str())
.header(
HOST,
HeaderValue::from_str(&config.target.authority)
.map_err(|error| error.to_string())?,
)
.body(Empty::new())
.map_err(|error| error.to_string())?;
let response = self
.sender
.send_request(request)
.await
.map_err(|error| error.to_string())?;
validate_response(&config.expected, response).await
}
async fn close(mut self) {
if let Some(driver) = self.driver.take() {
driver.abort();
let _ = driver.await;
}
}
}
impl Drop for Http1Connection {
fn drop(&mut self) {
if let Some(driver) = self.driver.take() {
driver.abort();
}
}
}
#[derive(Default)]
struct Http2Connections {
connections: Vec<Http2Connection>,
}
impl Http2Connections {
async fn connect(config: &ClientConfig, shape: ConnectionShape) -> Result<Self, String> {
let mut connections = Vec::with_capacity(shape.physical_connections);
for _ in 0..shape.physical_connections {
match Http2Connection::connect(config).await {
Ok(connection) => connections.push(connection),
Err(error) => {
close_http2_connections(connections).await;
return Err(error);
}
}
}
Ok(Self { connections })
}
fn sender(&self, connection: usize) -> http2::SendRequest<Empty<Bytes>> {
self.connections[connection].sender.clone()
}
fn successful_handshakes(&self) -> u64 {
self.connections.len() as u64
}
async fn close(&mut self) {
close_http2_connections(std::mem::take(&mut self.connections)).await;
}
}
struct Http2Connection {
sender: http2::SendRequest<Empty<Bytes>>,
driver: Option<JoinHandle<Result<(), hyper::Error>>>,
}
impl Http2Connection {
async fn connect(config: &ClientConfig) -> Result<Self, String> {
let limits = config.http2.ok_or("HTTP/2 client has no HTTP/2 limits")?;
let stream = tls::connect(
config.target.port,
config.timeout,
config.tls.as_ref(),
HttpVersion::H2,
)
.await?;
let mut builder = http2::Builder::new(TokioExecutor::new());
builder
.timer(TokioTimer::new())
.adaptive_window(false)
.initial_stream_window_size(limits.stream_window_bytes)
.initial_connection_window_size(limits.connection_window_bytes)
.initial_max_send_streams(limits.streams)
.max_concurrent_streams(0)
.max_header_list_size(HEADER_BYTES as u32)
.max_frame_size(HTTP2_FRAME_BYTES)
.max_send_buf_size(HTTP2_FRAME_BYTES as usize);
let (sender, connection) =
tokio::time::timeout(config.timeout, builder.handshake(TokioIo::new(stream)))
.await
.map_err(|_| "HTTP/2 handshake timed out".to_owned())?
.map_err(|error| error.to_string())?;
Ok(Self {
sender,
driver: Some(tokio::spawn(connection)),
})
}
async fn close(mut self) {
if let Some(driver) = self.driver.take() {
driver.abort();
let _ = driver.await;
}
}
}
impl Drop for Http2Connection {
fn drop(&mut self) {
if let Some(driver) = self.driver.take() {
driver.abort();
}
}
}
async fn close_http2_connections(connections: Vec<Http2Connection>) {
for connection in connections {
connection.close().await;
}
}
async fn http2_request(
sender: &mut http2::SendRequest<Empty<Bytes>>,
config: &ClientConfig,
) -> Result<(), String> {
sender.ready().await.map_err(|error| error.to_string())?;
let scheme = if config.target.tls { "https" } else { "http" };
let uri = format!("{scheme}://{}{}", config.target.authority, config.path);
let request = Request::builder()
.method("GET")
.version(Version::HTTP_2)
.uri(uri)
.body(Empty::new())
.map_err(|error| error.to_string())?;
let response = sender
.send_request(request)
.await
.map_err(|error| error.to_string())?;
validate_response(&config.expected, response).await
}
async fn validate_response(
expected: &response::ExpectedResponse,
response: hyper::Response<Incoming>,
) -> Result<(), String> {
if response.status() != StatusCode::OK {
return Err(format!("unexpected response status {}", response.status()));
}
let content_type = response
.headers()
.get(CONTENT_TYPE)
.ok_or("response has no Content-Type")?;
if content_type.as_bytes() != expected.content_type.as_bytes() {
return Err("response Content-Type does not match the benchmark contract".into());
}
let mut body = response.into_body();
let mut offset: usize = 0;
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|error| error.to_string())?;
let data = frame
.into_data()
.map_err(|_| "response contains unexpected trailers")?;
let end = offset.saturating_add(data.len());
if end > expected.body.len() || data.as_ref() != &expected.body[offset..end] {
return Err("response body does not match the benchmark contract".into());
}
offset = end;
}
if offset != expected.body.len() {
return Err("response body is shorter than the benchmark contract".into());
}
Ok(())
}
struct Results {
http_version: HttpVersion,
shape: ConnectionShape,
successful_handshakes: u64,
scheduled: u64,
sent: u64,
completed: u64,
succeeded: u64,
errors: u64,
dropped: u64,
dropped_capacity: u64,
dispatch_expired: u64,
elapsed: Duration,
latency: Histogram,
dispatch_lag: Histogram,
request_latency: Histogram,
}
impl Results {
fn new(
scheduled: u64,
http_version: HttpVersion,
shape: ConnectionShape,
successful_handshakes: u64,
) -> Self {
Self {
http_version,
shape,
successful_handshakes,
scheduled,
sent: 0,
completed: 0,
succeeded: 0,
errors: 0,
dropped: 0,
dropped_capacity: 0,
dispatch_expired: 0,
elapsed: Duration::ZERO,
latency: Histogram::new(),
dispatch_lag: Histogram::new(),
request_latency: Histogram::new(),
}
}
fn observe(&mut self, event: Event, ready: &mut VecDeque<usize>) {
match event {
Event::Ready(id) => ready.push_back(id),
Event::Complete(completion) => {
self.completed += 1;
self.successful_handshakes += completion.successful_handshakes;
self.dispatch_lag.record(completion.dispatch_lag_us);
if completion.succeeded {
self.succeeded += 1;
self.latency.record(completion.latency_us);
self.request_latency.record(completion.request_latency_us);
} else {
self.errors += 1;
}
}
}
}
fn json(&self) -> String {
let elapsed = self.elapsed.as_secs_f64();
let qps = if elapsed == 0.0 {
0.0
} else {
self.succeeded as f64 / elapsed
};
let latency = percentiles_json(&self.latency);
let dispatch_lag = percentiles_json(&self.dispatch_lag);
let request_latency = percentiles_json(&self.request_latency);
format!(
concat!(
"{{\"http_version\":\"{}\",\"physical_connections\":{},",
"\"streams_per_connection\":{},\"request_slots\":{},",
"\"successful_handshakes\":{},",
"\"scheduled\":{},\"sent\":{},\"completed\":{},",
"\"succeeded\":{},\"errors\":{},\"dropped\":{},",
"\"dropped_capacity\":{},\"dispatch_expired\":{},",
"\"latency_samples\":{},\"dispatch_lag_samples\":{},",
"\"request_latency_samples\":{},",
"\"elapsed_seconds\":{:.6},\"actual_success_qps\":{:.3},",
"\"latency_ms\":{},\"dispatch_lag_ms\":{},",
"\"request_latency_ms\":{}}}"
),
self.http_version.as_str(),
self.shape.physical_connections,
self.shape.streams_per_connection,
self.shape.request_slots,
self.successful_handshakes,
self.scheduled,
self.sent,
self.completed,
self.succeeded,
self.errors,
self.dropped,
self.dropped_capacity,
self.dispatch_expired,
self.latency.count(),
self.dispatch_lag.count(),
self.request_latency.count(),
elapsed,
qps,
latency,
dispatch_lag,
request_latency,
)
}
}
fn percentiles_json(histogram: &Histogram) -> String {
if histogram.count() == 0 {
return "null".into();
}
format!(
"{{\"sample_count\":{},\"p50\":{:.3},\"p95\":{:.3},\"p99\":{:.3}}}",
histogram.count(),
histogram.percentile_ms(50),
histogram.percentile_ms(95),
histogram.percentile_ms(99),
)
}
fn micros(duration: Duration) -> u64 {
duration.as_micros().min(u128::from(u64::MAX)) as u64
}
fn number(value: &str, name: &str, minimum: u64, maximum: u64) -> Result<u64, String> {
let parsed = value
.parse::<u64>()
.map_err(|_| format!("{name} must be an integer"))?;
if !(minimum..=maximum).contains(&parsed) {
return Err(format!("{name} must be between {minimum} and {maximum}"));
}
Ok(parsed)
}
fn decimal_duration(value: &str, name: &str, maximum: f64) -> Result<Duration, String> {
let seconds = value
.parse::<f64>()
.map_err(|_| format!("{name} must be a number"))?;
if !seconds.is_finite() || seconds <= 0.0 || seconds > maximum {
return Err(format!(
"{name} must be greater than 0 and at most {maximum}"
));
}
Ok(Duration::from_secs_f64(seconds))
}
@@ -0,0 +1,67 @@
const EXACT_BUCKETS: usize = 1_024;
const SUB_BUCKETS: usize = 64;
const BUCKETS: usize = 4_096;
pub struct Histogram {
buckets: Box<[u64; BUCKETS]>,
count: u64,
}
impl Histogram {
pub fn new() -> Self {
Self {
buckets: Box::new([0; BUCKETS]),
count: 0,
}
}
pub fn record(&mut self, microseconds: u64) {
self.buckets[index(microseconds)] += 1;
self.count += 1;
}
pub fn count(&self) -> u64 {
self.count
}
pub fn percentile_ms(&self, percentile: u64) -> f64 {
if self.count == 0 {
return 0.0;
}
let target = self.count.saturating_mul(percentile).div_ceil(100);
let mut observed = 0;
for (index, count) in self.buckets.iter().enumerate() {
observed += count;
if observed >= target {
return upper_bound(index) as f64 / 1_000.0;
}
}
upper_bound(BUCKETS - 1) as f64 / 1_000.0
}
}
fn index(microseconds: u64) -> usize {
if microseconds < EXACT_BUCKETS as u64 {
return microseconds as usize;
}
let exponent = 63 - microseconds.leading_zeros() as usize;
let lower = 1_u64 << exponent;
let step = (lower / SUB_BUCKETS as u64).max(1);
let sub = ((microseconds - lower) / step).min((SUB_BUCKETS - 1) as u64) as usize;
(EXACT_BUCKETS + (exponent - 10) * SUB_BUCKETS + sub).min(BUCKETS - 1)
}
fn upper_bound(index: usize) -> u64 {
if index < EXACT_BUCKETS {
return index as u64;
}
let offset = index - EXACT_BUCKETS;
let exponent = 10 + offset / SUB_BUCKETS;
if exponent >= 63 {
return u64::MAX;
}
let sub = offset % SUB_BUCKETS;
let lower = 1_u64 << exponent;
let step = lower / SUB_BUCKETS as u64;
lower.saturating_add(step.saturating_mul(sub as u64 + 1))
}
@@ -0,0 +1,117 @@
use bytes::Bytes;
use http_body_util::Full;
use hyper::header::{CONTENT_TYPE, HeaderValue};
use hyper::{Response, StatusCode};
pub const PLAIN_PATH: &str = "/plain";
pub const JSON_PATH: &str = "/json";
pub const BYTES_PATH: &str = "/bytes";
pub const ORM_COUNT_PATH: &str = "/benchmark/item/count";
const PLAIN_BODY: &[u8] = b"hello";
const JSON_BODY: &[u8] = br#"{"ok":true}"#;
#[derive(Clone, Debug)]
pub struct ExpectedResponse {
pub body: Bytes,
pub content_type: &'static str,
}
impl ExpectedResponse {
pub fn for_path(path: &str, orm_count: Option<usize>) -> Result<Self, String> {
match path {
PLAIN_PATH => Ok(Self {
body: Bytes::from_static(PLAIN_BODY),
content_type: "text/plain",
}),
JSON_PATH => Ok(Self {
body: Bytes::from_static(JSON_BODY),
content_type: "application/json",
}),
BYTES_PATH => Ok(Self {
body: Bytes::from(vec![b'x'; 65_536]),
content_type: "application/octet-stream",
}),
ORM_COUNT_PATH => Ok(Self {
body: Bytes::from(format!(
"{{\"code\":0,\"message\":\"ok\",\"data\":{}}}",
orm_count.ok_or("benchmark.orm_count is required for ORM count requests")?
)),
content_type: "application/json; charset=utf-8",
}),
_ => Err(format!("unsupported benchmark path '{path}'")),
}
}
}
pub struct HyperResponses {
bytes: Bytes,
}
impl HyperResponses {
pub fn new() -> Self {
Self {
bytes: Bytes::from(vec![b'x'; 65_536]),
}
}
pub fn response(&self, path: &str) -> Response<Full<Bytes>> {
let (body, content_type) = match path {
PLAIN_PATH => (Bytes::from_static(PLAIN_BODY), "text/plain"),
JSON_PATH => (Bytes::from_static(JSON_BODY), "application/json"),
BYTES_PATH => (self.bytes.clone(), "application/octet-stream"),
_ => {
let mut response = Response::new(Full::new(Bytes::new()));
*response.status_mut() = StatusCode::NOT_FOUND;
return response;
}
};
let mut response = Response::new(Full::new(body));
response
.headers_mut()
.insert(CONTENT_TYPE, HeaderValue::from_static(content_type));
response
}
}
#[derive(Clone)]
pub struct RuntimeResponses {
bytes: dever_runtime::bytes::Bytes,
}
impl RuntimeResponses {
pub fn new() -> Self {
Self {
bytes: dever_runtime::bytes::Bytes::new(vec![b'x'; 65_536]),
}
}
pub fn response(&self, path: &str) -> dever_runtime::http::Response {
let (body, content_type) = match path {
PLAIN_PATH => (
dever_runtime::bytes::Bytes::from_text("hello"),
"text/plain",
),
JSON_PATH => (
dever_runtime::bytes::Bytes::from_text(r#"{"ok":true}"#),
"application/json",
),
BYTES_PATH => (self.bytes.clone(), "application/octet-stream"),
_ => {
return dever_runtime::http::Response {
status: 404,
headers: vec![],
body: dever_runtime::bytes::Bytes::new(vec![]),
};
}
};
dever_runtime::http::Response {
status: 200,
headers: vec![dever_runtime::http::Header {
name: "content-type".into(),
value: dever_runtime::bytes::Bytes::from_text(content_type),
}],
body,
}
}
}
@@ -0,0 +1,369 @@
use std::future::Future;
use std::io::{self, Write};
use std::pin::Pin;
use std::sync::Arc;
use bytes::{Bytes, BytesMut};
use http_body_util::{BodyExt, Full};
use hyper::body::Incoming;
use hyper::header::{CONNECTION, HOST, UPGRADE};
use hyper::server::conn::{http1, http2};
use hyper::service::service_fn;
use hyper::{Method, Request, Response, StatusCode, Version};
use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer};
use tokio::net::TcpListener;
use tokio::runtime::Builder;
use tokio::sync::{Semaphore, watch};
use tokio::task::JoinSet;
use tokio::time::{Instant, timeout, timeout_at};
use crate::config::{
BODY_BYTES, HEADER_BYTES, HTTP2_FRAME_BYTES, Http2Config, HttpVersion, ServiceConfig,
ServiceKind,
};
use crate::{response, tls};
pub fn run(kind: ServiceKind) -> Result<(), String> {
let settings = crate::settings::Settings::load()?;
let config = ServiceConfig::from_settings(&settings, kind)?;
if kind.uses_runtime() {
run_runtime(config, kind)
} else {
run_hyper(config, kind)
}
}
fn run_runtime(config: ServiceConfig, kind: ServiceKind) -> Result<(), String> {
let server_tls = config
.cert_path
.as_deref()
.zip(config.key_path.as_deref())
.map(|(cert, key)| tls::runtime_server(cert, key))
.transpose()?;
let runtime = dever_runtime::task::RuntimeConfig {
worker_threads: config.workers,
max_blocking_threads: 2,
task_capacity: config.task_capacity,
};
dever_runtime::task::run_entry_with(runtime, async move {
let responses = response::RuntimeResponses::new();
let listener = dever_runtime::net::listen("127.0.0.1", 0).await?;
let port = dever_runtime::net::port(&listener)?;
ready(u16::try_from(port).map_err(|_| "listener returned an invalid port")?)?;
let limits = dever_runtime::http::Limits {
header_bytes: HEADER_BYTES as i64,
body_bytes: BODY_BYTES as i64,
timeout_ms: config.timeout.as_millis() as i64,
connections: config.connections as i64,
http2: config.http2.map(|limits| dever_runtime::http::Http2Limits {
streams: limits.streams as i64,
stream_window_bytes: i64::from(limits.stream_window_bytes),
connection_window_bytes: i64::from(limits.connection_window_bytes),
}),
};
let owned = listener.clone();
let route = move |request: dever_runtime::http::Request| {
let response = responses.response(&request.target);
async move { Ok(response) }
};
let server: Pin<Box<dyn Future<Output = Result<(), String>> + Send>> = match server_tls {
Some(tls) if kind.uses_tls() => {
Box::pin(dever_runtime::http::serve_tls(route, owned, limits, tls))
}
_ => Box::pin(dever_runtime::http::serve(route, owned, limits)),
};
tokio::pin!(server);
tokio::select! {
result = server.as_mut() => result,
_ = tokio::time::sleep(config.lifetime) => {
listener.close()?;
timeout(config.timeout, server.as_mut())
.await
.map_err(|_| "runtime server shutdown timed out".to_owned())?
}
}
})
}
fn run_hyper(config: ServiceConfig, kind: ServiceKind) -> Result<(), String> {
let server_tls = config
.cert_path
.as_deref()
.zip(config.key_path.as_deref())
.map(|(cert, key)| tls::server(cert, key, kind.http_version()))
.transpose()?;
let runtime = Builder::new_multi_thread()
.worker_threads(config.workers)
.max_blocking_threads(1)
.enable_io()
.enable_time()
.build()
.map_err(|error| format!("cannot start benchmark runtime: {error}"))?;
runtime.block_on(hyper_server(
config,
kind.http_version(),
kind.uses_tls().then_some(server_tls).flatten(),
))
}
async fn hyper_server(
config: ServiceConfig,
http_version: HttpVersion,
server_tls: Option<Arc<rustls::ServerConfig>>,
) -> Result<(), String> {
let responses = Arc::new(response::HyperResponses::new());
let listener = TcpListener::bind(("127.0.0.1", 0))
.await
.map_err(|error| error.to_string())?;
ready(
listener
.local_addr()
.map_err(|error| error.to_string())?
.port(),
)?;
let connections = Arc::new(Semaphore::new(config.connections));
let (shutdown, shutdown_receiver) = watch::channel(false);
let mut tasks = JoinSet::new();
let deadline = Instant::now() + config.lifetime;
loop {
while let Some(result) = tasks.try_join_next() {
result.map_err(|error| format!("connection task failed: {error}"))??;
}
let connections = connections.clone();
let accepted = timeout_at(deadline, async {
let permit = connections
.acquire_owned()
.await
.map_err(|_| "connection limit closed".to_owned())?;
let (stream, _) = listener.accept().await.map_err(|error| error.to_string())?;
stream
.set_nodelay(true)
.map_err(|error| error.to_string())?;
Ok::<_, String>((permit, stream))
})
.await;
let Ok(accepted) = accepted else {
break;
};
let (permit, stream) = accepted?;
let connection_config = config.clone();
let connection_tls = server_tls.clone();
let connection_shutdown = shutdown_receiver.clone();
let connection_responses = responses.clone();
tasks.spawn(async move {
let _permit = permit;
let stream = match tls::accept(
stream,
connection_config.timeout,
connection_tls.as_ref(),
http_version,
)
.await
{
Ok(stream) => stream,
Err(_) => return Ok(()),
};
serve_connection(
stream,
connection_config.timeout,
connection_shutdown,
connection_responses,
http_version,
connection_config.http2,
)
.await
});
}
let _ = shutdown.send(true);
if timeout(config.timeout, drain_connections(&mut tasks))
.await
.is_err()
{
tasks.abort_all();
while tasks.join_next().await.is_some() {}
return Err("hyper server shutdown timed out".into());
}
Ok(())
}
async fn serve_connection(
stream: tls::BoxedStream,
operation_timeout: std::time::Duration,
shutdown: watch::Receiver<bool>,
responses: Arc<response::HyperResponses>,
http_version: HttpVersion,
http2_config: Option<Http2Config>,
) -> Result<(), String> {
match http_version {
HttpVersion::H1 => serve_http1(stream, operation_timeout, shutdown, responses).await,
HttpVersion::H2 => {
let limits = http2_config.ok_or("HTTP/2 service has no HTTP/2 limits")?;
serve_http2(stream, operation_timeout, shutdown, responses, limits).await
}
}
}
async fn serve_http1(
stream: tls::BoxedStream,
operation_timeout: std::time::Duration,
mut shutdown: watch::Receiver<bool>,
responses: Arc<response::HyperResponses>,
) -> Result<(), String> {
let service = service_fn(move |request| {
dispatch(
request,
operation_timeout,
responses.clone(),
HttpVersion::H1,
)
});
let mut builder = http1::Builder::new();
builder
.timer(TokioTimer::new())
.header_read_timeout(operation_timeout)
.max_buf_size(HEADER_BYTES);
let connection = builder.serve_connection(TokioIo::new(stream), service);
tokio::pin!(connection);
let result = tokio::select! {
result = connection.as_mut() => result,
_ = shutdown.changed() => {
connection.as_mut().graceful_shutdown();
timeout(operation_timeout, connection.as_mut())
.await
.map_err(|_| "connection shutdown timed out".to_owned())?
}
};
// Peer disconnects and malformed requests end only their own connection.
let _ = result;
Ok(())
}
async fn serve_http2(
stream: tls::BoxedStream,
operation_timeout: std::time::Duration,
mut shutdown: watch::Receiver<bool>,
responses: Arc<response::HyperResponses>,
limits: Http2Config,
) -> Result<(), String> {
let service = service_fn(move |request| {
dispatch(
request,
operation_timeout,
responses.clone(),
HttpVersion::H2,
)
});
let mut builder = http2::Builder::new(TokioExecutor::new());
builder
.timer(TokioTimer::new())
.adaptive_window(false)
.initial_stream_window_size(limits.stream_window_bytes)
.initial_connection_window_size(limits.connection_window_bytes)
.max_concurrent_streams(limits.streams as u32)
.max_header_list_size(HEADER_BYTES as u32)
.max_frame_size(HTTP2_FRAME_BYTES)
.max_send_buf_size(HTTP2_FRAME_BYTES as usize);
let connection = builder.serve_connection(TokioIo::new(stream), service);
tokio::pin!(connection);
let result = tokio::select! {
result = connection.as_mut() => result,
_ = shutdown.changed() => {
connection.as_mut().graceful_shutdown();
timeout(operation_timeout, connection.as_mut())
.await
.map_err(|_| "connection shutdown timed out".to_owned())?
}
};
// Peer disconnects and malformed requests end only their own connection.
let _ = result;
Ok(())
}
async fn drain_connections(tasks: &mut JoinSet<Result<(), String>>) -> Result<(), String> {
while let Some(result) = tasks.join_next().await {
result.map_err(|error| format!("connection task failed: {error}"))??;
}
Ok(())
}
async fn dispatch(
request: Request<Incoming>,
operation_timeout: std::time::Duration,
responses: Arc<response::HyperResponses>,
http_version: HttpVersion,
) -> Result<Response<Full<Bytes>>, std::convert::Infallible> {
let response = timeout(
operation_timeout,
read_and_route(request, responses, http_version),
)
.await
.unwrap_or_else(|_| reject(StatusCode::REQUEST_TIMEOUT, http_version));
Ok(response)
}
async fn read_and_route(
request: Request<Incoming>,
responses: Arc<response::HyperResponses>,
http_version: HttpVersion,
) -> Response<Full<Bytes>> {
if request.method() == Method::CONNECT || request.headers().contains_key(UPGRADE) {
return reject(StatusCode::NOT_IMPLEMENTED, http_version);
}
if request.version() == Version::HTTP_11 && request.headers().get_all(HOST).iter().count() != 1
{
return reject(StatusCode::BAD_REQUEST, http_version);
}
if header_size(&request) > HEADER_BYTES {
return reject(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE, http_version);
}
let (head, mut body) = request.into_parts();
let mut received = BytesMut::new();
while let Some(frame) = body.frame().await {
let Ok(frame) = frame else {
return reject(StatusCode::BAD_REQUEST, http_version);
};
if frame.is_trailers() {
return reject(StatusCode::NOT_IMPLEMENTED, http_version);
}
if let Ok(data) = frame.into_data() {
if received.len().saturating_add(data.len()) > BODY_BYTES {
return reject(StatusCode::PAYLOAD_TOO_LARGE, http_version);
}
received.extend_from_slice(&data);
}
}
responses.response(head.uri.path())
}
fn header_size(request: &Request<Incoming>) -> usize {
let start = request.method().as_str().len() + request.uri().to_string().len() + 14;
request
.headers()
.iter()
.fold(start, |total, (name, value)| {
total
.saturating_add(name.as_str().len())
.saturating_add(value.as_bytes().len())
.saturating_add(4)
})
}
fn reject(status: StatusCode, http_version: HttpVersion) -> Response<Full<Bytes>> {
let mut response = Response::new(Full::new(Bytes::new()));
*response.status_mut() = status;
if http_version == HttpVersion::H1 {
response
.headers_mut()
.insert(CONNECTION, hyper::header::HeaderValue::from_static("close"));
}
response
}
fn ready(port: u16) -> Result<(), String> {
let mut output = io::stdout().lock();
writeln!(output, "READY|{port}").map_err(|error| error.to_string())?;
output.flush().map_err(|error| error.to_string())
}
@@ -0,0 +1,128 @@
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, pem::PemObject};
use rustls::{ClientConfig, RootCertStore, ServerConfig};
use tokio::io::{AsyncRead, AsyncWrite};
use tokio::net::TcpStream;
use tokio_rustls::{TlsAcceptor, TlsConnector};
use crate::config::HttpVersion;
pub trait Stream: AsyncRead + AsyncWrite {}
impl<T: AsyncRead + AsyncWrite> Stream for T {}
pub type BoxedStream = Box<dyn Stream + Unpin + Send>;
pub fn runtime_server(
cert_path: &str,
key_path: &str,
) -> Result<dever_runtime::tls::ServerTls, String> {
let certificate = std::fs::read(cert_path)
.map_err(|error| format!("cannot read certificate {cert_path}: {error}"))?;
let key = std::fs::read(key_path)
.map_err(|error| format!("cannot read private key {key_path}: {error}"))?;
dever_runtime::tls::server(
&dever_runtime::bytes::Bytes::new(certificate),
&dever_runtime::bytes::Bytes::new(key),
)
}
pub fn server(
cert_path: &str,
key_path: &str,
http_version: HttpVersion,
) -> Result<Arc<ServerConfig>, String> {
let certificates = certificates(cert_path)?;
let key_bytes = std::fs::read(key_path)
.map_err(|error| format!("cannot read private key {key_path}: {error}"))?;
let key = PrivateKeyDer::from_pem_slice(&key_bytes).map_err(|error| error.to_string())?;
let mut config = ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(certificates, key)
.map_err(|error| error.to_string())?;
config.session_storage = rustls::server::ServerSessionMemoryCache::new(32);
config.send_tls13_tickets = 1;
config.alpn_protocols = vec![http_version.alpn().to_vec()];
Ok(Arc::new(config))
}
pub fn client(ca_path: &str, http_version: HttpVersion) -> Result<Arc<ClientConfig>, String> {
let mut roots = RootCertStore::empty();
for certificate in certificates(ca_path)? {
roots.add(certificate).map_err(|error| error.to_string())?;
}
let mut config = ClientConfig::builder()
.with_root_certificates(roots)
.with_no_client_auth();
config.resumption = rustls::client::Resumption::in_memory_sessions(32);
config.alpn_protocols = vec![http_version.alpn().to_vec()];
Ok(Arc::new(config))
}
pub async fn connect(
port: u16,
timeout: Duration,
tls: Option<&Arc<ClientConfig>>,
http_version: HttpVersion,
) -> Result<BoxedStream, String> {
let tcp = tokio::time::timeout(timeout, TcpStream::connect(("127.0.0.1", port)))
.await
.map_err(|_| "TCP connection timed out".to_owned())?
.map_err(|error| error.to_string())?;
tcp.set_nodelay(true).map_err(|error| error.to_string())?;
let Some(config) = tls else {
return Ok(Box::new(tcp));
};
let name = ServerName::try_from("localhost").map_err(|error| error.to_string())?;
let stream = tokio::time::timeout(
timeout,
TlsConnector::from(config.clone()).connect(name, tcp),
)
.await
.map_err(|_| "TLS handshake timed out".to_owned())?
.map_err(|error| error.to_string())?;
verify_alpn(stream.get_ref().1.alpn_protocol(), http_version)?;
Ok(Box::new(stream))
}
pub async fn accept(
tcp: TcpStream,
timeout: Duration,
tls: Option<&Arc<ServerConfig>>,
http_version: HttpVersion,
) -> Result<BoxedStream, String> {
let Some(config) = tls else {
return Ok(Box::new(tcp));
};
let stream = tokio::time::timeout(timeout, TlsAcceptor::from(config.clone()).accept(tcp))
.await
.map_err(|_| "TLS handshake timed out".to_owned())?
.map_err(|error| error.to_string())?;
verify_alpn(stream.get_ref().1.alpn_protocol(), http_version)?;
Ok(Box::new(stream))
}
fn verify_alpn(negotiated: Option<&[u8]>, http_version: HttpVersion) -> Result<(), String> {
if http_version == HttpVersion::H2 && negotiated != Some(http_version.alpn()) {
return Err("TLS peer did not negotiate h2".into());
}
Ok(())
}
fn certificates(path: impl AsRef<Path>) -> Result<Vec<CertificateDer<'static>>, String> {
let path = path.as_ref();
let bytes = std::fs::read(path)
.map_err(|error| format!("cannot read certificate {}: {error}", path.display()))?;
let certificates = CertificateDer::pem_slice_iter(&bytes)
.collect::<Result<Vec<_>, _>>()
.map_err(|error| error.to_string())?;
if certificates.is_empty() {
return Err(format!(
"certificate {} contains no certificates",
path.display()
));
}
Ok(certificates)
}