Initial Dever 0.0.1

test/rust/runtime/upload/headers.rs

@@ -0,0 +1,73 @@
+use super::*;
+
+#[test]
+fn quoted_filename_preserves_semicolons_and_escaped_quotes() {
+    let directory = Directory::new();
+    let bytes = String::from_utf8(content(b"data"))
+        .unwrap()
+        .replace("filename=image.bin", r#"filename="semi;with\"quote.bin""#)
+        .into_bytes();
+    Runtime::new(TaskLimits::default())
+        .unwrap()
+        .run(async |scope| {
+            let mut parsed = upload::decode(
+                body(&bytes, 1),
+                "multipart/form-data; boundary=\"boundary\"",
+                &directory.0,
+                &Limits::default(),
+                scope,
+            )
+            .await
+            .unwrap();
+            let file = parsed.take("file").unwrap();
+            assert_eq!(file.filename(), "semi;with\"quote.bin");
+            file.close();
+            assert_eq!(directory.temporary_count(), 0);
+        })
+        .unwrap();
+}
+
+#[cfg(unix)]
+#[test]
+fn parent_symlinks_cannot_redirect_temporary_or_stored_uploads() {
+    let directory = Directory::new();
+    let outside = Directory::new();
+    let bytes = content(b"data");
+    std::os::unix::fs::symlink(&outside.0, directory.0.join("linked")).unwrap();
+    Runtime::new(TaskLimits::default())
+        .unwrap()
+        .run(async |scope| {
+            for path in [directory.0.join("linked"), directory.0.join("linked/data")] {
+                assert!(
+                    upload::decode(
+                        body(&bytes, 7),
+                        "multipart/form-data; boundary=boundary",
+                        &path,
+                        &Limits::default(),
+                        scope
+                    )
+                    .await
+                    .is_err()
+                );
+                assert_eq!(std::fs::read_dir(&outside.0).unwrap().count(), 0);
+            }
+            let mut parsed = upload::decode(
+                body(&bytes, 7),
+                "multipart/form-data; boundary=boundary",
+                &directory.0,
+                &Limits::default(),
+                scope,
+            )
+            .await
+            .unwrap();
+            std::os::unix::fs::symlink(&outside.0, directory.0.join("upload")).unwrap();
+            assert!(
+                upload::put(parsed.take("file").unwrap(), scope)
+                    .await
+                    .is_err()
+            );
+            assert_eq!(std::fs::read_dir(&outside.0).unwrap().count(), 0);
+            assert_eq!(directory.temporary_count(), 0);
+        })
+        .unwrap();
+}

test/rust/runtime/upload/mod.rs

@@ -0,0 +1,302 @@
+use bytes::Bytes;
+use dever_runtime::{
+    task::{Limits as TaskLimits, Runtime},
+    upload::{self, Limits},
+};
+use futures_util::stream::{self, StreamExt};
+use http_body_util::StreamBody;
+use hyper::body::Frame;
+use std::{
+    convert::Infallible,
+    path::PathBuf,
+    sync::atomic::{AtomicUsize, Ordering},
+    time::Duration,
+};
+
+mod headers;
+mod performance;
+mod transport;
+
+struct Directory(PathBuf);
+
+#[derive(Debug, PartialEq, serde::Deserialize)]
+#[serde(deny_unknown_fields)]
+struct Fields {
+    caption: String,
+    count: i64,
+}
+
+impl Directory {
+    fn new() -> Self {
+        static NEXT: AtomicUsize = AtomicUsize::new(0);
+        let path = std::env::temp_dir().join(format!(
+            "dever-upload-{}-{}",
+            std::process::id(),
+            NEXT.fetch_add(1, Ordering::Relaxed)
+        ));
+        std::fs::create_dir(&path).unwrap();
+        Self(path)
+    }
+
+    fn temporary_count(&self) -> usize {
+        std::fs::read_dir(self.0.join("tmp")).map_or(0, |files| files.count())
+    }
+}
+
+impl Drop for Directory {
+    fn drop(&mut self) {
+        std::fs::remove_dir_all(&self.0).unwrap();
+    }
+}
+
+fn content(file: &[u8]) -> Vec<u8> {
+    let mut body = b"--boundary\r\nContent-Disposition: form-data; name=caption\r\n\r\nhello\r\n--boundary\r\nContent-Disposition: form-data; name=count\r\n\r\n7\r\n--boundary\r\nContent-Disposition: form-data; name=file; filename=image.bin\r\nContent-Type: application/octet-stream\r\n\r\n".to_vec();
+    body.extend_from_slice(file);
+    body.extend_from_slice(b"\r\n--boundary--\r\n");
+    body
+}
+
+fn body(
+    bytes: &[u8],
+    size: usize,
+) -> impl hyper::body::Body<Data = Bytes, Error = Infallible> + Unpin + Send + 'static {
+    let chunks = bytes
+        .chunks(size)
+        .map(|chunk| Ok(Frame::data(Bytes::copy_from_slice(chunk))))
+        .collect::<Vec<_>>();
+    StreamBody::new(stream::iter(chunks))
+}
+
+#[test]
+fn fragmented_files_preserve_false_boundaries_and_transfer_once() {
+    let directory = Directory::new();
+    let mut payload = vec![b'x'; 170_001];
+    payload.extend_from_slice(b"\r\n--boundaryX\r\n--boundary--X\r\n--boundary-");
+    let bytes = content(&payload);
+    Runtime::new(TaskLimits::default())
+        .unwrap()
+        .run(async |scope| {
+            for size in [1, 7, 16_384, 250_000] {
+                let mut decoded = upload::decode(
+                    body(&bytes, size),
+                    "multipart/form-data; boundary=boundary",
+                    &directory.0,
+                    &Limits::default(),
+                    scope,
+                )
+                .await
+                .unwrap();
+                let file = decoded.take("file").unwrap();
+                assert_eq!(file.filename(), "image.bin");
+                assert_eq!(file.content_type(), "application/octet-stream");
+                assert_eq!(file.size(), payload.len() as i64);
+                assert_eq!(
+                    decoded
+                        .finish()
+                        .unwrap()
+                        .decode::<Fields>(dever_runtime::wire::decode)
+                        .unwrap(),
+                    Fields {
+                        caption: "hello".into(),
+                        count: 7
+                    }
+                );
+                let key = upload::put(file, scope).await.unwrap();
+                assert_eq!(
+                    std::fs::read(directory.0.join("upload").join(key.to_text())).unwrap(),
+                    payload
+                );
+                assert_eq!(directory.temporary_count(), 0);
+            }
+        })
+        .unwrap();
+}
+
+#[test]
+fn invalid_or_limited_bodies_remove_every_temporary_file() {
+    let directory = Directory::new();
+    let bytes = content(&vec![b'x'; 20_000]);
+    Runtime::new(TaskLimits::default())
+        .unwrap()
+        .run(async |scope| {
+            let invalid = [
+                bytes[..bytes.len() - 8].to_vec(),
+                [bytes.clone(), b"epilogue".to_vec()].concat(),
+                String::from_utf8(bytes.clone())
+                    .unwrap()
+                    .replace("image.bin", "../escape")
+                    .into_bytes(),
+            ];
+            for bytes in invalid {
+                assert!(
+                    upload::decode(
+                        body(&bytes, 7),
+                        "multipart/form-data; boundary=boundary",
+                        &directory.0,
+                        &Limits::default(),
+                        scope
+                    )
+                    .await
+                    .is_err()
+                );
+                assert_eq!(directory.temporary_count(), 0);
+            }
+            for limits in [
+                Limits {
+                    total_bytes: 10_000,
+                    file_bytes: 10_000,
+                    ..Limits::default()
+                },
+                Limits {
+                    file_bytes: 10_000,
+                    ..Limits::default()
+                },
+                Limits {
+                    field_bytes: 2,
+                    ..Limits::default()
+                },
+                Limits {
+                    parts: 2,
+                    fields: 2,
+                    ..Limits::default()
+                },
+                Limits {
+                    header_bytes: 10,
+                    ..Limits::default()
+                },
+            ] {
+                assert!(matches!(
+                    upload::decode(
+                        body(&bytes, 7),
+                        "multipart/form-data; boundary=boundary",
+                        &directory.0,
+                        &limits,
+                        scope
+                    )
+                    .await,
+                    Err(upload::Error::Limit)
+                ));
+                assert_eq!(directory.temporary_count(), 0);
+            }
+        })
+        .unwrap();
+}
+
+#[test]
+fn duplicate_names_across_fields_and_files_remove_temporary_files() {
+    let directory = Directory::new();
+    Runtime::new(TaskLimits::default())
+        .unwrap()
+        .run(async |scope| {
+            for parts in [
+                [("same", false), ("same", false)],
+                [("same", false), ("same", true)],
+                [("same", true), ("same", false)],
+                [("same", true), ("same", true)],
+                [("first", true), ("second", true)],
+            ] {
+                let mut source = String::new();
+                for (name, file) in parts {
+                    let filename = if file { "; filename=example.txt" } else { "" };
+                    source.push_str(&format!(
+                        "--boundary\r\nContent-Disposition: form-data; name={name}{filename}\r\n\r\nvalue\r\n"
+                    ));
+                }
+                source.push_str("--boundary--\r\n");
+                assert!(matches!(
+                    upload::decode(
+                        body(source.as_bytes(), 7),
+                        "multipart/form-data; boundary=boundary",
+                        &directory.0,
+                        &Limits::default(),
+                        scope,
+                    )
+                    .await,
+                    Err(upload::Error::Invalid)
+                ));
+                assert_eq!(directory.temporary_count(), 0);
+            }
+        })
+        .unwrap();
+}
+
+#[test]
+fn cancellation_while_waiting_for_more_file_bytes_removes_the_file() {
+    let directory = Directory::new();
+    let path = directory.0.clone();
+    let mut bytes = content(&vec![b'x'; 32_768]);
+    bytes.truncate(bytes.len() - 16);
+    Runtime::new(TaskLimits::default())
+        .unwrap()
+        .run(async |scope| {
+            let task = scope
+                .spawn(move |child| async move {
+                    let frames = stream::once(std::future::ready(Ok::<_, Infallible>(
+                        Frame::data(Bytes::from(bytes)),
+                    )))
+                    .chain(stream::pending());
+                    upload::decode(
+                        StreamBody::new(frames),
+                        "multipart/form-data; boundary=boundary",
+                        &path,
+                        &Limits::default(),
+                        &child,
+                    )
+                    .await
+                })
+                .unwrap();
+            tokio::time::timeout(Duration::from_secs(3), async {
+                while directory.temporary_count() == 0 {
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await
+            .unwrap();
+            task.cancel();
+            assert!(matches!(
+                task.wait().await,
+                Err(dever_runtime::task::Error::Cancelled) | Ok(Err(upload::Error::Cancelled))
+            ));
+            assert_eq!(directory.temporary_count(), 0);
+        })
+        .unwrap();
+}
+
+#[test]
+fn dropping_unconsumed_file_and_failed_storage_remove_the_temporary() {
+    let directory = Directory::new();
+    let bytes = content(b"data");
+    Runtime::new(TaskLimits::default())
+        .unwrap()
+        .run(async |scope| {
+            let parsed = upload::decode(
+                body(&bytes, 7),
+                "multipart/form-data; boundary=boundary",
+                &directory.0,
+                &Limits::default(),
+                scope,
+            )
+            .await
+            .unwrap();
+            assert_eq!(directory.temporary_count(), 1);
+            drop(parsed);
+            assert_eq!(directory.temporary_count(), 0);
+            std::fs::write(directory.0.join("upload"), b"not a directory").unwrap();
+            let mut parsed = upload::decode(
+                body(&bytes, 7),
+                "multipart/form-data; boundary=boundary",
+                &directory.0,
+                &Limits::default(),
+                scope,
+            )
+            .await
+            .unwrap();
+            assert!(
+                upload::put(parsed.take("file").unwrap(), scope)
+                    .await
+                    .is_err()
+            );
+            assert_eq!(directory.temporary_count(), 0);
+        })
+        .unwrap();
+}

test/rust/runtime/upload/performance.rs

@@ -0,0 +1,54 @@
+use super::*;
+use std::{collections::BTreeMap, hint::black_box, time::Instant};
+
+#[test]
+#[ignore = "explicit release performance probe"]
+fn multipart_field_cost() {
+    let directory = Directory::new();
+    Runtime::new(TaskLimits::default())
+        .unwrap()
+        .run(async |scope| {
+            println!("\ncase,sample,iterations,elapsed_ns,checksum");
+            for count in [1, 16, 64] {
+                let mut source = String::new();
+                for index in 0..count {
+                    source.push_str(&format!(
+                        "--boundary\r\nContent-Disposition: form-data; name=field{index}\r\n\r\n{}\r\n",
+                        "a".repeat(64)
+                    ));
+                }
+                source.push_str("--boundary--\r\n");
+                let bytes = Bytes::from(source);
+                let limits = Limits {
+                    fields: count,
+                    parts: count,
+                    ..Limits::default()
+                };
+                for sample in 0..15 {
+                    let started = Instant::now();
+                    let mut checksum = 0;
+                    for _ in 0..1000 {
+                        let multipart = upload::decode(
+                            http_body_util::Full::new(black_box(bytes.clone())),
+                            "multipart/form-data; boundary=boundary",
+                            &directory.0,
+                            &limits,
+                            scope,
+                        )
+                        .await
+                        .unwrap();
+                        let fields = multipart
+                            .finish()
+                            .unwrap()
+                            .decode::<BTreeMap<String, String>>(dever_runtime::wire::decode)
+                            .unwrap();
+                        checksum += black_box(fields.len());
+                    }
+                    let elapsed = started.elapsed().as_nanos();
+                    assert_eq!(checksum, count * 1000);
+                    println!("multipart_{count},{sample},1000,{elapsed},{checksum}");
+                }
+            }
+        })
+        .unwrap();
+}

test/rust/runtime/upload/transport.rs

@@ -0,0 +1,85 @@
+use super::*;
+use dever_runtime::{api, http, net};
+use std::sync::Arc;
+
+#[test]
+fn http1_and_http2_stream_uploads_above_the_json_body_limit() {
+    let directory = Directory::new();
+    Runtime::new(TaskLimits::default())
+        .unwrap()
+        .run(async |scope| {
+            for http2 in [false, true] {
+                let listener = net::listen("127.0.0.1", 0, scope).await.unwrap();
+                let port = listener.port().unwrap();
+                let incoming = listener.clone();
+                let data = Arc::new(directory.0.clone());
+                let transport = http::Limits {
+                    body_bytes: 1024,
+                    http2: http2.then(Default::default),
+                    ..Default::default()
+                };
+                let client_limits = http::Limits {
+                    body_bytes: 1_048_576,
+                    ..transport.clone()
+                };
+                let server = scope
+                    .spawn(move |child| async move {
+                        api::serve_upload::<_, _, true>(
+                            &incoming,
+                            &transport,
+                            &child,
+                            None,
+                            move |request, body, child, _| {
+                                let data = data.clone();
+                                async move {
+                                    let mut parsed = upload::read(
+                                        &request,
+                                        body,
+                                        &data,
+                                        &Limits::default(),
+                                        &child,
+                                    )
+                                    .await
+                                    .unwrap();
+                                    let file = parsed.take("file").unwrap();
+                                    let size = file.size();
+                                    parsed
+                                        .finish()
+                                        .unwrap()
+                                        .decode::<Fields>(dever_runtime::wire::decode)
+                                        .unwrap();
+                                    file.close();
+                                    Ok(api::response(size.to_string()))
+                                }
+                            },
+                        )
+                        .await
+                    })
+                    .unwrap();
+                let request = http::Request {
+                    method: "POST".into(),
+                    target: "/upload".into(),
+                    headers: vec![
+                        http::Header {
+                            name: "host".into(),
+                            value: b"fixture.test".to_vec().into(),
+                        },
+                        http::Header {
+                            name: "content-type".into(),
+                            value: b"multipart/form-data; boundary=boundary".to_vec().into(),
+                        },
+                    ],
+                    body: content(&vec![b'x'; 170_001]).into(),
+                };
+                let response = http::send("127.0.0.1", port, request, &client_limits)
+                    .await
+                    .unwrap();
+                assert_eq!(response.status, 200);
+                assert_eq!(response.body.as_ref(), b"170001");
+                assert_eq!(directory.temporary_count(), 0);
+                listener.close().unwrap();
+                server.wait().await.unwrap().unwrap();
+            }
+        })
+        .unwrap();
+}