From 9413d08580273edb0a6fa92338ff397bf49b864d Mon Sep 17 00:00:00 2001 From: Christine Dodrill Date: Fri, 30 Oct 2020 12:20:09 -0400 Subject: [PATCH] add rocket_upload code --- Cargo.lock | 261 ++++++++++++++++++++++++++++++++++- Cargo.toml | 7 + Rocket.toml.example | 1 + lib/rocket_upload/Cargo.toml | 11 ++ lib/rocket_upload/LICENSE | 21 +++ lib/rocket_upload/src/lib.rs | 168 ++++++++++++++++++++++ 6 files changed, 468 insertions(+), 1 deletion(-) create mode 100644 lib/rocket_upload/Cargo.toml create mode 100644 lib/rocket_upload/LICENSE create mode 100644 lib/rocket_upload/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 8c434c9..533b846 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,6 +70,15 @@ dependencies = [ "opaque-debug 0.2.3", ] +[[package]] +name = "aho-corasick" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b476ce7103678b0c6d3d395dbbae31d48ff910bd28be979ba5d48c6351131d0d" +dependencies = [ + "memchr", +] + [[package]] name = "ansi_term" version = "0.11.0" @@ -88,6 +97,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "ascii" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97be891acc47ca214468e09425d02cef3af2c94d0d82081cd02061f996802f14" + [[package]] name = "atty" version = "0.2.14" @@ -204,6 +219,16 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "buf_redux" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" +dependencies = [ + "memchr", + "safemem", +] + [[package]] name = "bumpalo" version = "3.4.0" @@ -254,6 +279,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "chunked_transfer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498d20a7aaf62625b9bf26e637cf7736417cde1d0c99f1d04d1170229a85cf87" + [[package]] name = "chunked_transfer" version = "1.3.0" @@ -676,6 +707,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "groupable" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32619942b8be646939eaf3db0602b39f5229b74575b67efc897811ded1db4e57" + [[package]] name = "hashbrown" version = "0.9.1" @@ -834,6 +871,22 @@ dependencies = [ "libc", ] +[[package]] +name = "iron" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6d308ca2d884650a8bf9ed2ff4cb13fbb2207b71f64cda11dc9b892067295e8" +dependencies = [ + "hyper", + "log 0.3.9", + "mime_guess", + "modifier", + "num_cpus", + "plugin", + "typemap", + "url 1.7.2", +] + [[package]] name = "itoa" version = "0.4.6" @@ -968,6 +1021,18 @@ dependencies = [ "log 0.3.9", ] +[[package]] +name = "mime_guess" +version = "1.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216929a5ee4dd316b1702eedf5e74548c123d370f47841ceaac38ca154690ca3" +dependencies = [ + "mime", + "phf", + "phf_codegen", + "unicase", +] + [[package]] name = "miniz_oxide" version = "0.4.3" @@ -1021,6 +1086,44 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "modifier" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f5c9112cb662acd3b204077e0de5bc66305fa8df65c8019d5adb10e9ab6e58" + +[[package]] +name = "multipart" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136eed74cadb9edd2651ffba732b19a450316b680e4f48d6c79e905799e19d01" +dependencies = [ + "buf_redux", + "httparse", + "hyper", + "iron", + "log 0.4.11", + "mime", + "mime_guess", + "nickel", + "quick-error", + "rand 0.6.5", + "safemem", + "tempfile", + "tiny_http", + "twoway", +] + +[[package]] +name = "mustache" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51956ef1c5d20a1384524d91e616fb44dfc7d8f249bf696d49c97dd3289ecab5" +dependencies = [ + "log 0.3.9", + "serde", +] + [[package]] name = "names" version = "0.11.0" @@ -1041,6 +1144,27 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "nickel" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5061a832728db2dacb61cefe0ce303b58f85764ec680e71d9138229640a46d9" +dependencies = [ + "groupable", + "hyper", + "lazy_static", + "log 0.3.9", + "modifier", + "mustache", + "plugin", + "regex", + "serde", + "serde_json", + "time 0.1.44", + "typemap", + "url 1.7.2", +] + [[package]] name = "notify" version = "4.0.15" @@ -1178,12 +1302,60 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "phf" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +dependencies = [ + "phf_shared", + "rand 0.6.5", +] + +[[package]] +name = "phf_shared" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" +dependencies = [ + "siphasher", + "unicase", +] + [[package]] name = "pin-project-lite" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" +[[package]] +name = "plugin" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a6a0dc3910bc8db877ffed8e457763b317cf880df4ae19109b9f77d277cf6e0" +dependencies = [ + "typemap", +] + [[package]] name = "polyval" version = "0.3.3" @@ -1255,6 +1427,12 @@ dependencies = [ "percent-encoding 2.1.0", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "0.6.13" @@ -1475,7 +1653,10 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" dependencies = [ + "aho-corasick", + "memchr", "regex-syntax", + "thread_local", ] [[package]] @@ -1494,6 +1675,15 @@ version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "ring" version = "0.16.15" @@ -1608,6 +1798,14 @@ dependencies = [ "url 2.1.1", ] +[[package]] +name = "rocket_upload" +version = "0.1.0" +dependencies = [ + "multipart", + "rocket", +] + [[package]] name = "rustc-demangle" version = "0.1.18" @@ -1788,6 +1986,12 @@ dependencies = [ "loom", ] +[[package]] +name = "siphasher" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" + [[package]] name = "slab" version = "0.4.2" @@ -1904,6 +2108,20 @@ dependencies = [ "unicode-xid 0.2.1", ] +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "rand 0.7.3", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.9", +] + [[package]] name = "thiserror" version = "1.0.21" @@ -1982,6 +2200,19 @@ dependencies = [ "syn 1.0.48", ] +[[package]] +name = "tiny_http" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1661fa0a44c95d01604bd05c66732a446c657efb62b5164a7a083a3b552b4951" +dependencies = [ + "ascii", + "chrono", + "chunked_transfer 0.3.1", + "log 0.4.11", + "url 1.7.2", +] + [[package]] name = "tinyvec" version = "0.3.4" @@ -2088,12 +2319,30 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" +[[package]] +name = "twoway" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" +dependencies = [ + "memchr", +] + [[package]] name = "typeable" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1410f6f91f21d1612654e7cc69193b0334f909dcf2c790c4826254fbb86f8887" +[[package]] +name = "typemap" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "653be63c80a3296da5551e1bfd2cca35227e13cdd08c6668903ae2f4f77aa1f6" +dependencies = [ + "unsafe-any", +] + [[package]] name = "typenum" version = "1.12.0" @@ -2149,6 +2398,15 @@ dependencies = [ "subtle 2.3.0", ] +[[package]] +name = "unsafe-any" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f" +dependencies = [ + "traitobject", +] + [[package]] name = "untrusted" version = "0.7.1" @@ -2162,7 +2420,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed21e32e4e3ff89891022affaa7091c3a164d5049cb3872f1cf0fd6ccd9fc8f7" dependencies = [ "base64 0.13.0", - "chunked_transfer", + "chunked_transfer 1.3.0", "cookie 0.14.2", "cookie_store", "encoding", @@ -2318,6 +2576,7 @@ dependencies = [ "rocket", "rocket_contrib", "rocket_oauth2", + "rocket_upload", "serde", "serde_json", "sha2 0.9.1", diff --git a/Cargo.toml b/Cargo.toml index 19cf272..32d38b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,14 @@ tracing-log = "0.1" ureq = { version = "1", features = ["json", "charset"] } uuid = { version = "0.7", features = ["serde", "v4"] } +rocket_upload = { path = "./lib/rocket_upload" } + [dependencies.rocket_contrib] version = "0.4" default-features = false features = ["json", "diesel_postgres_pool", "uuid", "helmet"] + +[workspace] +members = [ + "./lib/rocket_upload" +] diff --git a/Rocket.toml.example b/Rocket.toml.example index 6ea1fb0..4fdba6a 100644 --- a/Rocket.toml.example +++ b/Rocket.toml.example @@ -1,5 +1,6 @@ [global.limits] json = 5242880 +forms = 5242880 [global.oauth.gitea] provider = { auth_uri = "https://tulpa.dev/login/oauth/authorize", token_uri = "https://tulpa.dev/login/oauth/access_token" } diff --git a/lib/rocket_upload/Cargo.toml b/lib/rocket_upload/Cargo.toml new file mode 100644 index 0000000..0516c48 --- /dev/null +++ b/lib/rocket_upload/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rocket_upload" +version = "0.1.0" +authors = ["Christine Dodrill "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rocket = "0.4" +multipart = "0.16.1" diff --git a/lib/rocket_upload/LICENSE b/lib/rocket_upload/LICENSE new file mode 100644 index 0000000..47294a8 --- /dev/null +++ b/lib/rocket_upload/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 magiclen.org (Ron Li) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lib/rocket_upload/src/lib.rs b/lib/rocket_upload/src/lib.rs new file mode 100644 index 0000000..4c5df72 --- /dev/null +++ b/lib/rocket_upload/src/lib.rs @@ -0,0 +1,168 @@ +use std::io::{Cursor, Read, Write}; +// use std::string; +// use std::sync::Arc; +use std::fs::{self, File}; +use std::path::{Path}; + +use rocket::{Request, Data, Outcome, Outcome::*}; +use rocket::data::{self, FromDataSimple}; +use rocket::http::{Status /*, ContentType*/}; + +use multipart::server::Multipart; +// use multipart::server::MultipartData; + +#[derive(Debug)] +pub struct TextPart { + pub key:String, + pub value:String, +} +#[derive(Debug)] +pub struct FilePart { + pub name:String, + pub path:String, + pub filename:String, +} +#[derive(Debug)] +pub struct MultipartDatas { + pub texts: Vec, + pub files: Vec, +} + +impl FilePart { + pub fn persist(&self, p:&Path){ + let s = Path::join(p, &self.filename); + fs::copy(Path::new(&self.path), &s).unwrap(); + } +} +impl Drop for FilePart { + fn drop(&mut self){ + fs::remove_file(Path::new(&self.path)).unwrap(); + } +} +const TMP_PATH:&str = "/tmp/rust_upload/"; + +impl<'t> FromDataSimple for MultipartDatas { + type Error = String; + + fn from_data(request: &Request, data: Data) -> data::Outcome { + + let ct = request.headers().get_one("Content-Type").expect("no content-type"); + let idx = ct.find("boundary=").expect("no boundary"); + let boundary = &ct[(idx + "boundary=".len())..]; + + let mut d = Vec::new(); + data.stream_to(&mut d).expect("Unable to read"); + + let mut mp = Multipart::with_body(Cursor::new(d), boundary); + let mut texts=Vec::new(); + let mut files=Vec::new(); + + let mut buffer = [0u8; 4096]; + + let mut err_out: Option> = None; + + mp.foreach_entry(|entry| { + //println!("part.headers: {:?}",entry.headers); + let mut data = entry.data; + if entry.headers.filename == None { + // let str=proc_text(entry.data); + let mut text_buffer = Vec::new(); + + loop { + let c = match data.read(&mut buffer) { + Ok(c) => c, + Err(err) => { + err_out = Some(Failure((Status::UnprocessableEntity, format!("{:?}", err)))); + return; + } + }; + + if c == 0 { + break; + } + + text_buffer.extend_from_slice(&buffer[..c]); + } + + let text = match String::from_utf8(text_buffer) { + Ok(s) => s, + Err(_err) => { + err_out = Some(Failure((Status::UnprocessableEntity, + ": Data can not read as UTF-8".into()))); + return; + } + }; + // println!("data: {}", str); + // println!("name: {:?}", entry.headers.name); + texts.push(TextPart{ key:entry.headers.name.to_string(), value:text}); + } else { + // let str=proc_file(entry.headers.filename.clone().unwrap(), entry.data); + let filename = entry.headers.filename.clone().unwrap(); + if !Path::new(TMP_PATH).exists() { + fs::create_dir_all(TMP_PATH).unwrap(); + } + + let target_path = Path::join(Path::new(TMP_PATH), &filename); + + let mut file = match File::create(&target_path) { + Ok(f) => f, + Err(err) => { + err_out = Some(Failure((Status::InternalServerError, format!("{:?}", err)))); + return; + } + }; + + let mut sum_c = 0u64; + + loop { + let c = match data.read(&mut buffer) { + Ok(c) => c, + Err(err) => { + try_delete(&target_path); + err_out = Some(Failure((Status::UnprocessableEntity, format!("{:?}", err)))); + return; + } + }; + + if c == 0 { + break; + } + + sum_c = sum_c + c as u64; + + match file.write(&buffer[..c]) { + Ok(_) => (), + Err(err) => { + try_delete(&target_path); + err_out = Some(Failure((Status::InternalServerError, format!("{:?}", err)))); + return; + } + } + } + + // println!("filename: {:?}", entry.headers.name); + files.push(FilePart{ + name: entry.headers.name.to_string(), + path: String::from(TMP_PATH) + &filename, + filename: entry.headers.filename.clone().unwrap() + }) + } + //println!("part.data: {:?}",entry.data); + }).unwrap(); + if let Some(failed) = err_out { + return failed; + } else { + let v=MultipartDatas{ + texts: texts, + files: files, + }; + return Outcome::Success(v); + } + } +} + +#[inline] +fn try_delete>(path: P) { + if fs::remove_file(path.as_ref()).is_err() {} +} +