From f13603368885624046a169c49947524ccd1a8e7f Mon Sep 17 00:00:00 2001 From: Christine Dodrill Date: Fri, 30 Oct 2020 17:18:52 -0400 Subject: [PATCH] better uploads --- Cargo.lock | 52 +++++++++++++++++++++++++++++ Cargo.toml | 12 ++++--- lib/rocket_upload/Cargo.toml | 1 + lib/rocket_upload/src/lib.rs | 23 ++++++------- src/api/handler.rs | 49 +++++++++++++++++++++++++-- src/api/mod.rs | 20 +++++++---- src/b2.rs | 64 +++++++++++++++++++++++++++--------- src/main.rs | 3 +- 8 files changed, 182 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c3a09ee..1f40fab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,18 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + [[package]] name = "ascii" version = "0.8.7" @@ -180,6 +192,21 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "blake3" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9ff35b701f3914bdb8fad3368d822c766ef2858b2583198e41639b936f09d3f" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if 0.1.10", + "constant_time_eq", + "crypto-mac 0.8.0", + "digest 0.9.0", +] + [[package]] name = "block-buffer" version = "0.7.3" @@ -347,6 +374,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "cookie" version = "0.11.3" @@ -422,6 +455,16 @@ dependencies = [ "subtle 1.0.0", ] +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array 0.14.4", + "subtle 2.3.0", +] + [[package]] name = "crypto-mac" version = "0.9.1" @@ -857,6 +900,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + [[package]] name = "hkdf" version = "0.8.0" @@ -2171,6 +2220,7 @@ dependencies = [ name = "rocket_upload" version = "0.1.0" dependencies = [ + "elfs", "mime 0.3.16", "multipart", "rocket", @@ -3092,10 +3142,12 @@ checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" name = "wasmcloud-api" version = "0.1.0" dependencies = [ + "blake3", "chrono", "color-eyre", "diesel", "elfs", + "hex", "hmac 0.9.0", "jwt", "lazy_static", diff --git a/Cargo.toml b/Cargo.toml index 73381ac..0ac1ea8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,23 +7,25 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +blake3 = "0.3" chrono = { version = "0.4", features = ["serde"] } color-eyre = "0.5" diesel = { version = "1", features = ["postgres", "r2d2", "uuidv07", "chrono"] } elfs = "0" -lazy_static = "1.4" -jwt = "0.11" +hex = "0" hmac = "0.9" -sha2 = "0.9" +jwt = "0.11" +lazy_static = "1.4" raze = "0.2" rocket = "0.4" rocket_oauth2 = "0.4" -serde = { version = "^1", features = ["derive"] } serde_json = "^1" +serde = { version = "^1", features = ["derive"] } +sha2 = "0.9" thiserror = "1" tracing = "0.1" -tracing-subscriber = "0.2" tracing-log = "0.1" +tracing-subscriber = "0.2" ureq = { version = "1", features = ["json", "charset"] } uuid = { version = "0.7", features = ["serde", "v4"] } diff --git a/lib/rocket_upload/Cargo.toml b/lib/rocket_upload/Cargo.toml index ca4ad91..9a58756 100644 --- a/lib/rocket_upload/Cargo.toml +++ b/lib/rocket_upload/Cargo.toml @@ -11,3 +11,4 @@ rocket = "0.4" mime = "0.3" multipart = "0.17" tracing = "0.1" +elfs = "0" diff --git a/lib/rocket_upload/src/lib.rs b/lib/rocket_upload/src/lib.rs index 49ab0c2..dc3298c 100644 --- a/lib/rocket_upload/src/lib.rs +++ b/lib/rocket_upload/src/lib.rs @@ -1,12 +1,12 @@ -use std::io::{Cursor, Read, Write}; -use std::fs::{self, File}; -use std::path::Path; +use multipart::server::Multipart; use rocket::data::{self, FromDataSimple}; use rocket::http::Status; use rocket::{Data, Outcome, Outcome::*, Request}; -use multipart::{server::Multipart}; +use std::fs::{self, File}; +use std::io::{Cursor, Read, Write}; +use std::path::Path; -pub use mime::{Mime}; +pub use mime::Mime; #[derive(Debug)] pub struct TextPart { @@ -37,7 +37,7 @@ impl Drop for FilePart { fs::remove_file(Path::new(&self.path)).unwrap(); } } -const TMP_PATH: &str = "/tmp/rust_upload/"; +const TMP_PATH: &str = "/tmp/wasmcloud_upload/"; impl<'t> FromDataSimple for MultipartDatas { type Error = String; @@ -60,9 +60,10 @@ impl<'t> FromDataSimple for MultipartDatas { let mut buffer = [0u8; 4096]; let mut err_out: Option> = None; + let temp_folder = format!("{}{}/", TMP_PATH, elfs::next()); mp.foreach_entry(|entry| { - tracing::debug!("part.headers: {:?}",entry.headers); + tracing::debug!("part.headers: {:?}", entry.headers); let mut data = entry.data; if entry.headers.filename == None { let mut text_buffer = Vec::new(); @@ -101,11 +102,11 @@ impl<'t> FromDataSimple for MultipartDatas { }); } else { let filename = entry.headers.filename.clone().unwrap(); - if !Path::new(TMP_PATH).exists() { - fs::create_dir_all(TMP_PATH).unwrap(); + if !Path::new(&temp_folder).exists() { + fs::create_dir_all(&temp_folder).unwrap(); } - let target_path = Path::join(Path::new(TMP_PATH), &filename); + let target_path = Path::join(Path::new(&temp_folder), &filename); let mut file = match File::create(&target_path) { Ok(f) => f, @@ -149,7 +150,7 @@ impl<'t> FromDataSimple for MultipartDatas { tracing::debug!("filename: {:?}", entry.headers.name); files.push(FilePart { name: entry.headers.name.to_string(), - path: String::from(TMP_PATH) + &filename, + path: format!("{}{}", temp_folder, filename), filename: entry.headers.filename.clone().unwrap(), content_type: entry.headers.content_type.clone(), }) diff --git a/src/api/handler.rs b/src/api/handler.rs index 351679d..5f46515 100644 --- a/src/api/handler.rs +++ b/src/api/handler.rs @@ -1,8 +1,10 @@ use super::{Error, Result}; -use crate::{models, schema, MainDatabase}; +use crate::{b2, models, schema, MainDatabase}; use chrono::prelude::*; use diesel::prelude::*; +use rocket::http::ContentType; use rocket_contrib::{json::Json, uuid::Uuid}; +use rocket_upload::MultipartDatas; use schema::handlers::dsl::*; use serde::Deserialize; @@ -154,7 +156,50 @@ pub fn create_config( .get_result::(&*conn) .map_err(Error::Database)?; - let _ = cfg.iter().inspect(|kv| info!(name = kv.key_name.as_str(), "config created")); + let _ = cfg + .iter() + .inspect(|kv| info!(name = kv.key_name.as_str(), "config created")); Ok(()) } + +#[instrument(skip(conn, data, ct), err)] +#[post("/handler//upload", data = "")] +pub fn upload_version( + user: models::User, + hdl_id: Uuid, + ct: &ContentType, + data: MultipartDatas, + conn: MainDatabase, +) -> Result> { + let uuid = hdl_id.into_inner(); + + let handler = handlers + .find(uuid) + .get_result::(&*conn) + .map_err(Error::Database)?; + + if handler.user_id != user.id { + return Err(Error::LackPermissions); + } + + if data.files.len() != 1 { + return Err(Error::IncorrectFilecount(1)); + } + + let file = data.files.get(0).ok_or(Error::IncorrectFilecount(1))?; + let ct = file + .content_type + .clone() + .ok_or(Error::IncorrectFilecount(1))?; + let upload_url = b2::upload(file.path.clone().into(), ct)?; + + let handler = diesel::update(handlers.filter(id.eq(handler.id))) + .set(current_version.eq(Some(upload_url.clone()))) + .get_result(&*conn) + .map_err(Error::Database)?; + + info!(url = upload_url.as_str(), "uploaded new version of handler"); + + Ok(Json(handler)) +} diff --git a/src/api/mod.rs b/src/api/mod.rs index 6fecda5..c5e13bb 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -31,6 +31,9 @@ pub enum Error { #[error("backblaze error: {0:?}")] Backblaze(raze::Error), + + #[error("incorrect number of files uploaded (wanted {0})")] + IncorrectFilecount(usize), } impl<'a> Responder<'a> for Error { @@ -53,13 +56,16 @@ impl<'a> Responder<'a> for Error { .sized_body(Cursor::new(format!("{}", why))) .ok() } - Error::Backblaze(why) => { - Response::build() - .header(ContentType::Plain) - .status(Status::InternalServerError) - .sized_body(Cursor::new(format!("b2 error: {:?}", why))).ok() - } - + Error::Backblaze(why) => Response::build() + .header(ContentType::Plain) + .status(Status::InternalServerError) + .sized_body(Cursor::new(format!("b2 error: {:?}", why))) + .ok(), + Error::IncorrectFilecount(_) => Response::build() + .header(ContentType::Plain) + .status(Status::BadRequest) + .sized_body(Cursor::new(format!("{}", self))) + .ok(), } } } diff --git a/src/b2.rs b/src/b2.rs index 32a0b6c..83a5094 100644 --- a/src/b2.rs +++ b/src/b2.rs @@ -1,23 +1,59 @@ use crate::api::Error::Backblaze; -use color_eyre::eyre::{eyre, Result}; +use blake3::Hasher; +use color_eyre::eyre::Result; use lazy_static::lazy_static; use raze::{ api::*, - util::{self, ReadHashAtEnd, ReadThrottled}, + util::{self, ReadHashAtEnd}, }; use reqwest::blocking::ClientBuilder; use rocket_upload::Mime; -use std::{env, fs, path::PathBuf}; +use std::{ + env, fs, + io::{self, Read}, + path::PathBuf, +}; lazy_static! { pub static ref CREDS: String = env::var("B2_CREDFILE") .expect("B2_CREDFILE to be populated") .to_string(); - pub static ref BUCKET_NAME: String = env::var("B2_MODULE_BUCKET_NAME") - .expect("B2_MODULE_BUCKET_NAME to be populated") + pub static ref BUCKET_ID: String = env::var("B2_MODULE_BUCKET_ID") + .expect("B2_MODULE_BUCKET_ID to be populated") .to_string(); } +fn hash(filename: &PathBuf) -> Result<(String, u64)> { + let mut fin = fs::File::open(filename)?; + let mut hasher = Hasher::new(); + let size = copy_wide(&mut fin, &mut hasher)?; + let hash = hasher.finalize(); + let hash = hash.as_bytes(); + let hash = hex::encode(&hash); + + Ok((hash, size)) +} + +// A 16 KiB buffer is enough to take advantage of all the SIMD instruction sets +// that we support, but `std::io::copy` currently uses 8 KiB. Most platforms +// can support at least 64 KiB, and there's some performance benefit to using +// bigger reads, so that's what we use here. +fn copy_wide(mut reader: impl Read, hasher: &mut blake3::Hasher) -> io::Result { + let mut buffer = [0; 65536]; + let mut total = 0; + loop { + match reader.read(&mut buffer) { + Ok(0) => return Ok(total), + Ok(n) => { + hasher.update(&buffer[..n]); + total += n as u64; + } + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + } + } +} + #[instrument(err)] pub fn upload(filename: PathBuf, content_type: Mime) -> Result { let client = ClientBuilder::new() @@ -26,10 +62,11 @@ pub fn upload(filename: PathBuf, content_type: Mime) -> Result { .build()?; let auth = util::authenticate_from_file(&client, CREDS.clone()).map_err(Backblaze)?; - let upauth = b2_get_upload_url(&client, &auth, "bucket_id").map_err(Backblaze)?; + let upauth = b2_get_upload_url(&client, &auth, BUCKET_ID.clone()).map_err(Backblaze)?; let fin = fs::File::open(filename.clone())?; let meta = fin.metadata()?; - let size = meta.len(); + let (hash, size) = hash(&filename)?; + let hash = format!("{}.wasm", hash); let modf = meta .modified() .unwrap() @@ -37,14 +74,10 @@ pub fn upload(filename: PathBuf, content_type: Mime) -> Result { .as_secs() * 1000; let ct = content_type.to_string(); - let filepath = filename - .file_name() - .ok_or(eyre!("wanted file_name to work"))? - .to_str() - .ok_or(eyre!("filename is somehow not utf-8, what"))?; + debug!(hash = hash.as_str(), size = size, "uploading to b2"); let param = FileParameters { - file_path: filepath.clone(), + file_path: hash.as_str(), file_size: size, content_type: Some(&ct), content_sha1: Sha1Variant::HexAtEnd, @@ -53,9 +86,8 @@ pub fn upload(filename: PathBuf, content_type: Mime) -> Result { let reader = fin; let reader = ReadHashAtEnd::wrap(reader); - let reader = ReadThrottled::wrap(reader, 5000); - let resp = b2_upload_file(&client, &upauth, reader, param).map_err(Backblaze)?; + b2_upload_file(&client, &upauth, reader, param).map_err(Backblaze)?; - Ok(format!("b2://{}/{}", *BUCKET_NAME, filepath)) + Ok(format!("b2://{}", hash)) } diff --git a/src/main.rs b/src/main.rs index 8b21660..6e7258e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -42,7 +42,7 @@ fn main() -> Result<()> { // evaluated and will kill the program if JWT_SECRET is not found. let _ = *jwt::SECRET; let _ = *b2::CREDS; - let _ = *b2::BUCKET_NAME; + let _ = *b2::BUCKET_ID; rocket::ignite() .attach(OAuth2::::fairing("gitea")) @@ -57,6 +57,7 @@ fn main() -> Result<()> { api::handler::delete, api::handler::get_config, api::handler::create_config, + api::handler::upload_version, api::user::whoami, api::user::get, api::token::list,