wasmcloud
/
wasmcloud
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

better uploads

This commit is contained in:
Cadey Ratio 2020-10-30 17:18:52 -04:00
parent 732f951ee3
commit f136033688
8 changed files with 182 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
Cargo.lock generated
View File

@ -97,6 +97,18 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "arrayref"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
[[package]]
name = "arrayvec"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
[[package]]
name = "ascii"
version = "0.8.7"
@ -180,6 +192,21 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "blake3"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9ff35b701f3914bdb8fad3368d822c766ef2858b2583198e41639b936f09d3f"
dependencies = [
"arrayref",
"arrayvec",
"cc",
"cfg-if 0.1.10",
"constant_time_eq",
"crypto-mac 0.8.0",
"digest 0.9.0",
]
[[package]]
name = "block-buffer"
version = "0.7.3"
@ -347,6 +374,12 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2"
[[package]]
name = "constant_time_eq"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
[[package]]
name = "cookie"
version = "0.11.3"
@ -422,6 +455,16 @@ dependencies = [
"subtle 1.0.0",
]
[[package]]
name = "crypto-mac"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab"
dependencies = [
"generic-array 0.14.4",
"subtle 2.3.0",
]
[[package]]
name = "crypto-mac"
version = "0.9.1"
@ -857,6 +900,12 @@ dependencies = [
"libc",
]
[[package]]
name = "hex"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35"
[[package]]
name = "hkdf"
version = "0.8.0"
@ -2171,6 +2220,7 @@ dependencies = [
name = "rocket_upload"
version = "0.1.0"
dependencies = [
"elfs",
"mime 0.3.16",
"multipart",
"rocket",
@ -3092,10 +3142,12 @@ checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307"
name = "wasmcloud-api"
version = "0.1.0"
dependencies = [
"blake3",
"chrono",
"color-eyre",
"diesel",
"elfs",
"hex",
"hmac 0.9.0",
"jwt",
"lazy_static",

12
Cargo.toml
View File

@ -7,23 +7,25 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
blake3 = "0.3"
chrono = { version = "0.4", features = ["serde"] }
color-eyre = "0.5"
diesel = { version = "1", features = ["postgres", "r2d2", "uuidv07", "chrono"] }
elfs = "0"
lazy_static = "1.4"
jwt = "0.11"
hex = "0"
hmac = "0.9"
sha2 = "0.9"
jwt = "0.11"
lazy_static = "1.4"
raze = "0.2"
rocket = "0.4"
rocket_oauth2 = "0.4"
serde = { version = "^1", features = ["derive"] }
serde_json = "^1"
serde = { version = "^1", features = ["derive"] }
sha2 = "0.9"
thiserror = "1"
tracing = "0.1"
tracing-subscriber = "0.2"
tracing-log = "0.1"
tracing-subscriber = "0.2"
ureq = { version = "1", features = ["json", "charset"] }
uuid = { version = "0.7", features = ["serde", "v4"] }

1
lib/rocket_upload/Cargo.toml
View File

@ -11,3 +11,4 @@ rocket = "0.4"
mime = "0.3"
multipart = "0.17"
tracing = "0.1"
elfs = "0"

23
lib/rocket_upload/src/lib.rs
View File

@ -1,12 +1,12 @@
use std::io::{Cursor, Read, Write};
use std::fs::{self, File};
use std::path::Path;
use multipart::server::Multipart;
use rocket::data::{self, FromDataSimple};
use rocket::http::Status;
use rocket::{Data, Outcome, Outcome::*, Request};
use multipart::{server::Multipart};
use std::fs::{self, File};
use std::io::{Cursor, Read, Write};
use std::path::Path;
pub use mime::{Mime};
pub use mime::Mime;
#[derive(Debug)]
pub struct TextPart {
@ -37,7 +37,7 @@ impl Drop for FilePart {
fs::remove_file(Path::new(&self.path)).unwrap();
}
}
const TMP_PATH: &str = "/tmp/rust_upload/";
const TMP_PATH: &str = "/tmp/wasmcloud_upload/";
impl<'t> FromDataSimple for MultipartDatas {
type Error = String;
@ -60,9 +60,10 @@ impl<'t> FromDataSimple for MultipartDatas {
let mut buffer = [0u8; 4096];
let mut err_out: Option<Outcome<_, (Status, _), _>> = None;
let temp_folder = format!("{}{}/", TMP_PATH, elfs::next());
mp.foreach_entry(|entry| {
tracing::debug!("part.headers: {:?}",entry.headers);
tracing::debug!("part.headers: {:?}", entry.headers);
let mut data = entry.data;
if entry.headers.filename == None {
let mut text_buffer = Vec::new();
@ -101,11 +102,11 @@ impl<'t> FromDataSimple for MultipartDatas {
});
} else {
let filename = entry.headers.filename.clone().unwrap();
if !Path::new(TMP_PATH).exists() {
fs::create_dir_all(TMP_PATH).unwrap();
if !Path::new(&temp_folder).exists() {
fs::create_dir_all(&temp_folder).unwrap();
}
let target_path = Path::join(Path::new(TMP_PATH), &filename);
let target_path = Path::join(Path::new(&temp_folder), &filename);
let mut file = match File::create(&target_path) {
Ok(f) => f,
@ -149,7 +150,7 @@ impl<'t> FromDataSimple for MultipartDatas {
tracing::debug!("filename: {:?}", entry.headers.name);
files.push(FilePart {
name: entry.headers.name.to_string(),
path: String::from(TMP_PATH) + &filename,
path: format!("{}{}", temp_folder, filename),
filename: entry.headers.filename.clone().unwrap(),
content_type: entry.headers.content_type.clone(),
})

49
src/api/handler.rs
View File

@ -1,8 +1,10 @@
use super::{Error, Result};
use crate::{models, schema, MainDatabase};
use crate::{b2, models, schema, MainDatabase};
use chrono::prelude::*;
use diesel::prelude::*;
use rocket::http::ContentType;
use rocket_contrib::{json::Json, uuid::Uuid};
use rocket_upload::MultipartDatas;
use schema::handlers::dsl::*;
use serde::Deserialize;
@ -154,7 +156,50 @@ pub fn create_config(
.get_result::<models::HandlerConfig>(&*conn)
.map_err(Error::Database)?;
let _ = cfg.iter().inspect(|kv| info!(name = kv.key_name.as_str(), "config created"));
let _ = cfg
.iter()
.inspect(|kv| info!(name = kv.key_name.as_str(), "config created"));
Ok(())
}
#[instrument(skip(conn, data, ct), err)]
#[post("/handler/<hdl_id>/upload", data = "<data>")]
pub fn upload_version(
user: models::User,
hdl_id: Uuid,
ct: &ContentType,
data: MultipartDatas,
conn: MainDatabase,
) -> Result<Json<models::Handler>> {
let uuid = hdl_id.into_inner();
let handler = handlers
.find(uuid)
.get_result::<models::Handler>(&*conn)
.map_err(Error::Database)?;
if handler.user_id != user.id {
return Err(Error::LackPermissions);
}
if data.files.len() != 1 {
return Err(Error::IncorrectFilecount(1));
}
let file = data.files.get(0).ok_or(Error::IncorrectFilecount(1))?;
let ct = file
.content_type
.clone()
.ok_or(Error::IncorrectFilecount(1))?;
let upload_url = b2::upload(file.path.clone().into(), ct)?;
let handler = diesel::update(handlers.filter(id.eq(handler.id)))
.set(current_version.eq(Some(upload_url.clone())))
.get_result(&*conn)
.map_err(Error::Database)?;
info!(url = upload_url.as_str(), "uploaded new version of handler");
Ok(Json(handler))
}

20
src/api/mod.rs
View File

@ -31,6 +31,9 @@ pub enum Error {
#[error("backblaze error: {0:?}")]
Backblaze(raze::Error),
#[error("incorrect number of files uploaded (wanted {0})")]
IncorrectFilecount(usize),
}
impl<'a> Responder<'a> for Error {
@ -53,13 +56,16 @@ impl<'a> Responder<'a> for Error {
.sized_body(Cursor::new(format!("{}", why)))
.ok()
}
Error::Backblaze(why) => {
Response::build()
.header(ContentType::Plain)
.status(Status::InternalServerError)
.sized_body(Cursor::new(format!("b2 error: {:?}", why))).ok()
}
Error::Backblaze(why) => Response::build()
.header(ContentType::Plain)
.status(Status::InternalServerError)
.sized_body(Cursor::new(format!("b2 error: {:?}", why)))
.ok(),
Error::IncorrectFilecount(_) => Response::build()
.header(ContentType::Plain)
.status(Status::BadRequest)
.sized_body(Cursor::new(format!("{}", self)))
.ok(),
}
}
}

64
src/b2.rs
View File

@ -1,23 +1,59 @@
use crate::api::Error::Backblaze;
use color_eyre::eyre::{eyre, Result};
use blake3::Hasher;
use color_eyre::eyre::Result;
use lazy_static::lazy_static;
use raze::{
api::*,
util::{self, ReadHashAtEnd, ReadThrottled},
util::{self, ReadHashAtEnd},
};
use reqwest::blocking::ClientBuilder;
use rocket_upload::Mime;
use std::{env, fs, path::PathBuf};
use std::{
env, fs,
io::{self, Read},
path::PathBuf,
};
lazy_static! {
pub static ref CREDS: String = env::var("B2_CREDFILE")
.expect("B2_CREDFILE to be populated")
.to_string();
pub static ref BUCKET_NAME: String = env::var("B2_MODULE_BUCKET_NAME")
.expect("B2_MODULE_BUCKET_NAME to be populated")
pub static ref BUCKET_ID: String = env::var("B2_MODULE_BUCKET_ID")
.expect("B2_MODULE_BUCKET_ID to be populated")
.to_string();
}
fn hash(filename: &PathBuf) -> Result<(String, u64)> {
let mut fin = fs::File::open(filename)?;
let mut hasher = Hasher::new();
let size = copy_wide(&mut fin, &mut hasher)?;
let hash = hasher.finalize();
let hash = hash.as_bytes();
let hash = hex::encode(&hash);
Ok((hash, size))
}
// A 16 KiB buffer is enough to take advantage of all the SIMD instruction sets
// that we support, but `std::io::copy` currently uses 8 KiB. Most platforms
// can support at least 64 KiB, and there's some performance benefit to using
// bigger reads, so that's what we use here.
fn copy_wide(mut reader: impl Read, hasher: &mut blake3::Hasher) -> io::Result<u64> {
let mut buffer = [0; 65536];
let mut total = 0;
loop {
match reader.read(&mut buffer) {
Ok(0) => return Ok(total),
Ok(n) => {
hasher.update(&buffer[..n]);
total += n as u64;
}
Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(e) => return Err(e),
}
}
}
#[instrument(err)]
pub fn upload(filename: PathBuf, content_type: Mime) -> Result<String> {
let client = ClientBuilder::new()
@ -26,10 +62,11 @@ pub fn upload(filename: PathBuf, content_type: Mime) -> Result<String> {
.build()?;
let auth = util::authenticate_from_file(&client, CREDS.clone()).map_err(Backblaze)?;
let upauth = b2_get_upload_url(&client, &auth, "bucket_id").map_err(Backblaze)?;
let upauth = b2_get_upload_url(&client, &auth, BUCKET_ID.clone()).map_err(Backblaze)?;
let fin = fs::File::open(filename.clone())?;
let meta = fin.metadata()?;
let size = meta.len();
let (hash, size) = hash(&filename)?;
let hash = format!("{}.wasm", hash);
let modf = meta
.modified()
.unwrap()
@ -37,14 +74,10 @@ pub fn upload(filename: PathBuf, content_type: Mime) -> Result<String> {
.as_secs()
* 1000;
let ct = content_type.to_string();
let filepath = filename
.file_name()
.ok_or(eyre!("wanted file_name to work"))?
.to_str()
.ok_or(eyre!("filename is somehow not utf-8, what"))?;
debug!(hash = hash.as_str(), size = size, "uploading to b2");
let param = FileParameters {
file_path: filepath.clone(),
file_path: hash.as_str(),
file_size: size,
content_type: Some(&ct),
content_sha1: Sha1Variant::HexAtEnd,
@ -53,9 +86,8 @@ pub fn upload(filename: PathBuf, content_type: Mime) -> Result<String> {
let reader = fin;
let reader = ReadHashAtEnd::wrap(reader);
let reader = ReadThrottled::wrap(reader, 5000);
let resp = b2_upload_file(&client, &upauth, reader, param).map_err(Backblaze)?;
b2_upload_file(&client, &upauth, reader, param).map_err(Backblaze)?;
Ok(format!("b2://{}/{}", *BUCKET_NAME, filepath))
Ok(format!("b2://{}", hash))
}

3
src/main.rs
View File

@ -42,7 +42,7 @@ fn main() -> Result<()> {
// evaluated and will kill the program if JWT_SECRET is not found.
let _ = *jwt::SECRET;
let _ = *b2::CREDS;
let _ = *b2::BUCKET_NAME;
let _ = *b2::BUCKET_ID;
rocket::ignite()
.attach(OAuth2::<Gitea>::fairing("gitea"))
@ -57,6 +57,7 @@ fn main() -> Result<()> {
api::handler::delete,
api::handler::get_config,
api::handler::create_config,
api::handler::upload_version,
api::user::whoami,
api::user::get,
api::token::list,