From 732f951ee31fdbc6638934ad82ff156454d3833b Mon Sep 17 00:00:00 2001 From: Christine Dodrill Date: Fri, 30 Oct 2020 14:09:35 -0400 Subject: [PATCH] upload webassembly modules --- .gitignore | 3 + Cargo.lock | 569 ++++++++++++++++++++++++++++++++++- Cargo.toml | 6 + lib/rocket_upload/Cargo.toml | 4 +- lib/rocket_upload/src/lib.rs | 268 +++++++++-------- shell.nix | 17 +- src/api/mod.rs | 10 + src/b2.rs | 61 ++++ src/main.rs | 11 + var/secret/.gitkeep | 0 10 files changed, 797 insertions(+), 152 deletions(-) create mode 100644 src/b2.rs create mode 100644 var/secret/.gitkeep diff --git a/.gitignore b/.gitignore index e6f2d77..1c968f7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ # Rocket Rocket.toml + +var/secret/* +!var/secret/.gitkeep diff --git a/Cargo.lock b/Cargo.lock index 533b846..c3a09ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -247,6 +247,12 @@ version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + [[package]] name = "cc" version = "1.0.61" @@ -384,6 +390,22 @@ dependencies = [ "url 2.1.1", ] +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + [[package]] name = "cpuid-bool" version = "0.1.2" @@ -492,6 +514,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "dtoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" + [[package]] name = "elfs" version = "0.1.0" @@ -565,6 +593,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" +[[package]] +name = "encoding_rs" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" +dependencies = [ + "cfg-if 0.1.10", +] + [[package]] name = "error-chain" version = "0.12.4" @@ -602,6 +639,27 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "fsevent" version = "0.4.0" @@ -643,6 +701,57 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +[[package]] +name = "futures-channel" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0448174b01148032eed37ac4aed28963aaaa8cfa93569a08e5b479bbc6c2c151" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18eaa56102984bed2c88ea39026cff3ce3b4c7f508ca970cedf2450ea10d4e46" + +[[package]] +name = "futures-io" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1798854a4727ff944a7b12aa999f58ce7aa81db80d2dfaaf2ba06f065ddd2b" + +[[package]] +name = "futures-sink" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e3ca3f17d6e8804ae5d3df7a7d35b2b3a6fe89dac84b31872720fc3060a0b11" + +[[package]] +name = "futures-task" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d502af37186c4fef99453df03e374683f8a1eec9dcc1e66b3b82dc8278ce3c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abcb44342f62e6f3e8ac427b8aa815f724fd705dfad060b18ac7866c15bb8e34" +dependencies = [ + "futures-core", + "futures-io", + "futures-task", + "memchr", + "pin-project 1.0.1", + "pin-utils", + "slab", +] + [[package]] name = "generator" version = "0.6.23" @@ -713,6 +822,26 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32619942b8be646939eaf3db0602b39f5229b74575b67efc897811ded1db4e57" +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", + "tracing-futures", +] + [[package]] name = "hashbrown" version = "0.9.1" @@ -758,12 +887,39 @@ dependencies = [ "digest 0.9.0", ] +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes", + "http", +] + [[package]] name = "httparse" version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + [[package]] name = "hyper" version = "0.10.16" @@ -774,27 +930,64 @@ dependencies = [ "httparse", "language-tags", "log 0.3.9", - "mime", + "mime 0.2.6", "num_cpus", "time 0.1.44", "traitobject", "typeable", - "unicase", + "unicase 1.4.2", "url 1.7.2", ] +[[package]] +name = "hyper" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project 0.4.27", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper-sync-rustls" version = "0.3.0-rc.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cb014c4ea00486e2b62860b5e15229d37516d4924177218beafbf46583de3ab" dependencies = [ - "hyper", + "hyper 0.10.16", "rustls 0.17.0", "webpki", "webpki-roots 0.19.0", ] +[[package]] +name = "hyper-tls" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +dependencies = [ + "bytes", + "hyper 0.13.8", + "native-tls", + "tokio", + "tokio-tls", +] + [[package]] name = "idna" version = "0.1.5" @@ -871,15 +1064,21 @@ dependencies = [ "libc", ] +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + [[package]] name = "iron" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6d308ca2d884650a8bf9ed2ff4cb13fbb2207b71f64cda11dc9b892067295e8" dependencies = [ - "hyper", + "hyper 0.10.16", "log 0.3.9", - "mime_guess", + "mime_guess 1.8.8", "modifier", "num_cpus", "plugin", @@ -1021,16 +1220,32 @@ dependencies = [ "log 0.3.9", ] +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "mime_guess" version = "1.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "216929a5ee4dd316b1702eedf5e74548c123d370f47841ceaac38ca154690ca3" dependencies = [ - "mime", + "mime 0.2.6", "phf", "phf_codegen", - "unicase", + "unicase 1.4.2", +] + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime 0.3.16", + "unicase 2.6.0", ] [[package]] @@ -1094,17 +1309,17 @@ checksum = "41f5c9112cb662acd3b204077e0de5bc66305fa8df65c8019d5adb10e9ab6e58" [[package]] name = "multipart" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136eed74cadb9edd2651ffba732b19a450316b680e4f48d6c79e905799e19d01" +checksum = "8209c33c951f07387a8497841122fc6f712165e3f9bda3e6be4645b58188f676" dependencies = [ "buf_redux", "httparse", - "hyper", + "hyper 0.10.16", "iron", "log 0.4.11", - "mime", - "mime_guess", + "mime 0.3.16", + "mime_guess 2.0.3", "nickel", "quick-error", "rand 0.6.5", @@ -1133,6 +1348,24 @@ dependencies = [ "rand 0.3.23", ] +[[package]] +name = "native-tls" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b0d88c06fe90d5ee94048ba40409ef1d9315d86f6f38c2efdaad4fb50c58b2d" +dependencies = [ + "lazy_static", + "libc", + "log 0.4.11", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "net2" version = "0.2.35" @@ -1151,7 +1384,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5061a832728db2dacb61cefe0ce303b58f85764ec680e71d9138229640a46d9" dependencies = [ "groupable", - "hyper", + "hyper 0.10.16", "lazy_static", "log 0.3.9", "modifier", @@ -1236,6 +1469,39 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" +dependencies = [ + "bitflags", + "cfg-if 0.1.10", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" +dependencies = [ + "autocfg 1.0.1", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "owo-colors" version = "1.1.3" @@ -1338,7 +1604,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" dependencies = [ "siphasher", - "unicase", + "unicase 1.4.2", +] + +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841" +dependencies = [ + "pin-project-internal 1.0.1", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -1347,6 +1653,18 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + [[package]] name = "plugin" version = "0.2.6" @@ -1632,6 +1950,22 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "raze" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5d006770fcce615a00501888b3b61a83b6fdf81d570b1e05cf86615919189ca" +dependencies = [ + "base64 0.12.3", + "hyper 0.13.8", + "reqwest", + "serde", + "serde_derive", + "serde_json", + "sha1", + "url 2.1.1", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -1684,6 +2018,41 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "reqwest" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e" +dependencies = [ + "base64 0.12.3", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper 0.13.8", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log 0.4.11", + "mime 0.3.16", + "mime_guess 2.0.3", + "native-tls", + "percent-encoding 2.1.0", + "pin-project-lite", + "serde", + "serde_urlencoded", + "tokio", + "tokio-tls", + "url 2.1.1", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.16.15" @@ -1772,7 +2141,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aff5a5480175f2f553a876b251e9350c74196128806d176da3a51c82aab5428" dependencies = [ "cookie 0.11.3", - "hyper", + "hyper 0.10.16", "indexmap", "pear", "percent-encoding 1.0.1", @@ -1789,7 +2158,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee8d685f84c5873e507b97325f08fb8797c7e146fc068ee1d4aac56942d365a6" dependencies = [ "base64 0.12.3", - "hyper", + "hyper 0.10.16", "hyper-sync-rustls", "log 0.4.11", "rand 0.7.3", @@ -1802,8 +2171,10 @@ dependencies = [ name = "rocket_upload" version = "0.1.0" dependencies = [ + "mime 0.3.16", "multipart", "rocket", + "tracing", ] [[package]] @@ -1868,6 +2239,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + [[package]] name = "scheduled-thread-pool" version = "0.2.5" @@ -1899,6 +2280,29 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "0.9.0" @@ -1945,6 +2349,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +dependencies = [ + "dtoa", + "itoa", + "serde", + "url 2.1.1", +] + [[package]] name = "sha1" version = "0.6.0" @@ -2004,6 +2420,18 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" +[[package]] +name = "socket2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "redox_syscall", + "winapi 0.3.9", +] + [[package]] name = "spin" version = "0.5.2" @@ -2219,6 +2647,48 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" +[[package]] +name = "tokio" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "memchr", + "mio", + "num_cpus", + "pin-project-lite", + "slab", +] + +[[package]] +name = "tokio-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log 0.4.11", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.4.10" @@ -2228,6 +2698,12 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + [[package]] name = "tracing" version = "0.1.21" @@ -2235,6 +2711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" dependencies = [ "cfg-if 0.1.10", + "log 0.4.11", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2270,6 +2747,16 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + [[package]] name = "tracing-log" version = "0.1.1" @@ -2319,6 +2806,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "twoway" version = "0.1.8" @@ -2358,6 +2851,15 @@ dependencies = [ "version_check 0.1.5", ] +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check 0.9.2", +] + [[package]] name = "unicode-bidi" version = "0.3.4" @@ -2496,6 +2998,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log 0.4.11", + "try-lock", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -2515,6 +3027,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" dependencies = [ "cfg-if 0.1.10", + "serde", + "serde_json", "wasm-bindgen-macro", ] @@ -2533,6 +3047,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" +dependencies = [ + "cfg-if 0.1.10", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.68" @@ -2573,6 +3099,8 @@ dependencies = [ "hmac 0.9.0", "jwt", "lazy_static", + "raze", + "reqwest", "rocket", "rocket_contrib", "rocket_oauth2", @@ -2669,6 +3197,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "ws2_32-sys" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 32d38b1..73381ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ lazy_static = "1.4" jwt = "0.11" hmac = "0.9" sha2 = "0.9" +raze = "0.2" rocket = "0.4" rocket_oauth2 = "0.4" serde = { version = "^1", features = ["derive"] } @@ -33,6 +34,11 @@ version = "0.4" default-features = false features = ["json", "diesel_postgres_pool", "uuid", "helmet"] +[dependencies.reqwest] +version = "0.10" +default-features = false +features = [ "blocking" ] + [workspace] members = [ "./lib/rocket_upload" diff --git a/lib/rocket_upload/Cargo.toml b/lib/rocket_upload/Cargo.toml index 0516c48..ca4ad91 100644 --- a/lib/rocket_upload/Cargo.toml +++ b/lib/rocket_upload/Cargo.toml @@ -8,4 +8,6 @@ edition = "2018" [dependencies] rocket = "0.4" -multipart = "0.16.1" +mime = "0.3" +multipart = "0.17" +tracing = "0.1" diff --git a/lib/rocket_upload/src/lib.rs b/lib/rocket_upload/src/lib.rs index 4c5df72..49ab0c2 100644 --- a/lib/rocket_upload/src/lib.rs +++ b/lib/rocket_upload/src/lib.rs @@ -1,168 +1,174 @@ use std::io::{Cursor, Read, Write}; -// use std::string; -// use std::sync::Arc; use std::fs::{self, File}; -use std::path::{Path}; - -use rocket::{Request, Data, Outcome, Outcome::*}; +use std::path::Path; use rocket::data::{self, FromDataSimple}; -use rocket::http::{Status /*, ContentType*/}; +use rocket::http::Status; +use rocket::{Data, Outcome, Outcome::*, Request}; +use multipart::{server::Multipart}; -use multipart::server::Multipart; -// use multipart::server::MultipartData; +pub use mime::{Mime}; #[derive(Debug)] pub struct TextPart { - pub key:String, - pub value:String, + pub key: String, + pub value: String, } #[derive(Debug)] pub struct FilePart { - pub name:String, - pub path:String, - pub filename:String, + pub name: String, + pub path: String, + pub filename: String, + pub content_type: Option, } #[derive(Debug)] pub struct MultipartDatas { - pub texts: Vec, - pub files: Vec, + pub texts: Vec, + pub files: Vec, } impl FilePart { - pub fn persist(&self, p:&Path){ - let s = Path::join(p, &self.filename); - fs::copy(Path::new(&self.path), &s).unwrap(); - } + pub fn persist(&self, p: &Path) { + let s = Path::join(p, &self.filename); + fs::copy(Path::new(&self.path), &s).unwrap(); + } } impl Drop for FilePart { - fn drop(&mut self){ - fs::remove_file(Path::new(&self.path)).unwrap(); - } + fn drop(&mut self) { + fs::remove_file(Path::new(&self.path)).unwrap(); + } } -const TMP_PATH:&str = "/tmp/rust_upload/"; +const TMP_PATH: &str = "/tmp/rust_upload/"; impl<'t> FromDataSimple for MultipartDatas { - type Error = String; + type Error = String; - fn from_data(request: &Request, data: Data) -> data::Outcome { + fn from_data(request: &Request, data: Data) -> data::Outcome { + let ct = request + .headers() + .get_one("Content-Type") + .expect("no content-type"); + let idx = ct.find("boundary=").expect("no boundary"); + let boundary = &ct[(idx + "boundary=".len())..]; - let ct = request.headers().get_one("Content-Type").expect("no content-type"); - let idx = ct.find("boundary=").expect("no boundary"); - let boundary = &ct[(idx + "boundary=".len())..]; - - let mut d = Vec::new(); - data.stream_to(&mut d).expect("Unable to read"); + let mut d = Vec::new(); + data.stream_to(&mut d).expect("Unable to read"); - let mut mp = Multipart::with_body(Cursor::new(d), boundary); - let mut texts=Vec::new(); - let mut files=Vec::new(); + let mut mp = Multipart::with_body(Cursor::new(d), boundary); + let mut texts = Vec::new(); + let mut files = Vec::new(); - let mut buffer = [0u8; 4096]; + let mut buffer = [0u8; 4096]; - let mut err_out: Option> = None; + let mut err_out: Option> = None; - mp.foreach_entry(|entry| { - //println!("part.headers: {:?}",entry.headers); - let mut data = entry.data; - if entry.headers.filename == None { - // let str=proc_text(entry.data); - let mut text_buffer = Vec::new(); - - loop { - let c = match data.read(&mut buffer) { - Ok(c) => c, - Err(err) => { - err_out = Some(Failure((Status::UnprocessableEntity, format!("{:?}", err)))); - return; - } - }; - - if c == 0 { - break; - } - - text_buffer.extend_from_slice(&buffer[..c]); - } - - let text = match String::from_utf8(text_buffer) { - Ok(s) => s, - Err(_err) => { - err_out = Some(Failure((Status::UnprocessableEntity, - ": Data can not read as UTF-8".into()))); - return; - } - }; - // println!("data: {}", str); - // println!("name: {:?}", entry.headers.name); - texts.push(TextPart{ key:entry.headers.name.to_string(), value:text}); - } else { - // let str=proc_file(entry.headers.filename.clone().unwrap(), entry.data); - let filename = entry.headers.filename.clone().unwrap(); - if !Path::new(TMP_PATH).exists() { - fs::create_dir_all(TMP_PATH).unwrap(); - } - - let target_path = Path::join(Path::new(TMP_PATH), &filename); - - let mut file = match File::create(&target_path) { - Ok(f) => f, - Err(err) => { - err_out = Some(Failure((Status::InternalServerError, format!("{:?}", err)))); - return; - } - }; - - let mut sum_c = 0u64; - - loop { - let c = match data.read(&mut buffer) { - Ok(c) => c, - Err(err) => { - try_delete(&target_path); - err_out = Some(Failure((Status::UnprocessableEntity, format!("{:?}", err)))); - return; + mp.foreach_entry(|entry| { + tracing::debug!("part.headers: {:?}",entry.headers); + let mut data = entry.data; + if entry.headers.filename == None { + let mut text_buffer = Vec::new(); + + loop { + let c = match data.read(&mut buffer) { + Ok(c) => c, + Err(err) => { + err_out = + Some(Failure((Status::UnprocessableEntity, format!("{:?}", err)))); + return; + } + }; + + if c == 0 { + break; + } + + text_buffer.extend_from_slice(&buffer[..c]); } - }; - - if c == 0 { - break; + + let text = match String::from_utf8(text_buffer) { + Ok(s) => s, + Err(_err) => { + err_out = Some(Failure(( + Status::UnprocessableEntity, + ": Data can not read as UTF-8".into(), + ))); + return; + } + }; + tracing::debug!("name: {:?}", entry.headers.name); + texts.push(TextPart { + key: entry.headers.name.to_string(), + value: text, + }); + } else { + let filename = entry.headers.filename.clone().unwrap(); + if !Path::new(TMP_PATH).exists() { + fs::create_dir_all(TMP_PATH).unwrap(); + } + + let target_path = Path::join(Path::new(TMP_PATH), &filename); + + let mut file = match File::create(&target_path) { + Ok(f) => f, + Err(err) => { + err_out = + Some(Failure((Status::InternalServerError, format!("{:?}", err)))); + return; + } + }; + + let mut sum_c = 0u64; + + loop { + let c = match data.read(&mut buffer) { + Ok(c) => c, + Err(err) => { + try_delete(&target_path); + err_out = + Some(Failure((Status::UnprocessableEntity, format!("{:?}", err)))); + return; + } + }; + + if c == 0 { + break; + } + + sum_c = sum_c + c as u64; + + match file.write(&buffer[..c]) { + Ok(_) => (), + Err(err) => { + try_delete(&target_path); + err_out = + Some(Failure((Status::InternalServerError, format!("{:?}", err)))); + return; + } + } + } + + tracing::debug!("filename: {:?}", entry.headers.name); + files.push(FilePart { + name: entry.headers.name.to_string(), + path: String::from(TMP_PATH) + &filename, + filename: entry.headers.filename.clone().unwrap(), + content_type: entry.headers.content_type.clone(), + }) } - - sum_c = sum_c + c as u64; - - match file.write(&buffer[..c]) { - Ok(_) => (), - Err(err) => { - try_delete(&target_path); - err_out = Some(Failure((Status::InternalServerError, format!("{:?}", err)))); - return; - } - } - } - - // println!("filename: {:?}", entry.headers.name); - files.push(FilePart{ - name: entry.headers.name.to_string(), - path: String::from(TMP_PATH) + &filename, - filename: entry.headers.filename.clone().unwrap() }) - } - //println!("part.data: {:?}",entry.data); - }).unwrap(); - if let Some(failed) = err_out { - return failed; - } else { - let v=MultipartDatas{ - texts: texts, - files: files, - }; - return Outcome::Success(v); + .unwrap(); + if let Some(failed) = err_out { + return failed; + } else { + let v = MultipartDatas { + texts: texts, + files: files, + }; + return Outcome::Success(v); + } } - } } #[inline] fn try_delete>(path: P) { if fs::remove_file(path.as_ref()).is_err() {} } - diff --git a/shell.nix b/shell.nix index e573367..a6ef57d 100644 --- a/shell.nix +++ b/shell.nix @@ -3,11 +3,20 @@ let rust = import ./nix/rust.nix { inherit sources; }; pkgs = import sources.nixpkgs { }; in pkgs.mkShell rec { - buildInputs = with pkgs; [ rust diesel-cli postgresql pgcli cargo-watch ]; + buildInputs = with pkgs; [ + rust + diesel-cli + postgresql + pgcli + cargo-watch + pkg-config + openssl + ]; - RUST_LOG = "info,wasmcloud_api=debug,wasmcloud_api::*=debug"; + B2_CREDFILE = "./var/secret/b2-creds.txt"; + B2_MODULE_BUCKET_NAME = "wasmcloud-modules"; + RUST_LOG = "info,wasmcloud_api=debug"; DATABASE_URL = "postgresql://postgres:hunter2@localhost:5432/wasmcloud"; - ROCKET_DATABASES = '' - { main_data = { url = "${DATABASE_URL}" } }''; + ROCKET_DATABASES = ''{ main_data = { url = "${DATABASE_URL}" } }''; JWT_SECRET = "hunter2"; } diff --git a/src/api/mod.rs b/src/api/mod.rs index 2324e9c..6fecda5 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -28,6 +28,9 @@ pub enum Error { #[error("external dependency failed: {0}")] ExternalDependencyFailed(Report), + + #[error("backblaze error: {0:?}")] + Backblaze(raze::Error), } impl<'a> Responder<'a> for Error { @@ -50,6 +53,13 @@ impl<'a> Responder<'a> for Error { .sized_body(Cursor::new(format!("{}", why))) .ok() } + Error::Backblaze(why) => { + Response::build() + .header(ContentType::Plain) + .status(Status::InternalServerError) + .sized_body(Cursor::new(format!("b2 error: {:?}", why))).ok() + } + } } } diff --git a/src/b2.rs b/src/b2.rs new file mode 100644 index 0000000..32a0b6c --- /dev/null +++ b/src/b2.rs @@ -0,0 +1,61 @@ +use crate::api::Error::Backblaze; +use color_eyre::eyre::{eyre, Result}; +use lazy_static::lazy_static; +use raze::{ + api::*, + util::{self, ReadHashAtEnd, ReadThrottled}, +}; +use reqwest::blocking::ClientBuilder; +use rocket_upload::Mime; +use std::{env, fs, path::PathBuf}; + +lazy_static! { + pub static ref CREDS: String = env::var("B2_CREDFILE") + .expect("B2_CREDFILE to be populated") + .to_string(); + pub static ref BUCKET_NAME: String = env::var("B2_MODULE_BUCKET_NAME") + .expect("B2_MODULE_BUCKET_NAME to be populated") + .to_string(); +} + +#[instrument(err)] +pub fn upload(filename: PathBuf, content_type: Mime) -> Result { + let client = ClientBuilder::new() + .timeout(None) + .user_agent(crate::APP_USER_AGENT) + .build()?; + + let auth = util::authenticate_from_file(&client, CREDS.clone()).map_err(Backblaze)?; + let upauth = b2_get_upload_url(&client, &auth, "bucket_id").map_err(Backblaze)?; + let fin = fs::File::open(filename.clone())?; + let meta = fin.metadata()?; + let size = meta.len(); + let modf = meta + .modified() + .unwrap() + .duration_since(std::time::UNIX_EPOCH)? + .as_secs() + * 1000; + let ct = content_type.to_string(); + let filepath = filename + .file_name() + .ok_or(eyre!("wanted file_name to work"))? + .to_str() + .ok_or(eyre!("filename is somehow not utf-8, what"))?; + + let param = FileParameters { + file_path: filepath.clone(), + file_size: size, + content_type: Some(&ct), + content_sha1: Sha1Variant::HexAtEnd, + last_modified_millis: modf, + }; + + let reader = fin; + let reader = ReadHashAtEnd::wrap(reader); + let reader = ReadThrottled::wrap(reader, 5000); + + let resp = b2_upload_file(&client, &upauth, reader, param).map_err(Backblaze)?; + + Ok(format!("b2://{}/{}", *BUCKET_NAME, filepath)) +} diff --git a/src/main.rs b/src/main.rs index 9e74af1..8b21660 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,7 @@ use rocket_contrib::helmet::SpaceHelmet; use rocket_oauth2::OAuth2; pub mod api; +pub mod b2; pub mod gitea; pub mod jwt; pub mod models; @@ -25,6 +26,14 @@ pub struct MainDatabase(PgConnection); pub struct Gitea; +// Name your user agent after your app? +pub static APP_USER_AGENT: &str = concat!( + env!("CARGO_PKG_NAME"), + "/", + env!("CARGO_PKG_VERSION"), + " +https://tulpa.dev/wasmcloud/api", +); + fn main() -> Result<()> { color_eyre::install()?; tracing_subscriber::fmt::init(); @@ -32,6 +41,8 @@ fn main() -> Result<()> { // XXX(Xe): This looks ineffectual, however it forces jwt::SECRET to be // evaluated and will kill the program if JWT_SECRET is not found. let _ = *jwt::SECRET; + let _ = *b2::CREDS; + let _ = *b2::BUCKET_NAME; rocket::ignite() .attach(OAuth2::::fairing("gitea")) diff --git a/var/secret/.gitkeep b/var/secret/.gitkeep new file mode 100644 index 0000000..e69de29