diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 8965b5c..2df5970 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -192,7 +192,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" dependencies = [ "addr2line", - "cfg-if", + "cfg-if 0.1.10", "libc", "miniz_oxide", "object", @@ -314,6 +314,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "chrono" version = "0.4.15" @@ -431,7 +437,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -654,6 +660,24 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" +[[package]] +name = "encoding_rs" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "encoding_rs_io" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83" +dependencies = [ + "encoding_rs", +] + [[package]] name = "error-chain" version = "0.12.4" @@ -707,7 +731,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ed85775dcc68644b5c950ac06a2b23768d3bc9390464151aaf27136998dcf9e" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "redox_syscall", "winapi 0.3.9", @@ -840,7 +864,7 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -1151,7 +1175,7 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -1225,6 +1249,8 @@ dependencies = [ "color-eyre", "diesel", "diesel_migrations", + "encoding_rs", + "encoding_rs_io", "futures-io", "hex", "log 0.4.11", @@ -1242,6 +1268,7 @@ dependencies = [ "scraper", "sdnotify", "serde", + "serde-xml-rs", "serde_json", "thiserror", "tracing", @@ -1313,7 +1340,7 @@ version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "fuchsia-zircon", "fuchsia-zircon-sys", "iovec", @@ -1356,7 +1383,7 @@ version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "winapi 0.3.9", ] @@ -1455,7 +1482,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 0.1.10", "foreign-types", "lazy_static", "libc", @@ -1498,7 +1525,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "cloudabi 0.1.0", "instant", "libc", @@ -1661,7 +1688,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ec3341498978de3bfd12d1b22f1af1de22818f5473a11e8a6ef997989e3a212" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "universal-hash", ] @@ -1721,7 +1748,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30d70cf4412832bcac9cffe27906f4a66e450d323525e977168c70d1b36120ae" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "fnv", "lazy_static", "libc", @@ -2317,6 +2344,18 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-xml-rs" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efe415925cf3d0bbb2fc47d09b56ce03eef51c5d56846468a39bcc293c7a846c" +dependencies = [ + "log 0.4.11", + "serde", + "thiserror", + "xml-rs", +] + [[package]] name = "serde_derive" version = "1.0.116" @@ -2729,7 +2768,7 @@ version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d79ca061b032d6ce30c660fded31189ca0b9922bf483cd70759f13a2d86786c" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "tracing-attributes", "tracing-core", ] @@ -3029,7 +3068,7 @@ version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "wasm-bindgen-macro", ] @@ -3159,6 +3198,12 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "xml-rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" + [[package]] name = "yansi" version = "0.5.0" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 8ec21aa..0f2380c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -11,6 +11,8 @@ askama_rocket = "0.10" chrono = { version = "0.4", features = ["serde"] } color-eyre = "0.5" diesel_migrations = "1" +encoding_rs = "0.8.26" +encoding_rs_io = "0.1" futures-io = "0.3" hex = "0.4" log = "0.4" @@ -28,6 +30,7 @@ scraper = "0.12.0" sdnotify = { version = "0.1", default-features = false } serde_json = "^1" serde = { version = "1", features = ["derive"] } +serde-xml-rs = "0.4.0" thiserror = "1" tracing = "0.1" tracing-log = "0.1" diff --git a/backend/migrations/2020-12-22-003510_weather/down.sql b/backend/migrations/2020-12-22-003510_weather/down.sql new file mode 100644 index 0000000..696e959 --- /dev/null +++ b/backend/migrations/2020-12-22-003510_weather/down.sql @@ -0,0 +1,2 @@ +DROP INDEX weather_ts_region; +DROP TABLE weather; diff --git a/backend/migrations/2020-12-22-003510_weather/up.sql b/backend/migrations/2020-12-22-003510_weather/up.sql new file mode 100644 index 0000000..2d7dfaa --- /dev/null +++ b/backend/migrations/2020-12-22-003510_weather/up.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS weather + ( ts TIMESTAMP NOT NULL PRIMARY KEY + , region TEXT NOT NULL + , body BLOB NOT NULL -- JSON-encoded weather data + ); + +CREATE UNIQUE INDEX weather_ts_region + ON weather(ts, region); diff --git a/backend/src/bin/weatherscrape.rs b/backend/src/bin/weatherscrape.rs new file mode 100644 index 0000000..fcf3799 --- /dev/null +++ b/backend/src/bin/weatherscrape.rs @@ -0,0 +1,40 @@ +#[macro_use] +extern crate tracing; + +use color_eyre::eyre::Result; +use encoding_rs::WINDOWS_1252; +use encoding_rs_io::DecodeReaderBytesBuilder; +use serde_xml_rs::from_reader; + +use mi::*; + +pub const WEATHER_URL: &'static str = + "https://dd.weather.gc.ca/citypage_weather/xml/QC/s0000635_e.xml"; + +fn main() -> Result<()> { + color_eyre::install()?; + tracing_subscriber::fmt::init(); + + info!("{} weather importer starting up", mi::APPLICATION_NAME); + + let resp = ureq::get(WEATHER_URL).set("User-Agent", WEATHER_URL).call(); + + if !resp.ok() { + panic!( + "{}", + match resp.synthetic_error() { + Some(why) => why.to_string(), + None => resp.status_line().to_string(), + } + ); + } + + let fin = DecodeReaderBytesBuilder::new() + .encoding(Some(WINDOWS_1252)) + .build(resp.into_reader()); + let data: web::canada_weather::SiteData = from_reader(fin)?; + + println!("{:#?}", data); + + Ok(()) +} diff --git a/backend/src/schema.rs b/backend/src/schema.rs index aa54508..ddf019a 100644 --- a/backend/src/schema.rs +++ b/backend/src/schema.rs @@ -22,6 +22,14 @@ table! { } } +table! { + weather (ts) { + ts -> Timestamp, + region -> Text, + body -> Binary, + } +} + table! { webmentions (id) { id -> Text, @@ -37,5 +45,6 @@ allow_tables_to_appear_in_same_query!( blogposts, members, switches, + weather, webmentions, ); diff --git a/backend/src/web/canada_weather/mod.rs b/backend/src/web/canada_weather/mod.rs new file mode 100644 index 0000000..47ceb81 --- /dev/null +++ b/backend/src/web/canada_weather/mod.rs @@ -0,0 +1,10 @@ +pub mod types; +pub use types::SiteData; + +/// The credit string for this data. +/// +/// XXX(acli): the license [here](https://dd.weather.gc.ca/doc/LICENCE_GENERAL.txt) +/// demands that we include this string somewhere in data derived from this API. +/// This must be manually included in each response to remain within the scope +/// of the license. +pub const DATA_SOURCE: &'static str = "Data Source: Environment and Climate Change Canada"; diff --git a/backend/src/web/canada_weather/types.rs b/backend/src/web/canada_weather/types.rs new file mode 100644 index 0000000..c7721e9 --- /dev/null +++ b/backend/src/web/canada_weather/types.rs @@ -0,0 +1,236 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct SiteData { + pub license: String, + pub date_time: Vec, + pub location: Location, + pub warnings: Option, + pub current_conditions: CurrentConditions, + pub forecast_group: ForecastGroup, + // TODO(acli): hourly forecasts are not implemented yet. + pub yesterday_conditions: Yesterday, + pub rise_set: RiseSet, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Name { + pub name: String, + #[serde(rename = "$value")] + pub value: u8, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct DateTime { + pub name: String, + pub zone: String, + #[serde(rename = "UTCOffset")] + pub utc_offset: String, + pub year: u16, + pub month: Name, + pub day: Name, + pub hour: u8, + pub minute: u8, + pub time_stamp: String, + pub text_summary: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Location { + pub continent: String, + pub country: CodeName, + pub province: CodeName, + pub name: CodeName, + pub region: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct CodeName { + pub code: String, + pub lat: Option, + pub lon: Option, + #[serde(rename = "$value")] + pub name: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct IconCode { + pub format: String, + #[serde(rename = "$value")] + pub value: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct MetricWithUnits { + pub units: Option, + pub unit_type: Option, + pub change: Option, + pub tendency: Option, + #[serde(rename = "$value")] + pub value: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Wind { + pub speed: MetricWithUnits, + pub gust: MetricWithUnits, + pub direction: String, + pub bearing: MetricWithUnits, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct CurrentConditions { + pub station: CodeName, + pub date_time: Vec, + pub condition: String, + pub icon_code: IconCode, + pub temperature: MetricWithUnits, + pub dewpoint: MetricWithUnits, + pub wind_chill: Option, + pub pressure: MetricWithUnits, + pub visibility: MetricWithUnits, + pub relative_humidity: MetricWithUnits, + pub wind: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Warnings { + pub url: String, + pub event: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Event { + pub r#type: String, + pub priority: String, + pub description: String, + pub date_time: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct ForecastGroup { + pub date_time: Vec, + pub regional_normals: RegionalNormals, + pub forecast: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct RegionalNormals { + pub text_summary: String, + pub temperature: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Period { + pub text_forecast_name: String, + #[serde(rename = "$value")] + pub value: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct CloudPrecip { + pub text_summary: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Forecast { + pub period: Period, + pub text_summary: String, + pub cloud_precip: CloudPrecip, + pub abbreviated_forecast: AbbreviatedForecast, + pub temperatures: Temperatures, + pub winds: Winds, + pub precipitation: Option, + pub uv: Option, + pub relative_humidity: MetricWithUnits, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct AbbreviatedForecast { + pub icon_code: IconCode, + pub pop: MetricWithUnits, + pub text_summary: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Temperatures { + pub text_summary: String, + pub temperature: MetricWithUnits, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Winds { + pub text_summary: Option, + pub wind: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct PrecipitationType { + pub start: String, + pub end: String, + #[serde(rename = "$value")] + pub value: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Accumulation { + pub name: String, + pub amount: MetricWithUnits, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Precipitation { + pub text_summary: Option, + pub precip_type: Vec, + pub accumulation: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Visibility { + pub cause: String, + pub text_summary: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct UVIndex { + pub category: String, + pub index: String, + pub text_summary: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Yesterday { + pub temperature: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct RiseSet { + pub disclaimer: String, + pub date_time: Vec, +} diff --git a/backend/src/web/mod.rs b/backend/src/web/mod.rs index b587101..c46bacf 100644 --- a/backend/src/web/mod.rs +++ b/backend/src/web/mod.rs @@ -1,4 +1,5 @@ pub mod bridgy; +pub mod canada_weather; pub mod discord_webhook; pub mod mastodon; pub mod pluralkit;