From 233ea76204ea5bc9a7d8f12816a9525b7a732bc5 Mon Sep 17 00:00:00 2001 From: Christine Dodrill Date: Wed, 2 Dec 2020 16:16:58 -0500 Subject: [PATCH] add webmention support (#274) * add webmention support Signed-off-by: Christine Dodrill * add webmention integration post Signed-off-by: Christine Dodrill --- Cargo.lock | 18 ++++++ Cargo.toml | 5 +- blog/webmention-support-2020-12-02.markdown | 54 ++++++++++++++++++ config.dhall | 2 + lib/mi/Cargo.toml | 22 +++++++ lib/mi/src/lib.rs | 63 +++++++++++++++++++++ nix/rust.nix | 10 ++++ nix/sources.json | 12 ++++ shell.nix | 6 +- site.dhall | 9 ++- src/app/mod.rs | 11 +++- src/post/mod.rs | 33 +++++++---- templates/blogpost.rs.html | 12 ++++ 13 files changed, 233 insertions(+), 24 deletions(-) create mode 100644 blog/webmention-support-2020-12-02.markdown create mode 100644 lib/mi/Cargo.toml create mode 100644 lib/mi/src/lib.rs create mode 100644 nix/rust.nix diff --git a/Cargo.lock b/Cargo.lock index 6a4a3f4..3b31adb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1060,6 +1060,23 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +[[package]] +name = "mi" +version = "0.1.0" +dependencies = [ + "chrono", + "color-eyre", + "envy", + "pretty_env_logger", + "reqwest", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "tracing-futures", +] + [[package]] name = "mime" version = "0.3.16" @@ -2757,6 +2774,7 @@ dependencies = [ "kankyo", "lazy_static", "log", + "mi", "mime", "patreon", "pfacts", diff --git a/Cargo.toml b/Cargo.toml index c9748eb..7bf70a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ url = "2" # workspace dependencies go_vanity = { path = "./lib/go_vanity" } jsonfeed = { path = "./lib/jsonfeed" } +mi = { path = "./lib/mi" } patreon = { path = "./lib/patreon" } [build-dependencies] @@ -51,7 +52,5 @@ pretty_env_logger = "0" [workspace] members = [ - "./lib/go_vanity", - "./lib/jsonfeed", - "./lib/patreon" + "./lib/*", ] diff --git a/blog/webmention-support-2020-12-02.markdown b/blog/webmention-support-2020-12-02.markdown new file mode 100644 index 0000000..4b9f869 --- /dev/null +++ b/blog/webmention-support-2020-12-02.markdown @@ -0,0 +1,54 @@ +--- +title: "Site Update: WebMention Support" +date: 2020-12-02 +tags: + - indieweb +--- + +# Site Update: WebMention Support + +Recently in my [Various Updates](/blog/various-updates-2020-11-18) post I +announced that my website had gotten +[WebMention](https://www.w3.org/TR/webmention/) support. Today I implemented +WebMention integration into blog articles, allowing you to see where my articles +are mentioned across the internet. This will not work with every single mention +of my site, but if your publishing platform supports sending WebMentions, then +you will see them show up on the next deploy of my site. + +Thanks to the work of the folks at [Bridgy](https://brid.gy/), I have been able +to also keep track of mentions of my content across Twitter, Reddit and +Mastodon. My WebMention service will also attempt to resolve Bridgy mention +links to their original sources as much as it can. Hopefully this should allow +you to post my articles as normal across those networks and have those mentions +be recorded without having to do anything else. + +As I mentioned before, this is implemented on top of +[mi](https://github.com/Xe/mi). mi receives mentions sent to +`https://mi.within.website/api/webmention/accept` and will return a reference +URL in the `Location` header. This will return JSON-formatted data about the +mention. Here is an example: + +```console +$ curl https://mi.within.website/api/webmention/01ERGGEG7DCKRH3R7DH4BXZ6R9 | jq +{ + "id": "01ERGGEG7DCKRH3R7DH4BXZ6R9", + "source_url": "https://maya.land/responses/2020/12/01/i-think-this-blog-post-might-have-been.html", + "target_url": "https://christine.website/blog/toast-sandwich-recipe-2019-12-02", + "title": null +} +``` + +This is all of the information I store about each WebMention. I am working on +title detection (using the +[readability](https://github.com/jangernert/readability) algorithm), however I +am unable to run JavaScript on my scraper server. Content that is JavaScript +only may not be able to be scraped like this. + +--- + +Many thanks to [Chris Aldrich](https://boffosocko.com/2020/12/01/55781873/) for +inspiring me to push this feature to the end. Any articles that don't have any +WebMentions yet will link to the [WebMention +spec](https://www.w3.org/TR/webmention/). + +Be well. diff --git a/config.dhall b/config.dhall index 192955d..9ca7af8 100644 --- a/config.dhall +++ b/config.dhall @@ -17,6 +17,7 @@ let Config = , clackSet : List Text , resumeFname : Text , webMentionEndpoint : Text + , miToken : Text } , default = { signalboost = [] : List Person.Type @@ -24,6 +25,7 @@ let Config = , clackSet = [ "Ashlynn" ] , resumeFname = "./static/resume/resume.md" , webMentionEndpoint = defaultWebMentionEndpoint + , miToken = "${env:MI_TOKEN as Text ? ""}" } } diff --git a/lib/mi/Cargo.toml b/lib/mi/Cargo.toml new file mode 100644 index 0000000..1d0e716 --- /dev/null +++ b/lib/mi/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "mi" +version = "0.1.0" +authors = ["Christine Dodrill "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +chrono = { version = "0.4", features = ["serde"] } +color-eyre = "0.5" +reqwest = { version = "0.10", features = ["json"] } +serde_json = "1.0" +serde = { version = "1", features = ["derive"] } +thiserror = "1" +tracing = "0.1" +tracing-futures = "0.2" + +[dev-dependencies] +tokio = { version = "0.2", features = ["macros"] } +envy = "0.4" +pretty_env_logger = "0" diff --git a/lib/mi/src/lib.rs b/lib/mi/src/lib.rs new file mode 100644 index 0000000..ec9d459 --- /dev/null +++ b/lib/mi/src/lib.rs @@ -0,0 +1,63 @@ +use color_eyre::eyre::Result; +use reqwest::header; +use serde::Deserialize; +use tracing::instrument; + +const USER_AGENT_BASE: &str = concat!( + "library/", + env!("CARGO_PKG_NAME"), + "/", + env!("CARGO_PKG_VERSION") +); + +pub struct Client { + cli: reqwest::Client, + base_url: String, +} + +impl Client { + pub fn new(token: String, user_agent: String) -> Result { + let mut headers = header::HeaderMap::new(); + headers.insert( + header::AUTHORIZATION, + header::HeaderValue::from_str(&token.clone())?, + ); + + let cli = reqwest::Client::builder() + .user_agent(&format!("{} {}", user_agent, USER_AGENT_BASE)) + .default_headers(headers) + .build()?; + + Ok(Self { + cli: cli, + base_url: "https://mi.within.website".to_string(), + }) + } + + #[instrument(skip(self))] + pub async fn mentioners(&self, url: String) -> Result> { + Ok(self + .cli + .get(&format!("{}/api/webmention/for", self.base_url)) + .query(&[("target", &url)]) + .send() + .await? + .error_for_status()? + .json() + .await?) + } +} + +#[derive(Debug, Deserialize, Eq, PartialEq, Clone)] +pub struct WebMention { + pub source: String, + pub title: Option, +} + +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/nix/rust.nix b/nix/rust.nix new file mode 100644 index 0000000..725f042 --- /dev/null +++ b/nix/rust.nix @@ -0,0 +1,10 @@ +{ sources ? import ./sources.nix }: + +let + pkgs = + import sources.nixpkgs { overlays = [ (import sources.nixpkgs-mozilla) ]; }; + channel = "nightly"; + date = "2020-11-25"; + targets = [ ]; + chan = pkgs.latest.rustChannels.stable.rust; +in chan diff --git a/nix/sources.json b/nix/sources.json index 02fb939..80067d5 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -47,6 +47,18 @@ "url": "https://github.com/NixOS/nixpkgs-channels/archive/502845c3e31ef3de0e424f3fcb09217df2ce6df6.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, + "nixpkgs-mozilla": { + "branch": "master", + "description": "mozilla related nixpkgs (extends nixos/nixpkgs repo)", + "homepage": null, + "owner": "mozilla", + "repo": "nixpkgs-mozilla", + "rev": "8c007b60731c07dd7a052cce508de3bb1ae849b4", + "sha256": "1zybp62zz0h077zm2zmqs2wcg3whg6jqaah9hcl1gv4x8af4zhs6", + "type": "tarball", + "url": "https://github.com/mozilla/nixpkgs-mozilla/archive/8c007b60731c07dd7a052cce508de3bb1ae849b4.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "xepkgs": { "branch": "master", "ref": "master", diff --git a/shell.nix b/shell.nix index fbd55ce..1d1c111 100644 --- a/shell.nix +++ b/shell.nix @@ -5,16 +5,14 @@ let dhall-yaml = dhallpkgs.dhall-yaml-simple; dhall = dhallpkgs.dhall-simple; xepkgs = import sources.xepkgs { inherit pkgs; }; + rust = import ./nix/rust.nix { }; in with pkgs; with xepkgs; mkShell { buildInputs = [ # Rust - cargo + rust cargo-watch - rls - rustc - rustfmt # system dependencies openssl diff --git a/site.dhall b/site.dhall index 9e19183..a4689d3 100644 --- a/site.dhall +++ b/site.dhall @@ -1,5 +1,4 @@ -let kms = - https://tulpa.dev/cadey/kubermemes/raw/branch/master/k8s/package.dhall +let kms = https://tulpa.dev/cadey/kubermemes/raw/branch/master/k8s/package.dhall let kubernetes = https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/master/1.15/package.dhall @@ -28,13 +27,17 @@ let vars , name = "PATREON_REFRESH_TOKEN" , value = Some env:PATREON_REFRESH_TOKEN as Text } + , kubernetes.EnvVar::{ + , name = "MI_TOKEN" + , value = Some env:MI_TOKEN as Text + } ] in kms.app.make kms.app.Config::{ , name = "christinewebsite" , appPort = 3030 - , image = image + , image , replicas = 2 , domain = "christine.website" , leIssuer = "prod" diff --git a/src/app/mod.rs b/src/app/mod.rs index c18e121..7cb0044 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -16,6 +16,8 @@ pub struct Config { pub(crate) resume_fname: PathBuf, #[serde(rename = "webMentionEndpoint")] pub(crate) webmention_url: String, + #[serde(rename = "miToken")] + pub(crate) mi_token: String, } #[instrument] @@ -58,6 +60,7 @@ pub struct State { pub jf: jsonfeed::Feed, pub sitemap: Vec, pub patrons: Option, + pub mi: mi::Client, } pub async fn init(cfg: PathBuf) -> Result { @@ -65,9 +68,10 @@ pub async fn init(cfg: PathBuf) -> Result { let sb = cfg.signalboost.clone(); let resume = fs::read_to_string(cfg.resume_fname.clone())?; let resume: String = markdown::render(&resume)?; - let blog = crate::post::load("blog")?; - let gallery = crate::post::load("gallery")?; - let talks = crate::post::load("talks")?; + let mi = mi::Client::new(cfg.mi_token.clone(), crate::APPLICATION_NAME.to_string())?; + let blog = crate::post::load("blog", Some(&mi)).await?; + let gallery = crate::post::load("gallery", None).await?; + let talks = crate::post::load("talks", None).await?; let mut everything: Vec = vec![]; { @@ -122,6 +126,7 @@ pub async fn init(cfg: PathBuf) -> Result { urlwriter.end()?; Ok(State { + mi: mi, cfg: cfg, signalboost: sb, resume: resume, diff --git a/src/post/mod.rs b/src/post/mod.rs index c0062a4..c66e79c 100644 --- a/src/post/mod.rs +++ b/src/post/mod.rs @@ -12,6 +12,7 @@ pub struct Post { pub body: String, pub body_html: String, pub date: DateTime, + pub mentions: Vec, } impl Into for Post { @@ -70,7 +71,7 @@ impl Post { } } -pub fn load(dir: &str) -> Result> { +pub async fn load(dir: &str, mi: Option<&mi::Client>) -> Result> { let mut result: Vec = vec![]; for path in glob(&format!("{}/*.markdown", dir))?.filter_map(Result::ok) { @@ -81,10 +82,19 @@ pub fn load(dir: &str) -> Result> { .wrap_err_with(|| format!("can't parse frontmatter of {:?}", path))?; let markup = &body[content_offset..]; let date = NaiveDate::parse_from_str(&fm.clone().date, "%Y-%m-%d")?; + let link = format!("{}/{}", dir, path.file_stem().unwrap().to_str().unwrap()); + let mentions: Vec = match mi { + None => vec![], + Some(mi) => mi + .mentioners(format!("https://christine.website/{}", link)) + .await + .map_err(|why| tracing::error!("error: can't load mentions for {}: {}", link, why)) + .unwrap_or(vec![]), + }; result.push(Post { front_matter: fm, - link: format!("{}/{}", dir, path.file_stem().unwrap().to_str().unwrap()), + link: link, body: markup.to_string(), body_html: crate::app::markdown::render(&markup) .wrap_err_with(|| format!("can't parse markdown for {:?}", path))?, @@ -96,6 +106,7 @@ pub fn load(dir: &str) -> Result> { .with_timezone(&Utc) .into() }, + mentions: mentions, }) } @@ -113,23 +124,23 @@ mod tests { use super::*; use color_eyre::eyre::Result; - #[test] - fn blog() { + #[tokio::test] + async fn blog() { let _ = pretty_env_logger::try_init(); - load("blog").expect("posts to load"); + load("blog", None).await.expect("posts to load"); } - #[test] - fn gallery() -> Result<()> { + #[tokio::test] + async fn gallery() -> Result<()> { let _ = pretty_env_logger::try_init(); - load("gallery")?; + load("gallery", None).await?; Ok(()) } - #[test] - fn talks() -> Result<()> { + #[tokio::test] + async fn talks() -> Result<()> { let _ = pretty_env_logger::try_init(); - load("talks")?; + load("talks", None).await?; Ok(()) } } diff --git a/templates/blogpost.rs.html b/templates/blogpost.rs.html index 29cfd74..dd3a9da 100644 --- a/templates/blogpost.rs.html +++ b/templates/blogpost.rs.html @@ -62,6 +62,18 @@

Tags: @for tag in post.front_matter.tags.as_ref().unwrap() { @tag }

} +@if post.mentions.len() != 0 { +

This post was WebMentioned at the following URLs: +

+

+} else { +

This post was not WebMentioned yet. You could be the first!

+} +

The art for Mara was drawn by Selicre.