add webmention support (#274)

* add webmention support

Signed-off-by: Christine Dodrill <me@christine.website>

* add webmention integration post

Signed-off-by: Christine Dodrill <me@christine.website>

Cargo.lock

@@ -1060,6 +1060,23 @@ version = "2.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
 
+[[package]]
+name = "mi"
+version = "0.1.0"
+dependencies = [
+ "chrono",
+ "color-eyre",
+ "envy",
+ "pretty_env_logger",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "tracing-futures",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.16"
@@ -2757,6 +2774,7 @@ dependencies = [
  "kankyo",
  "lazy_static",
  "log",
+ "mi",
  "mime",
  "patreon",
  "pfacts",

Cargo.toml

@@ -37,6 +37,7 @@ url = "2"
 # workspace dependencies
 go_vanity = { path = "./lib/go_vanity" }
 jsonfeed = { path = "./lib/jsonfeed" }
+mi = { path = "./lib/mi" }
 patreon = { path = "./lib/patreon" }
 
 [build-dependencies]
@@ -51,7 +52,5 @@ pretty_env_logger = "0"
 
 [workspace]
 members = [
-  "./lib/go_vanity",
+  "./lib/*",
-  "./lib/jsonfeed",
-  "./lib/patreon"
 ]

blog/webmention-support-2020-12-02.markdown

@@ -0,0 +1,54 @@
+---
+title: "Site Update: WebMention Support"
+date: 2020-12-02
+tags:
+  - indieweb
+---
+
+# Site Update: WebMention Support
+
+Recently in my [Various Updates](/blog/various-updates-2020-11-18) post I
+announced that my website had gotten
+[WebMention](https://www.w3.org/TR/webmention/) support. Today I implemented
+WebMention integration into blog articles, allowing you to see where my articles
+are mentioned across the internet. This will not work with every single mention
+of my site, but if your publishing platform supports sending WebMentions, then
+you will see them show up on the next deploy of my site.
+
+Thanks to the work of the folks at [Bridgy](https://brid.gy/), I have been able
+to also keep track of mentions of my content across Twitter, Reddit and
+Mastodon. My WebMention service will also attempt to resolve Bridgy mention
+links to their original sources as much as it can. Hopefully this should allow
+you to post my articles as normal across those networks and have those mentions
+be recorded without having to do anything else.
+
+As I mentioned before, this is implemented on top of
+[mi](https://github.com/Xe/mi). mi receives mentions sent to
+`https://mi.within.website/api/webmention/accept` and will return a reference
+URL in the `Location` header. This will return JSON-formatted data about the
+mention. Here is an example:
+
+```console
+$ curl https://mi.within.website/api/webmention/01ERGGEG7DCKRH3R7DH4BXZ6R9 | jq
+{
+  "id": "01ERGGEG7DCKRH3R7DH4BXZ6R9",
+  "source_url": "https://maya.land/responses/2020/12/01/i-think-this-blog-post-might-have-been.html",
+  "target_url": "https://christine.website/blog/toast-sandwich-recipe-2019-12-02",
+  "title": null
+}
+```
+
+This is all of the information I store about each WebMention. I am working on
+title detection (using the
+[readability](https://github.com/jangernert/readability) algorithm), however I
+am unable to run JavaScript on my scraper server. Content that is JavaScript
+only may not be able to be scraped like this.
+
+---
+
+Many thanks to [Chris Aldrich](https://boffosocko.com/2020/12/01/55781873/) for
+inspiring me to push this feature to the end. Any articles that don't have any
+WebMentions yet will link to the [WebMention
+spec](https://www.w3.org/TR/webmention/).
+
+Be well.

config.dhall

@@ -17,6 +17,7 @@ let Config =
           , clackSet : List Text
           , resumeFname : Text
           , webMentionEndpoint : Text
+          , miToken : Text
           }
       , default =
         { signalboost = [] : List Person.Type
@@ -24,6 +25,7 @@ let Config =
         , clackSet = [ "Ashlynn" ]
         , resumeFname = "./static/resume/resume.md"
         , webMentionEndpoint = defaultWebMentionEndpoint
+        , miToken = "${env:MI_TOKEN as Text ? ""}"
         }
       }
 

lib/mi/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "mi"
+version = "0.1.0"
+authors = ["Christine Dodrill <me@christine.website>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+chrono = { version = "0.4", features = ["serde"] }
+color-eyre = "0.5"
+reqwest = { version = "0.10", features = ["json"] }
+serde_json = "1.0"
+serde = { version = "1", features = ["derive"] }
+thiserror = "1"
+tracing = "0.1"
+tracing-futures = "0.2"
+
+[dev-dependencies]
+tokio = { version = "0.2", features = ["macros"] }
+envy = "0.4"
+pretty_env_logger = "0"

lib/mi/src/lib.rs

@@ -0,0 +1,63 @@
+use color_eyre::eyre::Result;
+use reqwest::header;
+use serde::Deserialize;
+use tracing::instrument;
+
+const USER_AGENT_BASE: &str = concat!(
+    "library/",
+    env!("CARGO_PKG_NAME"),
+    "/",
+    env!("CARGO_PKG_VERSION")
+);
+
+pub struct Client {
+    cli: reqwest::Client,
+    base_url: String,
+}
+
+impl Client {
+    pub fn new(token: String, user_agent: String) -> Result<Self> {
+        let mut headers = header::HeaderMap::new();
+        headers.insert(
+            header::AUTHORIZATION,
+            header::HeaderValue::from_str(&token.clone())?,
+        );
+
+        let cli = reqwest::Client::builder()
+            .user_agent(&format!("{} {}", user_agent, USER_AGENT_BASE))
+            .default_headers(headers)
+            .build()?;
+
+        Ok(Self {
+            cli: cli,
+            base_url: "https://mi.within.website".to_string(),
+        })
+    }
+
+    #[instrument(skip(self))]
+    pub async fn mentioners(&self, url: String) -> Result<Vec<WebMention>> {
+        Ok(self
+            .cli
+            .get(&format!("{}/api/webmention/for", self.base_url))
+            .query(&[("target", &url)])
+            .send()
+            .await?
+            .error_for_status()?
+            .json()
+            .await?)
+    }
+}
+
+#[derive(Debug, Deserialize, Eq, PartialEq, Clone)]
+pub struct WebMention {
+    pub source: String,
+    pub title: Option<String>,
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn it_works() {
+        assert_eq!(2 + 2, 4);
+    }
+}

nix/rust.nix

@@ -0,0 +1,10 @@
+{ sources ? import ./sources.nix }:
+
+let
+  pkgs =
+    import sources.nixpkgs { overlays = [ (import sources.nixpkgs-mozilla) ]; };
+  channel = "nightly";
+  date = "2020-11-25";
+  targets = [ ];
+  chan = pkgs.latest.rustChannels.stable.rust;
+in chan

nix/sources.json

@@ -47,6 +47,18 @@
         "url": "https://github.com/NixOS/nixpkgs-channels/archive/502845c3e31ef3de0e424f3fcb09217df2ce6df6.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
+    "nixpkgs-mozilla": {
+        "branch": "master",
+        "description": "mozilla related nixpkgs (extends nixos/nixpkgs repo)",
+        "homepage": null,
+        "owner": "mozilla",
+        "repo": "nixpkgs-mozilla",
+        "rev": "8c007b60731c07dd7a052cce508de3bb1ae849b4",
+        "sha256": "1zybp62zz0h077zm2zmqs2wcg3whg6jqaah9hcl1gv4x8af4zhs6",
+        "type": "tarball",
+        "url": "https://github.com/mozilla/nixpkgs-mozilla/archive/8c007b60731c07dd7a052cce508de3bb1ae849b4.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
     "xepkgs": {
         "branch": "master",
         "ref": "master",

shell.nix

@@ -5,16 +5,14 @@ let
   dhall-yaml = dhallpkgs.dhall-yaml-simple;
   dhall = dhallpkgs.dhall-simple;
   xepkgs = import sources.xepkgs { inherit pkgs; };
+  rust = import ./nix/rust.nix { };
 in with pkgs;
 with xepkgs;
 mkShell {
   buildInputs = [
     # Rust
-    cargo
+    rust
     cargo-watch
-    rls
-    rustc
-    rustfmt
 
     # system dependencies
     openssl

site.dhall

@@ -1,5 +1,4 @@
-let kms =
+let kms = https://tulpa.dev/cadey/kubermemes/raw/branch/master/k8s/package.dhall
-      https://tulpa.dev/cadey/kubermemes/raw/branch/master/k8s/package.dhall
 
 let kubernetes =
       https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/master/1.15/package.dhall
@@ -28,13 +27,17 @@ let vars
         , name = "PATREON_REFRESH_TOKEN"
         , value = Some env:PATREON_REFRESH_TOKEN as Text
         }
+      , kubernetes.EnvVar::{
+        , name = "MI_TOKEN"
+        , value = Some env:MI_TOKEN as Text
+        }
       ]
 
 in  kms.app.make
       kms.app.Config::{
       , name = "christinewebsite"
       , appPort = 3030
-      , image = image
+      , image
       , replicas = 2
       , domain = "christine.website"
       , leIssuer = "prod"

src/app/mod.rs

@@ -16,6 +16,8 @@ pub struct Config {
     pub(crate) resume_fname: PathBuf,
     #[serde(rename = "webMentionEndpoint")]
     pub(crate) webmention_url: String,
+    #[serde(rename = "miToken")]
+    pub(crate) mi_token: String,
 }
 
 #[instrument]
@@ -58,6 +60,7 @@ pub struct State {
     pub jf: jsonfeed::Feed,
     pub sitemap: Vec<u8>,
     pub patrons: Option<patreon::Users>,
+    pub mi: mi::Client,
 }
 
 pub async fn init(cfg: PathBuf) -> Result<State> {
@@ -65,9 +68,10 @@ pub async fn init(cfg: PathBuf) -> Result<State> {
     let sb = cfg.signalboost.clone();
     let resume = fs::read_to_string(cfg.resume_fname.clone())?;
     let resume: String = markdown::render(&resume)?;
-    let blog = crate::post::load("blog")?;
+    let mi = mi::Client::new(cfg.mi_token.clone(), crate::APPLICATION_NAME.to_string())?;
-    let gallery = crate::post::load("gallery")?;
+    let blog = crate::post::load("blog", Some(&mi)).await?;
-    let talks = crate::post::load("talks")?;
+    let gallery = crate::post::load("gallery", None).await?;
+    let talks = crate::post::load("talks", None).await?;
     let mut everything: Vec<Post> = vec![];
 
     {
@@ -122,6 +126,7 @@ pub async fn init(cfg: PathBuf) -> Result<State> {
     urlwriter.end()?;
 
     Ok(State {
+        mi: mi,
         cfg: cfg,
         signalboost: sb,
         resume: resume,

src/post/mod.rs

@@ -12,6 +12,7 @@ pub struct Post {
     pub body: String,
     pub body_html: String,
     pub date: DateTime<FixedOffset>,
+    pub mentions: Vec<mi::WebMention>,
 }
 
 impl Into<jsonfeed::Item> for Post {
@@ -70,7 +71,7 @@ impl Post {
     }
 }
 
-pub fn load(dir: &str) -> Result<Vec<Post>> {
+pub async fn load(dir: &str, mi: Option<&mi::Client>) -> Result<Vec<Post>> {
     let mut result: Vec<Post> = vec![];
 
     for path in glob(&format!("{}/*.markdown", dir))?.filter_map(Result::ok) {
@@ -81,10 +82,19 @@ pub fn load(dir: &str) -> Result<Vec<Post>> {
             .wrap_err_with(|| format!("can't parse frontmatter of {:?}", path))?;
         let markup = &body[content_offset..];
         let date = NaiveDate::parse_from_str(&fm.clone().date, "%Y-%m-%d")?;
+        let link = format!("{}/{}", dir, path.file_stem().unwrap().to_str().unwrap());
+        let mentions: Vec<mi::WebMention> = match mi {
+            None => vec![],
+            Some(mi) => mi
+                .mentioners(format!("https://christine.website/{}", link))
+                .await
+                .map_err(|why| tracing::error!("error: can't load mentions for {}: {}", link, why))
+                .unwrap_or(vec![]),
+        };
 
         result.push(Post {
             front_matter: fm,
-            link: format!("{}/{}", dir, path.file_stem().unwrap().to_str().unwrap()),
+            link: link,
             body: markup.to_string(),
             body_html: crate::app::markdown::render(&markup)
                 .wrap_err_with(|| format!("can't parse markdown for {:?}", path))?,
@@ -96,6 +106,7 @@ pub fn load(dir: &str) -> Result<Vec<Post>> {
                 .with_timezone(&Utc)
                 .into()
             },
+            mentions: mentions,
         })
     }
 
@@ -113,23 +124,23 @@ mod tests {
     use super::*;
     use color_eyre::eyre::Result;
 
-    #[test]
+    #[tokio::test]
-    fn blog() {
+    async fn blog() {
         let _ = pretty_env_logger::try_init();
-        load("blog").expect("posts to load");
+        load("blog", None).await.expect("posts to load");
     }
 
-    #[test]
+    #[tokio::test]
-    fn gallery() -> Result<()> {
+    async fn gallery() -> Result<()> {
         let _ = pretty_env_logger::try_init();
-        load("gallery")?;
+        load("gallery", None).await?;
         Ok(())
     }
 
-    #[test]
+    #[tokio::test]
-    fn talks() -> Result<()> {
+    async fn talks() -> Result<()> {
         let _ = pretty_env_logger::try_init();
-        load("talks")?;
+        load("talks", None).await?;
         Ok(())
     }
 }

templates/blogpost.rs.html

@@ -62,6 +62,18 @@
     <p>Tags: @for tag in post.front_matter.tags.as_ref().unwrap() { <code>@tag</code> }</p>
 }
 
+@if post.mentions.len() != 0 {
+    <p>This post was <a href="https://www.w3.org/TR/webmention/">WebMention</a>ed at the following URLs:
+        <ul>
+            @for mention in post.mentions {
+            <li><a href="@mention.source">@mention.title.unwrap_or(mention.source)</a></li>
+            }
+        </ul>
+    </p>
+} else {
+    <p>This post was not <a href="https://www.w3.org/TR/webmention/">WebMention</a>ed yet. You could be the first!</p>
+}
+
 <p>The art for Mara was drawn by <a href="https://selic.re/">Selicre</a>.</p>
 
 <script>