Browse Source

add webmention support (#274)

* add webmention support

Signed-off-by: Christine Dodrill <[email protected]>

* add webmention integration post

Signed-off-by: Christine Dodrill <[email protected]>
pull/275/head
Cadey Ratio 10 months ago
committed by GitHub
parent
commit
233ea76204
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 18
      Cargo.lock
  2. 5
      Cargo.toml
  3. 54
      blog/webmention-support-2020-12-02.markdown
  4. 2
      config.dhall
  5. 22
      lib/mi/Cargo.toml
  6. 63
      lib/mi/src/lib.rs
  7. 10
      nix/rust.nix
  8. 12
      nix/sources.json
  9. 6
      shell.nix
  10. 9
      site.dhall
  11. 11
      src/app/mod.rs
  12. 33
      src/post/mod.rs
  13. 12
      templates/blogpost.rs.html

18
Cargo.lock

@ -1060,6 +1060,23 @@ version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "mi"
version = "0.1.0"
dependencies = [
"chrono",
"color-eyre",
"envy",
"pretty_env_logger",
"reqwest",
"serde",
"serde_json",
"thiserror",
"tokio",
"tracing",
"tracing-futures",
]
[[package]]
name = "mime"
version = "0.3.16"
@ -2757,6 +2774,7 @@ dependencies = [
"kankyo",
"lazy_static",
"log",
"mi",
"mime",
"patreon",
"pfacts",

5
Cargo.toml

@ -37,6 +37,7 @@ url = "2"
# workspace dependencies
go_vanity = { path = "./lib/go_vanity" }
jsonfeed = { path = "./lib/jsonfeed" }
mi = { path = "./lib/mi" }
patreon = { path = "./lib/patreon" }
[build-dependencies]
@ -51,7 +52,5 @@ pretty_env_logger = "0"
[workspace]
members = [
"./lib/go_vanity",
"./lib/jsonfeed",
"./lib/patreon"
"./lib/*",
]

54
blog/webmention-support-2020-12-02.markdown

@ -0,0 +1,54 @@
---
title: "Site Update: WebMention Support"
date: 2020-12-02
tags:
- indieweb
---
# Site Update: WebMention Support
Recently in my [Various Updates](/blog/various-updates-2020-11-18) post I
announced that my website had gotten
[WebMention](https://www.w3.org/TR/webmention/) support. Today I implemented
WebMention integration into blog articles, allowing you to see where my articles
are mentioned across the internet. This will not work with every single mention
of my site, but if your publishing platform supports sending WebMentions, then
you will see them show up on the next deploy of my site.
Thanks to the work of the folks at [Bridgy](https://brid.gy/), I have been able
to also keep track of mentions of my content across Twitter, Reddit and
Mastodon. My WebMention service will also attempt to resolve Bridgy mention
links to their original sources as much as it can. Hopefully this should allow
you to post my articles as normal across those networks and have those mentions
be recorded without having to do anything else.
As I mentioned before, this is implemented on top of
[mi](https://github.com/Xe/mi). mi receives mentions sent to
`https://mi.within.website/api/webmention/accept` and will return a reference
URL in the `Location` header. This will return JSON-formatted data about the
mention. Here is an example:
```console
$ curl https://mi.within.website/api/webmention/01ERGGEG7DCKRH3R7DH4BXZ6R9 | jq
{
"id": "01ERGGEG7DCKRH3R7DH4BXZ6R9",
"source_url": "https://maya.land/responses/2020/12/01/i-think-this-blog-post-might-have-been.html",
"target_url": "https://christine.website/blog/toast-sandwich-recipe-2019-12-02",
"title": null
}
```
This is all of the information I store about each WebMention. I am working on
title detection (using the
[readability](https://github.com/jangernert/readability) algorithm), however I
am unable to run JavaScript on my scraper server. Content that is JavaScript
only may not be able to be scraped like this.
---
Many thanks to [Chris Aldrich](https://boffosocko.com/2020/12/01/55781873/) for
inspiring me to push this feature to the end. Any articles that don't have any
WebMentions yet will link to the [WebMention
spec](https://www.w3.org/TR/webmention/).
Be well.

2
config.dhall

@ -17,6 +17,7 @@ let Config =
, clackSet : List Text
, resumeFname : Text
, webMentionEndpoint : Text
, miToken : Text
}
, default =
{ signalboost = [] : List Person.Type
@ -24,6 +25,7 @@ let Config =
, clackSet = [ "Ashlynn" ]
, resumeFname = "./static/resume/resume.md"
, webMentionEndpoint = defaultWebMentionEndpoint
, miToken = "${env:MI_TOKEN as Text ? ""}"
}
}

22
lib/mi/Cargo.toml

@ -0,0 +1,22 @@
[package]
name = "mi"
version = "0.1.0"
authors = ["Christine Dodrill <[email protected]>"]
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
chrono = { version = "0.4", features = ["serde"] }
color-eyre = "0.5"
reqwest = { version = "0.10", features = ["json"] }
serde_json = "1.0"
serde = { version = "1", features = ["derive"] }
thiserror = "1"
tracing = "0.1"
tracing-futures = "0.2"
[dev-dependencies]
tokio = { version = "0.2", features = ["macros"] }
envy = "0.4"
pretty_env_logger = "0"

63
lib/mi/src/lib.rs

@ -0,0 +1,63 @@
use color_eyre::eyre::Result;
use reqwest::header;
use serde::Deserialize;
use tracing::instrument;
const USER_AGENT_BASE: &str = concat!(
"library/",
env!("CARGO_PKG_NAME"),
"/",
env!("CARGO_PKG_VERSION")
);
pub struct Client {
cli: reqwest::Client,
base_url: String,
}
impl Client {
pub fn new(token: String, user_agent: String) -> Result<Self> {
let mut headers = header::HeaderMap::new();
headers.insert(
header::AUTHORIZATION,
header::HeaderValue::from_str(&token.clone())?,
);
let cli = reqwest::Client::builder()
.user_agent(&format!("{} {}", user_agent, USER_AGENT_BASE))
.default_headers(headers)
.build()?;
Ok(Self {
cli: cli,
base_url: "https://mi.within.website".to_string(),
})
}
#[instrument(skip(self))]
pub async fn mentioners(&self, url: String) -> Result<Vec<WebMention>> {
Ok(self
.cli
.get(&format!("{}/api/webmention/for", self.base_url))
.query(&[("target", &url)])
.send()
.await?
.error_for_status()?
.json()
.await?)
}
}
#[derive(Debug, Deserialize, Eq, PartialEq, Clone)]
pub struct WebMention {
pub source: String,
pub title: Option<String>,
}
#[cfg(test)]
mod tests {
#[test]
fn it_works() {
assert_eq!(2 + 2, 4);
}
}

10
nix/rust.nix

@ -0,0 +1,10 @@
{ sources ? import ./sources.nix }:
let
pkgs =
import sources.nixpkgs { overlays = [ (import sources.nixpkgs-mozilla) ]; };
channel = "nightly";
date = "2020-11-25";
targets = [ ];
chan = pkgs.latest.rustChannels.stable.rust;
in chan

12
nix/sources.json

@ -47,6 +47,18 @@
"url": "https://github.com/NixOS/nixpkgs-channels/archive/502845c3e31ef3de0e424f3fcb09217df2ce6df6.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixpkgs-mozilla": {
"branch": "master",
"description": "mozilla related nixpkgs (extends nixos/nixpkgs repo)",
"homepage": null,
"owner": "mozilla",
"repo": "nixpkgs-mozilla",
"rev": "8c007b60731c07dd7a052cce508de3bb1ae849b4",
"sha256": "1zybp62zz0h077zm2zmqs2wcg3whg6jqaah9hcl1gv4x8af4zhs6",
"type": "tarball",
"url": "https://github.com/mozilla/nixpkgs-mozilla/archive/8c007b60731c07dd7a052cce508de3bb1ae849b4.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"xepkgs": {
"branch": "master",
"ref": "master",

6
shell.nix

@ -5,16 +5,14 @@ let
dhall-yaml = dhallpkgs.dhall-yaml-simple;
dhall = dhallpkgs.dhall-simple;
xepkgs = import sources.xepkgs { inherit pkgs; };
rust = import ./nix/rust.nix { };
in with pkgs;
with xepkgs;
mkShell {
buildInputs = [
# Rust
cargo
rust
cargo-watch
rls
rustc
rustfmt
# system dependencies
openssl

9
site.dhall

@ -1,5 +1,4 @@
let kms =
https://tulpa.dev/cadey/kubermemes/raw/branch/master/k8s/package.dhall
let kms = https://tulpa.dev/cadey/kubermemes/raw/branch/master/k8s/package.dhall
let kubernetes =
https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/master/1.15/package.dhall
@ -28,13 +27,17 @@ let vars
, name = "PATREON_REFRESH_TOKEN"
, value = Some env:PATREON_REFRESH_TOKEN as Text
}
, kubernetes.EnvVar::{
, name = "MI_TOKEN"
, value = Some env:MI_TOKEN as Text
}
]
in kms.app.make
kms.app.Config::{
, name = "christinewebsite"
, appPort = 3030
, image = image
, image
, replicas = 2
, domain = "christine.website"
, leIssuer = "prod"

11
src/app/mod.rs

@ -16,6 +16,8 @@ pub struct Config {
pub(crate) resume_fname: PathBuf,
#[serde(rename = "webMentionEndpoint")]
pub(crate) webmention_url: String,
#[serde(rename = "miToken")]
pub(crate) mi_token: String,
}
#[instrument]
@ -58,6 +60,7 @@ pub struct State {
pub jf: jsonfeed::Feed,
pub sitemap: Vec<u8>,
pub patrons: Option<patreon::Users>,
pub mi: mi::Client,
}
pub async fn init(cfg: PathBuf) -> Result<State> {
@ -65,9 +68,10 @@ pub async fn init(cfg: PathBuf) -> Result<State> {
let sb = cfg.signalboost.clone();
let resume = fs::read_to_string(cfg.resume_fname.clone())?;
let resume: String = markdown::render(&resume)?;
let blog = crate::post::load("blog")?;
let gallery = crate::post::load("gallery")?;
let talks = crate::post::load("talks")?;
let mi = mi::Client::new(cfg.mi_token.clone(), crate::APPLICATION_NAME.to_string())?;
let blog = crate::post::load("blog", Some(&mi)).await?;
let gallery = crate::post::load("gallery", None).await?;
let talks = crate::post::load("talks", None).await?;
let mut everything: Vec<Post> = vec![];
{
@ -122,6 +126,7 @@ pub async fn init(cfg: PathBuf) -> Result<State> {
urlwriter.end()?;
Ok(State {
mi: mi,
cfg: cfg,
signalboost: sb,
resume: resume,

33
src/post/mod.rs

@ -12,6 +12,7 @@ pub struct Post {
pub body: String,
pub body_html: String,
pub date: DateTime<FixedOffset>,
pub mentions: Vec<mi::WebMention>,
}
impl Into<jsonfeed::Item> for Post {
@ -70,7 +71,7 @@ impl Post {
}
}
pub fn load(dir: &str) -> Result<Vec<Post>> {
pub async fn load(dir: &str, mi: Option<&mi::Client>) -> Result<Vec<Post>> {
let mut result: Vec<Post> = vec![];
for path in glob(&format!("{}/*.markdown", dir))?.filter_map(Result::ok) {
@ -81,10 +82,19 @@ pub fn load(dir: &str) -> Result<Vec<Post>> {
.wrap_err_with(|| format!("can't parse frontmatter of {:?}", path))?;
let markup = &body[content_offset..];
let date = NaiveDate::parse_from_str(&fm.clone().date, "%Y-%m-%d")?;
let link = format!("{}/{}", dir, path.file_stem().unwrap().to_str().unwrap());
let mentions: Vec<mi::WebMention> = match mi {
None => vec![],
Some(mi) => mi
.mentioners(format!("https://christine.website/{}", link))
.await
.map_err(|why| tracing::error!("error: can't load mentions for {}: {}", link, why))
.unwrap_or(vec![]),
};
result.push(Post {
front_matter: fm,
link: format!("{}/{}", dir, path.file_stem().unwrap().to_str().unwrap()),
link: link,
body: markup.to_string(),
body_html: crate::app::markdown::render(&markup)
.wrap_err_with(|| format!("can't parse markdown for {:?}", path))?,
@ -96,6 +106,7 @@ pub fn load(dir: &str) -> Result<Vec<Post>> {
.with_timezone(&Utc)
.into()
},
mentions: mentions,
})
}
@ -113,23 +124,23 @@ mod tests {
use super::*;
use color_eyre::eyre::Result;
#[test]
fn blog() {
#[tokio::test]
async fn blog() {
let _ = pretty_env_logger::try_init();
load("blog").expect("posts to load");
load("blog", None).await.expect("posts to load");
}
#[test]
fn gallery() -> Result<()> {
#[tokio::test]
async fn gallery() -> Result<()> {
let _ = pretty_env_logger::try_init();
load("gallery")?;
load("gallery", None).await?;
Ok(())
}
#[test]
fn talks() -> Result<()> {
#[tokio::test]
async fn talks() -> Result<()> {
let _ = pretty_env_logger::try_init();
load("talks")?;
load("talks", None).await?;
Ok(())
}
}

12
templates/blogpost.rs.html

@ -62,6 +62,18 @@
<p>Tags: @for tag in post.front_matter.tags.as_ref().unwrap() { <code>@tag</code> }</p>
}
@if post.mentions.len() != 0 {
<p>This post was <a href="https://www.w3.org/TR/webmention/">WebMention</a>ed at the following URLs:
<ul>
@for mention in post.mentions {
<li><a href="@mention.source">@mention.title.unwrap_or(mention.source)</a></li>
}
</ul>
</p>
} else {
<p>This post was not <a href="https://www.w3.org/TR/webmention/">WebMention</a>ed yet. You could be the first!</p>
}
<p>The art for Mara was drawn by <a href="https://selic.re/">Selicre</a>.</p>
<script>

Loading…
Cancel
Save