Xeorg
/
site
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add webmention support (#274) 

* add webmention support

Signed-off-by: Christine Dodrill <me@christine.website>

* add webmention integration post

Signed-off-by: Christine Dodrill <me@christine.website>
This commit is contained in:
Cadey Ratio 2020-12-02 16:16:58 -05:00 committed by GitHub
parent d35f62351f
commit 233ea76204
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 233 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Cargo.lock generated
View File

@ -1060,6 +1060,23 @@ version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "mi"
version = "0.1.0"
dependencies = [
"chrono",
"color-eyre",
"envy",
"pretty_env_logger",
"reqwest",
"serde",
"serde_json",
"thiserror",
"tokio",
"tracing",
"tracing-futures",
]
[[package]]
name = "mime"
version = "0.3.16"
@ -2757,6 +2774,7 @@ dependencies = [
"kankyo",
"lazy_static",
"log",
"mi",
"mime",
"patreon",
"pfacts",

5
Cargo.toml
View File

@ -37,6 +37,7 @@ url = "2"
# workspace dependencies
go_vanity = { path = "./lib/go_vanity" }
jsonfeed = { path = "./lib/jsonfeed" }
mi = { path = "./lib/mi" }
patreon = { path = "./lib/patreon" }
[build-dependencies]
@ -51,7 +52,5 @@ pretty_env_logger = "0"
[workspace]
members = [
"./lib/go_vanity",
"./lib/jsonfeed",
"./lib/patreon"
"./lib/*",
]

54
blog/webmention-support-2020-12-02.markdown Normal file
View File

@ -0,0 +1,54 @@
---
title: "Site Update: WebMention Support"
date: 2020-12-02
tags:
- indieweb
---
# Site Update: WebMention Support
Recently in my [Various Updates](/blog/various-updates-2020-11-18) post I
announced that my website had gotten
[WebMention](https://www.w3.org/TR/webmention/) support. Today I implemented
WebMention integration into blog articles, allowing you to see where my articles
are mentioned across the internet. This will not work with every single mention
of my site, but if your publishing platform supports sending WebMentions, then
you will see them show up on the next deploy of my site.
Thanks to the work of the folks at [Bridgy](https://brid.gy/), I have been able
to also keep track of mentions of my content across Twitter, Reddit and
Mastodon. My WebMention service will also attempt to resolve Bridgy mention
links to their original sources as much as it can. Hopefully this should allow
you to post my articles as normal across those networks and have those mentions
be recorded without having to do anything else.
As I mentioned before, this is implemented on top of
[mi](https://github.com/Xe/mi). mi receives mentions sent to
`https://mi.within.website/api/webmention/accept` and will return a reference
URL in the `Location` header. This will return JSON-formatted data about the
mention. Here is an example:
```console
$ curl https://mi.within.website/api/webmention/01ERGGEG7DCKRH3R7DH4BXZ6R9 | jq
{
"id": "01ERGGEG7DCKRH3R7DH4BXZ6R9",
"source_url": "https://maya.land/responses/2020/12/01/i-think-this-blog-post-might-have-been.html",
"target_url": "https://christine.website/blog/toast-sandwich-recipe-2019-12-02",
"title": null
}
```
This is all of the information I store about each WebMention. I am working on
title detection (using the
[readability](https://github.com/jangernert/readability) algorithm), however I
am unable to run JavaScript on my scraper server. Content that is JavaScript
only may not be able to be scraped like this.
---
Many thanks to [Chris Aldrich](https://boffosocko.com/2020/12/01/55781873/) for
inspiring me to push this feature to the end. Any articles that don't have any
WebMentions yet will link to the [WebMention
spec](https://www.w3.org/TR/webmention/).
Be well.

2
config.dhall
View File

@ -17,6 +17,7 @@ let Config =
, clackSet : List Text
, resumeFname : Text
, webMentionEndpoint : Text
, miToken : Text
}
, default =
{ signalboost = [] : List Person.Type
@ -24,6 +25,7 @@ let Config =
, clackSet = [ "Ashlynn" ]
, resumeFname = "./static/resume/resume.md"
, webMentionEndpoint = defaultWebMentionEndpoint
, miToken = "${env:MI_TOKEN as Text ? ""}"
}
}

22
lib/mi/Cargo.toml Normal file
View File

@ -0,0 +1,22 @@
[package]
name = "mi"
version = "0.1.0"
authors = ["Christine Dodrill <me@christine.website>"]
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
chrono = { version = "0.4", features = ["serde"] }
color-eyre = "0.5"
reqwest = { version = "0.10", features = ["json"] }
serde_json = "1.0"
serde = { version = "1", features = ["derive"] }
thiserror = "1"
tracing = "0.1"
tracing-futures = "0.2"
[dev-dependencies]
tokio = { version = "0.2", features = ["macros"] }
envy = "0.4"
pretty_env_logger = "0"

63
lib/mi/src/lib.rs Normal file
View File

@ -0,0 +1,63 @@
use color_eyre::eyre::Result;
use reqwest::header;
use serde::Deserialize;
use tracing::instrument;
const USER_AGENT_BASE: &str = concat!(
"library/",
env!("CARGO_PKG_NAME"),
"/",
env!("CARGO_PKG_VERSION")
);
pub struct Client {
cli: reqwest::Client,
base_url: String,
}
impl Client {
pub fn new(token: String, user_agent: String) -> Result<Self> {
let mut headers = header::HeaderMap::new();
headers.insert(
header::AUTHORIZATION,
header::HeaderValue::from_str(&token.clone())?,
);
let cli = reqwest::Client::builder()
.user_agent(&format!("{} {}", user_agent, USER_AGENT_BASE))
.default_headers(headers)
.build()?;
Ok(Self {
cli: cli,
base_url: "https://mi.within.website".to_string(),
})
}
#[instrument(skip(self))]
pub async fn mentioners(&self, url: String) -> Result<Vec<WebMention>> {
Ok(self
.cli
.get(&format!("{}/api/webmention/for", self.base_url))
.query(&[("target", &url)])
.send()
.await?
.error_for_status()?
.json()
.await?)
}
}
#[derive(Debug, Deserialize, Eq, PartialEq, Clone)]
pub struct WebMention {
pub source: String,
pub title: Option<String>,
}
#[cfg(test)]
mod tests {
#[test]
fn it_works() {
assert_eq!(2 + 2, 4);
}
}

10
nix/rust.nix Normal file
View File

@ -0,0 +1,10 @@
{ sources ? import ./sources.nix }:
let
pkgs =
import sources.nixpkgs { overlays = [ (import sources.nixpkgs-mozilla) ]; };
channel = "nightly";
date = "2020-11-25";
targets = [ ];
chan = pkgs.latest.rustChannels.stable.rust;
in chan

12
nix/sources.json
View File

@ -47,6 +47,18 @@
"url": "https://github.com/NixOS/nixpkgs-channels/archive/502845c3e31ef3de0e424f3fcb09217df2ce6df6.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixpkgs-mozilla": {
"branch": "master",
"description": "mozilla related nixpkgs (extends nixos/nixpkgs repo)",
"homepage": null,
"owner": "mozilla",
"repo": "nixpkgs-mozilla",
"rev": "8c007b60731c07dd7a052cce508de3bb1ae849b4",
"sha256": "1zybp62zz0h077zm2zmqs2wcg3whg6jqaah9hcl1gv4x8af4zhs6",
"type": "tarball",
"url": "https://github.com/mozilla/nixpkgs-mozilla/archive/8c007b60731c07dd7a052cce508de3bb1ae849b4.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"xepkgs": {
"branch": "master",
"ref": "master",

6
shell.nix
View File

@ -5,16 +5,14 @@ let
dhall-yaml = dhallpkgs.dhall-yaml-simple;
dhall = dhallpkgs.dhall-simple;
xepkgs = import sources.xepkgs { inherit pkgs; };
rust = import ./nix/rust.nix { };
in with pkgs;
with xepkgs;
mkShell {
buildInputs = [
# Rust
cargo
rust
cargo-watch
rls
rustc
rustfmt
# system dependencies
openssl

9
site.dhall
View File

@ -1,5 +1,4 @@
let kms =
https://tulpa.dev/cadey/kubermemes/raw/branch/master/k8s/package.dhall
let kms = https://tulpa.dev/cadey/kubermemes/raw/branch/master/k8s/package.dhall
let kubernetes =
https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/master/1.15/package.dhall
@ -28,13 +27,17 @@ let vars
, name = "PATREON_REFRESH_TOKEN"
, value = Some env:PATREON_REFRESH_TOKEN as Text
}
, kubernetes.EnvVar::{
, name = "MI_TOKEN"
, value = Some env:MI_TOKEN as Text
}
]
in kms.app.make
kms.app.Config::{
, name = "christinewebsite"
, appPort = 3030
, image = image
, image
, replicas = 2
, domain = "christine.website"
, leIssuer = "prod"

11
src/app/mod.rs
View File

@ -16,6 +16,8 @@ pub struct Config {
pub(crate) resume_fname: PathBuf,
#[serde(rename = "webMentionEndpoint")]
pub(crate) webmention_url: String,
#[serde(rename = "miToken")]
pub(crate) mi_token: String,
}
#[instrument]
@ -58,6 +60,7 @@ pub struct State {
pub jf: jsonfeed::Feed,
pub sitemap: Vec<u8>,
pub patrons: Option<patreon::Users>,
pub mi: mi::Client,
}
pub async fn init(cfg: PathBuf) -> Result<State> {
@ -65,9 +68,10 @@ pub async fn init(cfg: PathBuf) -> Result<State> {
let sb = cfg.signalboost.clone();
let resume = fs::read_to_string(cfg.resume_fname.clone())?;
let resume: String = markdown::render(&resume)?;
let blog = crate::post::load("blog")?;
let gallery = crate::post::load("gallery")?;
let talks = crate::post::load("talks")?;
let mi = mi::Client::new(cfg.mi_token.clone(), crate::APPLICATION_NAME.to_string())?;
let blog = crate::post::load("blog", Some(&mi)).await?;
let gallery = crate::post::load("gallery", None).await?;
let talks = crate::post::load("talks", None).await?;
let mut everything: Vec<Post> = vec![];
{
@ -122,6 +126,7 @@ pub async fn init(cfg: PathBuf) -> Result<State> {
urlwriter.end()?;
Ok(State {
mi: mi,
cfg: cfg,
signalboost: sb,
resume: resume,

33
src/post/mod.rs
View File

@ -12,6 +12,7 @@ pub struct Post {
pub body: String,
pub body_html: String,
pub date: DateTime<FixedOffset>,
pub mentions: Vec<mi::WebMention>,
}
impl Into<jsonfeed::Item> for Post {
@ -70,7 +71,7 @@ impl Post {
}
}
pub fn load(dir: &str) -> Result<Vec<Post>> {
pub async fn load(dir: &str, mi: Option<&mi::Client>) -> Result<Vec<Post>> {
let mut result: Vec<Post> = vec![];
for path in glob(&format!("{}/*.markdown", dir))?.filter_map(Result::ok) {
@ -81,10 +82,19 @@ pub fn load(dir: &str) -> Result<Vec<Post>> {
.wrap_err_with(|| format!("can't parse frontmatter of {:?}", path))?;
let markup = &body[content_offset..];
let date = NaiveDate::parse_from_str(&fm.clone().date, "%Y-%m-%d")?;
let link = format!("{}/{}", dir, path.file_stem().unwrap().to_str().unwrap());
let mentions: Vec<mi::WebMention> = match mi {
None => vec![],
Some(mi) => mi
.mentioners(format!("https://christine.website/{}", link))
.await
.map_err(|why| tracing::error!("error: can't load mentions for {}: {}", link, why))
.unwrap_or(vec![]),
};
result.push(Post {
front_matter: fm,
link: format!("{}/{}", dir, path.file_stem().unwrap().to_str().unwrap()),
link: link,
body: markup.to_string(),
body_html: crate::app::markdown::render(&markup)
.wrap_err_with(|| format!("can't parse markdown for {:?}", path))?,
@ -96,6 +106,7 @@ pub fn load(dir: &str) -> Result<Vec<Post>> {
.with_timezone(&Utc)
.into()
},
mentions: mentions,
})
}
@ -113,23 +124,23 @@ mod tests {
use super::*;
use color_eyre::eyre::Result;
#[test]
fn blog() {
#[tokio::test]
async fn blog() {
let _ = pretty_env_logger::try_init();
load("blog").expect("posts to load");
load("blog", None).await.expect("posts to load");
}
#[test]
fn gallery() -> Result<()> {
#[tokio::test]
async fn gallery() -> Result<()> {
let _ = pretty_env_logger::try_init();
load("gallery")?;
load("gallery", None).await?;
Ok(())
}
#[test]
fn talks() -> Result<()> {
#[tokio::test]
async fn talks() -> Result<()> {
let _ = pretty_env_logger::try_init();
load("talks")?;
load("talks", None).await?;
Ok(())
}
}

12
templates/blogpost.rs.html
View File

@ -62,6 +62,18 @@
<p>Tags: @for tag in post.front_matter.tags.as_ref().unwrap() { <code>@tag</code> }</p>
}
@if post.mentions.len() != 0 {
<p>This post was <a href="https://www.w3.org/TR/webmention/">WebMention</a>ed at the following URLs:
<ul>
@for mention in post.mentions {
<li><a href="@mention.source">@mention.title.unwrap_or(mention.source)</a></li>
}
</ul>
</p>
} else {
<p>This post was not <a href="https://www.w3.org/TR/webmention/">WebMention</a>ed yet. You could be the first!</p>
}
<p>The art for Mara was drawn by <a href="https://selic.re/">Selicre</a>.</p>
<script>