Merge pull request 'Add support for alt-text in preformatted blocks' (#15) from alch_emii/maj-prs:alt-text into main

Reviewed-on: cadey/maj#15

.drone.yml

@@ -7,9 +7,6 @@ steps:
     pull: always
     commands:
       - cargo test --all --all-features
-    when:
-      event:
-        - push
 
   - name: auto-release
     image: xena/gitea-release
@@ -25,6 +22,10 @@ steps:
         - push
       branch:
         - main
+trigger:
+  event:
+    - push
+    - pull_request
 
 ---
 
@@ -39,6 +40,6 @@ steps:
     environment:
       CARGO_TOKEN:
         from_secret: CARGO_TOKEN
-    when:
+trigger:
   event:
     - tag

Cargo.toml

@@ -10,8 +10,6 @@ repository = "https://tulpa.dev/cadey/maj"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-std = { version = "1.6", optional = true }
-async-tls = { default-features = false, optional = true, version = "0" }
 async-trait = { version = "0", optional = true }
 log = "0.4"
 mime_guess = "2.0"
@@ -19,11 +17,11 @@ num = "0.2"
 num-derive = "0.3"
 num-traits = "0.2"
 once_cell = "1.4"
-rustls = { version = "0.18", optional = true, features = ["dangerous_configuration"] }
+rustls = { version = "0.19", optional = true, features = ["dangerous_configuration"] }
 structopt = "0.3"
 thiserror = "1"
-tokio-rustls = { version = "0.14", features = ["dangerous_configuration"], optional = true }
+tokio-rustls = { version = "0.21", features = ["dangerous_configuration"] }
-tokio = { version = "0.2", features = ["full"], optional = true }
+tokio = { version = "0.3", features = ["full"] }
 url = "2"
 webpki-roots = { version = "0.20", optional = true }
 webpki = { version = "0.21.0", optional = true }
@@ -37,12 +35,8 @@ pretty_env_logger = "0.4"
 default = ["client", "server"]
 
 client = [
-  "tokio-rustls",
   "webpki",
   "webpki-roots",
-  "tokio",
-  "async-std",
-  "async-tls/client"
 ]
 
 server = [
@@ -50,8 +44,6 @@ server = [
   "webpki",
   "webpki-roots",
   "async-trait",
-  "async-std",
-  "async-tls/server"
 ]
 
 [workspace]

gemtext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "gemtext"
-version = "0.2.0"
+version = "0.2.1"
 authors = ["Christine Dodrill <me@christine.website>"]
 edition = "2018"
 license = "0BSD"

gemtext/src/lib.rs

@@ -18,6 +18,28 @@ impl Builder {
         self
     }
 
+    /// Append a single blank line to the document
+    ///
+    /// This is equivilent to calling [`text()`] with an empty string, or pushing a blank
+    /// [`Node`]
+    ///
+    /// ```
+    /// # use gemtext::Builder;
+    /// let greeting = Builder::new()
+    ///     .text("Hello")
+    ///     .blank_line()
+    ///     .text("universe")
+    ///     .to_string();
+    ///
+    /// assert_eq!(greeting.trim(), "Hello\n\nuniverse");
+    /// ```
+    ///
+    /// [`text()`]: Self::text()
+    pub fn blank_line(mut self) -> Self {
+        self.nodes.push(Node::blank());
+        self
+    }
+
     pub fn link<T: Into<String>>(mut self, to: T, name: Option<String>) -> Builder {
         self.nodes.push(Node::Link {
             to: to.into(),
@@ -26,8 +48,12 @@ impl Builder {
         self
     }
 
-    pub fn preformatted<T: Into<String>>(mut self, data: T) -> Builder {
+    pub fn preformatted<A, T>(mut self, alt_text: A, data: T) -> Builder
-        self.nodes.push(Node::Preformatted(data.into()));
+    where
+        A: Into<String>,
+        T: Into<String>,
+    {
+        self.nodes.push(Node::Preformatted { alt: alt_text.into(), body: data.into() });
         self
     }
 
@@ -54,11 +80,53 @@ impl Builder {
     }
 }
 
+impl ToString for Builder {
+    /// Render a document to a string
+    ///
+    /// This produces a text/gemini compliant text document, represented as a string
+    fn to_string(&self) -> String {
+        let len: usize = self.nodes.iter().map(Node::estimate_len).sum(); // sum up node lengths
+        let mut bytes = Vec::with_capacity(len + self.nodes.len()); // add in inter-node newlines
+        render(self, &mut bytes).unwrap(); // Writing to a string shouldn't produce errors
+
+        unsafe {
+            // This is safe because bytes is composed of Strings.  We could have this as
+            // pure safe code by replicating the `render()` method and switching it to use
+            // a fmt::Write (or even `String::push()`)instead of a io::Write, but this has
+            // the same effect, with much DRYer code.
+            String::from_utf8_unchecked(bytes)
+        }
+    }
+}
+
+impl AsRef<[Node]> for Builder {
+    /// Get a reference to the internal node list of this builder
+    fn as_ref(&self) -> &[Node] {
+        self.nodes.as_ref()
+    }
+}
+
+impl AsMut<[Node]> for Builder {
+    /// Get a mutable reference to the internal node list of this builder
+    fn as_mut(&mut self) -> &mut [Node] {
+        self.nodes.as_mut()
+    }
+}
+
+impl From<Builder> for Vec<Node> {
+    /// Convert into a collection of [`Node`]s.
+    ///
+    /// Equivilent to calling [`Builder::build()`]
+    fn from(builder: Builder) -> Self {
+        builder.build()
+    }
+}
+
 /// Render a set of nodes as a document to a writer.
-pub fn render(nodes: Vec<Node>, out: &mut impl Write) -> io::Result<()> {
+pub fn render(nodes: impl AsRef<[Node]>, out: &mut impl Write) -> io::Result<()> {
     use Node::*;
 
-    for node in nodes {
+    for node in nodes.as_ref() {
         match node {
             Text(body) => {
                 let special_prefixes = ["=>", "```", "#", "*", ">"];
@@ -71,8 +139,8 @@ pub fn render(nodes: Vec<Node>, out: &mut impl Write) -> io::Result<()> {
                 Some(name) => write!(out, "=> {} {}\n", to, name)?,
                 None => write!(out, "=> {}\n", to)?,
             },
-            Preformatted(body) => write!(out, "```\n{}\n```\n", body)?,
+            Preformatted { alt, body } => write!(out, "```{}\n{}\n```\n", alt, body)?,
-            Heading { level, body } => write!(out, "{} {}\n", "#".repeat(level as usize), body)?,
+            Heading { level, body } => write!(out, "{} {}\n", "#".repeat(*level as usize), body)?,
             ListItem(body) => write!(out, "* {}\n", body)?,
             Quote(body) => write!(out, "> {}\n", body)?,
         };
@@ -82,7 +150,7 @@ pub fn render(nodes: Vec<Node>, out: &mut impl Write) -> io::Result<()> {
 }
 
 /// Individual nodes of the document. Each node correlates to a line in the file.
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Clone)]
 pub enum Node {
     /// Text lines are the most fundamental line type - any line which does not
     /// match the definition of another line type defined below defaults to
@@ -123,7 +191,19 @@ pub enum Node {
     /// (e.g. Python) should be able to be copied and pasted from the client into
     /// a file and interpreted/compiled without any problems arising from the
     /// client's manner of displaying them.
-    Preformatted(String),
+    ///
+    /// The first preformatted toggle of a document is often followed by a short
+    /// string, which acts as alt-text for the preformatted block.  This is also
+    /// often used to denote the language of code in a block of text.  For example,
+    /// a block starting with the text `\`\`\`rust` may be interpreted as rust
+    /// code, and a block starting with `\`\`\` An ascii art owl` would be
+    /// described aptly to visually impaired users using a screen reader.  The alt
+    /// text may be separated from the toggle by whitespace.  `gemtext` currently
+    /// renders alt text without this separation.
+    ///
+    /// To create a preformatted block with no alt text, simply pass a zero-length
+    /// string as alt text.
+    Preformatted { alt: String, body: String },
 
     /// Lines beginning with "#" are heading lines. Heading lines consist of one,
     /// two or three consecutive "#" characters, followed by optional whitespace,
@@ -166,24 +246,73 @@ impl Node {
     pub fn blank() -> Node {
         Node::Text("".to_string())
     }
+
+    /// Cheaply estimate the length of this node
+    ///
+    /// This measures length in bytes, *not characters*.  So if the user includes
+    /// non-ascii characters, a single one of these characters may add several bytes to
+    /// the length, despite only displaying as one character.
+    ///
+    /// This does include any newlines, but not any trailing newlines.  For example, a
+    /// preformatted text block containing a single line reading "trans rights! 🏳️‍⚧️"
+    /// would have a length of 30:  3 backticks, a newline, the text (including 16 bytes
+    /// for the trans flag), another newline, and another 3 backticks.
+    ///
+    /// ```
+    /// # use gemtext::Node;
+    /// let simple_text = Node::Text(String::from("Henlo worl"));
+    /// let linky_link = Node::Link { to: "gemini://cetacean.club/maj/".to_string(), name: Some("Maj".to_string()) };
+    /// let human_rights = Node::Preformatted {
+    ///     alt: "".to_string(),
+    ///     body: "trans rights! 🏳️‍⚧️".to_string(),
+    /// };
+    ///
+    /// assert_eq!(
+    ///     simple_text.estimate_len(),
+    ///     "Henlo worl".as_bytes().len()
+    /// );
+    /// assert_eq!(
+    ///     linky_link.estimate_len(),
+    ///     "=> gemini://cetacean.club/maj/ Maj".as_bytes().len()
+    /// );
+    /// assert_eq!(
+    ///     human_rights.estimate_len(),
+    ///     "```\ntrans rights! 🏳️‍⚧️\n```".as_bytes().len()
+    /// );
+    /// ```
+    pub fn estimate_len(&self) -> usize {
+        match self {
+            Self::Text(text) => text.len(),
+            Self::Link { to, name } => 3 + to.as_bytes().len() +
+                name.as_ref().map(|n| n.as_bytes().len() + 1).unwrap_or(0),
+            Self::Preformatted { alt, body } => alt.as_bytes().len()
+                + body.as_bytes().len() + 8,
+            Self::Heading { level, body } => *level as usize + 1 + body.as_bytes().len(),
+            Self::ListItem(item) | Self::Quote(item)=> 2 + item.as_bytes().len(),
+        }
+    }
 }
 
 pub fn parse(doc: &str) -> Vec<Node> {
     let mut result: Vec<Node> = vec![];
     let mut collect_preformatted: bool = false;
     let mut preformatted_buffer: Vec<u8> = vec![];
+    let mut alt = "";
 
     for line in doc.lines() {
-        if line.starts_with("```") {
+        if let Some(trailing) = line.strip_prefix("```") {
             collect_preformatted = !collect_preformatted;
             if !collect_preformatted {
-                result.push(Node::Preformatted(
+                result.push(Node::Preformatted {
-                    String::from_utf8(preformatted_buffer)
+                    alt: alt.to_string(),
+                    body: String::from_utf8(preformatted_buffer)
                         .unwrap()
                         .trim_end()
                         .to_string(),
-                ));
+                });
                 preformatted_buffer = vec![];
+            } else {
+                alt = trailing.trim();
             }
             continue;
         }
@@ -282,13 +411,13 @@ mod tests {
     #[test]
     fn preformatted() {
         let _ = pretty_env_logger::try_init();
-        let msg = "```\n\
+        let msg = "```hi there\n\
-                   hi there\n\
+                   obi-wan kenobi\n\
                    ```\n\
                    \n\
                    Test\n";
         let expected: Vec<Node> = vec![
-            Node::Preformatted("hi there".to_string()),
+            Node::Preformatted{ alt: "hi there".to_string(), body: "obi-wan kenobi".to_string() },
             Node::Text(String::new()),
             Node::Text("Test".to_string()),
         ];
@@ -338,7 +467,7 @@ mod tests {
         let _ = pretty_env_logger::try_init();
         let msg = include_str!("../../testdata/ambig_preformatted.gmi");
         let expected: Vec<Node> = vec![
-            Node::Preformatted("FOO".to_string()),
+            Node::Preformatted { alt: "foo".to_string(), body: "FOO".to_string() },
             Node::Text("Foo bar".to_string()),
         ];
         assert_eq!(expected, parse(msg));

majc/Cargo.toml

@@ -11,7 +11,7 @@ cursive = "0.15"
 log = "0.4"
 url = "2"
 webpki = "0.21.0"
-rustls = { version = "0.18", features = ["dangerous_configuration"] }
+rustls = { version = "0.19", features = ["dangerous_configuration"] }
 smol = { version = "0.3", features = ["tokio02"] }
 
 maj = { path = ".." }

majc/src/gemini.rs

@@ -224,7 +224,7 @@ pub fn render(body: &str) -> StyledString {
                     styled.append(StyledString::styled(name, Style::from(Effect::Underline)))
                 }
             },
-            Preformatted(data) => styled.append(StyledString::plain(data)),
+            Preformatted { body, .. } => styled.append(StyledString::plain(body)),
             Heading { level, body } => styled.append(StyledString::styled(
                 format!("{} {}", "#".repeat(level as usize), body),
                 Style::from(Effect::Bold),

pilno/karnycukta/Cargo.toml

@@ -8,15 +8,14 @@ edition = "2018"
 
 [dependencies]
 anyhow = "1"
-async-std = "1.5"
 async-trait = "0"
 atom_syndication = "0.9"
 chrono = "*"
 log = "0"
 pretty_env_logger = "0.4"
 webpki = "0.21.0"
-rustls = { version = "0.18", features = ["dangerous_configuration"] }
+rustls = { version = "0.19", features = ["dangerous_configuration"] }
 structopt = "0.3"
-tokio = { version = "0.2", features = ["full"] }
+tokio = { version = "0.3", features = ["full"] }
 
 maj = { path = "../..", features = ["server", "client"], default-features = false }

pilno/karnycukta/src/zbasu/mod.rs

@@ -25,7 +25,7 @@ fn gem_to_md(tcana: Vec<Node>, out: &mut impl Write) -> io::Result<()> {
                 Some(name) => write!(out, "[{}]({})\n\n", name, to)?,
                 None => write!(out, "[{0}]({0})", to)?,
             },
-            Preformatted(body) => write!(out, "```\n{}\n```\n\n", body)?,
+            Preformatted { alt, body } => write!(out, "```{}\n{}\n```\n\n", alt, body)?,
             Heading { level, body } => {
                 write!(out, "##{} {}\n\n", "#".repeat(level as usize), body)?
             }

shell.nix

@@ -6,8 +6,6 @@ let
     "https://github.com/nix-community/NUR/archive/master.tar.gz") {
       inherit pkgs;
     };
-  tex = with pkgs;
-    texlive.combine { inherit (texlive) scheme-medium bitter titlesec; };
 in pkgs.mkShell {
   buildInputs = with pkgs; [
     pkgs.latest.rustChannels.stable.rust
@@ -15,10 +13,6 @@ in pkgs.mkShell {
 
     pkg-config
     ncurses
-
-    kindlegen
-    nur.repos.mic92.pandoc-bin
-    tex
   ];
 
   RUST_LOG="info,majsite=debug,majsite::server=debug";

site/Cargo.toml

@@ -9,7 +9,6 @@ build = "build.rs"
 
 [dependencies]
 anyhow = "1"
-async-std = "1.5"
 async-trait = "0"
 dnd_dice_roller = "0.3"
 env_logger = "0"
@@ -17,13 +16,14 @@ log = "0"
 mime = "0.3.0"
 percent-encoding = "2"
 rand = "0"
-rustls = { version = "0.18", features = ["dangerous_configuration"] }
+rustls = { version = "0.19", features = ["dangerous_configuration"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 smol = { version = "0.3", features = ["tokio02"] }
 structopt = "0.3"
 url = "2"
 warp = "0.2"
+tokio = { version = "0.3", features = ["rt"] }
 
 maj = { path = "..", features = ["server"], default-features = false }
 

site/src/http.rs

@@ -80,7 +80,12 @@ fn gemtext_to_html(inp: Vec<u8>) -> (String, impl ToHtml) {
                 name.as_ref().or(Some(&to.to_string())).unwrap()
             )
             .unwrap(),
-            Preformatted(body) => write!(buf, "<code><pre>{}</pre></code>", body).unwrap(),
+            Preformatted { alt, body } => write!(
+                buf,
+                "<code><pre title = \"{}\">{}</pre></code>",
+                alt.replace("\"", "\\\"").replace("\\", "\\\\"),
+                body
+            ).unwrap(),
             ListItem(body) => write!(buf, "<li>{}</li>", body).unwrap(),
             Quote(body) => write!(buf, "<blockquote>{}</blockquote>", body).unwrap(),
         }

site/src/main.rs

@@ -1,8 +1,8 @@
-use async_std::task;
 use rustls::{
     internal::pemfile::{certs, pkcs8_private_keys},
     AllowAnyAnonymousOrAuthenticatedClient, Certificate, PrivateKey, RootCertStore, ServerConfig,
 };
+
 use std::{
     fs::File,
     io::{self, BufReader},
@@ -64,7 +64,8 @@ fn load_keys(path: &Path) -> io::Result<Vec<PrivateKey>> {
         .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid key"))
 }
 
-fn main() -> Result<(), maj::server::Error> {
+#[tokio::main]
+async fn main() -> Result<(), maj::server::Error> {
     env_logger::init();
     let opts = Options::from_args();
     let certs = load_certs(&opts.cert).unwrap();
@@ -96,7 +97,7 @@ fn main() -> Result<(), maj::server::Error> {
         thread::spawn(move || http::run(h.clone(), port));
     }
 
-    task::block_on(maj::server::serve(h.clone(), config, opts.host, opts.port))?;
+    maj::server::serve(h.clone(), config, opts.host, opts.port).await?;
 
     Ok(())
 }

site/src/server.rs

@@ -56,7 +56,7 @@ async fn dice(req: Request) -> Result<Response, Error> {
                 .text("")
                 .text(format!("You rolled {} and you got:", dice))
                 .text("")
-                .preformatted(format!("{}", dice_roll(dice)?))
+                .preformatted("", format!("{}", dice_roll(dice)?))
                 .text("")
                 .link("/dice", Some("Do another roll".to_string()));
 

src/server/files.rs

@@ -26,7 +26,7 @@ impl MajHandler for Handler {
 
         log::debug!("opening file {:?}", path);
 
-        match async_std::fs::metadata(&path).await {
+        match tokio::fs::metadata(&path).await {
             Ok(stat) => {
                 if stat.is_dir() {
                     if r.url.as_str().ends_with('/') {
@@ -43,9 +43,9 @@ impl MajHandler for Handler {
             }
         }
 
-        let mut file = async_std::fs::File::open(&path).await?;
+        let mut file = tokio::fs::File::open(&path).await?;
         let mut buf: Vec<u8> = Vec::new();
-        async_std::io::copy(&mut file, &mut buf).await?;
+        tokio::io::copy(&mut file, &mut buf).await?;
 
         // Send header.
         if path.extension() == Some(OsStr::new("gmi"))

src/server/mod.rs

@@ -1,11 +1,10 @@
 use crate::{Response, StatusCode};
-use async_std::{
+use tokio::{
-    io::prelude::*,
+    prelude::*,
     net::{TcpListener, TcpStream},
-    stream::StreamExt,
     task,
 };
-use async_tls::TlsAcceptor;
+use tokio_rustls::TlsAcceptor;
 use async_trait::async_trait;
 use rustls::Certificate;
 use std::{error::Error as StdError, net::SocketAddr, sync::Arc};
@@ -53,12 +52,10 @@ where
 {
     let cfg = Arc::new(cfg);
     let listener = TcpListener::bind(&format!("{}:{}", host, port)).await?;
-    let mut incoming = listener.incoming();
     let acceptor = Arc::new(TlsAcceptor::from(cfg.clone()));
-    while let Some(Ok(stream)) = incoming.next().await {
+    while let Ok((stream, addr)) = listener.accept().await {
         let h = h.clone();
         let acceptor = acceptor.clone();
-        let addr = stream.peer_addr().unwrap();
         let port = port.clone();
 
         task::spawn(handle_request(h, stream, acceptor, addr, port));
@@ -83,7 +80,7 @@ async fn handle_request(
             if let Some(u_port) = url.port() {
                 if port != u_port {
                     let _ = write_header(
-                        &mut stream,
+                        stream,
                         StatusCode::ProxyRequestRefused,
                         "Cannot proxy to that URL",
                     )
@@ -117,7 +114,7 @@ async fn handle_request(
     Ok(())
 }
 
-pub async fn write_header<W: Write + Unpin>(
+pub async fn write_header<W: AsyncWrite + Unpin>(
     mut stream: W,
     status: StatusCode,
     meta: &str,
@@ -129,7 +126,7 @@ pub async fn write_header<W: Write + Unpin>(
 }
 
 /// Return the URL requested by the client.
-async fn parse_request<R: Read + Unpin>(mut stream: R) -> Result<Url> {
+async fn parse_request<R: AsyncRead + Unpin>(mut stream: R) -> Result<Url> {
     // Because requests are limited to 1024 bytes (plus 2 bytes for CRLF), we
     // can use a fixed-sized buffer on the stack, avoiding allocations and
     // copying, and stopping bad clients from making us use too much memory.
@@ -167,7 +164,7 @@ async fn handle<T>(
     stream: &mut T,
     addr: SocketAddr,
 ) where
-    T: Write + Unpin,
+    T: AsyncWrite + Unpin,
 {
     let u = req.url.clone();
     match h.handle(req).await {