Merge branch 'main' into cache-better

.github/workflows/nix.yml

@@ -15,4 +15,4 @@ jobs:
       - uses: cachix/cachix-action@v7
         with:
           name: xe
-      - run: nix-build --no-out-link
+      - run: nix build --no-link

.github/workflows/rust.yml

@@ -1,26 +0,0 @@
-name: Rust
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-env:
-  CARGO_TERM_COLOR: always
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Build
-        run: cargo build --all
-      - name: Run tests
-        run: |
-          cargo test
-          (cd lib/jsonfeed && cargo test)
-          (cd lib/patreon && cargo test)
-        env:
-          PATREON_ACCESS_TOKEN: ${{ secrets.PATREON_ACCESS_TOKEN }}
-          PATREON_CLIENT_ID: ${{ secrets.PATREON_CLIENT_ID }}
-          PATREON_CLIENT_SECRET: ${{ secrets.PATREON_CLIENT_SECRET }}
-          PATREON_REFRESH_TOKEN: ${{ secrets.PATREON_REFRESH_TOKEN }}
-          out: "/yolo"

blog/borg-backup-2021-01-09.markdown

@@ -0,0 +1,178 @@
+---
+title: "How to Set Up Borg Backup on NixOS"
+date: 2021-01-09
+series: howto
+tags:
+  - nixos
+  - borgbackup
+---
+
+# How to Set Up Borg Backup on NixOS
+
+[Borg Backup](https://www.borgbackup.org/) is a encrypted, compressed,
+deduplicated backup program for multiple platforms including Linux. This
+combined with the [NixOS options for configuring
+Borg Backup](https://search.nixos.org/options?channel=20.09&show=services.borgbackup.jobs.%3Cname%3E.paths&from=0&size=30&sort=relevance&query=services.borgbackup.jobs)
+allows you to backup on a schedule and restore from those backups when you need
+to.
+
+Borg Backup works with local files, remote servers and there are even [cloud
+hosts](https://www.borgbackup.org/support/commercial.html) that specialize in
+hosting your backups. In this post we will cover how to set up a backup job on a
+server using [BorgBase](https://www.borgbase.com/)'s free tier to host the
+backup files.
+
+## Setup
+
+You will need a few things:
+
+- A free BorgBase account
+- A server running NixOS
+- A list of folders to back up
+- A list of folders to NOT back up
+
+First, we will need to create a SSH key for root to use when connecting to
+BorgBase. Open a shell as root on the server and make a `borgbackup` folder in
+root's home directory:
+
+```shell
+mkdir borgbackup
+cd borgbackup
+```
+
+Then create a SSH key that will be used to connect to BorgBase:
+
+```shell
+ssh-keygen -f ssh_key -t ed25519 -C "Borg Backup"
+```
+
+Ignore the SSH key password because at this time the automated Borg Backup job
+doesn't allow the use of password-protected SSH keys.
+
+Now we need to create an encryption passphrase for the backup repository. Run
+this command to generate one using [xkcdpass](https://pypi.org/project/xkcdpass/):
+
+```shell
+nix-shell -p python39Packages.xkcdpass --run 'xkcdpass -n 12' > passphrase
+```
+
+[You can do whatever you want to generate a suitable passphrase, however
+xkcdpass is proven to be <a href="https://xkcd.com/936/">more random</a> than
+most other password generators.](conversation://Mara/hacker)
+
+## BorgBase Setup
+
+Now that we have the basic requirements out of the way, let's configure BorgBase
+to use that SSH key. In the BorgBase UI click on the Account tab in the upper
+right and open the SSH key management window. Click on Add Key and paste in the
+contents of `./ssh_key.pub`. Name it after the hostname of the server you are
+working on. Click Add Key and then go back to the Repositories tab in the upper
+right.
+
+Click New Repo and name it after the hostname of the server you are working on.
+Select the key you just created to have full access. Choose the region of the
+backup volume and then click Add Repository.
+
+On the main page copy the repository path with the copy icon next to your
+repository in the list. You will need this below. Attempt to SSH into the backup
+repo in order to have ssh recognize the server's host key:
+
+```shell
+ssh -i ./ssh_key o6h6zl22@o6h6zl22.repo.borgbase.com
+```
+
+Then accept the host key and press control-c to terminate the SSH connection.
+
+## NixOS Configuration
+
+In your `configuration.nix` file, add the following block:
+
+```nix
+services.borgbackup.jobs."borgbase" = {
+  paths = [
+    "/var/lib"
+    "/srv"
+    "/home"
+  ];
+  exclude = [
+    # very large paths
+    "/var/lib/docker"
+    "/var/lib/systemd"
+    "/var/lib/libvirt"
+    
+    # temporary files created by cargo and `go build`
+    "**/target"
+    "/home/*/go/bin"
+    "/home/*/go/pkg"
+  ];
+  repo = "o6h6zl22@o6h6zl22.repo.borgbase.com:repo";
+  encryption = {
+    mode = "repokey-blake2";
+    passCommand = "cat /root/borgbackup/passphrase";
+  };
+  environment.BORG_RSH = "ssh -i /root/borgbackup/ssh_key";
+  compression = "auto,lzma";
+  startAt = "daily";
+};
+```
+
+Customize the paths and exclude lists to your needs. Once you are satisfied,
+rebuild your NixOS system using `nixos-rebuild`:
+
+```shell
+nixos-rebuild switch
+```
+
+And then you can fire off an initial backup job with this command:
+
+```shell
+systemctl start borgbackup-job-borgbase.service
+```
+
+Monitor the job with this command:
+
+```shell
+journalctl -fu borgbackup-job-borgbase.service
+```
+
+The first backup job will always take the longest to run. Every incremental
+backup after that will get smaller and smaller. By default, the system will
+create new backup snapshots every night at midnight local time.
+
+## Restoring Files
+
+To restore files, first figure out when you want to restore the files from.
+NixOS includes a wrapper script for each Borg job you define. you can mount your
+backup archive using this command:
+
+```
+mkdir mount
+borg-job-borgbase mount o6h6zl22@o6h6zl22.repo.borgbase.com:repo ./mount
+```
+
+Then you can explore the backup (and with it each incremental snapshot) to
+your heart's content and copy files out manually. You can look through each
+folder and copy out what you need.
+
+When you are done you can unmount it with this command:
+
+```
+borg-job-borgbase umount /root/borgbase/mount
+```
+
+---
+
+And that's it! You can get more fancy with nixops using a setup [like
+this](https://github.com/Xe/nixos-configs/blob/master/common/services/backup.nix).
+In general though, you can get away with this setup. It may be a good idea to
+copy down the encryption passphrase onto paper and put it in a safe space like a
+safety deposit box.
+
+For more information about Borg Backup on NixOS, see [the relevant chapter of
+the NixOS
+manual](https://nixos.org/manual/nixos/stable/index.html#module-borgbase) or
+[the list of borgbackup
+options](https://search.nixos.org/options?channel=20.09&query=services.borgbackup.jobs)
+that you can pick from.
+
+I hope this is able to help.

default.nix

@@ -18,6 +18,7 @@ let
 
   xesite = naersk.buildPackage {
     inherit src;
+    doCheck = true;
     buildInputs = [ pkg-config openssl git ];
     remapPathPrefix = true;
   };

k8s/job.yml

@@ -1,31 +0,0 @@
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: christinewebsite-ping
-  namespace: apps
-  labels:
-    app: christinewebsite
-spec:
-  template:
-    spec:
-      containers:
-      - name: ping-bing
-        image: xena/alpine
-        command:
-          - "busybox"
-          - "wget"
-          - "-O"
-          - "-"
-          - "-q"
-          - "https://www.bing.com/ping?sitemap=https://christine.website/sitemap.xml"
-      - name: ping-google
-        image: xena/alpine
-        command:
-          - "busybox"
-          - "wget"
-          - "-O"
-          - "-"
-          - "-q"
-          - "https://www.google.com/ping?sitemap=https://christine.website/sitemap.xml"
-      restartPolicy: Never
-  backoffLimit: 4

nix/sources.json

@@ -5,10 +5,10 @@
         "homepage": "",
         "owner": "justinwoo",
         "repo": "easy-dhall-nix",
-        "rev": "8f0840b7c94dfba4d5fbbb8709ba6df2fe6a2158",
+        "rev": "eae7f64c4d6c70681e5a56c84198236930ba425e",
-        "sha256": "0rf0fx80kh2825sa07hg36d98r1gnjwrfdw2n2d9y8kkhvd03ppx",
+        "sha256": "1y2x15v8a679vlpxazjpibfwajp6zph60f8wjcm4xflbvazk0dx7",
         "type": "tarball",
-        "url": "https://github.com/justinwoo/easy-dhall-nix/archive/8f0840b7c94dfba4d5fbbb8709ba6df2fe6a2158.tar.gz",
+        "url": "https://github.com/justinwoo/easy-dhall-nix/archive/eae7f64c4d6c70681e5a56c84198236930ba425e.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "naersk": {
@@ -17,10 +17,10 @@
         "homepage": "",
         "owner": "nmattia",
         "repo": "naersk",
-        "rev": "8d562105f960c487cfc013c0e666c1951b03d392",
+        "rev": "a76924cbbb17c387e5ae4998a4721d88a3ac95c0",
-        "sha256": "16a1k5wamp67sqc341rlbmppm2xldfl207k9yhksgcvd4rw3iy20",
+        "sha256": "09b5g2krf8mfpajgz2bgapkv3dpimg0qx1nfpjafcrsk0fhxmqay",
         "type": "tarball",
-        "url": "https://github.com/nmattia/naersk/archive/8d562105f960c487cfc013c0e666c1951b03d392.tar.gz",
+        "url": "https://github.com/nmattia/naersk/archive/a76924cbbb17c387e5ae4998a4721d88a3ac95c0.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "niv": {
@@ -29,10 +29,10 @@
         "homepage": "https://github.com/nmattia/niv",
         "owner": "nmattia",
         "repo": "niv",
-        "rev": "ba57d5a29b4e0f2085917010380ef3ddc3cf380f",
+        "rev": "94dadba1a3a6a2f0b8ca2963e49daeec5d4e3098",
-        "sha256": "1kpsvc53x821cmjg1khvp1nz7906gczq8mp83664cr15h94sh8i4",
+        "sha256": "1y2h9wl7w60maa2m4xw9231xdr325xynzpph8xr4j5vsznygv986",
         "type": "tarball",
-        "url": "https://github.com/nmattia/niv/archive/ba57d5a29b4e0f2085917010380ef3ddc3cf380f.tar.gz",
+        "url": "https://github.com/nmattia/niv/archive/94dadba1a3a6a2f0b8ca2963e49daeec5d4e3098.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixpkgs": {

scripts/release.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env nix-shell
-#! nix-shell -p doctl -p kubectl -p curl -i bash
-#! nix-shell -I nixpkgs=https://releases.nixos.org/nixpkgs/nixpkgs-21.03pre252431.4f3475b113c/nixexprs.tar.xz
-
-nix-env -if ./nix/dhall-yaml.nix
-doctl kubernetes cluster kubeconfig save kubermemes
-dhall-to-yaml-ng < ./site.dhall | kubectl apply -n apps -f -
-kubectl rollout status -n apps deployment/christinewebsite
-kubectl apply -f ./k8s/job.yml
-sleep 10
-kubectl delete -f ./k8s/job.yml
-curl --http1.1 -H "Authorization: $MI_TOKEN" https://mi.within.website/api/blog/refresh -XPOST

src/build.rs

@@ -8,6 +8,11 @@ fn main() -> Result<()> {
         .args(&["rev-parse", "HEAD"])
         .output()
         .unwrap();
+
+    if std::env::var("out").is_err() {
+        println!("cargo:rustc-env=out=/yolo");
+    }
+
     let git_hash = String::from_utf8(output.stdout).unwrap();
     println!(
         "cargo:rustc-env=GITHUB_SHA={}",

src/main.rs

@@ -39,6 +39,21 @@ async fn main() -> Result<()> {
         .await?,
     );
 
+    match sdnotify::SdNotify::from_env() {
+        Ok(ref mut n) => {
+            n.notify_ready().map_err(|why| {
+                error!("can't signal readiness to systemd: {}", why);
+                why
+            })?;
+            n.set_status(format!("hosting {} posts", state.clone().everything.len()))
+                .map_err(|why| {
+                    error!("can't signal status to systemd: {}", why);
+                    why
+                })?;
+        }
+        Err(why) => error!("not running under systemd with Type=notify: {}", why),
+    }
+
     let healthcheck = warp::get().and(warp::path(".within").and(warp::path("health")).map(|| "OK"));
 
     let base = warp::path!("blog" / ..);
@@ -207,12 +222,6 @@ async fn main() -> Result<()> {
         .with(warp::log(APPLICATION_NAME))
         .recover(handlers::rejection);
 
-    if let Ok(ref mut n) = sdnotify::SdNotify::from_env() {
-        let _ = n
-            .notify_ready()
-            .map_err(|why| error!("can't signal readiness to systemd: {}", why));
-    }
-
     warp::serve(site)
         .run((
             [0, 0, 0, 0],