From 1bd858680de4bb9dc3ffaa274eb93aec56012a0f Mon Sep 17 00:00:00 2001 From: Christine Dodrill Date: Sat, 9 Jan 2021 17:16:30 -0500 Subject: [PATCH] Borgbackup nixos post (#291) * fix the systemd notify code Signed-off-by: Christine Dodrill * remove k8s baktag Signed-off-by: Christine Dodrill * borg backup post Signed-off-by: Christine Dodrill * fix build Signed-off-by: Christine Dodrill --- blog/borg-backup-2021-01-09.markdown | 178 +++++++++++++++++++++++++++ k8s/job.yml | 31 ----- scripts/release.sh | 12 -- site.dhall | 45 ------- src/build.rs | 5 + src/main.rs | 21 +++- 6 files changed, 198 insertions(+), 94 deletions(-) create mode 100644 blog/borg-backup-2021-01-09.markdown delete mode 100644 k8s/job.yml delete mode 100755 scripts/release.sh delete mode 100644 site.dhall diff --git a/blog/borg-backup-2021-01-09.markdown b/blog/borg-backup-2021-01-09.markdown new file mode 100644 index 0000000..3d0c1fb --- /dev/null +++ b/blog/borg-backup-2021-01-09.markdown @@ -0,0 +1,178 @@ +--- +title: "How to Set Up Borg Backup on NixOS" +date: 2021-01-09 +series: howto +tags: + - nixos + - borgbackup +--- + +# How to Set Up Borg Backup on NixOS + +[Borg Backup](https://www.borgbackup.org/) is a encrypted, compressed, +deduplicated backup program for multiple platforms including Linux. This +combined with the [NixOS options for configuring +Borg Backup](https://search.nixos.org/options?channel=20.09&show=services.borgbackup.jobs.%3Cname%3E.paths&from=0&size=30&sort=relevance&query=services.borgbackup.jobs) +allows you to backup on a schedule and restore from those backups when you need +to. + +Borg Backup works with local files, remote servers and there are even [cloud +hosts](https://www.borgbackup.org/support/commercial.html) that specialize in +hosting your backups. In this post we will cover how to set up a backup job on a +server using [BorgBase](https://www.borgbase.com/)'s free tier to host the +backup files. + +## Setup + +You will need a few things: + +- A free BorgBase account +- A server running NixOS +- A list of folders to back up +- A list of folders to NOT back up + +First, we will need to create a SSH key for root to use when connecting to +BorgBase. Open a shell as root on the server and make a `borgbackup` folder in +root's home directory: + +```shell +mkdir borgbackup +cd borgbackup +``` + +Then create a SSH key that will be used to connect to BorgBase: + +```shell +ssh-keygen -f ssh_key -t ed25519 -C "Borg Backup" +``` + +Ignore the SSH key password because at this time the automated Borg Backup job +doesn't allow the use of password-protected SSH keys. + +Now we need to create an encryption passphrase for the backup repository. Run +this command to generate one using [xkcdpass](https://pypi.org/project/xkcdpass/): + +```shell +nix-shell -p python39Packages.xkcdpass --run 'xkcdpass -n 12' > passphrase +``` + +[You can do whatever you want to generate a suitable passphrase, however +xkcdpass is proven to be more random than +most other password generators.](conversation://Mara/hacker) + +## BorgBase Setup + +Now that we have the basic requirements out of the way, let's configure BorgBase +to use that SSH key. In the BorgBase UI click on the Account tab in the upper +right and open the SSH key management window. Click on Add Key and paste in the +contents of `./ssh_key.pub`. Name it after the hostname of the server you are +working on. Click Add Key and then go back to the Repositories tab in the upper +right. + +Click New Repo and name it after the hostname of the server you are working on. +Select the key you just created to have full access. Choose the region of the +backup volume and then click Add Repository. + +On the main page copy the repository path with the copy icon next to your +repository in the list. You will need this below. Attempt to SSH into the backup +repo in order to have ssh recognize the server's host key: + +```shell +ssh -i ./ssh_key o6h6zl22@o6h6zl22.repo.borgbase.com +``` + +Then accept the host key and press control-c to terminate the SSH connection. + +## NixOS Configuration + +In your `configuration.nix` file, add the following block: + +```nix +services.borgbackup.jobs."borgbase" = { + paths = [ + "/var/lib" + "/srv" + "/home" + ]; + exclude = [ + # very large paths + "/var/lib/docker" + "/var/lib/systemd" + "/var/lib/libvirt" + + # temporary files created by cargo and `go build` + "**/target" + "/home/*/go/bin" + "/home/*/go/pkg" + ]; + repo = "o6h6zl22@o6h6zl22.repo.borgbase.com:repo"; + encryption = { + mode = "repokey-blake2"; + passCommand = "cat /root/borgbackup/passphrase"; + }; + environment.BORG_RSH = "ssh -i /root/borgbackup/ssh_key"; + compression = "auto,lzma"; + startAt = "daily"; +}; +``` + +Customize the paths and exclude lists to your needs. Once you are satisfied, +rebuild your NixOS system using `nixos-rebuild`: + +```shell +nixos-rebuild switch +``` + +And then you can fire off an initial backup job with this command: + +```shell +systemctl start borgbackup-job-borgbase.service +``` + +Monitor the job with this command: + +```shell +journalctl -fu borgbackup-job-borgbase.service +``` + +The first backup job will always take the longest to run. Every incremental +backup after that will get smaller and smaller. By default, the system will +create new backup snapshots every night at midnight local time. + +## Restoring Files + +To restore files, first figure out when you want to restore the files from. +NixOS includes a wrapper script for each Borg job you define. you can mount your +backup archive using this command: + +``` +mkdir mount +borg-job-borgbase mount o6h6zl22@o6h6zl22.repo.borgbase.com:repo ./mount +``` + +Then you can explore the backup (and with it each incremental snapshot) to +your heart's content and copy files out manually. You can look through each +folder and copy out what you need. + +When you are done you can unmount it with this command: + +``` +borg-job-borgbase umount /root/borgbase/mount +``` + +--- + +And that's it! You can get more fancy with nixops using a setup [like +this](https://github.com/Xe/nixos-configs/blob/master/common/services/backup.nix). +In general though, you can get away with this setup. It may be a good idea to +copy down the encryption passphrase onto paper and put it in a safe space like a +safety deposit box. + +For more information about Borg Backup on NixOS, see [the relevant chapter of +the NixOS +manual](https://nixos.org/manual/nixos/stable/index.html#module-borgbase) or +[the list of borgbackup +options](https://search.nixos.org/options?channel=20.09&query=services.borgbackup.jobs) +that you can pick from. + +I hope this is able to help. diff --git a/k8s/job.yml b/k8s/job.yml deleted file mode 100644 index e5cf4eb..0000000 --- a/k8s/job.yml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: christinewebsite-ping - namespace: apps - labels: - app: christinewebsite -spec: - template: - spec: - containers: - - name: ping-bing - image: xena/alpine - command: - - "busybox" - - "wget" - - "-O" - - "-" - - "-q" - - "https://www.bing.com/ping?sitemap=https://christine.website/sitemap.xml" - - name: ping-google - image: xena/alpine - command: - - "busybox" - - "wget" - - "-O" - - "-" - - "-q" - - "https://www.google.com/ping?sitemap=https://christine.website/sitemap.xml" - restartPolicy: Never - backoffLimit: 4 diff --git a/scripts/release.sh b/scripts/release.sh deleted file mode 100755 index 4cdaf9d..0000000 --- a/scripts/release.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env nix-shell -#! nix-shell -p doctl -p kubectl -p curl -i bash -#! nix-shell -I nixpkgs=https://releases.nixos.org/nixpkgs/nixpkgs-21.03pre252431.4f3475b113c/nixexprs.tar.xz - -nix-env -if ./nix/dhall-yaml.nix -doctl kubernetes cluster kubeconfig save kubermemes -dhall-to-yaml-ng < ./site.dhall | kubectl apply -n apps -f - -kubectl rollout status -n apps deployment/christinewebsite -kubectl apply -f ./k8s/job.yml -sleep 10 -kubectl delete -f ./k8s/job.yml -curl --http1.1 -H "Authorization: $MI_TOKEN" https://mi.within.website/api/blog/refresh -XPOST diff --git a/site.dhall b/site.dhall deleted file mode 100644 index a4689d3..0000000 --- a/site.dhall +++ /dev/null @@ -1,45 +0,0 @@ -let kms = https://tulpa.dev/cadey/kubermemes/raw/branch/master/k8s/package.dhall - -let kubernetes = - https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/master/1.15/package.dhall - -let tag = env:GITHUB_SHA as Text ? "latest" - -let image = "ghcr.io/xe/site:${tag}" - -let vars - : List kubernetes.EnvVar.Type - = [ kubernetes.EnvVar::{ name = "PORT", value = Some "3030" } - , kubernetes.EnvVar::{ name = "RUST_LOG", value = Some "info" } - , kubernetes.EnvVar::{ - , name = "PATREON_CLIENT_ID" - , value = Some env:PATREON_CLIENT_ID as Text - } - , kubernetes.EnvVar::{ - , name = "PATREON_CLIENT_SECRET" - , value = Some env:PATREON_CLIENT_SECRET as Text - } - , kubernetes.EnvVar::{ - , name = "PATREON_ACCESS_TOKEN" - , value = Some env:PATREON_ACCESS_TOKEN as Text - } - , kubernetes.EnvVar::{ - , name = "PATREON_REFRESH_TOKEN" - , value = Some env:PATREON_REFRESH_TOKEN as Text - } - , kubernetes.EnvVar::{ - , name = "MI_TOKEN" - , value = Some env:MI_TOKEN as Text - } - ] - -in kms.app.make - kms.app.Config::{ - , name = "christinewebsite" - , appPort = 3030 - , image - , replicas = 2 - , domain = "christine.website" - , leIssuer = "prod" - , envVars = vars - } diff --git a/src/build.rs b/src/build.rs index 600de8a..7c3f12d 100644 --- a/src/build.rs +++ b/src/build.rs @@ -8,6 +8,11 @@ fn main() -> Result<()> { .args(&["rev-parse", "HEAD"]) .output() .unwrap(); + + if std::env::var("out").is_err() { + println!("cargo:rustc-env=out=/yolo"); + } + let git_hash = String::from_utf8(output.stdout).unwrap(); println!( "cargo:rustc-env=GITHUB_SHA={}", diff --git a/src/main.rs b/src/main.rs index 0d3b1ca..c05ac49 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,6 +39,21 @@ async fn main() -> Result<()> { .await?, ); + match sdnotify::SdNotify::from_env() { + Ok(ref mut n) => { + n.notify_ready().map_err(|why| { + error!("can't signal readiness to systemd: {}", why); + why + })?; + n.set_status(format!("hosting {} posts", state.clone().everything.len())) + .map_err(|why| { + error!("can't signal status to systemd: {}", why); + why + })?; + } + Err(why) => error!("not running under systemd with Type=notify: {}", why), + } + let healthcheck = warp::get().and(warp::path(".within").and(warp::path("health")).map(|| "OK")); let base = warp::path!("blog" / ..); @@ -164,12 +179,6 @@ async fn main() -> Result<()> { .with(warp::log(APPLICATION_NAME)) .recover(handlers::rejection); - if let Ok(ref mut n) = sdnotify::SdNotify::from_env() { - let _ = n - .notify_ready() - .map_err(|why| error!("can't signal readiness to systemd: {}", why)); - } - warp::serve(site) .run(( [0, 0, 0, 0],