Merge branch 'main' into cache-better
This commit is contained in:
GitHub
commit
ef3b80ede3
|
|
@ -15,4 +15,4 @@ jobs:
|
|||
- uses: cachix/cachix-action@v7
|
||||
with:
|
||||
name: xe
|
||||
- run: nix-build --no-out-link
|
||||
- run: nix build --no-link
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
name: Rust
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Build
|
||||
run: cargo build --all
|
||||
- name: Run tests
|
||||
run: |
|
||||
cargo test
|
||||
(cd lib/jsonfeed && cargo test)
|
||||
(cd lib/patreon && cargo test)
|
||||
env:
|
||||
PATREON_ACCESS_TOKEN: ${{ secrets.PATREON_ACCESS_TOKEN }}
|
||||
PATREON_CLIENT_ID: ${{ secrets.PATREON_CLIENT_ID }}
|
||||
PATREON_CLIENT_SECRET: ${{ secrets.PATREON_CLIENT_SECRET }}
|
||||
PATREON_REFRESH_TOKEN: ${{ secrets.PATREON_REFRESH_TOKEN }}
|
||||
out: "/yolo"
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,178 @@
|
|||
---
|
||||
title: "How to Set Up Borg Backup on NixOS"
|
||||
date: 2021-01-09
|
||||
series: howto
|
||||
tags:
|
||||
- nixos
|
||||
- borgbackup
|
||||
---
|
||||
|
||||
# How to Set Up Borg Backup on NixOS
|
||||
|
||||
[Borg Backup](https://www.borgbackup.org/) is a encrypted, compressed,
|
||||
deduplicated backup program for multiple platforms including Linux. This
|
||||
combined with the [NixOS options for configuring
|
||||
Borg Backup](https://search.nixos.org/options?channel=20.09&show=services.borgbackup.jobs.%3Cname%3E.paths&from=0&size=30&sort=relevance&query=services.borgbackup.jobs)
|
||||
allows you to backup on a schedule and restore from those backups when you need
|
||||
to.
|
||||
|
||||
Borg Backup works with local files, remote servers and there are even [cloud
|
||||
hosts](https://www.borgbackup.org/support/commercial.html) that specialize in
|
||||
hosting your backups. In this post we will cover how to set up a backup job on a
|
||||
server using [BorgBase](https://www.borgbase.com/)'s free tier to host the
|
||||
backup files.
|
||||
|
||||
## Setup
|
||||
|
||||
You will need a few things:
|
||||
|
||||
- A free BorgBase account
|
||||
- A server running NixOS
|
||||
- A list of folders to back up
|
||||
- A list of folders to NOT back up
|
||||
|
||||
First, we will need to create a SSH key for root to use when connecting to
|
||||
BorgBase. Open a shell as root on the server and make a `borgbackup` folder in
|
||||
root's home directory:
|
||||
|
||||
```shell
|
||||
mkdir borgbackup
|
||||
cd borgbackup
|
||||
```
|
||||
|
||||
Then create a SSH key that will be used to connect to BorgBase:
|
||||
|
||||
```shell
|
||||
ssh-keygen -f ssh_key -t ed25519 -C "Borg Backup"
|
||||
```
|
||||
|
||||
Ignore the SSH key password because at this time the automated Borg Backup job
|
||||
doesn't allow the use of password-protected SSH keys.
|
||||
|
||||
Now we need to create an encryption passphrase for the backup repository. Run
|
||||
this command to generate one using [xkcdpass](https://pypi.org/project/xkcdpass/):
|
||||
|
||||
```shell
|
||||
nix-shell -p python39Packages.xkcdpass --run 'xkcdpass -n 12' > passphrase
|
||||
```
|
||||
|
||||
[You can do whatever you want to generate a suitable passphrase, however
|
||||
xkcdpass is proven to be <a href="https://xkcd.com/936/">more random</a> than
|
||||
most other password generators.](conversation://Mara/hacker)
|
||||
|
||||
## BorgBase Setup
|
||||
|
||||
Now that we have the basic requirements out of the way, let's configure BorgBase
|
||||
to use that SSH key. In the BorgBase UI click on the Account tab in the upper
|
||||
right and open the SSH key management window. Click on Add Key and paste in the
|
||||
contents of `./ssh_key.pub`. Name it after the hostname of the server you are
|
||||
working on. Click Add Key and then go back to the Repositories tab in the upper
|
||||
right.
|
||||
|
||||
Click New Repo and name it after the hostname of the server you are working on.
|
||||
Select the key you just created to have full access. Choose the region of the
|
||||
backup volume and then click Add Repository.
|
||||
|
||||
On the main page copy the repository path with the copy icon next to your
|
||||
repository in the list. You will need this below. Attempt to SSH into the backup
|
||||
repo in order to have ssh recognize the server's host key:
|
||||
|
||||
```shell
|
||||
ssh -i ./ssh_key o6h6zl22@o6h6zl22.repo.borgbase.com
|
||||
```
|
||||
|
||||
Then accept the host key and press control-c to terminate the SSH connection.
|
||||
|
||||
## NixOS Configuration
|
||||
|
||||
In your `configuration.nix` file, add the following block:
|
||||
|
||||
```nix
|
||||
services.borgbackup.jobs."borgbase" = {
|
||||
paths = [
|
||||
"/var/lib"
|
||||
"/srv"
|
||||
"/home"
|
||||
];
|
||||
exclude = [
|
||||
# very large paths
|
||||
"/var/lib/docker"
|
||||
"/var/lib/systemd"
|
||||
"/var/lib/libvirt"
|
||||
|
||||
# temporary files created by cargo and `go build`
|
||||
"**/target"
|
||||
"/home/*/go/bin"
|
||||
"/home/*/go/pkg"
|
||||
];
|
||||
repo = "o6h6zl22@o6h6zl22.repo.borgbase.com:repo";
|
||||
encryption = {
|
||||
mode = "repokey-blake2";
|
||||
passCommand = "cat /root/borgbackup/passphrase";
|
||||
};
|
||||
environment.BORG_RSH = "ssh -i /root/borgbackup/ssh_key";
|
||||
compression = "auto,lzma";
|
||||
startAt = "daily";
|
||||
};
|
||||
```
|
||||
|
||||
Customize the paths and exclude lists to your needs. Once you are satisfied,
|
||||
rebuild your NixOS system using `nixos-rebuild`:
|
||||
|
||||
```shell
|
||||
nixos-rebuild switch
|
||||
```
|
||||
|
||||
And then you can fire off an initial backup job with this command:
|
||||
|
||||
```shell
|
||||
systemctl start borgbackup-job-borgbase.service
|
||||
```
|
||||
|
||||
Monitor the job with this command:
|
||||
|
||||
```shell
|
||||
journalctl -fu borgbackup-job-borgbase.service
|
||||
```
|
||||
|
||||
The first backup job will always take the longest to run. Every incremental
|
||||
backup after that will get smaller and smaller. By default, the system will
|
||||
create new backup snapshots every night at midnight local time.
|
||||
|
||||
## Restoring Files
|
||||
|
||||
To restore files, first figure out when you want to restore the files from.
|
||||
NixOS includes a wrapper script for each Borg job you define. you can mount your
|
||||
backup archive using this command:
|
||||
|
||||
```
|
||||
mkdir mount
|
||||
borg-job-borgbase mount o6h6zl22@o6h6zl22.repo.borgbase.com:repo ./mount
|
||||
```
|
||||
|
||||
Then you can explore the backup (and with it each incremental snapshot) to
|
||||
your heart's content and copy files out manually. You can look through each
|
||||
folder and copy out what you need.
|
||||
|
||||
When you are done you can unmount it with this command:
|
||||
|
||||
```
|
||||
borg-job-borgbase umount /root/borgbase/mount
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
And that's it! You can get more fancy with nixops using a setup [like
|
||||
this](https://github.com/Xe/nixos-configs/blob/master/common/services/backup.nix).
|
||||
In general though, you can get away with this setup. It may be a good idea to
|
||||
copy down the encryption passphrase onto paper and put it in a safe space like a
|
||||
safety deposit box.
|
||||
|
||||
For more information about Borg Backup on NixOS, see [the relevant chapter of
|
||||
the NixOS
|
||||
manual](https://nixos.org/manual/nixos/stable/index.html#module-borgbase) or
|
||||
[the list of borgbackup
|
||||
options](https://search.nixos.org/options?channel=20.09&query=services.borgbackup.jobs)
|
||||
that you can pick from.
|
||||
|
||||
I hope this is able to help.
|
||||
|
|
@ -18,6 +18,7 @@ let
|
|||
|
||||
xesite = naersk.buildPackage {
|
||||
inherit src;
|
||||
doCheck = true;
|
||||
buildInputs = [ pkg-config openssl git ];
|
||||
remapPathPrefix = true;
|
||||
};
|
||||
|
|
|
|||
31
k8s/job.yml
31
k8s/job.yml
|
|
@ -1,31 +0,0 @@
|
|||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: christinewebsite-ping
|
||||
namespace: apps
|
||||
labels:
|
||||
app: christinewebsite
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: ping-bing
|
||||
image: xena/alpine
|
||||
command:
|
||||
- "busybox"
|
||||
- "wget"
|
||||
- "-O"
|
||||
- "-"
|
||||
- "-q"
|
||||
- "https://www.bing.com/ping?sitemap=https://christine.website/sitemap.xml"
|
||||
- name: ping-google
|
||||
image: xena/alpine
|
||||
command:
|
||||
- "busybox"
|
||||
- "wget"
|
||||
- "-O"
|
||||
- "-"
|
||||
- "-q"
|
||||
- "https://www.google.com/ping?sitemap=https://christine.website/sitemap.xml"
|
||||
restartPolicy: Never
|
||||
backoffLimit: 4
|
||||
|
|
@ -5,10 +5,10 @@
|
|||
"homepage": "",
|
||||
"owner": "justinwoo",
|
||||
"repo": "easy-dhall-nix",
|
||||
"rev": "8f0840b7c94dfba4d5fbbb8709ba6df2fe6a2158",
|
||||
"sha256": "0rf0fx80kh2825sa07hg36d98r1gnjwrfdw2n2d9y8kkhvd03ppx",
|
||||
"rev": "eae7f64c4d6c70681e5a56c84198236930ba425e",
|
||||
"sha256": "1y2x15v8a679vlpxazjpibfwajp6zph60f8wjcm4xflbvazk0dx7",
|
||||
"type": "tarball",
|
||||
"url": "https://github.com/justinwoo/easy-dhall-nix/archive/8f0840b7c94dfba4d5fbbb8709ba6df2fe6a2158.tar.gz",
|
||||
"url": "https://github.com/justinwoo/easy-dhall-nix/archive/eae7f64c4d6c70681e5a56c84198236930ba425e.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
},
|
||||
"naersk": {
|
||||
|
|
@ -17,10 +17,10 @@
|
|||
"homepage": "",
|
||||
"owner": "nmattia",
|
||||
"repo": "naersk",
|
||||
"rev": "8d562105f960c487cfc013c0e666c1951b03d392",
|
||||
"sha256": "16a1k5wamp67sqc341rlbmppm2xldfl207k9yhksgcvd4rw3iy20",
|
||||
"rev": "a76924cbbb17c387e5ae4998a4721d88a3ac95c0",
|
||||
"sha256": "09b5g2krf8mfpajgz2bgapkv3dpimg0qx1nfpjafcrsk0fhxmqay",
|
||||
"type": "tarball",
|
||||
"url": "https://github.com/nmattia/naersk/archive/8d562105f960c487cfc013c0e666c1951b03d392.tar.gz",
|
||||
"url": "https://github.com/nmattia/naersk/archive/a76924cbbb17c387e5ae4998a4721d88a3ac95c0.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
},
|
||||
"niv": {
|
||||
|
|
@ -29,10 +29,10 @@
|
|||
"homepage": "https://github.com/nmattia/niv",
|
||||
"owner": "nmattia",
|
||||
"repo": "niv",
|
||||
"rev": "ba57d5a29b4e0f2085917010380ef3ddc3cf380f",
|
||||
"sha256": "1kpsvc53x821cmjg1khvp1nz7906gczq8mp83664cr15h94sh8i4",
|
||||
"rev": "94dadba1a3a6a2f0b8ca2963e49daeec5d4e3098",
|
||||
"sha256": "1y2h9wl7w60maa2m4xw9231xdr325xynzpph8xr4j5vsznygv986",
|
||||
"type": "tarball",
|
||||
"url": "https://github.com/nmattia/niv/archive/ba57d5a29b4e0f2085917010380ef3ddc3cf380f.tar.gz",
|
||||
"url": "https://github.com/nmattia/niv/archive/94dadba1a3a6a2f0b8ca2963e49daeec5d4e3098.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
},
|
||||
"nixpkgs": {
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
#!/usr/bin/env nix-shell
|
||||
#! nix-shell -p doctl -p kubectl -p curl -i bash
|
||||
#! nix-shell -I nixpkgs=https://releases.nixos.org/nixpkgs/nixpkgs-21.03pre252431.4f3475b113c/nixexprs.tar.xz
|
||||
|
||||
nix-env -if ./nix/dhall-yaml.nix
|
||||
doctl kubernetes cluster kubeconfig save kubermemes
|
||||
dhall-to-yaml-ng < ./site.dhall | kubectl apply -n apps -f -
|
||||
kubectl rollout status -n apps deployment/christinewebsite
|
||||
kubectl apply -f ./k8s/job.yml
|
||||
sleep 10
|
||||
kubectl delete -f ./k8s/job.yml
|
||||
curl --http1.1 -H "Authorization: $MI_TOKEN" https://mi.within.website/api/blog/refresh -XPOST
|
||||
|
|
@ -8,6 +8,11 @@ fn main() -> Result<()> {
|
|||
.args(&["rev-parse", "HEAD"])
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
if std::env::var("out").is_err() {
|
||||
println!("cargo:rustc-env=out=/yolo");
|
||||
}
|
||||
|
||||
let git_hash = String::from_utf8(output.stdout).unwrap();
|
||||
println!(
|
||||
"cargo:rustc-env=GITHUB_SHA={}",
|
||||
|
|
|
|||
21
src/main.rs
21
src/main.rs
|
|
@ -39,6 +39,21 @@ async fn main() -> Result<()> {
|
|||
.await?,
|
||||
);
|
||||
|
||||
match sdnotify::SdNotify::from_env() {
|
||||
Ok(ref mut n) => {
|
||||
n.notify_ready().map_err(|why| {
|
||||
error!("can't signal readiness to systemd: {}", why);
|
||||
why
|
||||
})?;
|
||||
n.set_status(format!("hosting {} posts", state.clone().everything.len()))
|
||||
.map_err(|why| {
|
||||
error!("can't signal status to systemd: {}", why);
|
||||
why
|
||||
})?;
|
||||
}
|
||||
Err(why) => error!("not running under systemd with Type=notify: {}", why),
|
||||
}
|
||||
|
||||
let healthcheck = warp::get().and(warp::path(".within").and(warp::path("health")).map(|| "OK"));
|
||||
|
||||
let base = warp::path!("blog" / ..);
|
||||
|
|
@ -207,12 +222,6 @@ async fn main() -> Result<()> {
|
|||
.with(warp::log(APPLICATION_NAME))
|
||||
.recover(handlers::rejection);
|
||||
|
||||
if let Ok(ref mut n) = sdnotify::SdNotify::from_env() {
|
||||
let _ = n
|
||||
.notify_ready()
|
||||
.map_err(|why| error!("can't signal readiness to systemd: {}", why));
|
||||
}
|
||||
|
||||
warp::serve(site)
|
||||
.run((
|
||||
[0, 0, 0, 0],
|
||||
|
|
|
|||
Loading…
Reference in New Issue