initial commit

Signed-off-by: Xe <me@christine.website>
2022-01-15 16:32:49 +00:00 · 2022-01-15 16:32:49 +00:00 · a114369956
commit a114369956
12 changed files with 2347 additions and 0 deletions
--- a/.envrc
+++ b/.envrc
@ -0,0 +1 @@
+eval "$(lorri direnv)"
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,3 @@
+var/
+*.db
+*.csv
--- a/go.mod
+++ b/go.mod
@ -0,0 +1,9 @@
+module tulpa.dev/cadey/twitchalitics
+
+go 1.16
+
+require (
+	github.com/tailscale/sqlite v0.0.0-20220107190256-d637a57026df // indirect
+	tailscale.com v1.20.1 // indirect
+	within.website/ln v0.9.1 // indirect
+)
--- a/go.sum
+++ b/go.sum
--- a/main.go
+++ b/main.go
@ -0,0 +1,111 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"database/sql/driver"
+	"embed"
+	"flag"
+	"log"
+	"net/http"
+	"os"
+
+	"github.com/tailscale/sqlite"
+	"tailscale.com/tsnet"
+	"tailscale.com/types/logger"
+	"within.website/ln"
+	"within.website/ln/ex"
+)
+
+const (
+	twitchDateFormat = `Mon Jan 02 2006` // Fri Dec 17 2021
+	sqliteDateFormat = "2006-01-02"      // 2006-02-01
+)
+
+var (
+	dbLoc    = flag.String("db", "var/twitchalitics.db", "path to SQLite database file")
+	hostname = flag.String("hostname", "twitchalitics", "hostname to use on your tailnet")
+	tsLogLoc = flag.String("ts-log-loc", "/mnt/fast/share/twitchalitics.ts.log", "path for Tailscale to dump logs")
+
+	//go:embed schema/*
+	schemaFS embed.FS
+)
+
+func getDB() (*sql.DB, error) {
+	connInitFunc := func(ctx context.Context, conn driver.ConnPrepareContext) error {
+		err := sqlite.ExecScript(conn.(sqlite.SQLConn), "PRAGMA journal_mode=WAL;")
+		if err != nil {
+			return err
+		}
+
+		schema, err := schemaFS.ReadFile("schema/schema.sql")
+		if err != nil {
+			return err
+		}
+
+		return sqlite.ExecScript(conn.(sqlite.SQLConn), string(schema))
+	}
+
+	db := sql.OpenDB(sqlite.Connector(*dbLoc, connInitFunc, nil))
+	err := db.Ping()
+	if err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
+func getLogFout(fname string) (func(), logger.Logf, error) {
+	os.Remove(fname)
+	fout, err := os.Create(fname)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	done := func() { fout.Close() }
+	lgr := log.New(fout, "", log.LstdFlags)
+	return done, lgr.Printf, nil
+}
+
+func main() {
+	os.Setenv("TAILSCALE_USE_WIP_CODE", "true")
+
+	flag.Parse()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	db, err := getDB()
+	if err != nil {
+		ln.FatalErr(ctx, err)
+	}
+	defer db.Close()
+
+	done, tsLgr, err := getLogFout(*tsLogLoc)
+	if err != nil {
+		ln.FatalErr(ctx, err)
+	}
+	defer done()
+
+	srv := &tsnet.Server{
+		Hostname: *hostname,
+		Logf:     tsLgr,
+	}
+
+	_ = srv
+
+	go plainHTTPRedirect(ctx, srv)
+	ln.FatalErr(ctx, NewServer(db, srv).ListenAndServe())
+}
+
+func plainHTTPRedirect(ctx context.Context, srv *tsnet.Server) {
+	l, err := srv.Listen("tcp", ":80")
+	if err != nil {
+		ln.FatalErr(ctx, err)
+	}
+
+	ln.Log(ctx, ln.Info("listening on :80 to forward to HTTPS"))
+
+	err = http.Serve(l, ex.HTTPLog(http.RedirectHandler("https://twitchalitics.shark-harmonic.ts.net", http.StatusPermanentRedirect)))
+	ln.FatalErr(ctx, err)
+}
--- a/main_test.go
+++ b/main_test.go
@ -0,0 +1,10 @@
+package main
+
+import "testing"
+
+func TestReadRowsFromFile(t *testing.T) {
+	_, err := ReadRowsFromFile("var/analytics-2021-12-05--2022-01-03.csv")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
--- a/schema/insert.sql
+++ b/schema/insert.sql
@ -0,0 +1,110 @@
+INSERT INTO twitch_revenue
+  ( date
+  , ad_break_minutes 
+  , ad_time_seconds_per_hour 
+  , average_viewers 
+  , chat_messages 
+  , chatters 
+  , clip_views 
+  , clips_created 
+  , follows 
+  , hosts_and_raids_viewers 
+  , live_views 
+  , max_viewers 
+  , minutes_watched 
+  , minutes_streamed 
+  , unique_viewers 
+  , sub_revenue 
+  , prime_revenue 
+  , gifted_subs_revenue 
+  , multi_month_gifted_subs_revenue 
+  , bits_revenue 
+  , ad_revenue 
+  , game_sales_revenue 
+  , extensions_revenue 
+  , bounties_revenue
+  , prime_subs
+  , total_paid_subs
+  , tier_one_subs
+  , total_gifted_subs
+  , gifted_tier_one_subs
+  , gifted_tier_two_subs
+  , gifted_tier_three_subs
+  , total_multi_month_gifted_subs
+  , multi_month_gifted_tier_one_subs
+  , multi_month_gifted_tier_two_subs
+  , multi_month_gifted_tier_three_subs
+  )
+VALUES
+  ( ?1
+  , ?2
+  , ?3
+  , ?4
+  , ?5
+  , ?6
+  , ?7
+  , ?8
+  , ?9
+  , ?10
+  , ?11
+  , ?12
+  , ?13
+  , ?14
+  , ?15
+  , ?16
+  , ?17
+  , ?18
+  , ?19
+  , ?20
+  , ?21
+  , ?22
+  , ?23
+  , ?24
+  , ?25
+  , ?26
+  , ?27
+  , ?28
+  , ?29
+  , ?30
+  , ?31
+  , ?32
+  , ?33
+  , ?34
+  , ?35
+  )
+ON CONFLICT DO
+  UPDATE SET
+    ad_break_minutes = ?2
+  , ad_time_seconds_per_hour = ?3
+  , average_viewers = ?4
+  , chat_messages = ?5
+  , chatters = ?6
+  , clip_views = ?7
+  , clips_created = ?8
+  , follows = ?9
+  , hosts_and_raids_viewers = ?10
+  , live_views = ?11
+  , max_viewers = ?12
+  , minutes_watched = ?13
+  , minutes_streamed = ?14
+  , unique_viewers = ?15
+  , sub_revenue = ?16
+  , prime_revenue = ?17
+  , gifted_subs_revenue = ?18
+  , multi_month_gifted_subs_revenue = ?19
+  , bits_revenue = ?20
+  , ad_revenue = ?21
+  , game_sales_revenue = ?22
+  , extensions_revenue = ?23
+  , bounties_revenue = ?24
+  , prime_subs = ?25
+  , total_paid_subs = ?26
+  , tier_one_subs = ?27
+  , total_gifted_subs = ?28
+  , gifted_tier_one_subs = ?29
+  , gifted_tier_two_subs = ?30
+  , gifted_tier_three_subs = ?31
+  , total_multi_month_gifted_subs = ?32
+  , multi_month_gifted_tier_one_subs = ?33
+  , multi_month_gifted_tier_two_subs = ?34
+  , multi_month_gifted_tier_three_subs = ?35
--- a/schema/schema.sql
+++ b/schema/schema.sql
@ -0,0 +1,39 @@
+CREATE TABLE IF NOT EXISTS twitch_revenue
+  ( date TEXT PRIMARY KEY
+  , ad_break_minutes INTEGER NOT NULL
+  , ad_time_seconds_per_hour REAL NOT NULL
+  , average_viewers REAL NOT NULL
+  , chat_messages INTEGER NOT NULL
+  , chatters INTEGER NOT NULL
+  , clip_views INTEGER NOT NULL
+  , clips_created INTEGER NOT NULL
+  , follows INTEGER NOT NULL
+  , hosts_and_raids_viewers INTEGER NOT NULL
+  , live_views INTEGER NOT NULL
+  , max_viewers INTEGER NOT NULL
+  , minutes_watched INTEGER NOT NULL
+  , minutes_streamed INTEGER NOT NULL
+  , unique_viewers INTEGER NOT NULL
+  , sub_revenue REAL NOT NULL
+  , prime_revenue REAL NOT NULL
+  , gifted_subs_revenue REAL NOT NULL
+  , multi_month_gifted_subs_revenue REAL NOT NULL
+  , bits_revenue REAL NOT NULL
+  , ad_revenue REAL NOT NULL
+  , game_sales_revenue REAL NOT NULL
+  , extensions_revenue REAL NOT NULL
+  , bounties_revenue REAL NOT NULL
+  , prime_subs INTEGER NOT NULL
+  , total_paid_subs INTEGER NOT NULL
+  , tier_one_subs INTEGER NOT NULL
+  , total_gifted_subs INTEGER NOT NULL
+  , gifted_tier_one_subs INTEGER NOT NULL
+  , gifted_tier_two_subs INTEGER NOT NULL
+  , gifted_tier_three_subs INTEGER NOT NULL
+  , total_multi_month_gifted_subs INTEGER NOT NULL
+  , multi_month_gifted_tier_one_subs INTEGER NOT NULL
+  , multi_month_gifted_tier_two_subs INTEGER NOT NULL
+  , multi_month_gifted_tier_three_subs INTEGER NOT NULL
+  -- generated columns
+  , total_revenue REAL GENERATED ALWAYS AS (sub_revenue + prime_revenue + gifted_subs_revenue + multi_month_gifted_subs_revenue + bits_revenue + ad_revenue + game_sales_revenue + extensions_revenue + bounties_revenue)
+  );
--- a/server.go
+++ b/server.go
@ -0,0 +1,87 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"database/sql"
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"tailscale.com/client/tailscale"
+	"tailscale.com/tsnet"
+	"within.website/ln"
+	"within.website/ln/ex"
+)
+
+type Server struct {
+	db  *sql.DB
+	mux *http.ServeMux
+	srv *tsnet.Server
+}
+
+func NewServer(db *sql.DB, srv *tsnet.Server) *Server {
+	mux := http.NewServeMux()
+
+	s := &Server{
+		db:  db,
+		mux: mux,
+		srv: srv,
+	}
+
+	mux.HandleFunc("/api/whois", s.whois)
+
+	return s
+}
+
+func (s *Server) ListenAndServe() error {
+	l, err := s.srv.Listen("tcp", ":443")
+	if err != nil {
+		return err
+	}
+
+	l = tls.NewListener(l, &tls.Config{
+		GetCertificate: func(chi *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			c, err := tailscale.GetCertificate(chi)
+			if err != nil {
+				ln.Error(context.Background(), err, ln.F{"remote_addr": chi.Conn.RemoteAddr().String()})
+			}
+			return c, err
+		},
+	})
+
+	hs := &http.Server{
+		IdleTimeout: 5 * time.Minute,
+		Handler:     ex.HTTPLog(s),
+	}
+
+	ln.Log(context.Background(), ln.Info("listening on https://twitchalitics.shark-harmonic.ts.net"))
+
+	return hs.Serve(l)
+}
+
+func (s Server) whois(w http.ResponseWriter, r *http.Request) {
+	userInfo, err := tailscale.WhoIs(r.Context(), r.RemoteAddr)
+	if err != nil {
+		http.Error(w, "can't get whois response: "+err.Error(), http.StatusInternalServerError)
+		ln.Error(r.Context(), err)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(userInfo)
+}
+
+func (s Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	userInfo, err := tailscale.WhoIs(r.Context(), r.RemoteAddr)
+	if err != nil {
+		http.Error(w, "can't get whois response: "+err.Error(), http.StatusInternalServerError)
+		ln.Error(r.Context(), err)
+		return
+	}
+
+	ctx := ln.WithF(r.Context(), ln.F{"username": userInfo.UserProfile.LoginName, "hostname": userInfo.Node.ComputedName, "os": userInfo.Node.Hostinfo.OS})
+	r = r.WithContext(ctx)
+
+	s.mux.ServeHTTP(w, r)
+}
--- a/shell.nix
+++ b/shell.nix
@ -0,0 +1,15 @@
+{ pkgs ?
+  import <nixpkgs> { overlays = [ (self: super: { go = super.go_1_17; }) ]; } }:
+
+pkgs.mkShell {
+  buildInputs = with pkgs; [
+    go
+    gopls
+    goimports
+    sqlite-interactive
+    pkg-config
+
+    # keep this line if you use bash
+    pkgs.bashInteractive
+  ];
+}
--- a/twitch.go
+++ b/twitch.go
@ -0,0 +1,354 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"encoding/csv"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+)
+
+func ImportCSV(ctx context.Context, db *sql.DB, dir string) error {
+	files, err := filepath.Glob(filepath.Join(dir, "*.csv"))
+	if err != nil {
+		return err
+	}
+
+	for _, fname := range files {
+		rows, err := ReadRowsFromFile(fname)
+		if err != nil {
+			return err
+		}
+
+		tx, err := db.BeginTx(ctx, nil)
+		if err != nil {
+			return err
+		}
+
+		for _, row := range rows {
+			err := row.Upsert(tx)
+			if err != nil {
+				tx.Rollback()
+				return err
+			}
+		}
+
+		err = tx.Commit()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type TwitchRevenueCSV struct {
+	Date                          time.Time
+	AdBreakMinutes                int
+	AdTimeSecondsPerHour          float64
+	AverageViewers                float64
+	ChatMessages                  int
+	Chatters                      int
+	ClipViews                     int
+	ClipsCreated                  int
+	Follows                       int
+	HostsAndRaidsViewers          int
+	LiveViews                     int
+	MaxViewers                    int
+	MinutesWatched                int
+	MinutesStreamed               int
+	UniqueViewers                 int
+	SubRevenue                    float64
+	PrimeRevenue                  float64
+	GiftedSubsRevenue             float64
+	MultiMonthGiftedSubsRevenue   float64
+	BitsRevenue                   float64
+	AdRevenue                     float64
+	GameSalesRevenue              float64
+	ExtensionsRevenue             float64
+	BountiesRevenue               float64
+	PrimeSubs                     int
+	TotalPaidSubs                 int
+	TierOneSubs                   int
+	TotalGiftedSubs               int
+	GiftedTierOneSubs             int
+	GiftedTierTwoSubs             int
+	GiftedTierThreeSubs           int
+	TotalMultiMonthGiftedSubs     int
+	MultiMonthGiftedTierOneSubs   int
+	MultiMonthGiftedTierTwoSubs   int
+	MultiMonthGiftedTierThreeSubs int
+}
+
+func (t TwitchRevenueCSV) Upsert(tx *sql.Tx) error {
+	qBytes, err := schemaFS.ReadFile("schema/insert.sql")
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec(
+		string(qBytes),
+		t.Date.Format(sqliteDateFormat),
+		t.AdBreakMinutes,
+		t.AdTimeSecondsPerHour,
+		t.AverageViewers,
+		t.ChatMessages,
+		t.Chatters,
+		t.ClipViews,
+		t.ClipsCreated,
+		t.Follows,
+		t.HostsAndRaidsViewers,
+		t.LiveViews,
+		t.MaxViewers,
+		t.MinutesWatched,
+		t.MinutesStreamed,
+		t.UniqueViewers,
+		t.SubRevenue,
+		t.PrimeRevenue,
+		t.GiftedSubsRevenue,
+		t.MultiMonthGiftedSubsRevenue,
+		t.BitsRevenue,
+		t.AdRevenue,
+		t.GameSalesRevenue,
+		t.ExtensionsRevenue,
+		t.BountiesRevenue,
+		t.PrimeSubs,
+		t.TotalPaidSubs,
+		t.TierOneSubs,
+		t.TotalGiftedSubs,
+		t.GiftedTierOneSubs,
+		t.GiftedTierTwoSubs,
+		t.GiftedTierThreeSubs,
+		t.TotalMultiMonthGiftedSubs,
+		t.MultiMonthGiftedTierOneSubs,
+		t.MultiMonthGiftedTierTwoSubs,
+		t.MultiMonthGiftedTierThreeSubs,
+	)
+
+	return err
+}
+
+func ReadRowsFromFile(fname string) ([]*TwitchRevenueCSV, error) {
+	var result []*TwitchRevenueCSV
+
+	fin, err := os.Open(fname)
+	if err != nil {
+		return nil, fmt.Errorf("can't open file: %w", err)
+	}
+	defer fin.Close()
+
+	rdr := csv.NewReader(fin)
+
+	_, err = rdr.Read() // discard header row
+	var n = 1
+
+	for {
+		row, err := rdr.Read()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+
+			return nil, fmt.Errorf("can't read csv row: %w", err)
+		}
+
+		val, err := ReadFromRow(row)
+		if err != nil {
+			return nil, fmt.Errorf("can't parse row %d %w", n, err)
+		}
+
+		result = append(result, val)
+	}
+
+	return result, nil
+}
+
+func ReadFromRow(inp []string) (*TwitchRevenueCSV, error) {
+	var (
+		result TwitchRevenueCSV
+		err    error
+		n      int
+	)
+
+	result.Date, err = time.Parse(twitchDateFormat, inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.AdBreakMinutes, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.AdTimeSecondsPerHour, err = strconv.ParseFloat(inp[n], 64)
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.AverageViewers, err = strconv.ParseFloat(inp[n], 64)
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.ChatMessages, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.Chatters, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.ClipViews, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.ClipsCreated, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.Follows, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.HostsAndRaidsViewers, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.LiveViews, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.MaxViewers, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.MinutesWatched, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.MinutesStreamed, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.UniqueViewers, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.SubRevenue, err = strconv.ParseFloat(inp[n], 64)
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.PrimeRevenue, err = strconv.ParseFloat(inp[n], 64)
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.GiftedSubsRevenue, err = strconv.ParseFloat(inp[n], 64)
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.MultiMonthGiftedSubsRevenue, err = strconv.ParseFloat(inp[n], 64)
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.BitsRevenue, err = strconv.ParseFloat(inp[n], 64)
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.AdRevenue, err = strconv.ParseFloat(inp[n], 64)
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.GameSalesRevenue, err = strconv.ParseFloat(inp[n], 64)
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.ExtensionsRevenue, err = strconv.ParseFloat(inp[n], 64)
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.BountiesRevenue, err = strconv.ParseFloat(inp[n], 64)
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.PrimeSubs, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.TotalPaidSubs, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.TierOneSubs, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.TotalGiftedSubs, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.GiftedTierOneSubs, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.GiftedTierTwoSubs, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.GiftedTierThreeSubs, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.TotalMultiMonthGiftedSubs, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.MultiMonthGiftedTierOneSubs, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.MultiMonthGiftedTierTwoSubs, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+	result.MultiMonthGiftedTierThreeSubs, err = strconv.Atoi(inp[n])
+	if err != nil {
+		return nil, err
+	}
+	n++
+
+	return &result, err
+}
--- a/var/.gitkeep
+++ b/var/.gitkeep