cadey
/
twitchalitics
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

initial commit 

Signed-off-by: Xe <me@christine.website>
This commit is contained in:
Cadey Ratio 2022-01-15 16:32:49 +00:00
commit a114369956
12 changed files with 2347 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.envrc Normal file
View File

@ -0,0 +1 @@
eval "$(lorri direnv)"

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
var/
*.db
*.csv

9
go.mod Normal file
View File

@ -0,0 +1,9 @@
module tulpa.dev/cadey/twitchalitics
go 1.16
require (
github.com/tailscale/sqlite v0.0.0-20220107190256-d637a57026df // indirect
tailscale.com v1.20.1 // indirect
within.website/ln v0.9.1 // indirect
)

1608
go.sum Normal file
View File

File diff suppressed because it is too large Load Diff

111
main.go Normal file
View File

@ -0,0 +1,111 @@
package main
import (
"context"
"database/sql"
"database/sql/driver"
"embed"
"flag"
"log"
"net/http"
"os"
"github.com/tailscale/sqlite"
"tailscale.com/tsnet"
"tailscale.com/types/logger"
"within.website/ln"
"within.website/ln/ex"
)
const (
twitchDateFormat = `Mon Jan 02 2006` // Fri Dec 17 2021
sqliteDateFormat = "2006-01-02" // 2006-02-01
)
var (
dbLoc = flag.String("db", "var/twitchalitics.db", "path to SQLite database file")
hostname = flag.String("hostname", "twitchalitics", "hostname to use on your tailnet")
tsLogLoc = flag.String("ts-log-loc", "/mnt/fast/share/twitchalitics.ts.log", "path for Tailscale to dump logs")
//go:embed schema/*
schemaFS embed.FS
)
func getDB() (*sql.DB, error) {
connInitFunc := func(ctx context.Context, conn driver.ConnPrepareContext) error {
err := sqlite.ExecScript(conn.(sqlite.SQLConn), "PRAGMA journal_mode=WAL;")
if err != nil {
return err
}
schema, err := schemaFS.ReadFile("schema/schema.sql")
if err != nil {
return err
}
return sqlite.ExecScript(conn.(sqlite.SQLConn), string(schema))
}
db := sql.OpenDB(sqlite.Connector(*dbLoc, connInitFunc, nil))
err := db.Ping()
if err != nil {
return nil, err
}
return db, nil
}
func getLogFout(fname string) (func(), logger.Logf, error) {
os.Remove(fname)
fout, err := os.Create(fname)
if err != nil {
return nil, nil, err
}
done := func() { fout.Close() }
lgr := log.New(fout, "", log.LstdFlags)
return done, lgr.Printf, nil
}
func main() {
os.Setenv("TAILSCALE_USE_WIP_CODE", "true")
flag.Parse()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
db, err := getDB()
if err != nil {
ln.FatalErr(ctx, err)
}
defer db.Close()
done, tsLgr, err := getLogFout(*tsLogLoc)
if err != nil {
ln.FatalErr(ctx, err)
}
defer done()
srv := &tsnet.Server{
Hostname: *hostname,
Logf: tsLgr,
}
_ = srv
go plainHTTPRedirect(ctx, srv)
ln.FatalErr(ctx, NewServer(db, srv).ListenAndServe())
}
func plainHTTPRedirect(ctx context.Context, srv *tsnet.Server) {
l, err := srv.Listen("tcp", ":80")
if err != nil {
ln.FatalErr(ctx, err)
}
ln.Log(ctx, ln.Info("listening on :80 to forward to HTTPS"))
err = http.Serve(l, ex.HTTPLog(http.RedirectHandler("https://twitchalitics.shark-harmonic.ts.net", http.StatusPermanentRedirect)))
ln.FatalErr(ctx, err)
}

10
main_test.go Normal file
View File

@ -0,0 +1,10 @@
package main
import "testing"
func TestReadRowsFromFile(t *testing.T) {
_, err := ReadRowsFromFile("var/analytics-2021-12-05--2022-01-03.csv")
if err != nil {
t.Fatal(err)
}
}

110
schema/insert.sql Normal file
View File

@ -0,0 +1,110 @@
INSERT INTO twitch_revenue
( date
, ad_break_minutes
, ad_time_seconds_per_hour
, average_viewers
, chat_messages
, chatters
, clip_views
, clips_created
, follows
, hosts_and_raids_viewers
, live_views
, max_viewers
, minutes_watched
, minutes_streamed
, unique_viewers
, sub_revenue
, prime_revenue
, gifted_subs_revenue
, multi_month_gifted_subs_revenue
, bits_revenue
, ad_revenue
, game_sales_revenue
, extensions_revenue
, bounties_revenue
, prime_subs
, total_paid_subs
, tier_one_subs
, total_gifted_subs
, gifted_tier_one_subs
, gifted_tier_two_subs
, gifted_tier_three_subs
, total_multi_month_gifted_subs
, multi_month_gifted_tier_one_subs
, multi_month_gifted_tier_two_subs
, multi_month_gifted_tier_three_subs
)
VALUES
( ?1
, ?2
, ?3
, ?4
, ?5
, ?6
, ?7
, ?8
, ?9
, ?10
, ?11
, ?12
, ?13
, ?14
, ?15
, ?16
, ?17
, ?18
, ?19
, ?20
, ?21
, ?22
, ?23
, ?24
, ?25
, ?26
, ?27
, ?28
, ?29
, ?30
, ?31
, ?32
, ?33
, ?34
, ?35
)
ON CONFLICT DO
UPDATE SET
ad_break_minutes = ?2
, ad_time_seconds_per_hour = ?3
, average_viewers = ?4
, chat_messages = ?5
, chatters = ?6
, clip_views = ?7
, clips_created = ?8
, follows = ?9
, hosts_and_raids_viewers = ?10
, live_views = ?11
, max_viewers = ?12
, minutes_watched = ?13
, minutes_streamed = ?14
, unique_viewers = ?15
, sub_revenue = ?16
, prime_revenue = ?17
, gifted_subs_revenue = ?18
, multi_month_gifted_subs_revenue = ?19
, bits_revenue = ?20
, ad_revenue = ?21
, game_sales_revenue = ?22
, extensions_revenue = ?23
, bounties_revenue = ?24
, prime_subs = ?25
, total_paid_subs = ?26
, tier_one_subs = ?27
, total_gifted_subs = ?28
, gifted_tier_one_subs = ?29
, gifted_tier_two_subs = ?30
, gifted_tier_three_subs = ?31
, total_multi_month_gifted_subs = ?32
, multi_month_gifted_tier_one_subs = ?33
, multi_month_gifted_tier_two_subs = ?34
, multi_month_gifted_tier_three_subs = ?35

39
schema/schema.sql Normal file
View File

@ -0,0 +1,39 @@
CREATE TABLE IF NOT EXISTS twitch_revenue
( date TEXT PRIMARY KEY
, ad_break_minutes INTEGER NOT NULL
, ad_time_seconds_per_hour REAL NOT NULL
, average_viewers REAL NOT NULL
, chat_messages INTEGER NOT NULL
, chatters INTEGER NOT NULL
, clip_views INTEGER NOT NULL
, clips_created INTEGER NOT NULL
, follows INTEGER NOT NULL
, hosts_and_raids_viewers INTEGER NOT NULL
, live_views INTEGER NOT NULL
, max_viewers INTEGER NOT NULL
, minutes_watched INTEGER NOT NULL
, minutes_streamed INTEGER NOT NULL
, unique_viewers INTEGER NOT NULL
, sub_revenue REAL NOT NULL
, prime_revenue REAL NOT NULL
, gifted_subs_revenue REAL NOT NULL
, multi_month_gifted_subs_revenue REAL NOT NULL
, bits_revenue REAL NOT NULL
, ad_revenue REAL NOT NULL
, game_sales_revenue REAL NOT NULL
, extensions_revenue REAL NOT NULL
, bounties_revenue REAL NOT NULL
, prime_subs INTEGER NOT NULL
, total_paid_subs INTEGER NOT NULL
, tier_one_subs INTEGER NOT NULL
, total_gifted_subs INTEGER NOT NULL
, gifted_tier_one_subs INTEGER NOT NULL
, gifted_tier_two_subs INTEGER NOT NULL
, gifted_tier_three_subs INTEGER NOT NULL
, total_multi_month_gifted_subs INTEGER NOT NULL
, multi_month_gifted_tier_one_subs INTEGER NOT NULL
, multi_month_gifted_tier_two_subs INTEGER NOT NULL
, multi_month_gifted_tier_three_subs INTEGER NOT NULL
-- generated columns
, total_revenue REAL GENERATED ALWAYS AS (sub_revenue + prime_revenue + gifted_subs_revenue + multi_month_gifted_subs_revenue + bits_revenue + ad_revenue + game_sales_revenue + extensions_revenue + bounties_revenue)
);

87
server.go Normal file
View File

@ -0,0 +1,87 @@
package main
import (
"context"
"crypto/tls"
"database/sql"
"encoding/json"
"net/http"
"time"
"tailscale.com/client/tailscale"
"tailscale.com/tsnet"
"within.website/ln"
"within.website/ln/ex"
)
type Server struct {
db *sql.DB
mux *http.ServeMux
srv *tsnet.Server
}
func NewServer(db *sql.DB, srv *tsnet.Server) *Server {
mux := http.NewServeMux()
s := &Server{
db: db,
mux: mux,
srv: srv,
}
mux.HandleFunc("/api/whois", s.whois)
return s
}
func (s *Server) ListenAndServe() error {
l, err := s.srv.Listen("tcp", ":443")
if err != nil {
return err
}
l = tls.NewListener(l, &tls.Config{
GetCertificate: func(chi *tls.ClientHelloInfo) (*tls.Certificate, error) {
c, err := tailscale.GetCertificate(chi)
if err != nil {
ln.Error(context.Background(), err, ln.F{"remote_addr": chi.Conn.RemoteAddr().String()})
}
return c, err
},
})
hs := &http.Server{
IdleTimeout: 5 * time.Minute,
Handler: ex.HTTPLog(s),
}
ln.Log(context.Background(), ln.Info("listening on https://twitchalitics.shark-harmonic.ts.net"))
return hs.Serve(l)
}
func (s Server) whois(w http.ResponseWriter, r *http.Request) {
userInfo, err := tailscale.WhoIs(r.Context(), r.RemoteAddr)
if err != nil {
http.Error(w, "can't get whois response: "+err.Error(), http.StatusInternalServerError)
ln.Error(r.Context(), err)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(userInfo)
}
func (s Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
userInfo, err := tailscale.WhoIs(r.Context(), r.RemoteAddr)
if err != nil {
http.Error(w, "can't get whois response: "+err.Error(), http.StatusInternalServerError)
ln.Error(r.Context(), err)
return
}
ctx := ln.WithF(r.Context(), ln.F{"username": userInfo.UserProfile.LoginName, "hostname": userInfo.Node.ComputedName, "os": userInfo.Node.Hostinfo.OS})
r = r.WithContext(ctx)
s.mux.ServeHTTP(w, r)
}

15
shell.nix Normal file
View File

@ -0,0 +1,15 @@
{ pkgs ?
import <nixpkgs> { overlays = [ (self: super: { go = super.go_1_17; }) ]; } }:
pkgs.mkShell {
buildInputs = with pkgs; [
go
gopls
goimports
sqlite-interactive
pkg-config
# keep this line if you use bash
pkgs.bashInteractive
];
}

354
twitch.go Normal file
View File

@ -0,0 +1,354 @@
package main
import (
"context"
"database/sql"
"encoding/csv"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"time"
)
func ImportCSV(ctx context.Context, db *sql.DB, dir string) error {
files, err := filepath.Glob(filepath.Join(dir, "*.csv"))
if err != nil {
return err
}
for _, fname := range files {
rows, err := ReadRowsFromFile(fname)
if err != nil {
return err
}
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
for _, row := range rows {
err := row.Upsert(tx)
if err != nil {
tx.Rollback()
return err
}
}
err = tx.Commit()
if err != nil {
return err
}
}
return nil
}
type TwitchRevenueCSV struct {
Date time.Time
AdBreakMinutes int
AdTimeSecondsPerHour float64
AverageViewers float64
ChatMessages int
Chatters int
ClipViews int
ClipsCreated int
Follows int
HostsAndRaidsViewers int
LiveViews int
MaxViewers int
MinutesWatched int
MinutesStreamed int
UniqueViewers int
SubRevenue float64
PrimeRevenue float64
GiftedSubsRevenue float64
MultiMonthGiftedSubsRevenue float64
BitsRevenue float64
AdRevenue float64
GameSalesRevenue float64
ExtensionsRevenue float64
BountiesRevenue float64
PrimeSubs int
TotalPaidSubs int
TierOneSubs int
TotalGiftedSubs int
GiftedTierOneSubs int
GiftedTierTwoSubs int
GiftedTierThreeSubs int
TotalMultiMonthGiftedSubs int
MultiMonthGiftedTierOneSubs int
MultiMonthGiftedTierTwoSubs int
MultiMonthGiftedTierThreeSubs int
}
func (t TwitchRevenueCSV) Upsert(tx *sql.Tx) error {
qBytes, err := schemaFS.ReadFile("schema/insert.sql")
if err != nil {
return err
}
_, err = tx.Exec(
string(qBytes),
t.Date.Format(sqliteDateFormat),
t.AdBreakMinutes,
t.AdTimeSecondsPerHour,
t.AverageViewers,
t.ChatMessages,
t.Chatters,
t.ClipViews,
t.ClipsCreated,
t.Follows,
t.HostsAndRaidsViewers,
t.LiveViews,
t.MaxViewers,
t.MinutesWatched,
t.MinutesStreamed,
t.UniqueViewers,
t.SubRevenue,
t.PrimeRevenue,
t.GiftedSubsRevenue,
t.MultiMonthGiftedSubsRevenue,
t.BitsRevenue,
t.AdRevenue,
t.GameSalesRevenue,
t.ExtensionsRevenue,
t.BountiesRevenue,
t.PrimeSubs,
t.TotalPaidSubs,
t.TierOneSubs,
t.TotalGiftedSubs,
t.GiftedTierOneSubs,
t.GiftedTierTwoSubs,
t.GiftedTierThreeSubs,
t.TotalMultiMonthGiftedSubs,
t.MultiMonthGiftedTierOneSubs,
t.MultiMonthGiftedTierTwoSubs,
t.MultiMonthGiftedTierThreeSubs,
)
return err
}
func ReadRowsFromFile(fname string) ([]*TwitchRevenueCSV, error) {
var result []*TwitchRevenueCSV
fin, err := os.Open(fname)
if err != nil {
return nil, fmt.Errorf("can't open file: %w", err)
}
defer fin.Close()
rdr := csv.NewReader(fin)
_, err = rdr.Read() // discard header row
var n = 1
for {
row, err := rdr.Read()
if err != nil {
if err == io.EOF {
break
}
return nil, fmt.Errorf("can't read csv row: %w", err)
}
val, err := ReadFromRow(row)
if err != nil {
return nil, fmt.Errorf("can't parse row %d %w", n, err)
}
result = append(result, val)
}
return result, nil
}
func ReadFromRow(inp []string) (*TwitchRevenueCSV, error) {
var (
result TwitchRevenueCSV
err error
n int
)
result.Date, err = time.Parse(twitchDateFormat, inp[n])
if err != nil {
return nil, err
}
n++
result.AdBreakMinutes, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.AdTimeSecondsPerHour, err = strconv.ParseFloat(inp[n], 64)
if err != nil {
return nil, err
}
n++
result.AverageViewers, err = strconv.ParseFloat(inp[n], 64)
if err != nil {
return nil, err
}
n++
result.ChatMessages, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.Chatters, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.ClipViews, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.ClipsCreated, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.Follows, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.HostsAndRaidsViewers, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.LiveViews, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.MaxViewers, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.MinutesWatched, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.MinutesStreamed, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.UniqueViewers, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.SubRevenue, err = strconv.ParseFloat(inp[n], 64)
if err != nil {
return nil, err
}
n++
result.PrimeRevenue, err = strconv.ParseFloat(inp[n], 64)
if err != nil {
return nil, err
}
n++
result.GiftedSubsRevenue, err = strconv.ParseFloat(inp[n], 64)
if err != nil {
return nil, err
}
n++
result.MultiMonthGiftedSubsRevenue, err = strconv.ParseFloat(inp[n], 64)
if err != nil {
return nil, err
}
n++
result.BitsRevenue, err = strconv.ParseFloat(inp[n], 64)
if err != nil {
return nil, err
}
n++
result.AdRevenue, err = strconv.ParseFloat(inp[n], 64)
if err != nil {
return nil, err
}
n++
result.GameSalesRevenue, err = strconv.ParseFloat(inp[n], 64)
if err != nil {
return nil, err
}
n++
result.ExtensionsRevenue, err = strconv.ParseFloat(inp[n], 64)
if err != nil {
return nil, err
}
n++
result.BountiesRevenue, err = strconv.ParseFloat(inp[n], 64)
if err != nil {
return nil, err
}
n++
result.PrimeSubs, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.TotalPaidSubs, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.TierOneSubs, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.TotalGiftedSubs, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.GiftedTierOneSubs, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.GiftedTierTwoSubs, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.GiftedTierThreeSubs, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.TotalMultiMonthGiftedSubs, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.MultiMonthGiftedTierOneSubs, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.MultiMonthGiftedTierTwoSubs, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
result.MultiMonthGiftedTierThreeSubs, err = strconv.Atoi(inp[n])
if err != nil {
return nil, err
}
n++
return &result, err
}

0
var/.gitkeep Normal file
View File