package main import ( "context" "database/sql" "encoding/csv" "fmt" "io" "os" "path/filepath" "strconv" "time" ) func ImportCSV(ctx context.Context, db *sql.DB, dir string) error { files, err := filepath.Glob(filepath.Join(dir, "*.csv")) if err != nil { return err } for _, fname := range files { fin, err := os.Open(fname) if err != nil { return fmt.Errorf("can't open file: %w", err) } defer fin.Close() rows, err := ReadRows(fin) if err != nil { return err } tx, err := db.BeginTx(ctx, nil) if err != nil { return err } for _, row := range rows { err := row.Upsert(tx) if err != nil { tx.Rollback() return err } } err = tx.Commit() if err != nil { return err } } return nil } type TwitchRevenueCSV struct { Date time.Time AdBreakMinutes int AdTimeSecondsPerHour float64 AverageViewers float64 ChatMessages int Chatters int ClipViews int ClipsCreated int Follows int HostsAndRaidsViewers int LiveViews int MaxViewers int MinutesWatched int MinutesStreamed int UniqueViewers int SubRevenue float64 PrimeRevenue float64 GiftedSubsRevenue float64 MultiMonthGiftedSubsRevenue float64 BitsRevenue float64 AdRevenue float64 GameSalesRevenue float64 ExtensionsRevenue float64 BountiesRevenue float64 PrimeSubs int TotalPaidSubs int TierOneSubs int TotalGiftedSubs int GiftedTierOneSubs int GiftedTierTwoSubs int GiftedTierThreeSubs int TotalMultiMonthGiftedSubs int MultiMonthGiftedTierOneSubs int MultiMonthGiftedTierTwoSubs int MultiMonthGiftedTierThreeSubs int } func (t TwitchRevenueCSV) Upsert(tx *sql.Tx) error { qBytes, err := schemaFS.ReadFile("schema/insert.sql") if err != nil { return err } _, err = tx.Exec( string(qBytes), t.Date.Format(sqliteDateFormat), t.AdBreakMinutes, t.AdTimeSecondsPerHour, t.AverageViewers, t.ChatMessages, t.Chatters, t.ClipViews, t.ClipsCreated, t.Follows, t.HostsAndRaidsViewers, t.LiveViews, t.MaxViewers, t.MinutesWatched, t.MinutesStreamed, t.UniqueViewers, t.SubRevenue, t.PrimeRevenue, t.GiftedSubsRevenue, t.MultiMonthGiftedSubsRevenue, t.BitsRevenue, t.AdRevenue, t.GameSalesRevenue, t.ExtensionsRevenue, t.BountiesRevenue, t.PrimeSubs, t.TotalPaidSubs, t.TierOneSubs, t.TotalGiftedSubs, t.GiftedTierOneSubs, t.GiftedTierTwoSubs, t.GiftedTierThreeSubs, t.TotalMultiMonthGiftedSubs, t.MultiMonthGiftedTierOneSubs, t.MultiMonthGiftedTierTwoSubs, t.MultiMonthGiftedTierThreeSubs, ) return err } func ReadRows(fin io.Reader) ([]*TwitchRevenueCSV, error) { var result []*TwitchRevenueCSV rdr := csv.NewReader(fin) _, err := rdr.Read() // discard header row if err != nil { return nil, err } var n = 1 for { row, err := rdr.Read() if err != nil { if err == io.EOF { break } return nil, fmt.Errorf("can't read csv row: %w", err) } val, err := ReadFromRow(row) if err != nil { return nil, fmt.Errorf("can't parse row %d %w", n, err) } result = append(result, val) } return result, nil } func ReadFromRow(inp []string) (*TwitchRevenueCSV, error) { var ( result TwitchRevenueCSV err error n int ) result.Date, err = time.Parse(twitchDateFormat, inp[n]) if err != nil { return nil, err } n++ result.AdBreakMinutes, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.AdTimeSecondsPerHour, err = strconv.ParseFloat(inp[n], 64) if err != nil { return nil, err } n++ result.AverageViewers, err = strconv.ParseFloat(inp[n], 64) if err != nil { return nil, err } n++ result.ChatMessages, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.Chatters, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.ClipViews, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.ClipsCreated, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.Follows, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.HostsAndRaidsViewers, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.LiveViews, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.MaxViewers, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.MinutesWatched, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.MinutesStreamed, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.UniqueViewers, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.SubRevenue, err = strconv.ParseFloat(inp[n], 64) if err != nil { return nil, err } n++ result.PrimeRevenue, err = strconv.ParseFloat(inp[n], 64) if err != nil { return nil, err } n++ result.GiftedSubsRevenue, err = strconv.ParseFloat(inp[n], 64) if err != nil { return nil, err } n++ result.MultiMonthGiftedSubsRevenue, err = strconv.ParseFloat(inp[n], 64) if err != nil { return nil, err } n++ result.BitsRevenue, err = strconv.ParseFloat(inp[n], 64) if err != nil { return nil, err } n++ result.AdRevenue, err = strconv.ParseFloat(inp[n], 64) if err != nil { return nil, err } n++ result.GameSalesRevenue, err = strconv.ParseFloat(inp[n], 64) if err != nil { return nil, err } n++ result.ExtensionsRevenue, err = strconv.ParseFloat(inp[n], 64) if err != nil { return nil, err } n++ result.BountiesRevenue, err = strconv.ParseFloat(inp[n], 64) if err != nil { return nil, err } n++ result.PrimeSubs, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.TotalPaidSubs, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.TierOneSubs, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.TotalGiftedSubs, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.GiftedTierOneSubs, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.GiftedTierTwoSubs, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.GiftedTierThreeSubs, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.TotalMultiMonthGiftedSubs, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.MultiMonthGiftedTierOneSubs, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.MultiMonthGiftedTierTwoSubs, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ result.MultiMonthGiftedTierThreeSubs, err = strconv.Atoi(inp[n]) if err != nil { return nil, err } n++ return &result, err }