package main import ( "bufio" "bytes" "fmt" "log" "os" "sort" "time" ) type LogLine struct { Time time.Time TimeTook time.Duration Host string } type Bucket struct { Time time.Time // only resolved to the minute Entries []*LogLine } const dateFormat = "2006-01-02T15:04:05.999999Z07:00" const outDateFormat = "2006-01-02T15:04:05" const outDateFormatNoSeconds = "2006-01-02T15:04" var ( seenHosts []string ) func ParseLogLine(line []byte) (*LogLine, error) { sl := bytes.Split(line, []byte(" ")) date := sl[0] lTime, err := time.Parse(dateFormat, string(date)) if err != nil { return nil, err } var sdur time.Duration var host string for i, section := range sl { switch i { case 0, 1: continue // not key->value pairs } set := bytes.Split(section, []byte("=")) if len(set) != 2 { log.Printf("invalid: %v", set) continue } k := string(set[0]) v := set[1] if k == "service" { dur, err := time.ParseDuration(string(v)) if err != nil { return nil, err } sdur = dur } if k == "host" { host = Shuck(string(v)) } } ll := &LogLine{ Time: lTime, TimeTook: sdur, Host: host, } return ll, nil } // Shuck removes the first and last character of a string, analogous to // shucking off the husk of an ear of corn. func Shuck(victim string) string { return victim[1 : len(victim)-1] } func main() { scanner := bufio.NewScanner(os.Stdin) var lastTime time.Time // active buckets, short var name because it's going to be referenced a lot ab := map[string]*Bucket{} for scanner.Scan() { line := scanner.Bytes() ll, err := ParseLogLine(line) if err != nil { log.Fatal(err) } if lastTime.IsZero() { year, month, day := ll.Time.Date() hour, minute, _ := ll.Time.Clock() lastTime = time.Date(year, month, day, hour, minute, 0, 0, time.UTC) } // last line minutes _, llm, _ := ll.Time.Clock() _, ltm, _ := lastTime.Clock() if llm != ltm { processBuckets(lastTime, ab) ab = map[string]*Bucket{} lastTime = lastTime.Add(time.Minute) } b := ab[ll.Host] if b == nil { year, month, day := ll.Time.Date() hour, minute, _ := ll.Time.Clock() b = &Bucket{ Time: time.Date(year, month, day, hour, minute, 0, 0, time.UTC), } ab[ll.Host] = b } b.Entries = append(b.Entries, ll) ab[ll.Host] = b } processBuckets(lastTime, ab) } func toMS(dur time.Duration) int64 { return dur.Nanoseconds() / 1000000 } func contains(host string) bool { for _, val := range seenHosts { if val == host { return true } } return false } func processBuckets(lt time.Time, set map[string]*Bucket) { for host, _ := range set { if !contains(host) { seenHosts = append(seenHosts, host) } } sort.Sort(sort.StringSlice(seenHosts)) log.Printf("%s printing %d buckets for %d hosts", lt.Format(outDateFormatNoSeconds), len(set), len(seenHosts)) for _, host := range seenHosts { bucket, ok := set[host] if !ok { fmt.Printf("%s,%s,0,0,0,0\n", lt.Format(outDateFormat), host) continue } var longest time.Duration var shortest time.Duration var total time.Duration for _, entry := range bucket.Entries { if shortest == 0 { shortest = entry.TimeTook } switch true { case entry.TimeTook > longest: longest = entry.TimeTook case shortest > entry.TimeTook: shortest = entry.TimeTook } total += entry.TimeTook } fmt.Printf("%s,%s,%d,%d,%d,%d\n", bucket.Time.Format(outDateFormat), host, len(bucket.Entries), toMS(total), toMS(shortest), toMS(longest)) } }