// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package timeseries implements a time series structure for stats collection. package timeseries // import "golang.org/x/net/internal/timeseries" import ( "fmt" "log" "time" ) const ( timeSeriesNumBuckets = 64 minuteHourSeriesNumBuckets = 60 ) var timeSeriesResolutions = []time.Duration{ 1 * time.Second, 10 * time.Second, 1 * time.Minute, 10 * time.Minute, 1 * time.Hour, 6 * time.Hour, 24 * time.Hour, // 1 day 7 * 24 * time.Hour, // 1 week 4 * 7 * 24 * time.Hour, // 4 weeks 16 * 7 * 24 * time.Hour, // 16 weeks } var minuteHourSeriesResolutions = []time.Duration{ 1 * time.Second, 1 * time.Minute, } // An Observable is a kind of data that can be aggregated in a time series. type Observable interface { Multiply(ratio float64) // Multiplies the data in self by a given ratio Add(other Observable) // Adds the data from a different observation to self Clear() // Clears the observation so it can be reused. CopyFrom(other Observable) // Copies the contents of a given observation to self } // Float attaches the methods of Observable to a float64. type Float float64 // NewFloat returns a Float. func NewFloat() Observable { f := Float(0) return &f } // String returns the float as a string. func (f *Float) String() string { return fmt.Sprintf("%g", f.Value()) } // Value returns the float's value. func (f *Float) Value() float64 { return float64(*f) } func (f *Float) Multiply(ratio float64) { *f *= Float(ratio) } func (f *Float) Add(other Observable) { o := other.(*Float) *f += *o } func (f *Float) Clear() { *f = 0 } func (f *Float) CopyFrom(other Observable) { o := other.(*Float) *f = *o } // A Clock tells the current time. type Clock interface { Time() time.Time } type defaultClock int var defaultClockInstance defaultClock func (defaultClock) Time() time.Time { return time.Now() } // Information kept per level. Each level consists of a circular list of // observations. The start of the level may be derived from end and the // len(buckets) * sizeInMillis. type tsLevel struct { oldest int // index to oldest bucketed Observable newest int // index to newest bucketed Observable end time.Time // end timestamp for this level size time.Duration // duration of the bucketed Observable buckets []Observable // collections of observations provider func() Observable // used for creating new Observable } func (l *tsLevel) Clear() { l.oldest = 0 l.newest = len(l.buckets) - 1 l.end = time.Time{} for i := range l.buckets { if l.buckets[i] != nil { l.buckets[i].Clear() l.buckets[i] = nil } } } func (l *tsLevel) InitLevel(size time.Duration, numBuckets int, f func() Observable) { l.size = size l.provider = f l.buckets = make([]Observable, numBuckets) } // Keeps a sequence of levels. Each level is responsible for storing data at // a given resolution. For example, the first level stores data at a one // minute resolution while the second level stores data at a one hour // resolution. // Each level is represented by a sequence of buckets. Each bucket spans an // interval equal to the resolution of the level. New observations are added // to the last bucket. type timeSeries struct { provider func() Observable // make more Observable numBuckets int // number of buckets in each level levels []*tsLevel // levels of bucketed Observable lastAdd time.Time // time of last Observable tracked total Observable // convenient aggregation of all Observable clock Clock // Clock for getting current time pending Observable // observations not yet bucketed pendingTime time.Time // what time are we keeping in pending dirty bool // if there are pending observations } // init initializes a level according to the supplied criteria. func (ts *timeSeries) init(resolutions []time.Duration, f func() Observable, numBuckets int, clock Clock) { ts.provider = f ts.numBuckets = numBuckets ts.clock = clock ts.levels = make([]*tsLevel, len(resolutions)) for i := range resolutions { if i > 0 && resolutions[i-1] >= resolutions[i] { log.Print("timeseries: resolutions must be monotonically increasing") break } newLevel := new(tsLevel) newLevel.InitLevel(resolutions[i], ts.numBuckets, ts.provider) ts.levels[i] = newLevel } ts.Clear() } // Clear removes all observations from the time series. func (ts *timeSeries) Clear() { ts.lastAdd = time.Time{} ts.total = ts.resetObservation(ts.total) ts.pending = ts.resetObservation(ts.pending) ts.pendingTime = time.Time{} ts.dirty = false for i := range ts.levels { ts.levels[i].Clear() } } // Add records an observation at the current time. func (ts *timeSeries) Add(observation Observable) { ts.AddWithTime(observation, ts.clock.Time()) } // AddWithTime records an observation at the specified time. func (ts *timeSeries) AddWithTime(observation Observable, t time.Time) { smallBucketDuration := ts.levels[0].size if t.After(ts.lastAdd) { ts.lastAdd = t } if t.After(ts.pendingTime) { ts.advance(t) ts.mergePendingUpdates() ts.pendingTime = ts.levels[0].end ts.pending.CopyFrom(observation) ts.dirty = true } else if t.After(ts.pendingTime.Add(-1 * smallBucketDuration)) { // The observation is close enough to go into the pending bucket. // This compensates for clock skewing and small scheduling delays // by letting the update stay in the fast path. ts.pending.Add(observation) ts.dirty = true } else { ts.mergeValue(observation, t) } } // mergeValue inserts the observation at the specified time in the past into all levels. func (ts *timeSeries) mergeValue(observation Observable, t time.Time) { for _, level := range ts.levels { index := (ts.numBuckets - 1) - int(level.end.Sub(t)/level.size) if 0 <= index && index < ts.numBuckets { bucketNumber := (level.oldest + index) % ts.numBuckets if level.buckets[bucketNumber] == nil { level.buckets[bucketNumber] = level.provider() } level.buckets[bucketNumber].Add(observation) } } ts.total.Add(observation) } // mergePendingUpdates applies the pending updates into all levels. func (ts *timeSeries) mergePendingUpdates() { if ts.dirty { ts.mergeValue(ts.pending, ts.pendingTime) ts.pending = ts.resetObservation(ts.pending) ts.dirty = false } } // advance cycles the buckets at each level until the latest bucket in // each level can hold the time specified. func (ts *timeSeries) advance(t time.Time) { if !t.After(ts.levels[0].end) { return } for i := 0; i < len(ts.levels); i++ { level := ts.levels[i] if !level.end.Before(t) { break } // If the time is sufficiently far, just clear the level and advance // directly. if !t.Before(level.end.Add(level.size * time.Duration(ts.numBuckets))) { for _, b := range level.buckets { ts.resetObservation(b) } level.end = time.Unix(0, (t.UnixNano()/level.size.Nanoseconds())*level.size.Nanoseconds()) } for t.After(level.end) { level.end = level.end.Add(level.size) level.newest = level.oldest level.oldest = (level.oldest + 1) % ts.numBuckets ts.resetObservation(level.buckets[level.newest]) } t = level.end } } // Latest returns the sum of the num latest buckets from the level. func (ts *timeSeries) Latest(level, num int) Observable { now := ts.clock.Time() if ts.levels[0].end.Before(now) { ts.advance(now) } ts.mergePendingUpdates() result := ts.provider() l := ts.levels[level] index := l.newest for i := 0; i < num; i++ { if l.buckets[index] != nil { result.Add(l.buckets[index]) } if index == 0 { index = ts.numBuckets } index-- } return result } // LatestBuckets returns a copy of the num latest buckets from level. func (ts *timeSeries) LatestBuckets(level, num int) []Observable { if level < 0 || level > len(ts.levels) { log.Print("timeseries: bad level argument: ", level) return nil } if num < 0 || num >= ts.numBuckets { log.Print("timeseries: bad num argument: ", num) return nil } results := make([]Observable, num) now := ts.clock.Time() if ts.levels[0].end.Before(now) { ts.advance(now) } ts.mergePendingUpdates() l := ts.levels[level] index := l.newest for i := 0; i < num; i++ { result := ts.provider() results[i] = result if l.buckets[index] != nil { result.CopyFrom(l.buckets[index]) } if index == 0 { index = ts.numBuckets } index -= 1 } return results } // ScaleBy updates observations by scaling by factor. func (ts *timeSeries) ScaleBy(factor float64) { for _, l := range ts.levels { for i := 0; i < ts.numBuckets; i++ { l.buckets[i].Multiply(factor) } } ts.total.Multiply(factor) ts.pending.Multiply(factor) } // Range returns the sum of observations added over the specified time range. // If start or finish times don't fall on bucket boundaries of the same // level, then return values are approximate answers. func (ts *timeSeries) Range(start, finish time.Time) Observable { return ts.ComputeRange(start, finish, 1)[0] } // Recent returns the sum of observations from the last delta. func (ts *timeSeries) Recent(delta time.Duration) Observable { now := ts.clock.Time() return ts.Range(now.Add(-delta), now) } // Total returns the total of all observations. func (ts *timeSeries) Total() Observable { ts.mergePendingUpdates() return ts.total } // ComputeRange computes a specified number of values into a slice using // the observations recorded over the specified time period. The return // values are approximate if the start or finish times don't fall on the // bucket boundaries at the same level or if the number of buckets spanning // the range is not an integral multiple of num. func (ts *timeSeries) ComputeRange(start, finish time.Time, num int) []Observable { if start.After(finish) { log.Printf("timeseries: start > finish, %v>%v", start, finish) return nil } if num < 0 { log.Printf("timeseries: num < 0, %v", num) return nil } results := make([]Observable, num) for _, l := range ts.levels { if !start.Before(l.end.Add(-l.size * time.Duration(ts.numBuckets))) { ts.extract(l, start, finish, num, results) return results } } // Failed to find a level that covers the desired range. So just // extract from the last level, even if it doesn't cover the entire // desired range. ts.extract(ts.levels[len(ts.levels)-1], start, finish, num, results) return results } // RecentList returns the specified number of values in slice over the most // recent time period of the specified range. func (ts *timeSeries) RecentList(delta time.Duration, num int) []Observable { if delta < 0 { return nil } now := ts.clock.Time() return ts.ComputeRange(now.Add(-delta), now, num) } // extract returns a slice of specified number of observations from a given // level over a given range. func (ts *timeSeries) extract(l *tsLevel, start, finish time.Time, num int, results []Observable) { ts.mergePendingUpdates() srcInterval := l.size dstInterval := finish.Sub(start) / time.Duration(num) dstStart := start srcStart := l.end.Add(-srcInterval * time.Duration(ts.numBuckets)) srcIndex := 0 // Where should scanning start? if dstStart.After(srcStart) { advance := dstStart.Sub(srcStart) / srcInterval srcIndex += int(advance) srcStart = srcStart.Add(advance * srcInterval) } // The i'th value is computed as show below. // interval = (finish/start)/num // i'th value = sum of observation in range // [ start + i * interval, // start + (i + 1) * interval ) for i := 0; i < num; i++ { results[i] = ts.resetObservation(results[i]) dstEnd := dstStart.Add(dstInterval) for srcIndex < ts.numBuckets && srcStart.Before(dstEnd) { srcEnd := srcStart.Add(srcInterval) if srcEnd.After(ts.lastAdd) { srcEnd = ts.lastAdd } if !srcEnd.Before(dstStart) { srcValue := l.buckets[(srcIndex+l.oldest)%ts.numBuckets] if !srcStart.Before(dstStart) && !srcEnd.After(dstEnd) { // dst completely contains src. if srcValue != nil { results[i].Add(srcValue) } } else { // dst partially overlaps src. overlapStart := maxTime(srcStart, dstStart) overlapEnd := minTime(srcEnd, dstEnd) base := srcEnd.Sub(srcStart) fraction := overlapEnd.Sub(overlapStart).Seconds() / base.Seconds() used := ts.provider() if srcValue != nil { used.CopyFrom(srcValue) } used.Multiply(fraction) results[i].Add(used) } if srcEnd.After(dstEnd) { break } } srcIndex++ srcStart = srcStart.Add(srcInterval) } dstStart = dstStart.Add(dstInterval) } } // resetObservation clears the content so the struct may be reused. func (ts *timeSeries) resetObservation(observation Observable) Observable { if observation == nil { observation = ts.provider() } else { observation.Clear() } return observation } // TimeSeries tracks data at granularities from 1 second to 16 weeks. type TimeSeries struct { timeSeries } // NewTimeSeries creates a new TimeSeries using the function provided for creating new Observable. func NewTimeSeries(f func() Observable) *TimeSeries { return NewTimeSeriesWithClock(f, defaultClockInstance) } // NewTimeSeriesWithClock creates a new TimeSeries using the function provided for creating new Observable and the clock for // assigning timestamps. func NewTimeSeriesWithClock(f func() Observable, clock Clock) *TimeSeries { ts := new(TimeSeries) ts.timeSeries.init(timeSeriesResolutions, f, timeSeriesNumBuckets, clock) return ts } // MinuteHourSeries tracks data at granularities of 1 minute and 1 hour. type MinuteHourSeries struct { timeSeries } // NewMinuteHourSeries creates a new MinuteHourSeries using the function provided for creating new Observable. func NewMinuteHourSeries(f func() Observable) *MinuteHourSeries { return NewMinuteHourSeriesWithClock(f, defaultClockInstance) } // NewMinuteHourSeriesWithClock creates a new MinuteHourSeries using the function provided for creating new Observable and the clock for // assigning timestamps. func NewMinuteHourSeriesWithClock(f func() Observable, clock Clock) *MinuteHourSeries { ts := new(MinuteHourSeries) ts.timeSeries.init(minuteHourSeriesResolutions, f, minuteHourSeriesNumBuckets, clock) return ts } func (ts *MinuteHourSeries) Minute() Observable { return ts.timeSeries.Latest(0, 60) } func (ts *MinuteHourSeries) Hour() Observable { return ts.timeSeries.Latest(1, 60) } func minTime(a, b time.Time) time.Time { if a.Before(b) { return a } return b } func maxTime(a, b time.Time) time.Time { if a.After(b) { return a } return b }