vsphere-influxdb-go/vendor/github.com/influxdata/influxdb/influxql/iterator.gen.go

11930 lines
310 KiB
Go

// Generated by tmpl
// https://github.com/benbjohnson/tmpl
//
// DO NOT EDIT!
// Source: iterator.gen.go.tmpl
package influxql
import (
"container/heap"
"encoding/binary"
"fmt"
"io"
"sort"
"sync"
"time"
"github.com/gogo/protobuf/proto"
internal "github.com/influxdata/influxdb/influxql/internal"
)
// DefaultStatsInterval is the default value for IteratorEncoder.StatsInterval.
const DefaultStatsInterval = 10 * time.Second
// FloatIterator represents a stream of float points.
type FloatIterator interface {
Iterator
Next() (*FloatPoint, error)
}
// newFloatIterators converts a slice of Iterator to a slice of FloatIterator.
// Drop and closes any iterator in itrs that is not a FloatIterator and cannot
// be cast to a FloatIterator.
func newFloatIterators(itrs []Iterator) []FloatIterator {
a := make([]FloatIterator, 0, len(itrs))
for _, itr := range itrs {
switch itr := itr.(type) {
case FloatIterator:
a = append(a, itr)
case IntegerIterator:
a = append(a, &integerFloatCastIterator{input: itr})
default:
itr.Close()
}
}
return a
}
// bufFloatIterator represents a buffered FloatIterator.
type bufFloatIterator struct {
itr FloatIterator
buf *FloatPoint
}
// newBufFloatIterator returns a buffered FloatIterator.
func newBufFloatIterator(itr FloatIterator) *bufFloatIterator {
return &bufFloatIterator{itr: itr}
}
// Stats returns statistics from the input iterator.
func (itr *bufFloatIterator) Stats() IteratorStats { return itr.itr.Stats() }
// Close closes the underlying iterator.
func (itr *bufFloatIterator) Close() error { return itr.itr.Close() }
// peek returns the next point without removing it from the iterator.
func (itr *bufFloatIterator) peek() (*FloatPoint, error) {
p, err := itr.Next()
if err != nil {
return nil, err
}
itr.unread(p)
return p, nil
}
// peekTime returns the time of the next point.
// Returns zero time if no more points available.
func (itr *bufFloatIterator) peekTime() (int64, error) {
p, err := itr.peek()
if p == nil || err != nil {
return ZeroTime, err
}
return p.Time, nil
}
// Next returns the current buffer, if exists, or calls the underlying iterator.
func (itr *bufFloatIterator) Next() (*FloatPoint, error) {
buf := itr.buf
if buf != nil {
itr.buf = nil
return buf, nil
}
return itr.itr.Next()
}
// NextInWindow returns the next value if it is between [startTime, endTime).
// If the next value is outside the range then it is moved to the buffer.
func (itr *bufFloatIterator) NextInWindow(startTime, endTime int64) (*FloatPoint, error) {
v, err := itr.Next()
if v == nil || err != nil {
return nil, err
} else if t := v.Time; t >= endTime || t < startTime {
itr.unread(v)
return nil, nil
}
return v, nil
}
// unread sets v to the buffer. It is read on the next call to Next().
func (itr *bufFloatIterator) unread(v *FloatPoint) { itr.buf = v }
// floatMergeIterator represents an iterator that combines multiple float iterators.
type floatMergeIterator struct {
inputs []FloatIterator
heap *floatMergeHeap
init bool
// Current iterator and window.
curr *floatMergeHeapItem
window struct {
name string
tags string
startTime int64
endTime int64
}
}
// newFloatMergeIterator returns a new instance of floatMergeIterator.
func newFloatMergeIterator(inputs []FloatIterator, opt IteratorOptions) *floatMergeIterator {
itr := &floatMergeIterator{
inputs: inputs,
heap: &floatMergeHeap{
items: make([]*floatMergeHeapItem, 0, len(inputs)),
opt: opt,
},
}
// Initialize heap items.
for _, input := range inputs {
// Wrap in buffer, ignore any inputs without anymore points.
bufInput := newBufFloatIterator(input)
// Append to the heap.
itr.heap.items = append(itr.heap.items, &floatMergeHeapItem{itr: bufInput})
}
return itr
}
// Stats returns an aggregation of stats from the underlying iterators.
func (itr *floatMergeIterator) Stats() IteratorStats {
var stats IteratorStats
for _, input := range itr.inputs {
stats.Add(input.Stats())
}
return stats
}
// Close closes the underlying iterators.
func (itr *floatMergeIterator) Close() error {
for _, input := range itr.inputs {
input.Close()
}
itr.curr = nil
itr.inputs = nil
itr.heap.items = nil
return nil
}
// Next returns the next point from the iterator.
func (itr *floatMergeIterator) Next() (*FloatPoint, error) {
// Initialize the heap. This needs to be done lazily on the first call to this iterator
// so that iterator initialization done through the Select() call returns quickly.
// Queries can only be interrupted after the Select() call completes so any operations
// done during iterator creation cannot be interrupted, which is why we do it here
// instead so an interrupt can happen while initializing the heap.
if !itr.init {
items := itr.heap.items
itr.heap.items = make([]*floatMergeHeapItem, 0, len(items))
for _, item := range items {
if p, err := item.itr.peek(); err != nil {
return nil, err
} else if p == nil {
continue
}
itr.heap.items = append(itr.heap.items, item)
}
heap.Init(itr.heap)
itr.init = true
}
for {
// Retrieve the next iterator if we don't have one.
if itr.curr == nil {
if len(itr.heap.items) == 0 {
return nil, nil
}
itr.curr = heap.Pop(itr.heap).(*floatMergeHeapItem)
// Read point and set current window.
p, err := itr.curr.itr.Next()
if err != nil {
return nil, err
}
tags := p.Tags.Subset(itr.heap.opt.Dimensions)
itr.window.name, itr.window.tags = p.Name, tags.ID()
itr.window.startTime, itr.window.endTime = itr.heap.opt.Window(p.Time)
return p, nil
}
// Read the next point from the current iterator.
p, err := itr.curr.itr.Next()
if err != nil {
return nil, err
}
// If there are no more points then remove iterator from heap and find next.
if p == nil {
itr.curr = nil
continue
}
// Check if the point is inside of our current window.
inWindow := true
if window := itr.window; window.name != p.Name {
inWindow = false
} else if tags := p.Tags.Subset(itr.heap.opt.Dimensions); window.tags != tags.ID() {
inWindow = false
} else if opt := itr.heap.opt; opt.Ascending && p.Time >= window.endTime {
inWindow = false
} else if !opt.Ascending && p.Time < window.startTime {
inWindow = false
}
// If it's outside our window then push iterator back on the heap and find new iterator.
if !inWindow {
itr.curr.itr.unread(p)
heap.Push(itr.heap, itr.curr)
itr.curr = nil
continue
}
return p, nil
}
}
// floatMergeHeap represents a heap of floatMergeHeapItems.
// Items are sorted by their next window and then by name/tags.
type floatMergeHeap struct {
opt IteratorOptions
items []*floatMergeHeapItem
}
func (h *floatMergeHeap) Len() int { return len(h.items) }
func (h *floatMergeHeap) Swap(i, j int) { h.items[i], h.items[j] = h.items[j], h.items[i] }
func (h *floatMergeHeap) Less(i, j int) bool {
x, err := h.items[i].itr.peek()
if err != nil {
return true
}
y, err := h.items[j].itr.peek()
if err != nil {
return false
}
if h.opt.Ascending {
if x.Name != y.Name {
return x.Name < y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); xTags.ID() != yTags.ID() {
return xTags.ID() < yTags.ID()
}
} else {
if x.Name != y.Name {
return x.Name > y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); xTags.ID() != yTags.ID() {
return xTags.ID() > yTags.ID()
}
}
xt, _ := h.opt.Window(x.Time)
yt, _ := h.opt.Window(y.Time)
if h.opt.Ascending {
return xt < yt
}
return xt > yt
}
func (h *floatMergeHeap) Push(x interface{}) {
h.items = append(h.items, x.(*floatMergeHeapItem))
}
func (h *floatMergeHeap) Pop() interface{} {
old := h.items
n := len(old)
item := old[n-1]
h.items = old[0 : n-1]
return item
}
type floatMergeHeapItem struct {
itr *bufFloatIterator
}
// floatSortedMergeIterator is an iterator that sorts and merges multiple iterators into one.
type floatSortedMergeIterator struct {
inputs []FloatIterator
heap *floatSortedMergeHeap
init bool
}
// newFloatSortedMergeIterator returns an instance of floatSortedMergeIterator.
func newFloatSortedMergeIterator(inputs []FloatIterator, opt IteratorOptions) Iterator {
itr := &floatSortedMergeIterator{
inputs: inputs,
heap: &floatSortedMergeHeap{
items: make([]*floatSortedMergeHeapItem, 0, len(inputs)),
opt: opt,
},
}
// Initialize heap items.
for _, input := range inputs {
// Append to the heap.
itr.heap.items = append(itr.heap.items, &floatSortedMergeHeapItem{itr: input})
}
return itr
}
// Stats returns an aggregation of stats from the underlying iterators.
func (itr *floatSortedMergeIterator) Stats() IteratorStats {
var stats IteratorStats
for _, input := range itr.inputs {
stats.Add(input.Stats())
}
return stats
}
// Close closes the underlying iterators.
func (itr *floatSortedMergeIterator) Close() error {
for _, input := range itr.inputs {
input.Close()
}
return nil
}
// Next returns the next points from the iterator.
func (itr *floatSortedMergeIterator) Next() (*FloatPoint, error) { return itr.pop() }
// pop returns the next point from the heap.
// Reads the next point from item's cursor and puts it back on the heap.
func (itr *floatSortedMergeIterator) pop() (*FloatPoint, error) {
// Initialize the heap. See the MergeIterator to see why this has to be done lazily.
if !itr.init {
items := itr.heap.items
itr.heap.items = make([]*floatSortedMergeHeapItem, 0, len(items))
for _, item := range items {
var err error
if item.point, err = item.itr.Next(); err != nil {
return nil, err
} else if item.point == nil {
continue
}
itr.heap.items = append(itr.heap.items, item)
}
heap.Init(itr.heap)
itr.init = true
}
if len(itr.heap.items) == 0 {
return nil, nil
}
// Read the next item from the heap.
item := heap.Pop(itr.heap).(*floatSortedMergeHeapItem)
if item.err != nil {
return nil, item.err
} else if item.point == nil {
return nil, nil
}
// Copy the point for return.
p := item.point.Clone()
// Read the next item from the cursor. Push back to heap if one exists.
if item.point, item.err = item.itr.Next(); item.point != nil {
heap.Push(itr.heap, item)
}
return p, nil
}
// floatSortedMergeHeap represents a heap of floatSortedMergeHeapItems.
type floatSortedMergeHeap struct {
opt IteratorOptions
items []*floatSortedMergeHeapItem
}
func (h *floatSortedMergeHeap) Len() int { return len(h.items) }
func (h *floatSortedMergeHeap) Swap(i, j int) { h.items[i], h.items[j] = h.items[j], h.items[i] }
func (h *floatSortedMergeHeap) Less(i, j int) bool {
x, y := h.items[i].point, h.items[j].point
if h.opt.Ascending {
if x.Name != y.Name {
return x.Name < y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); !xTags.Equals(&yTags) {
return xTags.ID() < yTags.ID()
}
return x.Time < y.Time
}
if x.Name != y.Name {
return x.Name > y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); !xTags.Equals(&yTags) {
return xTags.ID() > yTags.ID()
}
return x.Time > y.Time
}
func (h *floatSortedMergeHeap) Push(x interface{}) {
h.items = append(h.items, x.(*floatSortedMergeHeapItem))
}
func (h *floatSortedMergeHeap) Pop() interface{} {
old := h.items
n := len(old)
item := old[n-1]
h.items = old[0 : n-1]
return item
}
type floatSortedMergeHeapItem struct {
point *FloatPoint
err error
itr FloatIterator
}
// floatParallelIterator represents an iterator that pulls data in a separate goroutine.
type floatParallelIterator struct {
input FloatIterator
ch chan floatPointError
once sync.Once
closing chan struct{}
wg sync.WaitGroup
}
// newFloatParallelIterator returns a new instance of floatParallelIterator.
func newFloatParallelIterator(input FloatIterator) *floatParallelIterator {
itr := &floatParallelIterator{
input: input,
ch: make(chan floatPointError, 256),
closing: make(chan struct{}),
}
itr.wg.Add(1)
go itr.monitor()
return itr
}
// Stats returns stats from the underlying iterator.
func (itr *floatParallelIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the underlying iterators.
func (itr *floatParallelIterator) Close() error {
itr.once.Do(func() { close(itr.closing) })
itr.wg.Wait()
return itr.input.Close()
}
// Next returns the next point from the iterator.
func (itr *floatParallelIterator) Next() (*FloatPoint, error) {
v, ok := <-itr.ch
if !ok {
return nil, io.EOF
}
return v.point, v.err
}
// monitor runs in a separate goroutine and actively pulls the next point.
func (itr *floatParallelIterator) monitor() {
defer close(itr.ch)
defer itr.wg.Done()
for {
// Read next point.
p, err := itr.input.Next()
if p != nil {
p = p.Clone()
}
select {
case <-itr.closing:
return
case itr.ch <- floatPointError{point: p, err: err}:
}
}
}
type floatPointError struct {
point *FloatPoint
err error
}
// floatLimitIterator represents an iterator that limits points per group.
type floatLimitIterator struct {
input FloatIterator
opt IteratorOptions
n int
prev struct {
name string
tags Tags
}
}
// newFloatLimitIterator returns a new instance of floatLimitIterator.
func newFloatLimitIterator(input FloatIterator, opt IteratorOptions) *floatLimitIterator {
return &floatLimitIterator{
input: input,
opt: opt,
}
}
// Stats returns stats from the underlying iterator.
func (itr *floatLimitIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the underlying iterators.
func (itr *floatLimitIterator) Close() error { return itr.input.Close() }
// Next returns the next point from the iterator.
func (itr *floatLimitIterator) Next() (*FloatPoint, error) {
for {
p, err := itr.input.Next()
if p == nil || err != nil {
return nil, err
}
// Reset window and counter if a new window is encountered.
if p.Name != itr.prev.name || !p.Tags.Equals(&itr.prev.tags) {
itr.prev.name = p.Name
itr.prev.tags = p.Tags
itr.n = 0
}
// Increment counter.
itr.n++
// Read next point if not beyond the offset.
if itr.n <= itr.opt.Offset {
continue
}
// Read next point if we're beyond the limit.
if itr.opt.Limit > 0 && (itr.n-itr.opt.Offset) > itr.opt.Limit {
continue
}
return p, nil
}
}
type floatFillIterator struct {
input *bufFloatIterator
prev FloatPoint
startTime int64
endTime int64
auxFields []interface{}
init bool
opt IteratorOptions
window struct {
name string
tags Tags
time int64
offset int64
}
}
func newFloatFillIterator(input FloatIterator, expr Expr, opt IteratorOptions) *floatFillIterator {
if opt.Fill == NullFill {
if expr, ok := expr.(*Call); ok && expr.Name == "count" {
opt.Fill = NumberFill
opt.FillValue = float64(0)
}
}
var startTime, endTime int64
if opt.Ascending {
startTime, _ = opt.Window(opt.StartTime)
endTime, _ = opt.Window(opt.EndTime)
} else {
startTime, _ = opt.Window(opt.EndTime)
endTime, _ = opt.Window(opt.StartTime)
}
var auxFields []interface{}
if len(opt.Aux) > 0 {
auxFields = make([]interface{}, len(opt.Aux))
}
return &floatFillIterator{
input: newBufFloatIterator(input),
prev: FloatPoint{Nil: true},
startTime: startTime,
endTime: endTime,
auxFields: auxFields,
opt: opt,
}
}
func (itr *floatFillIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *floatFillIterator) Close() error { return itr.input.Close() }
func (itr *floatFillIterator) Next() (*FloatPoint, error) {
if !itr.init {
p, err := itr.input.peek()
if p == nil || err != nil {
return nil, err
}
itr.window.name, itr.window.tags = p.Name, p.Tags
itr.window.time = itr.startTime
if itr.opt.Location != nil {
_, itr.window.offset = itr.opt.Zone(itr.window.time)
}
itr.init = true
}
p, err := itr.input.Next()
if err != nil {
return nil, err
}
// Check if the next point is outside of our window or is nil.
for p == nil || p.Name != itr.window.name || p.Tags.ID() != itr.window.tags.ID() {
// If we are inside of an interval, unread the point and continue below to
// constructing a new point.
if itr.opt.Ascending {
if itr.window.time <= itr.endTime {
itr.input.unread(p)
p = nil
break
}
} else {
if itr.window.time >= itr.endTime {
itr.input.unread(p)
p = nil
break
}
}
// We are *not* in a current interval. If there is no next point,
// we are at the end of all intervals.
if p == nil {
return nil, nil
}
// Set the new interval.
itr.window.name, itr.window.tags = p.Name, p.Tags
itr.window.time = itr.startTime
if itr.opt.Location != nil {
_, itr.window.offset = itr.opt.Zone(itr.window.time)
}
itr.prev = FloatPoint{Nil: true}
break
}
// Check if the point is our next expected point.
if p == nil || (itr.opt.Ascending && p.Time > itr.window.time) || (!itr.opt.Ascending && p.Time < itr.window.time) {
if p != nil {
itr.input.unread(p)
}
p = &FloatPoint{
Name: itr.window.name,
Tags: itr.window.tags,
Time: itr.window.time,
Aux: itr.auxFields,
}
switch itr.opt.Fill {
case LinearFill:
if !itr.prev.Nil {
next, err := itr.input.peek()
if err != nil {
return nil, err
} else if next != nil && next.Name == itr.window.name && next.Tags.ID() == itr.window.tags.ID() {
interval := int64(itr.opt.Interval.Duration)
start := itr.window.time / interval
p.Value = linearFloat(start, itr.prev.Time/interval, next.Time/interval, itr.prev.Value, next.Value)
} else {
p.Nil = true
}
} else {
p.Nil = true
}
case NullFill:
p.Nil = true
case NumberFill:
p.Value = castToFloat(itr.opt.FillValue)
case PreviousFill:
if !itr.prev.Nil {
p.Value = itr.prev.Value
p.Nil = itr.prev.Nil
} else {
p.Nil = true
}
}
} else {
itr.prev = *p
}
// Advance the expected time. Do not advance to a new window here
// as there may be lingering points with the same timestamp in the previous
// window.
if itr.opt.Ascending {
itr.window.time += int64(itr.opt.Interval.Duration)
} else {
itr.window.time -= int64(itr.opt.Interval.Duration)
}
// Check to see if we have passed over an offset change and adjust the time
// to account for this new offset.
if itr.opt.Location != nil {
if _, offset := itr.opt.Zone(itr.window.time - 1); offset != itr.window.offset {
diff := itr.window.offset - offset
if abs(diff) < int64(itr.opt.Interval.Duration) {
itr.window.time += diff
}
itr.window.offset = offset
}
}
return p, nil
}
// floatIntervalIterator represents a float implementation of IntervalIterator.
type floatIntervalIterator struct {
input FloatIterator
opt IteratorOptions
}
func newFloatIntervalIterator(input FloatIterator, opt IteratorOptions) *floatIntervalIterator {
return &floatIntervalIterator{input: input, opt: opt}
}
func (itr *floatIntervalIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *floatIntervalIterator) Close() error { return itr.input.Close() }
func (itr *floatIntervalIterator) Next() (*FloatPoint, error) {
p, err := itr.input.Next()
if p == nil || err != nil {
return nil, err
}
p.Time, _ = itr.opt.Window(p.Time)
// If we see the minimum allowable time, set the time to zero so we don't
// break the default returned time for aggregate queries without times.
if p.Time == MinTime {
p.Time = 0
}
return p, nil
}
// floatInterruptIterator represents a float implementation of InterruptIterator.
type floatInterruptIterator struct {
input FloatIterator
closing <-chan struct{}
count int
}
func newFloatInterruptIterator(input FloatIterator, closing <-chan struct{}) *floatInterruptIterator {
return &floatInterruptIterator{input: input, closing: closing}
}
func (itr *floatInterruptIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *floatInterruptIterator) Close() error { return itr.input.Close() }
func (itr *floatInterruptIterator) Next() (*FloatPoint, error) {
// Only check if the channel is closed every N points. This
// intentionally checks on both 0 and N so that if the iterator
// has been interrupted before the first point is emitted it will
// not emit any points.
if itr.count&0xFF == 0xFF {
select {
case <-itr.closing:
return nil, itr.Close()
default:
// Reset iterator count to zero and fall through to emit the next point.
itr.count = 0
}
}
// Increment the counter for every point read.
itr.count++
return itr.input.Next()
}
// floatCloseInterruptIterator represents a float implementation of CloseInterruptIterator.
type floatCloseInterruptIterator struct {
input FloatIterator
closing <-chan struct{}
done chan struct{}
once sync.Once
}
func newFloatCloseInterruptIterator(input FloatIterator, closing <-chan struct{}) *floatCloseInterruptIterator {
itr := &floatCloseInterruptIterator{
input: input,
closing: closing,
done: make(chan struct{}),
}
go itr.monitor()
return itr
}
func (itr *floatCloseInterruptIterator) monitor() {
select {
case <-itr.closing:
itr.Close()
case <-itr.done:
}
}
func (itr *floatCloseInterruptIterator) Stats() IteratorStats {
return itr.input.Stats()
}
func (itr *floatCloseInterruptIterator) Close() error {
itr.once.Do(func() {
close(itr.done)
itr.input.Close()
})
return nil
}
func (itr *floatCloseInterruptIterator) Next() (*FloatPoint, error) {
p, err := itr.input.Next()
if err != nil {
// Check if the iterator was closed.
select {
case <-itr.done:
return nil, nil
default:
return nil, err
}
}
return p, nil
}
// auxFloatPoint represents a combination of a point and an error for the AuxIterator.
type auxFloatPoint struct {
point *FloatPoint
err error
}
// floatAuxIterator represents a float implementation of AuxIterator.
type floatAuxIterator struct {
input *bufFloatIterator
output chan auxFloatPoint
fields *auxIteratorFields
background bool
}
func newFloatAuxIterator(input FloatIterator, opt IteratorOptions) *floatAuxIterator {
return &floatAuxIterator{
input: newBufFloatIterator(input),
output: make(chan auxFloatPoint, 1),
fields: newAuxIteratorFields(opt),
}
}
func (itr *floatAuxIterator) Background() {
itr.background = true
itr.Start()
go DrainIterator(itr)
}
func (itr *floatAuxIterator) Start() { go itr.stream() }
func (itr *floatAuxIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *floatAuxIterator) Close() error { return itr.input.Close() }
func (itr *floatAuxIterator) Next() (*FloatPoint, error) {
p := <-itr.output
return p.point, p.err
}
func (itr *floatAuxIterator) Iterator(name string, typ DataType) Iterator {
return itr.fields.iterator(name, typ)
}
func (itr *floatAuxIterator) stream() {
for {
// Read next point.
p, err := itr.input.Next()
if err != nil {
itr.output <- auxFloatPoint{err: err}
itr.fields.sendError(err)
break
} else if p == nil {
break
}
// Send point to output and to each field iterator.
itr.output <- auxFloatPoint{point: p}
if ok := itr.fields.send(p); !ok && itr.background {
break
}
}
close(itr.output)
itr.fields.close()
}
// floatChanIterator represents a new instance of floatChanIterator.
type floatChanIterator struct {
buf struct {
i int
filled bool
points [2]FloatPoint
}
err error
cond *sync.Cond
done bool
}
func (itr *floatChanIterator) Stats() IteratorStats { return IteratorStats{} }
func (itr *floatChanIterator) Close() error {
itr.cond.L.Lock()
// Mark the channel iterator as done and signal all waiting goroutines to start again.
itr.done = true
itr.cond.Broadcast()
// Do not defer the unlock so we don't create an unnecessary allocation.
itr.cond.L.Unlock()
return nil
}
func (itr *floatChanIterator) setBuf(name string, tags Tags, time int64, value interface{}) bool {
itr.cond.L.Lock()
defer itr.cond.L.Unlock()
// Wait for either the iterator to be done (so we don't have to set the value)
// or for the buffer to have been read and ready for another write.
for !itr.done && itr.buf.filled {
itr.cond.Wait()
}
// Do not set the value and return false to signal that the iterator is closed.
// Do this after the above wait as the above for loop may have exited because
// the iterator was closed.
if itr.done {
return false
}
switch v := value.(type) {
case float64:
itr.buf.points[itr.buf.i] = FloatPoint{Name: name, Tags: tags, Time: time, Value: v}
case int64:
itr.buf.points[itr.buf.i] = FloatPoint{Name: name, Tags: tags, Time: time, Value: float64(v)}
default:
itr.buf.points[itr.buf.i] = FloatPoint{Name: name, Tags: tags, Time: time, Nil: true}
}
itr.buf.filled = true
// Signal to all waiting goroutines that a new value is ready to read.
itr.cond.Signal()
return true
}
func (itr *floatChanIterator) setErr(err error) {
itr.cond.L.Lock()
defer itr.cond.L.Unlock()
itr.err = err
// Signal to all waiting goroutines that a new value is ready to read.
itr.cond.Signal()
}
func (itr *floatChanIterator) Next() (*FloatPoint, error) {
itr.cond.L.Lock()
defer itr.cond.L.Unlock()
// Check for an error and return one if there.
if itr.err != nil {
return nil, itr.err
}
// Wait until either a value is available in the buffer or
// the iterator is closed.
for !itr.done && !itr.buf.filled {
itr.cond.Wait()
}
// Return nil once the channel is done and the buffer is empty.
if itr.done && !itr.buf.filled {
return nil, nil
}
// Always read from the buffer if it exists, even if the iterator
// is closed. This prevents the last value from being truncated by
// the parent iterator.
p := &itr.buf.points[itr.buf.i]
itr.buf.i = (itr.buf.i + 1) % len(itr.buf.points)
itr.buf.filled = false
itr.cond.Signal()
return p, nil
}
// floatReduceFloatIterator executes a reducer for every interval and buffers the result.
type floatReduceFloatIterator struct {
input *bufFloatIterator
create func() (FloatPointAggregator, FloatPointEmitter)
dims []string
opt IteratorOptions
points []FloatPoint
keepTags bool
}
func newFloatReduceFloatIterator(input FloatIterator, opt IteratorOptions, createFn func() (FloatPointAggregator, FloatPointEmitter)) *floatReduceFloatIterator {
return &floatReduceFloatIterator{
input: newBufFloatIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *floatReduceFloatIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *floatReduceFloatIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *floatReduceFloatIterator) Next() (*FloatPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// floatReduceFloatPoint stores the reduced data for a name/tag combination.
type floatReduceFloatPoint struct {
Name string
Tags Tags
Aggregator FloatPointAggregator
Emitter FloatPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *floatReduceFloatIterator) reduce() ([]FloatPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*floatReduceFloatPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &floatReduceFloatPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateFloat(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]FloatPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(floatPointsByTime(a)))
}
return a, nil
}
// floatStreamFloatIterator streams inputs into the iterator and emits points gradually.
type floatStreamFloatIterator struct {
input *bufFloatIterator
create func() (FloatPointAggregator, FloatPointEmitter)
dims []string
opt IteratorOptions
m map[string]*floatReduceFloatPoint
points []FloatPoint
}
// newFloatStreamFloatIterator returns a new instance of floatStreamFloatIterator.
func newFloatStreamFloatIterator(input FloatIterator, createFn func() (FloatPointAggregator, FloatPointEmitter), opt IteratorOptions) *floatStreamFloatIterator {
return &floatStreamFloatIterator{
input: newBufFloatIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*floatReduceFloatPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *floatStreamFloatIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *floatStreamFloatIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *floatStreamFloatIterator) Next() (*FloatPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *floatStreamFloatIterator) reduce() ([]FloatPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []FloatPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &floatReduceFloatPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateFloat(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// floatExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type floatExprIterator struct {
left *bufFloatIterator
right *bufFloatIterator
fn floatExprFunc
points []FloatPoint // must be size 2
storePrev bool
}
func newFloatExprIterator(left, right FloatIterator, opt IteratorOptions, fn func(a, b float64) float64) *floatExprIterator {
var points []FloatPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []FloatPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToFloat(opt.FillValue)
points = []FloatPoint{{Value: value}, {Value: value}}
}
return &floatExprIterator{
left: newBufFloatIterator(left),
right: newBufFloatIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *floatExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *floatExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *floatExprIterator) Next() (*FloatPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = 0
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = 0
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
if a.Nil {
return a, nil
} else if b.Nil {
return b, nil
}
a.Value = itr.fn(a.Value, b.Value)
return a, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *floatExprIterator) next() (a, b *FloatPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// floatExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type floatExprFunc func(a, b float64) float64
// floatReduceIntegerIterator executes a reducer for every interval and buffers the result.
type floatReduceIntegerIterator struct {
input *bufFloatIterator
create func() (FloatPointAggregator, IntegerPointEmitter)
dims []string
opt IteratorOptions
points []IntegerPoint
keepTags bool
}
func newFloatReduceIntegerIterator(input FloatIterator, opt IteratorOptions, createFn func() (FloatPointAggregator, IntegerPointEmitter)) *floatReduceIntegerIterator {
return &floatReduceIntegerIterator{
input: newBufFloatIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *floatReduceIntegerIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *floatReduceIntegerIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *floatReduceIntegerIterator) Next() (*IntegerPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// floatReduceIntegerPoint stores the reduced data for a name/tag combination.
type floatReduceIntegerPoint struct {
Name string
Tags Tags
Aggregator FloatPointAggregator
Emitter IntegerPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *floatReduceIntegerIterator) reduce() ([]IntegerPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*floatReduceIntegerPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &floatReduceIntegerPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateFloat(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]IntegerPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(integerPointsByTime(a)))
}
return a, nil
}
// floatStreamIntegerIterator streams inputs into the iterator and emits points gradually.
type floatStreamIntegerIterator struct {
input *bufFloatIterator
create func() (FloatPointAggregator, IntegerPointEmitter)
dims []string
opt IteratorOptions
m map[string]*floatReduceIntegerPoint
points []IntegerPoint
}
// newFloatStreamIntegerIterator returns a new instance of floatStreamIntegerIterator.
func newFloatStreamIntegerIterator(input FloatIterator, createFn func() (FloatPointAggregator, IntegerPointEmitter), opt IteratorOptions) *floatStreamIntegerIterator {
return &floatStreamIntegerIterator{
input: newBufFloatIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*floatReduceIntegerPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *floatStreamIntegerIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *floatStreamIntegerIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *floatStreamIntegerIterator) Next() (*IntegerPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *floatStreamIntegerIterator) reduce() ([]IntegerPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []IntegerPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &floatReduceIntegerPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateFloat(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// floatIntegerExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type floatIntegerExprIterator struct {
left *bufFloatIterator
right *bufFloatIterator
fn floatIntegerExprFunc
points []FloatPoint // must be size 2
storePrev bool
}
func newFloatIntegerExprIterator(left, right FloatIterator, opt IteratorOptions, fn func(a, b float64) int64) *floatIntegerExprIterator {
var points []FloatPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []FloatPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToFloat(opt.FillValue)
points = []FloatPoint{{Value: value}, {Value: value}}
}
return &floatIntegerExprIterator{
left: newBufFloatIterator(left),
right: newBufFloatIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *floatIntegerExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *floatIntegerExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *floatIntegerExprIterator) Next() (*IntegerPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = 0
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = 0
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
p := &IntegerPoint{
Name: a.Name,
Tags: a.Tags,
Time: a.Time,
Nil: a.Nil || b.Nil,
Aggregated: a.Aggregated,
}
if !p.Nil {
p.Value = itr.fn(a.Value, b.Value)
}
return p, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *floatIntegerExprIterator) next() (a, b *FloatPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// floatIntegerExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type floatIntegerExprFunc func(a, b float64) int64
// floatReduceStringIterator executes a reducer for every interval and buffers the result.
type floatReduceStringIterator struct {
input *bufFloatIterator
create func() (FloatPointAggregator, StringPointEmitter)
dims []string
opt IteratorOptions
points []StringPoint
keepTags bool
}
func newFloatReduceStringIterator(input FloatIterator, opt IteratorOptions, createFn func() (FloatPointAggregator, StringPointEmitter)) *floatReduceStringIterator {
return &floatReduceStringIterator{
input: newBufFloatIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *floatReduceStringIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *floatReduceStringIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *floatReduceStringIterator) Next() (*StringPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// floatReduceStringPoint stores the reduced data for a name/tag combination.
type floatReduceStringPoint struct {
Name string
Tags Tags
Aggregator FloatPointAggregator
Emitter StringPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *floatReduceStringIterator) reduce() ([]StringPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*floatReduceStringPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &floatReduceStringPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateFloat(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]StringPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(stringPointsByTime(a)))
}
return a, nil
}
// floatStreamStringIterator streams inputs into the iterator and emits points gradually.
type floatStreamStringIterator struct {
input *bufFloatIterator
create func() (FloatPointAggregator, StringPointEmitter)
dims []string
opt IteratorOptions
m map[string]*floatReduceStringPoint
points []StringPoint
}
// newFloatStreamStringIterator returns a new instance of floatStreamStringIterator.
func newFloatStreamStringIterator(input FloatIterator, createFn func() (FloatPointAggregator, StringPointEmitter), opt IteratorOptions) *floatStreamStringIterator {
return &floatStreamStringIterator{
input: newBufFloatIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*floatReduceStringPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *floatStreamStringIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *floatStreamStringIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *floatStreamStringIterator) Next() (*StringPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *floatStreamStringIterator) reduce() ([]StringPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []StringPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &floatReduceStringPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateFloat(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// floatStringExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type floatStringExprIterator struct {
left *bufFloatIterator
right *bufFloatIterator
fn floatStringExprFunc
points []FloatPoint // must be size 2
storePrev bool
}
func newFloatStringExprIterator(left, right FloatIterator, opt IteratorOptions, fn func(a, b float64) string) *floatStringExprIterator {
var points []FloatPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []FloatPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToFloat(opt.FillValue)
points = []FloatPoint{{Value: value}, {Value: value}}
}
return &floatStringExprIterator{
left: newBufFloatIterator(left),
right: newBufFloatIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *floatStringExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *floatStringExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *floatStringExprIterator) Next() (*StringPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = 0
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = 0
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
p := &StringPoint{
Name: a.Name,
Tags: a.Tags,
Time: a.Time,
Nil: a.Nil || b.Nil,
Aggregated: a.Aggregated,
}
if !p.Nil {
p.Value = itr.fn(a.Value, b.Value)
}
return p, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *floatStringExprIterator) next() (a, b *FloatPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// floatStringExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type floatStringExprFunc func(a, b float64) string
// floatReduceBooleanIterator executes a reducer for every interval and buffers the result.
type floatReduceBooleanIterator struct {
input *bufFloatIterator
create func() (FloatPointAggregator, BooleanPointEmitter)
dims []string
opt IteratorOptions
points []BooleanPoint
keepTags bool
}
func newFloatReduceBooleanIterator(input FloatIterator, opt IteratorOptions, createFn func() (FloatPointAggregator, BooleanPointEmitter)) *floatReduceBooleanIterator {
return &floatReduceBooleanIterator{
input: newBufFloatIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *floatReduceBooleanIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *floatReduceBooleanIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *floatReduceBooleanIterator) Next() (*BooleanPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// floatReduceBooleanPoint stores the reduced data for a name/tag combination.
type floatReduceBooleanPoint struct {
Name string
Tags Tags
Aggregator FloatPointAggregator
Emitter BooleanPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *floatReduceBooleanIterator) reduce() ([]BooleanPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*floatReduceBooleanPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &floatReduceBooleanPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateFloat(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]BooleanPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(booleanPointsByTime(a)))
}
return a, nil
}
// floatStreamBooleanIterator streams inputs into the iterator and emits points gradually.
type floatStreamBooleanIterator struct {
input *bufFloatIterator
create func() (FloatPointAggregator, BooleanPointEmitter)
dims []string
opt IteratorOptions
m map[string]*floatReduceBooleanPoint
points []BooleanPoint
}
// newFloatStreamBooleanIterator returns a new instance of floatStreamBooleanIterator.
func newFloatStreamBooleanIterator(input FloatIterator, createFn func() (FloatPointAggregator, BooleanPointEmitter), opt IteratorOptions) *floatStreamBooleanIterator {
return &floatStreamBooleanIterator{
input: newBufFloatIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*floatReduceBooleanPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *floatStreamBooleanIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *floatStreamBooleanIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *floatStreamBooleanIterator) Next() (*BooleanPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *floatStreamBooleanIterator) reduce() ([]BooleanPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []BooleanPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &floatReduceBooleanPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateFloat(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// floatBooleanExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type floatBooleanExprIterator struct {
left *bufFloatIterator
right *bufFloatIterator
fn floatBooleanExprFunc
points []FloatPoint // must be size 2
storePrev bool
}
func newFloatBooleanExprIterator(left, right FloatIterator, opt IteratorOptions, fn func(a, b float64) bool) *floatBooleanExprIterator {
var points []FloatPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []FloatPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToFloat(opt.FillValue)
points = []FloatPoint{{Value: value}, {Value: value}}
}
return &floatBooleanExprIterator{
left: newBufFloatIterator(left),
right: newBufFloatIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *floatBooleanExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *floatBooleanExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *floatBooleanExprIterator) Next() (*BooleanPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = 0
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = 0
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
p := &BooleanPoint{
Name: a.Name,
Tags: a.Tags,
Time: a.Time,
Nil: a.Nil || b.Nil,
Aggregated: a.Aggregated,
}
if !p.Nil {
p.Value = itr.fn(a.Value, b.Value)
}
return p, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *floatBooleanExprIterator) next() (a, b *FloatPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// floatBooleanExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type floatBooleanExprFunc func(a, b float64) bool
// floatTransformIterator executes a function to modify an existing point for every
// output of the input iterator.
type floatTransformIterator struct {
input FloatIterator
fn floatTransformFunc
}
// Stats returns stats from the input iterator.
func (itr *floatTransformIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *floatTransformIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *floatTransformIterator) Next() (*FloatPoint, error) {
p, err := itr.input.Next()
if err != nil {
return nil, err
} else if p != nil {
p = itr.fn(p)
}
return p, nil
}
// floatTransformFunc creates or modifies a point.
// The point passed in may be modified and returned rather than allocating a
// new point if possible.
type floatTransformFunc func(p *FloatPoint) *FloatPoint
// floatBoolTransformIterator executes a function to modify an existing point for every
// output of the input iterator.
type floatBoolTransformIterator struct {
input FloatIterator
fn floatBoolTransformFunc
}
// Stats returns stats from the input iterator.
func (itr *floatBoolTransformIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *floatBoolTransformIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *floatBoolTransformIterator) Next() (*BooleanPoint, error) {
p, err := itr.input.Next()
if err != nil {
return nil, err
} else if p != nil {
return itr.fn(p), nil
}
return nil, nil
}
// floatBoolTransformFunc creates or modifies a point.
// The point passed in may be modified and returned rather than allocating a
// new point if possible.
type floatBoolTransformFunc func(p *FloatPoint) *BooleanPoint
// floatDedupeIterator only outputs unique points.
// This differs from the DistinctIterator in that it compares all aux fields too.
// This iterator is relatively inefficient and should only be used on small
// datasets such as meta query results.
type floatDedupeIterator struct {
input FloatIterator
m map[string]struct{} // lookup of points already sent
}
type floatIteratorMapper struct {
e *Emitter
buf []interface{}
driver IteratorMap // which iterator to use for the primary value, can be nil
fields []IteratorMap // which iterator to use for an aux field
point FloatPoint
}
func newFloatIteratorMapper(itrs []Iterator, driver IteratorMap, fields []IteratorMap, opt IteratorOptions) *floatIteratorMapper {
e := NewEmitter(itrs, opt.Ascending, 0)
e.OmitTime = true
return &floatIteratorMapper{
e: e,
buf: make([]interface{}, len(itrs)),
driver: driver,
fields: fields,
point: FloatPoint{
Aux: make([]interface{}, len(fields)),
},
}
}
func (itr *floatIteratorMapper) Next() (*FloatPoint, error) {
t, name, tags, err := itr.e.loadBuf()
if err != nil || t == ZeroTime {
return nil, err
}
itr.point.Time = t
itr.point.Name = name
itr.point.Tags = tags
itr.e.readInto(t, name, tags, itr.buf)
if itr.driver != nil {
if v := itr.driver.Value(tags, itr.buf); v != nil {
if v, ok := v.(float64); ok {
itr.point.Value = v
itr.point.Nil = false
} else {
itr.point.Value = 0
itr.point.Nil = true
}
} else {
itr.point.Value = 0
itr.point.Nil = true
}
}
for i, f := range itr.fields {
itr.point.Aux[i] = f.Value(tags, itr.buf)
}
return &itr.point, nil
}
func (itr *floatIteratorMapper) Stats() IteratorStats {
stats := IteratorStats{}
for _, itr := range itr.e.itrs {
stats.Add(itr.Stats())
}
return stats
}
func (itr *floatIteratorMapper) Close() error {
return itr.e.Close()
}
type floatFilterIterator struct {
input FloatIterator
cond Expr
opt IteratorOptions
m map[string]interface{}
}
func newFloatFilterIterator(input FloatIterator, cond Expr, opt IteratorOptions) FloatIterator {
// Strip out time conditions from the WHERE clause.
// TODO(jsternberg): This should really be done for us when creating the IteratorOptions struct.
n := RewriteFunc(CloneExpr(cond), func(n Node) Node {
switch n := n.(type) {
case *BinaryExpr:
if n.LHS.String() == "time" {
return &BooleanLiteral{Val: true}
}
}
return n
})
cond, _ = n.(Expr)
if cond == nil {
return input
} else if n, ok := cond.(*BooleanLiteral); ok && n.Val {
return input
}
return &floatFilterIterator{
input: input,
cond: cond,
opt: opt,
m: make(map[string]interface{}),
}
}
func (itr *floatFilterIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *floatFilterIterator) Close() error { return itr.input.Close() }
func (itr *floatFilterIterator) Next() (*FloatPoint, error) {
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
}
for i, ref := range itr.opt.Aux {
itr.m[ref.Val] = p.Aux[i]
}
for k, v := range p.Tags.KeyValues() {
itr.m[k] = v
}
if !EvalBool(itr.cond, itr.m) {
continue
}
return p, nil
}
}
// newFloatDedupeIterator returns a new instance of floatDedupeIterator.
func newFloatDedupeIterator(input FloatIterator) *floatDedupeIterator {
return &floatDedupeIterator{
input: input,
m: make(map[string]struct{}),
}
}
// Stats returns stats from the input iterator.
func (itr *floatDedupeIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *floatDedupeIterator) Close() error { return itr.input.Close() }
// Next returns the next unique point from the input iterator.
func (itr *floatDedupeIterator) Next() (*FloatPoint, error) {
for {
// Read next point.
p, err := itr.input.Next()
if p == nil || err != nil {
return nil, err
}
// Serialize to bytes to store in lookup.
buf, err := proto.Marshal(encodeFloatPoint(p))
if err != nil {
return nil, err
}
// If the point has already been output then move to the next point.
if _, ok := itr.m[string(buf)]; ok {
continue
}
// Otherwise mark it as emitted and return point.
itr.m[string(buf)] = struct{}{}
return p, nil
}
}
// floatReaderIterator represents an iterator that streams from a reader.
type floatReaderIterator struct {
r io.Reader
dec *FloatPointDecoder
}
// newFloatReaderIterator returns a new instance of floatReaderIterator.
func newFloatReaderIterator(r io.Reader, stats IteratorStats) *floatReaderIterator {
dec := NewFloatPointDecoder(r)
dec.stats = stats
return &floatReaderIterator{
r: r,
dec: dec,
}
}
// Stats returns stats about points processed.
func (itr *floatReaderIterator) Stats() IteratorStats { return itr.dec.stats }
// Close closes the underlying reader, if applicable.
func (itr *floatReaderIterator) Close() error {
if r, ok := itr.r.(io.ReadCloser); ok {
return r.Close()
}
return nil
}
// Next returns the next point from the iterator.
func (itr *floatReaderIterator) Next() (*FloatPoint, error) {
// OPTIMIZE(benbjohnson): Reuse point on iterator.
// Unmarshal next point.
p := &FloatPoint{}
if err := itr.dec.DecodeFloatPoint(p); err == io.EOF {
return nil, nil
} else if err != nil {
return nil, err
}
return p, nil
}
// IntegerIterator represents a stream of integer points.
type IntegerIterator interface {
Iterator
Next() (*IntegerPoint, error)
}
// newIntegerIterators converts a slice of Iterator to a slice of IntegerIterator.
// Drop and closes any iterator in itrs that is not a IntegerIterator and cannot
// be cast to a IntegerIterator.
func newIntegerIterators(itrs []Iterator) []IntegerIterator {
a := make([]IntegerIterator, 0, len(itrs))
for _, itr := range itrs {
switch itr := itr.(type) {
case IntegerIterator:
a = append(a, itr)
default:
itr.Close()
}
}
return a
}
// bufIntegerIterator represents a buffered IntegerIterator.
type bufIntegerIterator struct {
itr IntegerIterator
buf *IntegerPoint
}
// newBufIntegerIterator returns a buffered IntegerIterator.
func newBufIntegerIterator(itr IntegerIterator) *bufIntegerIterator {
return &bufIntegerIterator{itr: itr}
}
// Stats returns statistics from the input iterator.
func (itr *bufIntegerIterator) Stats() IteratorStats { return itr.itr.Stats() }
// Close closes the underlying iterator.
func (itr *bufIntegerIterator) Close() error { return itr.itr.Close() }
// peek returns the next point without removing it from the iterator.
func (itr *bufIntegerIterator) peek() (*IntegerPoint, error) {
p, err := itr.Next()
if err != nil {
return nil, err
}
itr.unread(p)
return p, nil
}
// peekTime returns the time of the next point.
// Returns zero time if no more points available.
func (itr *bufIntegerIterator) peekTime() (int64, error) {
p, err := itr.peek()
if p == nil || err != nil {
return ZeroTime, err
}
return p.Time, nil
}
// Next returns the current buffer, if exists, or calls the underlying iterator.
func (itr *bufIntegerIterator) Next() (*IntegerPoint, error) {
buf := itr.buf
if buf != nil {
itr.buf = nil
return buf, nil
}
return itr.itr.Next()
}
// NextInWindow returns the next value if it is between [startTime, endTime).
// If the next value is outside the range then it is moved to the buffer.
func (itr *bufIntegerIterator) NextInWindow(startTime, endTime int64) (*IntegerPoint, error) {
v, err := itr.Next()
if v == nil || err != nil {
return nil, err
} else if t := v.Time; t >= endTime || t < startTime {
itr.unread(v)
return nil, nil
}
return v, nil
}
// unread sets v to the buffer. It is read on the next call to Next().
func (itr *bufIntegerIterator) unread(v *IntegerPoint) { itr.buf = v }
// integerMergeIterator represents an iterator that combines multiple integer iterators.
type integerMergeIterator struct {
inputs []IntegerIterator
heap *integerMergeHeap
init bool
// Current iterator and window.
curr *integerMergeHeapItem
window struct {
name string
tags string
startTime int64
endTime int64
}
}
// newIntegerMergeIterator returns a new instance of integerMergeIterator.
func newIntegerMergeIterator(inputs []IntegerIterator, opt IteratorOptions) *integerMergeIterator {
itr := &integerMergeIterator{
inputs: inputs,
heap: &integerMergeHeap{
items: make([]*integerMergeHeapItem, 0, len(inputs)),
opt: opt,
},
}
// Initialize heap items.
for _, input := range inputs {
// Wrap in buffer, ignore any inputs without anymore points.
bufInput := newBufIntegerIterator(input)
// Append to the heap.
itr.heap.items = append(itr.heap.items, &integerMergeHeapItem{itr: bufInput})
}
return itr
}
// Stats returns an aggregation of stats from the underlying iterators.
func (itr *integerMergeIterator) Stats() IteratorStats {
var stats IteratorStats
for _, input := range itr.inputs {
stats.Add(input.Stats())
}
return stats
}
// Close closes the underlying iterators.
func (itr *integerMergeIterator) Close() error {
for _, input := range itr.inputs {
input.Close()
}
itr.curr = nil
itr.inputs = nil
itr.heap.items = nil
return nil
}
// Next returns the next point from the iterator.
func (itr *integerMergeIterator) Next() (*IntegerPoint, error) {
// Initialize the heap. This needs to be done lazily on the first call to this iterator
// so that iterator initialization done through the Select() call returns quickly.
// Queries can only be interrupted after the Select() call completes so any operations
// done during iterator creation cannot be interrupted, which is why we do it here
// instead so an interrupt can happen while initializing the heap.
if !itr.init {
items := itr.heap.items
itr.heap.items = make([]*integerMergeHeapItem, 0, len(items))
for _, item := range items {
if p, err := item.itr.peek(); err != nil {
return nil, err
} else if p == nil {
continue
}
itr.heap.items = append(itr.heap.items, item)
}
heap.Init(itr.heap)
itr.init = true
}
for {
// Retrieve the next iterator if we don't have one.
if itr.curr == nil {
if len(itr.heap.items) == 0 {
return nil, nil
}
itr.curr = heap.Pop(itr.heap).(*integerMergeHeapItem)
// Read point and set current window.
p, err := itr.curr.itr.Next()
if err != nil {
return nil, err
}
tags := p.Tags.Subset(itr.heap.opt.Dimensions)
itr.window.name, itr.window.tags = p.Name, tags.ID()
itr.window.startTime, itr.window.endTime = itr.heap.opt.Window(p.Time)
return p, nil
}
// Read the next point from the current iterator.
p, err := itr.curr.itr.Next()
if err != nil {
return nil, err
}
// If there are no more points then remove iterator from heap and find next.
if p == nil {
itr.curr = nil
continue
}
// Check if the point is inside of our current window.
inWindow := true
if window := itr.window; window.name != p.Name {
inWindow = false
} else if tags := p.Tags.Subset(itr.heap.opt.Dimensions); window.tags != tags.ID() {
inWindow = false
} else if opt := itr.heap.opt; opt.Ascending && p.Time >= window.endTime {
inWindow = false
} else if !opt.Ascending && p.Time < window.startTime {
inWindow = false
}
// If it's outside our window then push iterator back on the heap and find new iterator.
if !inWindow {
itr.curr.itr.unread(p)
heap.Push(itr.heap, itr.curr)
itr.curr = nil
continue
}
return p, nil
}
}
// integerMergeHeap represents a heap of integerMergeHeapItems.
// Items are sorted by their next window and then by name/tags.
type integerMergeHeap struct {
opt IteratorOptions
items []*integerMergeHeapItem
}
func (h *integerMergeHeap) Len() int { return len(h.items) }
func (h *integerMergeHeap) Swap(i, j int) { h.items[i], h.items[j] = h.items[j], h.items[i] }
func (h *integerMergeHeap) Less(i, j int) bool {
x, err := h.items[i].itr.peek()
if err != nil {
return true
}
y, err := h.items[j].itr.peek()
if err != nil {
return false
}
if h.opt.Ascending {
if x.Name != y.Name {
return x.Name < y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); xTags.ID() != yTags.ID() {
return xTags.ID() < yTags.ID()
}
} else {
if x.Name != y.Name {
return x.Name > y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); xTags.ID() != yTags.ID() {
return xTags.ID() > yTags.ID()
}
}
xt, _ := h.opt.Window(x.Time)
yt, _ := h.opt.Window(y.Time)
if h.opt.Ascending {
return xt < yt
}
return xt > yt
}
func (h *integerMergeHeap) Push(x interface{}) {
h.items = append(h.items, x.(*integerMergeHeapItem))
}
func (h *integerMergeHeap) Pop() interface{} {
old := h.items
n := len(old)
item := old[n-1]
h.items = old[0 : n-1]
return item
}
type integerMergeHeapItem struct {
itr *bufIntegerIterator
}
// integerSortedMergeIterator is an iterator that sorts and merges multiple iterators into one.
type integerSortedMergeIterator struct {
inputs []IntegerIterator
heap *integerSortedMergeHeap
init bool
}
// newIntegerSortedMergeIterator returns an instance of integerSortedMergeIterator.
func newIntegerSortedMergeIterator(inputs []IntegerIterator, opt IteratorOptions) Iterator {
itr := &integerSortedMergeIterator{
inputs: inputs,
heap: &integerSortedMergeHeap{
items: make([]*integerSortedMergeHeapItem, 0, len(inputs)),
opt: opt,
},
}
// Initialize heap items.
for _, input := range inputs {
// Append to the heap.
itr.heap.items = append(itr.heap.items, &integerSortedMergeHeapItem{itr: input})
}
return itr
}
// Stats returns an aggregation of stats from the underlying iterators.
func (itr *integerSortedMergeIterator) Stats() IteratorStats {
var stats IteratorStats
for _, input := range itr.inputs {
stats.Add(input.Stats())
}
return stats
}
// Close closes the underlying iterators.
func (itr *integerSortedMergeIterator) Close() error {
for _, input := range itr.inputs {
input.Close()
}
return nil
}
// Next returns the next points from the iterator.
func (itr *integerSortedMergeIterator) Next() (*IntegerPoint, error) { return itr.pop() }
// pop returns the next point from the heap.
// Reads the next point from item's cursor and puts it back on the heap.
func (itr *integerSortedMergeIterator) pop() (*IntegerPoint, error) {
// Initialize the heap. See the MergeIterator to see why this has to be done lazily.
if !itr.init {
items := itr.heap.items
itr.heap.items = make([]*integerSortedMergeHeapItem, 0, len(items))
for _, item := range items {
var err error
if item.point, err = item.itr.Next(); err != nil {
return nil, err
} else if item.point == nil {
continue
}
itr.heap.items = append(itr.heap.items, item)
}
heap.Init(itr.heap)
itr.init = true
}
if len(itr.heap.items) == 0 {
return nil, nil
}
// Read the next item from the heap.
item := heap.Pop(itr.heap).(*integerSortedMergeHeapItem)
if item.err != nil {
return nil, item.err
} else if item.point == nil {
return nil, nil
}
// Copy the point for return.
p := item.point.Clone()
// Read the next item from the cursor. Push back to heap if one exists.
if item.point, item.err = item.itr.Next(); item.point != nil {
heap.Push(itr.heap, item)
}
return p, nil
}
// integerSortedMergeHeap represents a heap of integerSortedMergeHeapItems.
type integerSortedMergeHeap struct {
opt IteratorOptions
items []*integerSortedMergeHeapItem
}
func (h *integerSortedMergeHeap) Len() int { return len(h.items) }
func (h *integerSortedMergeHeap) Swap(i, j int) { h.items[i], h.items[j] = h.items[j], h.items[i] }
func (h *integerSortedMergeHeap) Less(i, j int) bool {
x, y := h.items[i].point, h.items[j].point
if h.opt.Ascending {
if x.Name != y.Name {
return x.Name < y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); !xTags.Equals(&yTags) {
return xTags.ID() < yTags.ID()
}
return x.Time < y.Time
}
if x.Name != y.Name {
return x.Name > y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); !xTags.Equals(&yTags) {
return xTags.ID() > yTags.ID()
}
return x.Time > y.Time
}
func (h *integerSortedMergeHeap) Push(x interface{}) {
h.items = append(h.items, x.(*integerSortedMergeHeapItem))
}
func (h *integerSortedMergeHeap) Pop() interface{} {
old := h.items
n := len(old)
item := old[n-1]
h.items = old[0 : n-1]
return item
}
type integerSortedMergeHeapItem struct {
point *IntegerPoint
err error
itr IntegerIterator
}
// integerParallelIterator represents an iterator that pulls data in a separate goroutine.
type integerParallelIterator struct {
input IntegerIterator
ch chan integerPointError
once sync.Once
closing chan struct{}
wg sync.WaitGroup
}
// newIntegerParallelIterator returns a new instance of integerParallelIterator.
func newIntegerParallelIterator(input IntegerIterator) *integerParallelIterator {
itr := &integerParallelIterator{
input: input,
ch: make(chan integerPointError, 256),
closing: make(chan struct{}),
}
itr.wg.Add(1)
go itr.monitor()
return itr
}
// Stats returns stats from the underlying iterator.
func (itr *integerParallelIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the underlying iterators.
func (itr *integerParallelIterator) Close() error {
itr.once.Do(func() { close(itr.closing) })
itr.wg.Wait()
return itr.input.Close()
}
// Next returns the next point from the iterator.
func (itr *integerParallelIterator) Next() (*IntegerPoint, error) {
v, ok := <-itr.ch
if !ok {
return nil, io.EOF
}
return v.point, v.err
}
// monitor runs in a separate goroutine and actively pulls the next point.
func (itr *integerParallelIterator) monitor() {
defer close(itr.ch)
defer itr.wg.Done()
for {
// Read next point.
p, err := itr.input.Next()
if p != nil {
p = p.Clone()
}
select {
case <-itr.closing:
return
case itr.ch <- integerPointError{point: p, err: err}:
}
}
}
type integerPointError struct {
point *IntegerPoint
err error
}
// integerLimitIterator represents an iterator that limits points per group.
type integerLimitIterator struct {
input IntegerIterator
opt IteratorOptions
n int
prev struct {
name string
tags Tags
}
}
// newIntegerLimitIterator returns a new instance of integerLimitIterator.
func newIntegerLimitIterator(input IntegerIterator, opt IteratorOptions) *integerLimitIterator {
return &integerLimitIterator{
input: input,
opt: opt,
}
}
// Stats returns stats from the underlying iterator.
func (itr *integerLimitIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the underlying iterators.
func (itr *integerLimitIterator) Close() error { return itr.input.Close() }
// Next returns the next point from the iterator.
func (itr *integerLimitIterator) Next() (*IntegerPoint, error) {
for {
p, err := itr.input.Next()
if p == nil || err != nil {
return nil, err
}
// Reset window and counter if a new window is encountered.
if p.Name != itr.prev.name || !p.Tags.Equals(&itr.prev.tags) {
itr.prev.name = p.Name
itr.prev.tags = p.Tags
itr.n = 0
}
// Increment counter.
itr.n++
// Read next point if not beyond the offset.
if itr.n <= itr.opt.Offset {
continue
}
// Read next point if we're beyond the limit.
if itr.opt.Limit > 0 && (itr.n-itr.opt.Offset) > itr.opt.Limit {
continue
}
return p, nil
}
}
type integerFillIterator struct {
input *bufIntegerIterator
prev IntegerPoint
startTime int64
endTime int64
auxFields []interface{}
init bool
opt IteratorOptions
window struct {
name string
tags Tags
time int64
offset int64
}
}
func newIntegerFillIterator(input IntegerIterator, expr Expr, opt IteratorOptions) *integerFillIterator {
if opt.Fill == NullFill {
if expr, ok := expr.(*Call); ok && expr.Name == "count" {
opt.Fill = NumberFill
opt.FillValue = int64(0)
}
}
var startTime, endTime int64
if opt.Ascending {
startTime, _ = opt.Window(opt.StartTime)
endTime, _ = opt.Window(opt.EndTime)
} else {
startTime, _ = opt.Window(opt.EndTime)
endTime, _ = opt.Window(opt.StartTime)
}
var auxFields []interface{}
if len(opt.Aux) > 0 {
auxFields = make([]interface{}, len(opt.Aux))
}
return &integerFillIterator{
input: newBufIntegerIterator(input),
prev: IntegerPoint{Nil: true},
startTime: startTime,
endTime: endTime,
auxFields: auxFields,
opt: opt,
}
}
func (itr *integerFillIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *integerFillIterator) Close() error { return itr.input.Close() }
func (itr *integerFillIterator) Next() (*IntegerPoint, error) {
if !itr.init {
p, err := itr.input.peek()
if p == nil || err != nil {
return nil, err
}
itr.window.name, itr.window.tags = p.Name, p.Tags
itr.window.time = itr.startTime
if itr.opt.Location != nil {
_, itr.window.offset = itr.opt.Zone(itr.window.time)
}
itr.init = true
}
p, err := itr.input.Next()
if err != nil {
return nil, err
}
// Check if the next point is outside of our window or is nil.
for p == nil || p.Name != itr.window.name || p.Tags.ID() != itr.window.tags.ID() {
// If we are inside of an interval, unread the point and continue below to
// constructing a new point.
if itr.opt.Ascending {
if itr.window.time <= itr.endTime {
itr.input.unread(p)
p = nil
break
}
} else {
if itr.window.time >= itr.endTime {
itr.input.unread(p)
p = nil
break
}
}
// We are *not* in a current interval. If there is no next point,
// we are at the end of all intervals.
if p == nil {
return nil, nil
}
// Set the new interval.
itr.window.name, itr.window.tags = p.Name, p.Tags
itr.window.time = itr.startTime
if itr.opt.Location != nil {
_, itr.window.offset = itr.opt.Zone(itr.window.time)
}
itr.prev = IntegerPoint{Nil: true}
break
}
// Check if the point is our next expected point.
if p == nil || (itr.opt.Ascending && p.Time > itr.window.time) || (!itr.opt.Ascending && p.Time < itr.window.time) {
if p != nil {
itr.input.unread(p)
}
p = &IntegerPoint{
Name: itr.window.name,
Tags: itr.window.tags,
Time: itr.window.time,
Aux: itr.auxFields,
}
switch itr.opt.Fill {
case LinearFill:
if !itr.prev.Nil {
next, err := itr.input.peek()
if err != nil {
return nil, err
} else if next != nil && next.Name == itr.window.name && next.Tags.ID() == itr.window.tags.ID() {
interval := int64(itr.opt.Interval.Duration)
start := itr.window.time / interval
p.Value = linearInteger(start, itr.prev.Time/interval, next.Time/interval, itr.prev.Value, next.Value)
} else {
p.Nil = true
}
} else {
p.Nil = true
}
case NullFill:
p.Nil = true
case NumberFill:
p.Value = castToInteger(itr.opt.FillValue)
case PreviousFill:
if !itr.prev.Nil {
p.Value = itr.prev.Value
p.Nil = itr.prev.Nil
} else {
p.Nil = true
}
}
} else {
itr.prev = *p
}
// Advance the expected time. Do not advance to a new window here
// as there may be lingering points with the same timestamp in the previous
// window.
if itr.opt.Ascending {
itr.window.time += int64(itr.opt.Interval.Duration)
} else {
itr.window.time -= int64(itr.opt.Interval.Duration)
}
// Check to see if we have passed over an offset change and adjust the time
// to account for this new offset.
if itr.opt.Location != nil {
if _, offset := itr.opt.Zone(itr.window.time - 1); offset != itr.window.offset {
diff := itr.window.offset - offset
if abs(diff) < int64(itr.opt.Interval.Duration) {
itr.window.time += diff
}
itr.window.offset = offset
}
}
return p, nil
}
// integerIntervalIterator represents a integer implementation of IntervalIterator.
type integerIntervalIterator struct {
input IntegerIterator
opt IteratorOptions
}
func newIntegerIntervalIterator(input IntegerIterator, opt IteratorOptions) *integerIntervalIterator {
return &integerIntervalIterator{input: input, opt: opt}
}
func (itr *integerIntervalIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *integerIntervalIterator) Close() error { return itr.input.Close() }
func (itr *integerIntervalIterator) Next() (*IntegerPoint, error) {
p, err := itr.input.Next()
if p == nil || err != nil {
return nil, err
}
p.Time, _ = itr.opt.Window(p.Time)
// If we see the minimum allowable time, set the time to zero so we don't
// break the default returned time for aggregate queries without times.
if p.Time == MinTime {
p.Time = 0
}
return p, nil
}
// integerInterruptIterator represents a integer implementation of InterruptIterator.
type integerInterruptIterator struct {
input IntegerIterator
closing <-chan struct{}
count int
}
func newIntegerInterruptIterator(input IntegerIterator, closing <-chan struct{}) *integerInterruptIterator {
return &integerInterruptIterator{input: input, closing: closing}
}
func (itr *integerInterruptIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *integerInterruptIterator) Close() error { return itr.input.Close() }
func (itr *integerInterruptIterator) Next() (*IntegerPoint, error) {
// Only check if the channel is closed every N points. This
// intentionally checks on both 0 and N so that if the iterator
// has been interrupted before the first point is emitted it will
// not emit any points.
if itr.count&0xFF == 0xFF {
select {
case <-itr.closing:
return nil, itr.Close()
default:
// Reset iterator count to zero and fall through to emit the next point.
itr.count = 0
}
}
// Increment the counter for every point read.
itr.count++
return itr.input.Next()
}
// integerCloseInterruptIterator represents a integer implementation of CloseInterruptIterator.
type integerCloseInterruptIterator struct {
input IntegerIterator
closing <-chan struct{}
done chan struct{}
once sync.Once
}
func newIntegerCloseInterruptIterator(input IntegerIterator, closing <-chan struct{}) *integerCloseInterruptIterator {
itr := &integerCloseInterruptIterator{
input: input,
closing: closing,
done: make(chan struct{}),
}
go itr.monitor()
return itr
}
func (itr *integerCloseInterruptIterator) monitor() {
select {
case <-itr.closing:
itr.Close()
case <-itr.done:
}
}
func (itr *integerCloseInterruptIterator) Stats() IteratorStats {
return itr.input.Stats()
}
func (itr *integerCloseInterruptIterator) Close() error {
itr.once.Do(func() {
close(itr.done)
itr.input.Close()
})
return nil
}
func (itr *integerCloseInterruptIterator) Next() (*IntegerPoint, error) {
p, err := itr.input.Next()
if err != nil {
// Check if the iterator was closed.
select {
case <-itr.done:
return nil, nil
default:
return nil, err
}
}
return p, nil
}
// auxIntegerPoint represents a combination of a point and an error for the AuxIterator.
type auxIntegerPoint struct {
point *IntegerPoint
err error
}
// integerAuxIterator represents a integer implementation of AuxIterator.
type integerAuxIterator struct {
input *bufIntegerIterator
output chan auxIntegerPoint
fields *auxIteratorFields
background bool
}
func newIntegerAuxIterator(input IntegerIterator, opt IteratorOptions) *integerAuxIterator {
return &integerAuxIterator{
input: newBufIntegerIterator(input),
output: make(chan auxIntegerPoint, 1),
fields: newAuxIteratorFields(opt),
}
}
func (itr *integerAuxIterator) Background() {
itr.background = true
itr.Start()
go DrainIterator(itr)
}
func (itr *integerAuxIterator) Start() { go itr.stream() }
func (itr *integerAuxIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *integerAuxIterator) Close() error { return itr.input.Close() }
func (itr *integerAuxIterator) Next() (*IntegerPoint, error) {
p := <-itr.output
return p.point, p.err
}
func (itr *integerAuxIterator) Iterator(name string, typ DataType) Iterator {
return itr.fields.iterator(name, typ)
}
func (itr *integerAuxIterator) stream() {
for {
// Read next point.
p, err := itr.input.Next()
if err != nil {
itr.output <- auxIntegerPoint{err: err}
itr.fields.sendError(err)
break
} else if p == nil {
break
}
// Send point to output and to each field iterator.
itr.output <- auxIntegerPoint{point: p}
if ok := itr.fields.send(p); !ok && itr.background {
break
}
}
close(itr.output)
itr.fields.close()
}
// integerChanIterator represents a new instance of integerChanIterator.
type integerChanIterator struct {
buf struct {
i int
filled bool
points [2]IntegerPoint
}
err error
cond *sync.Cond
done bool
}
func (itr *integerChanIterator) Stats() IteratorStats { return IteratorStats{} }
func (itr *integerChanIterator) Close() error {
itr.cond.L.Lock()
// Mark the channel iterator as done and signal all waiting goroutines to start again.
itr.done = true
itr.cond.Broadcast()
// Do not defer the unlock so we don't create an unnecessary allocation.
itr.cond.L.Unlock()
return nil
}
func (itr *integerChanIterator) setBuf(name string, tags Tags, time int64, value interface{}) bool {
itr.cond.L.Lock()
defer itr.cond.L.Unlock()
// Wait for either the iterator to be done (so we don't have to set the value)
// or for the buffer to have been read and ready for another write.
for !itr.done && itr.buf.filled {
itr.cond.Wait()
}
// Do not set the value and return false to signal that the iterator is closed.
// Do this after the above wait as the above for loop may have exited because
// the iterator was closed.
if itr.done {
return false
}
switch v := value.(type) {
case int64:
itr.buf.points[itr.buf.i] = IntegerPoint{Name: name, Tags: tags, Time: time, Value: v}
default:
itr.buf.points[itr.buf.i] = IntegerPoint{Name: name, Tags: tags, Time: time, Nil: true}
}
itr.buf.filled = true
// Signal to all waiting goroutines that a new value is ready to read.
itr.cond.Signal()
return true
}
func (itr *integerChanIterator) setErr(err error) {
itr.cond.L.Lock()
defer itr.cond.L.Unlock()
itr.err = err
// Signal to all waiting goroutines that a new value is ready to read.
itr.cond.Signal()
}
func (itr *integerChanIterator) Next() (*IntegerPoint, error) {
itr.cond.L.Lock()
defer itr.cond.L.Unlock()
// Check for an error and return one if there.
if itr.err != nil {
return nil, itr.err
}
// Wait until either a value is available in the buffer or
// the iterator is closed.
for !itr.done && !itr.buf.filled {
itr.cond.Wait()
}
// Return nil once the channel is done and the buffer is empty.
if itr.done && !itr.buf.filled {
return nil, nil
}
// Always read from the buffer if it exists, even if the iterator
// is closed. This prevents the last value from being truncated by
// the parent iterator.
p := &itr.buf.points[itr.buf.i]
itr.buf.i = (itr.buf.i + 1) % len(itr.buf.points)
itr.buf.filled = false
itr.cond.Signal()
return p, nil
}
// integerReduceFloatIterator executes a reducer for every interval and buffers the result.
type integerReduceFloatIterator struct {
input *bufIntegerIterator
create func() (IntegerPointAggregator, FloatPointEmitter)
dims []string
opt IteratorOptions
points []FloatPoint
keepTags bool
}
func newIntegerReduceFloatIterator(input IntegerIterator, opt IteratorOptions, createFn func() (IntegerPointAggregator, FloatPointEmitter)) *integerReduceFloatIterator {
return &integerReduceFloatIterator{
input: newBufIntegerIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *integerReduceFloatIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *integerReduceFloatIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *integerReduceFloatIterator) Next() (*FloatPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// integerReduceFloatPoint stores the reduced data for a name/tag combination.
type integerReduceFloatPoint struct {
Name string
Tags Tags
Aggregator IntegerPointAggregator
Emitter FloatPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *integerReduceFloatIterator) reduce() ([]FloatPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*integerReduceFloatPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &integerReduceFloatPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateInteger(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]FloatPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(floatPointsByTime(a)))
}
return a, nil
}
// integerStreamFloatIterator streams inputs into the iterator and emits points gradually.
type integerStreamFloatIterator struct {
input *bufIntegerIterator
create func() (IntegerPointAggregator, FloatPointEmitter)
dims []string
opt IteratorOptions
m map[string]*integerReduceFloatPoint
points []FloatPoint
}
// newIntegerStreamFloatIterator returns a new instance of integerStreamFloatIterator.
func newIntegerStreamFloatIterator(input IntegerIterator, createFn func() (IntegerPointAggregator, FloatPointEmitter), opt IteratorOptions) *integerStreamFloatIterator {
return &integerStreamFloatIterator{
input: newBufIntegerIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*integerReduceFloatPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *integerStreamFloatIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *integerStreamFloatIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *integerStreamFloatIterator) Next() (*FloatPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *integerStreamFloatIterator) reduce() ([]FloatPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []FloatPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &integerReduceFloatPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateInteger(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// integerFloatExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type integerFloatExprIterator struct {
left *bufIntegerIterator
right *bufIntegerIterator
fn integerFloatExprFunc
points []IntegerPoint // must be size 2
storePrev bool
}
func newIntegerFloatExprIterator(left, right IntegerIterator, opt IteratorOptions, fn func(a, b int64) float64) *integerFloatExprIterator {
var points []IntegerPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []IntegerPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToInteger(opt.FillValue)
points = []IntegerPoint{{Value: value}, {Value: value}}
}
return &integerFloatExprIterator{
left: newBufIntegerIterator(left),
right: newBufIntegerIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *integerFloatExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *integerFloatExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *integerFloatExprIterator) Next() (*FloatPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = 0
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = 0
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
p := &FloatPoint{
Name: a.Name,
Tags: a.Tags,
Time: a.Time,
Nil: a.Nil || b.Nil,
Aggregated: a.Aggregated,
}
if !p.Nil {
p.Value = itr.fn(a.Value, b.Value)
}
return p, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *integerFloatExprIterator) next() (a, b *IntegerPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// integerFloatExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type integerFloatExprFunc func(a, b int64) float64
// integerReduceIntegerIterator executes a reducer for every interval and buffers the result.
type integerReduceIntegerIterator struct {
input *bufIntegerIterator
create func() (IntegerPointAggregator, IntegerPointEmitter)
dims []string
opt IteratorOptions
points []IntegerPoint
keepTags bool
}
func newIntegerReduceIntegerIterator(input IntegerIterator, opt IteratorOptions, createFn func() (IntegerPointAggregator, IntegerPointEmitter)) *integerReduceIntegerIterator {
return &integerReduceIntegerIterator{
input: newBufIntegerIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *integerReduceIntegerIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *integerReduceIntegerIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *integerReduceIntegerIterator) Next() (*IntegerPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// integerReduceIntegerPoint stores the reduced data for a name/tag combination.
type integerReduceIntegerPoint struct {
Name string
Tags Tags
Aggregator IntegerPointAggregator
Emitter IntegerPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *integerReduceIntegerIterator) reduce() ([]IntegerPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*integerReduceIntegerPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &integerReduceIntegerPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateInteger(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]IntegerPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(integerPointsByTime(a)))
}
return a, nil
}
// integerStreamIntegerIterator streams inputs into the iterator and emits points gradually.
type integerStreamIntegerIterator struct {
input *bufIntegerIterator
create func() (IntegerPointAggregator, IntegerPointEmitter)
dims []string
opt IteratorOptions
m map[string]*integerReduceIntegerPoint
points []IntegerPoint
}
// newIntegerStreamIntegerIterator returns a new instance of integerStreamIntegerIterator.
func newIntegerStreamIntegerIterator(input IntegerIterator, createFn func() (IntegerPointAggregator, IntegerPointEmitter), opt IteratorOptions) *integerStreamIntegerIterator {
return &integerStreamIntegerIterator{
input: newBufIntegerIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*integerReduceIntegerPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *integerStreamIntegerIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *integerStreamIntegerIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *integerStreamIntegerIterator) Next() (*IntegerPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *integerStreamIntegerIterator) reduce() ([]IntegerPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []IntegerPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &integerReduceIntegerPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateInteger(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// integerExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type integerExprIterator struct {
left *bufIntegerIterator
right *bufIntegerIterator
fn integerExprFunc
points []IntegerPoint // must be size 2
storePrev bool
}
func newIntegerExprIterator(left, right IntegerIterator, opt IteratorOptions, fn func(a, b int64) int64) *integerExprIterator {
var points []IntegerPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []IntegerPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToInteger(opt.FillValue)
points = []IntegerPoint{{Value: value}, {Value: value}}
}
return &integerExprIterator{
left: newBufIntegerIterator(left),
right: newBufIntegerIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *integerExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *integerExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *integerExprIterator) Next() (*IntegerPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = 0
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = 0
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
if a.Nil {
return a, nil
} else if b.Nil {
return b, nil
}
a.Value = itr.fn(a.Value, b.Value)
return a, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *integerExprIterator) next() (a, b *IntegerPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// integerExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type integerExprFunc func(a, b int64) int64
// integerReduceStringIterator executes a reducer for every interval and buffers the result.
type integerReduceStringIterator struct {
input *bufIntegerIterator
create func() (IntegerPointAggregator, StringPointEmitter)
dims []string
opt IteratorOptions
points []StringPoint
keepTags bool
}
func newIntegerReduceStringIterator(input IntegerIterator, opt IteratorOptions, createFn func() (IntegerPointAggregator, StringPointEmitter)) *integerReduceStringIterator {
return &integerReduceStringIterator{
input: newBufIntegerIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *integerReduceStringIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *integerReduceStringIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *integerReduceStringIterator) Next() (*StringPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// integerReduceStringPoint stores the reduced data for a name/tag combination.
type integerReduceStringPoint struct {
Name string
Tags Tags
Aggregator IntegerPointAggregator
Emitter StringPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *integerReduceStringIterator) reduce() ([]StringPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*integerReduceStringPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &integerReduceStringPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateInteger(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]StringPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(stringPointsByTime(a)))
}
return a, nil
}
// integerStreamStringIterator streams inputs into the iterator and emits points gradually.
type integerStreamStringIterator struct {
input *bufIntegerIterator
create func() (IntegerPointAggregator, StringPointEmitter)
dims []string
opt IteratorOptions
m map[string]*integerReduceStringPoint
points []StringPoint
}
// newIntegerStreamStringIterator returns a new instance of integerStreamStringIterator.
func newIntegerStreamStringIterator(input IntegerIterator, createFn func() (IntegerPointAggregator, StringPointEmitter), opt IteratorOptions) *integerStreamStringIterator {
return &integerStreamStringIterator{
input: newBufIntegerIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*integerReduceStringPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *integerStreamStringIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *integerStreamStringIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *integerStreamStringIterator) Next() (*StringPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *integerStreamStringIterator) reduce() ([]StringPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []StringPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &integerReduceStringPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateInteger(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// integerStringExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type integerStringExprIterator struct {
left *bufIntegerIterator
right *bufIntegerIterator
fn integerStringExprFunc
points []IntegerPoint // must be size 2
storePrev bool
}
func newIntegerStringExprIterator(left, right IntegerIterator, opt IteratorOptions, fn func(a, b int64) string) *integerStringExprIterator {
var points []IntegerPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []IntegerPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToInteger(opt.FillValue)
points = []IntegerPoint{{Value: value}, {Value: value}}
}
return &integerStringExprIterator{
left: newBufIntegerIterator(left),
right: newBufIntegerIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *integerStringExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *integerStringExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *integerStringExprIterator) Next() (*StringPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = 0
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = 0
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
p := &StringPoint{
Name: a.Name,
Tags: a.Tags,
Time: a.Time,
Nil: a.Nil || b.Nil,
Aggregated: a.Aggregated,
}
if !p.Nil {
p.Value = itr.fn(a.Value, b.Value)
}
return p, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *integerStringExprIterator) next() (a, b *IntegerPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// integerStringExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type integerStringExprFunc func(a, b int64) string
// integerReduceBooleanIterator executes a reducer for every interval and buffers the result.
type integerReduceBooleanIterator struct {
input *bufIntegerIterator
create func() (IntegerPointAggregator, BooleanPointEmitter)
dims []string
opt IteratorOptions
points []BooleanPoint
keepTags bool
}
func newIntegerReduceBooleanIterator(input IntegerIterator, opt IteratorOptions, createFn func() (IntegerPointAggregator, BooleanPointEmitter)) *integerReduceBooleanIterator {
return &integerReduceBooleanIterator{
input: newBufIntegerIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *integerReduceBooleanIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *integerReduceBooleanIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *integerReduceBooleanIterator) Next() (*BooleanPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// integerReduceBooleanPoint stores the reduced data for a name/tag combination.
type integerReduceBooleanPoint struct {
Name string
Tags Tags
Aggregator IntegerPointAggregator
Emitter BooleanPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *integerReduceBooleanIterator) reduce() ([]BooleanPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*integerReduceBooleanPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &integerReduceBooleanPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateInteger(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]BooleanPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(booleanPointsByTime(a)))
}
return a, nil
}
// integerStreamBooleanIterator streams inputs into the iterator and emits points gradually.
type integerStreamBooleanIterator struct {
input *bufIntegerIterator
create func() (IntegerPointAggregator, BooleanPointEmitter)
dims []string
opt IteratorOptions
m map[string]*integerReduceBooleanPoint
points []BooleanPoint
}
// newIntegerStreamBooleanIterator returns a new instance of integerStreamBooleanIterator.
func newIntegerStreamBooleanIterator(input IntegerIterator, createFn func() (IntegerPointAggregator, BooleanPointEmitter), opt IteratorOptions) *integerStreamBooleanIterator {
return &integerStreamBooleanIterator{
input: newBufIntegerIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*integerReduceBooleanPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *integerStreamBooleanIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *integerStreamBooleanIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *integerStreamBooleanIterator) Next() (*BooleanPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *integerStreamBooleanIterator) reduce() ([]BooleanPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []BooleanPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &integerReduceBooleanPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateInteger(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// integerBooleanExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type integerBooleanExprIterator struct {
left *bufIntegerIterator
right *bufIntegerIterator
fn integerBooleanExprFunc
points []IntegerPoint // must be size 2
storePrev bool
}
func newIntegerBooleanExprIterator(left, right IntegerIterator, opt IteratorOptions, fn func(a, b int64) bool) *integerBooleanExprIterator {
var points []IntegerPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []IntegerPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToInteger(opt.FillValue)
points = []IntegerPoint{{Value: value}, {Value: value}}
}
return &integerBooleanExprIterator{
left: newBufIntegerIterator(left),
right: newBufIntegerIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *integerBooleanExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *integerBooleanExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *integerBooleanExprIterator) Next() (*BooleanPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = 0
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = 0
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
p := &BooleanPoint{
Name: a.Name,
Tags: a.Tags,
Time: a.Time,
Nil: a.Nil || b.Nil,
Aggregated: a.Aggregated,
}
if !p.Nil {
p.Value = itr.fn(a.Value, b.Value)
}
return p, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *integerBooleanExprIterator) next() (a, b *IntegerPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// integerBooleanExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type integerBooleanExprFunc func(a, b int64) bool
// integerTransformIterator executes a function to modify an existing point for every
// output of the input iterator.
type integerTransformIterator struct {
input IntegerIterator
fn integerTransformFunc
}
// Stats returns stats from the input iterator.
func (itr *integerTransformIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *integerTransformIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *integerTransformIterator) Next() (*IntegerPoint, error) {
p, err := itr.input.Next()
if err != nil {
return nil, err
} else if p != nil {
p = itr.fn(p)
}
return p, nil
}
// integerTransformFunc creates or modifies a point.
// The point passed in may be modified and returned rather than allocating a
// new point if possible.
type integerTransformFunc func(p *IntegerPoint) *IntegerPoint
// integerBoolTransformIterator executes a function to modify an existing point for every
// output of the input iterator.
type integerBoolTransformIterator struct {
input IntegerIterator
fn integerBoolTransformFunc
}
// Stats returns stats from the input iterator.
func (itr *integerBoolTransformIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *integerBoolTransformIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *integerBoolTransformIterator) Next() (*BooleanPoint, error) {
p, err := itr.input.Next()
if err != nil {
return nil, err
} else if p != nil {
return itr.fn(p), nil
}
return nil, nil
}
// integerBoolTransformFunc creates or modifies a point.
// The point passed in may be modified and returned rather than allocating a
// new point if possible.
type integerBoolTransformFunc func(p *IntegerPoint) *BooleanPoint
// integerDedupeIterator only outputs unique points.
// This differs from the DistinctIterator in that it compares all aux fields too.
// This iterator is relatively inefficient and should only be used on small
// datasets such as meta query results.
type integerDedupeIterator struct {
input IntegerIterator
m map[string]struct{} // lookup of points already sent
}
type integerIteratorMapper struct {
e *Emitter
buf []interface{}
driver IteratorMap // which iterator to use for the primary value, can be nil
fields []IteratorMap // which iterator to use for an aux field
point IntegerPoint
}
func newIntegerIteratorMapper(itrs []Iterator, driver IteratorMap, fields []IteratorMap, opt IteratorOptions) *integerIteratorMapper {
e := NewEmitter(itrs, opt.Ascending, 0)
e.OmitTime = true
return &integerIteratorMapper{
e: e,
buf: make([]interface{}, len(itrs)),
driver: driver,
fields: fields,
point: IntegerPoint{
Aux: make([]interface{}, len(fields)),
},
}
}
func (itr *integerIteratorMapper) Next() (*IntegerPoint, error) {
t, name, tags, err := itr.e.loadBuf()
if err != nil || t == ZeroTime {
return nil, err
}
itr.point.Time = t
itr.point.Name = name
itr.point.Tags = tags
itr.e.readInto(t, name, tags, itr.buf)
if itr.driver != nil {
if v := itr.driver.Value(tags, itr.buf); v != nil {
if v, ok := v.(int64); ok {
itr.point.Value = v
itr.point.Nil = false
} else {
itr.point.Value = 0
itr.point.Nil = true
}
} else {
itr.point.Value = 0
itr.point.Nil = true
}
}
for i, f := range itr.fields {
itr.point.Aux[i] = f.Value(tags, itr.buf)
}
return &itr.point, nil
}
func (itr *integerIteratorMapper) Stats() IteratorStats {
stats := IteratorStats{}
for _, itr := range itr.e.itrs {
stats.Add(itr.Stats())
}
return stats
}
func (itr *integerIteratorMapper) Close() error {
return itr.e.Close()
}
type integerFilterIterator struct {
input IntegerIterator
cond Expr
opt IteratorOptions
m map[string]interface{}
}
func newIntegerFilterIterator(input IntegerIterator, cond Expr, opt IteratorOptions) IntegerIterator {
// Strip out time conditions from the WHERE clause.
// TODO(jsternberg): This should really be done for us when creating the IteratorOptions struct.
n := RewriteFunc(CloneExpr(cond), func(n Node) Node {
switch n := n.(type) {
case *BinaryExpr:
if n.LHS.String() == "time" {
return &BooleanLiteral{Val: true}
}
}
return n
})
cond, _ = n.(Expr)
if cond == nil {
return input
} else if n, ok := cond.(*BooleanLiteral); ok && n.Val {
return input
}
return &integerFilterIterator{
input: input,
cond: cond,
opt: opt,
m: make(map[string]interface{}),
}
}
func (itr *integerFilterIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *integerFilterIterator) Close() error { return itr.input.Close() }
func (itr *integerFilterIterator) Next() (*IntegerPoint, error) {
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
}
for i, ref := range itr.opt.Aux {
itr.m[ref.Val] = p.Aux[i]
}
for k, v := range p.Tags.KeyValues() {
itr.m[k] = v
}
if !EvalBool(itr.cond, itr.m) {
continue
}
return p, nil
}
}
// newIntegerDedupeIterator returns a new instance of integerDedupeIterator.
func newIntegerDedupeIterator(input IntegerIterator) *integerDedupeIterator {
return &integerDedupeIterator{
input: input,
m: make(map[string]struct{}),
}
}
// Stats returns stats from the input iterator.
func (itr *integerDedupeIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *integerDedupeIterator) Close() error { return itr.input.Close() }
// Next returns the next unique point from the input iterator.
func (itr *integerDedupeIterator) Next() (*IntegerPoint, error) {
for {
// Read next point.
p, err := itr.input.Next()
if p == nil || err != nil {
return nil, err
}
// Serialize to bytes to store in lookup.
buf, err := proto.Marshal(encodeIntegerPoint(p))
if err != nil {
return nil, err
}
// If the point has already been output then move to the next point.
if _, ok := itr.m[string(buf)]; ok {
continue
}
// Otherwise mark it as emitted and return point.
itr.m[string(buf)] = struct{}{}
return p, nil
}
}
// integerReaderIterator represents an iterator that streams from a reader.
type integerReaderIterator struct {
r io.Reader
dec *IntegerPointDecoder
}
// newIntegerReaderIterator returns a new instance of integerReaderIterator.
func newIntegerReaderIterator(r io.Reader, stats IteratorStats) *integerReaderIterator {
dec := NewIntegerPointDecoder(r)
dec.stats = stats
return &integerReaderIterator{
r: r,
dec: dec,
}
}
// Stats returns stats about points processed.
func (itr *integerReaderIterator) Stats() IteratorStats { return itr.dec.stats }
// Close closes the underlying reader, if applicable.
func (itr *integerReaderIterator) Close() error {
if r, ok := itr.r.(io.ReadCloser); ok {
return r.Close()
}
return nil
}
// Next returns the next point from the iterator.
func (itr *integerReaderIterator) Next() (*IntegerPoint, error) {
// OPTIMIZE(benbjohnson): Reuse point on iterator.
// Unmarshal next point.
p := &IntegerPoint{}
if err := itr.dec.DecodeIntegerPoint(p); err == io.EOF {
return nil, nil
} else if err != nil {
return nil, err
}
return p, nil
}
// StringIterator represents a stream of string points.
type StringIterator interface {
Iterator
Next() (*StringPoint, error)
}
// newStringIterators converts a slice of Iterator to a slice of StringIterator.
// Drop and closes any iterator in itrs that is not a StringIterator and cannot
// be cast to a StringIterator.
func newStringIterators(itrs []Iterator) []StringIterator {
a := make([]StringIterator, 0, len(itrs))
for _, itr := range itrs {
switch itr := itr.(type) {
case StringIterator:
a = append(a, itr)
default:
itr.Close()
}
}
return a
}
// bufStringIterator represents a buffered StringIterator.
type bufStringIterator struct {
itr StringIterator
buf *StringPoint
}
// newBufStringIterator returns a buffered StringIterator.
func newBufStringIterator(itr StringIterator) *bufStringIterator {
return &bufStringIterator{itr: itr}
}
// Stats returns statistics from the input iterator.
func (itr *bufStringIterator) Stats() IteratorStats { return itr.itr.Stats() }
// Close closes the underlying iterator.
func (itr *bufStringIterator) Close() error { return itr.itr.Close() }
// peek returns the next point without removing it from the iterator.
func (itr *bufStringIterator) peek() (*StringPoint, error) {
p, err := itr.Next()
if err != nil {
return nil, err
}
itr.unread(p)
return p, nil
}
// peekTime returns the time of the next point.
// Returns zero time if no more points available.
func (itr *bufStringIterator) peekTime() (int64, error) {
p, err := itr.peek()
if p == nil || err != nil {
return ZeroTime, err
}
return p.Time, nil
}
// Next returns the current buffer, if exists, or calls the underlying iterator.
func (itr *bufStringIterator) Next() (*StringPoint, error) {
buf := itr.buf
if buf != nil {
itr.buf = nil
return buf, nil
}
return itr.itr.Next()
}
// NextInWindow returns the next value if it is between [startTime, endTime).
// If the next value is outside the range then it is moved to the buffer.
func (itr *bufStringIterator) NextInWindow(startTime, endTime int64) (*StringPoint, error) {
v, err := itr.Next()
if v == nil || err != nil {
return nil, err
} else if t := v.Time; t >= endTime || t < startTime {
itr.unread(v)
return nil, nil
}
return v, nil
}
// unread sets v to the buffer. It is read on the next call to Next().
func (itr *bufStringIterator) unread(v *StringPoint) { itr.buf = v }
// stringMergeIterator represents an iterator that combines multiple string iterators.
type stringMergeIterator struct {
inputs []StringIterator
heap *stringMergeHeap
init bool
// Current iterator and window.
curr *stringMergeHeapItem
window struct {
name string
tags string
startTime int64
endTime int64
}
}
// newStringMergeIterator returns a new instance of stringMergeIterator.
func newStringMergeIterator(inputs []StringIterator, opt IteratorOptions) *stringMergeIterator {
itr := &stringMergeIterator{
inputs: inputs,
heap: &stringMergeHeap{
items: make([]*stringMergeHeapItem, 0, len(inputs)),
opt: opt,
},
}
// Initialize heap items.
for _, input := range inputs {
// Wrap in buffer, ignore any inputs without anymore points.
bufInput := newBufStringIterator(input)
// Append to the heap.
itr.heap.items = append(itr.heap.items, &stringMergeHeapItem{itr: bufInput})
}
return itr
}
// Stats returns an aggregation of stats from the underlying iterators.
func (itr *stringMergeIterator) Stats() IteratorStats {
var stats IteratorStats
for _, input := range itr.inputs {
stats.Add(input.Stats())
}
return stats
}
// Close closes the underlying iterators.
func (itr *stringMergeIterator) Close() error {
for _, input := range itr.inputs {
input.Close()
}
itr.curr = nil
itr.inputs = nil
itr.heap.items = nil
return nil
}
// Next returns the next point from the iterator.
func (itr *stringMergeIterator) Next() (*StringPoint, error) {
// Initialize the heap. This needs to be done lazily on the first call to this iterator
// so that iterator initialization done through the Select() call returns quickly.
// Queries can only be interrupted after the Select() call completes so any operations
// done during iterator creation cannot be interrupted, which is why we do it here
// instead so an interrupt can happen while initializing the heap.
if !itr.init {
items := itr.heap.items
itr.heap.items = make([]*stringMergeHeapItem, 0, len(items))
for _, item := range items {
if p, err := item.itr.peek(); err != nil {
return nil, err
} else if p == nil {
continue
}
itr.heap.items = append(itr.heap.items, item)
}
heap.Init(itr.heap)
itr.init = true
}
for {
// Retrieve the next iterator if we don't have one.
if itr.curr == nil {
if len(itr.heap.items) == 0 {
return nil, nil
}
itr.curr = heap.Pop(itr.heap).(*stringMergeHeapItem)
// Read point and set current window.
p, err := itr.curr.itr.Next()
if err != nil {
return nil, err
}
tags := p.Tags.Subset(itr.heap.opt.Dimensions)
itr.window.name, itr.window.tags = p.Name, tags.ID()
itr.window.startTime, itr.window.endTime = itr.heap.opt.Window(p.Time)
return p, nil
}
// Read the next point from the current iterator.
p, err := itr.curr.itr.Next()
if err != nil {
return nil, err
}
// If there are no more points then remove iterator from heap and find next.
if p == nil {
itr.curr = nil
continue
}
// Check if the point is inside of our current window.
inWindow := true
if window := itr.window; window.name != p.Name {
inWindow = false
} else if tags := p.Tags.Subset(itr.heap.opt.Dimensions); window.tags != tags.ID() {
inWindow = false
} else if opt := itr.heap.opt; opt.Ascending && p.Time >= window.endTime {
inWindow = false
} else if !opt.Ascending && p.Time < window.startTime {
inWindow = false
}
// If it's outside our window then push iterator back on the heap and find new iterator.
if !inWindow {
itr.curr.itr.unread(p)
heap.Push(itr.heap, itr.curr)
itr.curr = nil
continue
}
return p, nil
}
}
// stringMergeHeap represents a heap of stringMergeHeapItems.
// Items are sorted by their next window and then by name/tags.
type stringMergeHeap struct {
opt IteratorOptions
items []*stringMergeHeapItem
}
func (h *stringMergeHeap) Len() int { return len(h.items) }
func (h *stringMergeHeap) Swap(i, j int) { h.items[i], h.items[j] = h.items[j], h.items[i] }
func (h *stringMergeHeap) Less(i, j int) bool {
x, err := h.items[i].itr.peek()
if err != nil {
return true
}
y, err := h.items[j].itr.peek()
if err != nil {
return false
}
if h.opt.Ascending {
if x.Name != y.Name {
return x.Name < y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); xTags.ID() != yTags.ID() {
return xTags.ID() < yTags.ID()
}
} else {
if x.Name != y.Name {
return x.Name > y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); xTags.ID() != yTags.ID() {
return xTags.ID() > yTags.ID()
}
}
xt, _ := h.opt.Window(x.Time)
yt, _ := h.opt.Window(y.Time)
if h.opt.Ascending {
return xt < yt
}
return xt > yt
}
func (h *stringMergeHeap) Push(x interface{}) {
h.items = append(h.items, x.(*stringMergeHeapItem))
}
func (h *stringMergeHeap) Pop() interface{} {
old := h.items
n := len(old)
item := old[n-1]
h.items = old[0 : n-1]
return item
}
type stringMergeHeapItem struct {
itr *bufStringIterator
}
// stringSortedMergeIterator is an iterator that sorts and merges multiple iterators into one.
type stringSortedMergeIterator struct {
inputs []StringIterator
heap *stringSortedMergeHeap
init bool
}
// newStringSortedMergeIterator returns an instance of stringSortedMergeIterator.
func newStringSortedMergeIterator(inputs []StringIterator, opt IteratorOptions) Iterator {
itr := &stringSortedMergeIterator{
inputs: inputs,
heap: &stringSortedMergeHeap{
items: make([]*stringSortedMergeHeapItem, 0, len(inputs)),
opt: opt,
},
}
// Initialize heap items.
for _, input := range inputs {
// Append to the heap.
itr.heap.items = append(itr.heap.items, &stringSortedMergeHeapItem{itr: input})
}
return itr
}
// Stats returns an aggregation of stats from the underlying iterators.
func (itr *stringSortedMergeIterator) Stats() IteratorStats {
var stats IteratorStats
for _, input := range itr.inputs {
stats.Add(input.Stats())
}
return stats
}
// Close closes the underlying iterators.
func (itr *stringSortedMergeIterator) Close() error {
for _, input := range itr.inputs {
input.Close()
}
return nil
}
// Next returns the next points from the iterator.
func (itr *stringSortedMergeIterator) Next() (*StringPoint, error) { return itr.pop() }
// pop returns the next point from the heap.
// Reads the next point from item's cursor and puts it back on the heap.
func (itr *stringSortedMergeIterator) pop() (*StringPoint, error) {
// Initialize the heap. See the MergeIterator to see why this has to be done lazily.
if !itr.init {
items := itr.heap.items
itr.heap.items = make([]*stringSortedMergeHeapItem, 0, len(items))
for _, item := range items {
var err error
if item.point, err = item.itr.Next(); err != nil {
return nil, err
} else if item.point == nil {
continue
}
itr.heap.items = append(itr.heap.items, item)
}
heap.Init(itr.heap)
itr.init = true
}
if len(itr.heap.items) == 0 {
return nil, nil
}
// Read the next item from the heap.
item := heap.Pop(itr.heap).(*stringSortedMergeHeapItem)
if item.err != nil {
return nil, item.err
} else if item.point == nil {
return nil, nil
}
// Copy the point for return.
p := item.point.Clone()
// Read the next item from the cursor. Push back to heap if one exists.
if item.point, item.err = item.itr.Next(); item.point != nil {
heap.Push(itr.heap, item)
}
return p, nil
}
// stringSortedMergeHeap represents a heap of stringSortedMergeHeapItems.
type stringSortedMergeHeap struct {
opt IteratorOptions
items []*stringSortedMergeHeapItem
}
func (h *stringSortedMergeHeap) Len() int { return len(h.items) }
func (h *stringSortedMergeHeap) Swap(i, j int) { h.items[i], h.items[j] = h.items[j], h.items[i] }
func (h *stringSortedMergeHeap) Less(i, j int) bool {
x, y := h.items[i].point, h.items[j].point
if h.opt.Ascending {
if x.Name != y.Name {
return x.Name < y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); !xTags.Equals(&yTags) {
return xTags.ID() < yTags.ID()
}
return x.Time < y.Time
}
if x.Name != y.Name {
return x.Name > y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); !xTags.Equals(&yTags) {
return xTags.ID() > yTags.ID()
}
return x.Time > y.Time
}
func (h *stringSortedMergeHeap) Push(x interface{}) {
h.items = append(h.items, x.(*stringSortedMergeHeapItem))
}
func (h *stringSortedMergeHeap) Pop() interface{} {
old := h.items
n := len(old)
item := old[n-1]
h.items = old[0 : n-1]
return item
}
type stringSortedMergeHeapItem struct {
point *StringPoint
err error
itr StringIterator
}
// stringParallelIterator represents an iterator that pulls data in a separate goroutine.
type stringParallelIterator struct {
input StringIterator
ch chan stringPointError
once sync.Once
closing chan struct{}
wg sync.WaitGroup
}
// newStringParallelIterator returns a new instance of stringParallelIterator.
func newStringParallelIterator(input StringIterator) *stringParallelIterator {
itr := &stringParallelIterator{
input: input,
ch: make(chan stringPointError, 256),
closing: make(chan struct{}),
}
itr.wg.Add(1)
go itr.monitor()
return itr
}
// Stats returns stats from the underlying iterator.
func (itr *stringParallelIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the underlying iterators.
func (itr *stringParallelIterator) Close() error {
itr.once.Do(func() { close(itr.closing) })
itr.wg.Wait()
return itr.input.Close()
}
// Next returns the next point from the iterator.
func (itr *stringParallelIterator) Next() (*StringPoint, error) {
v, ok := <-itr.ch
if !ok {
return nil, io.EOF
}
return v.point, v.err
}
// monitor runs in a separate goroutine and actively pulls the next point.
func (itr *stringParallelIterator) monitor() {
defer close(itr.ch)
defer itr.wg.Done()
for {
// Read next point.
p, err := itr.input.Next()
if p != nil {
p = p.Clone()
}
select {
case <-itr.closing:
return
case itr.ch <- stringPointError{point: p, err: err}:
}
}
}
type stringPointError struct {
point *StringPoint
err error
}
// stringLimitIterator represents an iterator that limits points per group.
type stringLimitIterator struct {
input StringIterator
opt IteratorOptions
n int
prev struct {
name string
tags Tags
}
}
// newStringLimitIterator returns a new instance of stringLimitIterator.
func newStringLimitIterator(input StringIterator, opt IteratorOptions) *stringLimitIterator {
return &stringLimitIterator{
input: input,
opt: opt,
}
}
// Stats returns stats from the underlying iterator.
func (itr *stringLimitIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the underlying iterators.
func (itr *stringLimitIterator) Close() error { return itr.input.Close() }
// Next returns the next point from the iterator.
func (itr *stringLimitIterator) Next() (*StringPoint, error) {
for {
p, err := itr.input.Next()
if p == nil || err != nil {
return nil, err
}
// Reset window and counter if a new window is encountered.
if p.Name != itr.prev.name || !p.Tags.Equals(&itr.prev.tags) {
itr.prev.name = p.Name
itr.prev.tags = p.Tags
itr.n = 0
}
// Increment counter.
itr.n++
// Read next point if not beyond the offset.
if itr.n <= itr.opt.Offset {
continue
}
// Read next point if we're beyond the limit.
if itr.opt.Limit > 0 && (itr.n-itr.opt.Offset) > itr.opt.Limit {
continue
}
return p, nil
}
}
type stringFillIterator struct {
input *bufStringIterator
prev StringPoint
startTime int64
endTime int64
auxFields []interface{}
init bool
opt IteratorOptions
window struct {
name string
tags Tags
time int64
offset int64
}
}
func newStringFillIterator(input StringIterator, expr Expr, opt IteratorOptions) *stringFillIterator {
if opt.Fill == NullFill {
if expr, ok := expr.(*Call); ok && expr.Name == "count" {
opt.Fill = NumberFill
opt.FillValue = ""
}
}
var startTime, endTime int64
if opt.Ascending {
startTime, _ = opt.Window(opt.StartTime)
endTime, _ = opt.Window(opt.EndTime)
} else {
startTime, _ = opt.Window(opt.EndTime)
endTime, _ = opt.Window(opt.StartTime)
}
var auxFields []interface{}
if len(opt.Aux) > 0 {
auxFields = make([]interface{}, len(opt.Aux))
}
return &stringFillIterator{
input: newBufStringIterator(input),
prev: StringPoint{Nil: true},
startTime: startTime,
endTime: endTime,
auxFields: auxFields,
opt: opt,
}
}
func (itr *stringFillIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *stringFillIterator) Close() error { return itr.input.Close() }
func (itr *stringFillIterator) Next() (*StringPoint, error) {
if !itr.init {
p, err := itr.input.peek()
if p == nil || err != nil {
return nil, err
}
itr.window.name, itr.window.tags = p.Name, p.Tags
itr.window.time = itr.startTime
if itr.opt.Location != nil {
_, itr.window.offset = itr.opt.Zone(itr.window.time)
}
itr.init = true
}
p, err := itr.input.Next()
if err != nil {
return nil, err
}
// Check if the next point is outside of our window or is nil.
for p == nil || p.Name != itr.window.name || p.Tags.ID() != itr.window.tags.ID() {
// If we are inside of an interval, unread the point and continue below to
// constructing a new point.
if itr.opt.Ascending {
if itr.window.time <= itr.endTime {
itr.input.unread(p)
p = nil
break
}
} else {
if itr.window.time >= itr.endTime {
itr.input.unread(p)
p = nil
break
}
}
// We are *not* in a current interval. If there is no next point,
// we are at the end of all intervals.
if p == nil {
return nil, nil
}
// Set the new interval.
itr.window.name, itr.window.tags = p.Name, p.Tags
itr.window.time = itr.startTime
if itr.opt.Location != nil {
_, itr.window.offset = itr.opt.Zone(itr.window.time)
}
itr.prev = StringPoint{Nil: true}
break
}
// Check if the point is our next expected point.
if p == nil || (itr.opt.Ascending && p.Time > itr.window.time) || (!itr.opt.Ascending && p.Time < itr.window.time) {
if p != nil {
itr.input.unread(p)
}
p = &StringPoint{
Name: itr.window.name,
Tags: itr.window.tags,
Time: itr.window.time,
Aux: itr.auxFields,
}
switch itr.opt.Fill {
case LinearFill:
fallthrough
case NullFill:
p.Nil = true
case NumberFill:
p.Value = castToString(itr.opt.FillValue)
case PreviousFill:
if !itr.prev.Nil {
p.Value = itr.prev.Value
p.Nil = itr.prev.Nil
} else {
p.Nil = true
}
}
} else {
itr.prev = *p
}
// Advance the expected time. Do not advance to a new window here
// as there may be lingering points with the same timestamp in the previous
// window.
if itr.opt.Ascending {
itr.window.time += int64(itr.opt.Interval.Duration)
} else {
itr.window.time -= int64(itr.opt.Interval.Duration)
}
// Check to see if we have passed over an offset change and adjust the time
// to account for this new offset.
if itr.opt.Location != nil {
if _, offset := itr.opt.Zone(itr.window.time - 1); offset != itr.window.offset {
diff := itr.window.offset - offset
if abs(diff) < int64(itr.opt.Interval.Duration) {
itr.window.time += diff
}
itr.window.offset = offset
}
}
return p, nil
}
// stringIntervalIterator represents a string implementation of IntervalIterator.
type stringIntervalIterator struct {
input StringIterator
opt IteratorOptions
}
func newStringIntervalIterator(input StringIterator, opt IteratorOptions) *stringIntervalIterator {
return &stringIntervalIterator{input: input, opt: opt}
}
func (itr *stringIntervalIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *stringIntervalIterator) Close() error { return itr.input.Close() }
func (itr *stringIntervalIterator) Next() (*StringPoint, error) {
p, err := itr.input.Next()
if p == nil || err != nil {
return nil, err
}
p.Time, _ = itr.opt.Window(p.Time)
// If we see the minimum allowable time, set the time to zero so we don't
// break the default returned time for aggregate queries without times.
if p.Time == MinTime {
p.Time = 0
}
return p, nil
}
// stringInterruptIterator represents a string implementation of InterruptIterator.
type stringInterruptIterator struct {
input StringIterator
closing <-chan struct{}
count int
}
func newStringInterruptIterator(input StringIterator, closing <-chan struct{}) *stringInterruptIterator {
return &stringInterruptIterator{input: input, closing: closing}
}
func (itr *stringInterruptIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *stringInterruptIterator) Close() error { return itr.input.Close() }
func (itr *stringInterruptIterator) Next() (*StringPoint, error) {
// Only check if the channel is closed every N points. This
// intentionally checks on both 0 and N so that if the iterator
// has been interrupted before the first point is emitted it will
// not emit any points.
if itr.count&0xFF == 0xFF {
select {
case <-itr.closing:
return nil, itr.Close()
default:
// Reset iterator count to zero and fall through to emit the next point.
itr.count = 0
}
}
// Increment the counter for every point read.
itr.count++
return itr.input.Next()
}
// stringCloseInterruptIterator represents a string implementation of CloseInterruptIterator.
type stringCloseInterruptIterator struct {
input StringIterator
closing <-chan struct{}
done chan struct{}
once sync.Once
}
func newStringCloseInterruptIterator(input StringIterator, closing <-chan struct{}) *stringCloseInterruptIterator {
itr := &stringCloseInterruptIterator{
input: input,
closing: closing,
done: make(chan struct{}),
}
go itr.monitor()
return itr
}
func (itr *stringCloseInterruptIterator) monitor() {
select {
case <-itr.closing:
itr.Close()
case <-itr.done:
}
}
func (itr *stringCloseInterruptIterator) Stats() IteratorStats {
return itr.input.Stats()
}
func (itr *stringCloseInterruptIterator) Close() error {
itr.once.Do(func() {
close(itr.done)
itr.input.Close()
})
return nil
}
func (itr *stringCloseInterruptIterator) Next() (*StringPoint, error) {
p, err := itr.input.Next()
if err != nil {
// Check if the iterator was closed.
select {
case <-itr.done:
return nil, nil
default:
return nil, err
}
}
return p, nil
}
// auxStringPoint represents a combination of a point and an error for the AuxIterator.
type auxStringPoint struct {
point *StringPoint
err error
}
// stringAuxIterator represents a string implementation of AuxIterator.
type stringAuxIterator struct {
input *bufStringIterator
output chan auxStringPoint
fields *auxIteratorFields
background bool
}
func newStringAuxIterator(input StringIterator, opt IteratorOptions) *stringAuxIterator {
return &stringAuxIterator{
input: newBufStringIterator(input),
output: make(chan auxStringPoint, 1),
fields: newAuxIteratorFields(opt),
}
}
func (itr *stringAuxIterator) Background() {
itr.background = true
itr.Start()
go DrainIterator(itr)
}
func (itr *stringAuxIterator) Start() { go itr.stream() }
func (itr *stringAuxIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *stringAuxIterator) Close() error { return itr.input.Close() }
func (itr *stringAuxIterator) Next() (*StringPoint, error) {
p := <-itr.output
return p.point, p.err
}
func (itr *stringAuxIterator) Iterator(name string, typ DataType) Iterator {
return itr.fields.iterator(name, typ)
}
func (itr *stringAuxIterator) stream() {
for {
// Read next point.
p, err := itr.input.Next()
if err != nil {
itr.output <- auxStringPoint{err: err}
itr.fields.sendError(err)
break
} else if p == nil {
break
}
// Send point to output and to each field iterator.
itr.output <- auxStringPoint{point: p}
if ok := itr.fields.send(p); !ok && itr.background {
break
}
}
close(itr.output)
itr.fields.close()
}
// stringChanIterator represents a new instance of stringChanIterator.
type stringChanIterator struct {
buf struct {
i int
filled bool
points [2]StringPoint
}
err error
cond *sync.Cond
done bool
}
func (itr *stringChanIterator) Stats() IteratorStats { return IteratorStats{} }
func (itr *stringChanIterator) Close() error {
itr.cond.L.Lock()
// Mark the channel iterator as done and signal all waiting goroutines to start again.
itr.done = true
itr.cond.Broadcast()
// Do not defer the unlock so we don't create an unnecessary allocation.
itr.cond.L.Unlock()
return nil
}
func (itr *stringChanIterator) setBuf(name string, tags Tags, time int64, value interface{}) bool {
itr.cond.L.Lock()
defer itr.cond.L.Unlock()
// Wait for either the iterator to be done (so we don't have to set the value)
// or for the buffer to have been read and ready for another write.
for !itr.done && itr.buf.filled {
itr.cond.Wait()
}
// Do not set the value and return false to signal that the iterator is closed.
// Do this after the above wait as the above for loop may have exited because
// the iterator was closed.
if itr.done {
return false
}
switch v := value.(type) {
case string:
itr.buf.points[itr.buf.i] = StringPoint{Name: name, Tags: tags, Time: time, Value: v}
default:
itr.buf.points[itr.buf.i] = StringPoint{Name: name, Tags: tags, Time: time, Nil: true}
}
itr.buf.filled = true
// Signal to all waiting goroutines that a new value is ready to read.
itr.cond.Signal()
return true
}
func (itr *stringChanIterator) setErr(err error) {
itr.cond.L.Lock()
defer itr.cond.L.Unlock()
itr.err = err
// Signal to all waiting goroutines that a new value is ready to read.
itr.cond.Signal()
}
func (itr *stringChanIterator) Next() (*StringPoint, error) {
itr.cond.L.Lock()
defer itr.cond.L.Unlock()
// Check for an error and return one if there.
if itr.err != nil {
return nil, itr.err
}
// Wait until either a value is available in the buffer or
// the iterator is closed.
for !itr.done && !itr.buf.filled {
itr.cond.Wait()
}
// Return nil once the channel is done and the buffer is empty.
if itr.done && !itr.buf.filled {
return nil, nil
}
// Always read from the buffer if it exists, even if the iterator
// is closed. This prevents the last value from being truncated by
// the parent iterator.
p := &itr.buf.points[itr.buf.i]
itr.buf.i = (itr.buf.i + 1) % len(itr.buf.points)
itr.buf.filled = false
itr.cond.Signal()
return p, nil
}
// stringReduceFloatIterator executes a reducer for every interval and buffers the result.
type stringReduceFloatIterator struct {
input *bufStringIterator
create func() (StringPointAggregator, FloatPointEmitter)
dims []string
opt IteratorOptions
points []FloatPoint
keepTags bool
}
func newStringReduceFloatIterator(input StringIterator, opt IteratorOptions, createFn func() (StringPointAggregator, FloatPointEmitter)) *stringReduceFloatIterator {
return &stringReduceFloatIterator{
input: newBufStringIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *stringReduceFloatIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *stringReduceFloatIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *stringReduceFloatIterator) Next() (*FloatPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// stringReduceFloatPoint stores the reduced data for a name/tag combination.
type stringReduceFloatPoint struct {
Name string
Tags Tags
Aggregator StringPointAggregator
Emitter FloatPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *stringReduceFloatIterator) reduce() ([]FloatPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*stringReduceFloatPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &stringReduceFloatPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateString(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]FloatPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(floatPointsByTime(a)))
}
return a, nil
}
// stringStreamFloatIterator streams inputs into the iterator and emits points gradually.
type stringStreamFloatIterator struct {
input *bufStringIterator
create func() (StringPointAggregator, FloatPointEmitter)
dims []string
opt IteratorOptions
m map[string]*stringReduceFloatPoint
points []FloatPoint
}
// newStringStreamFloatIterator returns a new instance of stringStreamFloatIterator.
func newStringStreamFloatIterator(input StringIterator, createFn func() (StringPointAggregator, FloatPointEmitter), opt IteratorOptions) *stringStreamFloatIterator {
return &stringStreamFloatIterator{
input: newBufStringIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*stringReduceFloatPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *stringStreamFloatIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *stringStreamFloatIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *stringStreamFloatIterator) Next() (*FloatPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *stringStreamFloatIterator) reduce() ([]FloatPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []FloatPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &stringReduceFloatPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateString(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// stringFloatExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type stringFloatExprIterator struct {
left *bufStringIterator
right *bufStringIterator
fn stringFloatExprFunc
points []StringPoint // must be size 2
storePrev bool
}
func newStringFloatExprIterator(left, right StringIterator, opt IteratorOptions, fn func(a, b string) float64) *stringFloatExprIterator {
var points []StringPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []StringPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToString(opt.FillValue)
points = []StringPoint{{Value: value}, {Value: value}}
}
return &stringFloatExprIterator{
left: newBufStringIterator(left),
right: newBufStringIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *stringFloatExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *stringFloatExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *stringFloatExprIterator) Next() (*FloatPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = ""
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = ""
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
p := &FloatPoint{
Name: a.Name,
Tags: a.Tags,
Time: a.Time,
Nil: a.Nil || b.Nil,
Aggregated: a.Aggregated,
}
if !p.Nil {
p.Value = itr.fn(a.Value, b.Value)
}
return p, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *stringFloatExprIterator) next() (a, b *StringPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// stringFloatExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type stringFloatExprFunc func(a, b string) float64
// stringReduceIntegerIterator executes a reducer for every interval and buffers the result.
type stringReduceIntegerIterator struct {
input *bufStringIterator
create func() (StringPointAggregator, IntegerPointEmitter)
dims []string
opt IteratorOptions
points []IntegerPoint
keepTags bool
}
func newStringReduceIntegerIterator(input StringIterator, opt IteratorOptions, createFn func() (StringPointAggregator, IntegerPointEmitter)) *stringReduceIntegerIterator {
return &stringReduceIntegerIterator{
input: newBufStringIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *stringReduceIntegerIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *stringReduceIntegerIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *stringReduceIntegerIterator) Next() (*IntegerPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// stringReduceIntegerPoint stores the reduced data for a name/tag combination.
type stringReduceIntegerPoint struct {
Name string
Tags Tags
Aggregator StringPointAggregator
Emitter IntegerPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *stringReduceIntegerIterator) reduce() ([]IntegerPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*stringReduceIntegerPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &stringReduceIntegerPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateString(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]IntegerPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(integerPointsByTime(a)))
}
return a, nil
}
// stringStreamIntegerIterator streams inputs into the iterator and emits points gradually.
type stringStreamIntegerIterator struct {
input *bufStringIterator
create func() (StringPointAggregator, IntegerPointEmitter)
dims []string
opt IteratorOptions
m map[string]*stringReduceIntegerPoint
points []IntegerPoint
}
// newStringStreamIntegerIterator returns a new instance of stringStreamIntegerIterator.
func newStringStreamIntegerIterator(input StringIterator, createFn func() (StringPointAggregator, IntegerPointEmitter), opt IteratorOptions) *stringStreamIntegerIterator {
return &stringStreamIntegerIterator{
input: newBufStringIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*stringReduceIntegerPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *stringStreamIntegerIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *stringStreamIntegerIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *stringStreamIntegerIterator) Next() (*IntegerPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *stringStreamIntegerIterator) reduce() ([]IntegerPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []IntegerPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &stringReduceIntegerPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateString(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// stringIntegerExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type stringIntegerExprIterator struct {
left *bufStringIterator
right *bufStringIterator
fn stringIntegerExprFunc
points []StringPoint // must be size 2
storePrev bool
}
func newStringIntegerExprIterator(left, right StringIterator, opt IteratorOptions, fn func(a, b string) int64) *stringIntegerExprIterator {
var points []StringPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []StringPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToString(opt.FillValue)
points = []StringPoint{{Value: value}, {Value: value}}
}
return &stringIntegerExprIterator{
left: newBufStringIterator(left),
right: newBufStringIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *stringIntegerExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *stringIntegerExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *stringIntegerExprIterator) Next() (*IntegerPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = ""
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = ""
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
p := &IntegerPoint{
Name: a.Name,
Tags: a.Tags,
Time: a.Time,
Nil: a.Nil || b.Nil,
Aggregated: a.Aggregated,
}
if !p.Nil {
p.Value = itr.fn(a.Value, b.Value)
}
return p, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *stringIntegerExprIterator) next() (a, b *StringPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// stringIntegerExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type stringIntegerExprFunc func(a, b string) int64
// stringReduceStringIterator executes a reducer for every interval and buffers the result.
type stringReduceStringIterator struct {
input *bufStringIterator
create func() (StringPointAggregator, StringPointEmitter)
dims []string
opt IteratorOptions
points []StringPoint
keepTags bool
}
func newStringReduceStringIterator(input StringIterator, opt IteratorOptions, createFn func() (StringPointAggregator, StringPointEmitter)) *stringReduceStringIterator {
return &stringReduceStringIterator{
input: newBufStringIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *stringReduceStringIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *stringReduceStringIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *stringReduceStringIterator) Next() (*StringPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// stringReduceStringPoint stores the reduced data for a name/tag combination.
type stringReduceStringPoint struct {
Name string
Tags Tags
Aggregator StringPointAggregator
Emitter StringPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *stringReduceStringIterator) reduce() ([]StringPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*stringReduceStringPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &stringReduceStringPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateString(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]StringPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(stringPointsByTime(a)))
}
return a, nil
}
// stringStreamStringIterator streams inputs into the iterator and emits points gradually.
type stringStreamStringIterator struct {
input *bufStringIterator
create func() (StringPointAggregator, StringPointEmitter)
dims []string
opt IteratorOptions
m map[string]*stringReduceStringPoint
points []StringPoint
}
// newStringStreamStringIterator returns a new instance of stringStreamStringIterator.
func newStringStreamStringIterator(input StringIterator, createFn func() (StringPointAggregator, StringPointEmitter), opt IteratorOptions) *stringStreamStringIterator {
return &stringStreamStringIterator{
input: newBufStringIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*stringReduceStringPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *stringStreamStringIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *stringStreamStringIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *stringStreamStringIterator) Next() (*StringPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *stringStreamStringIterator) reduce() ([]StringPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []StringPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &stringReduceStringPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateString(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// stringExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type stringExprIterator struct {
left *bufStringIterator
right *bufStringIterator
fn stringExprFunc
points []StringPoint // must be size 2
storePrev bool
}
func newStringExprIterator(left, right StringIterator, opt IteratorOptions, fn func(a, b string) string) *stringExprIterator {
var points []StringPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []StringPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToString(opt.FillValue)
points = []StringPoint{{Value: value}, {Value: value}}
}
return &stringExprIterator{
left: newBufStringIterator(left),
right: newBufStringIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *stringExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *stringExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *stringExprIterator) Next() (*StringPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = ""
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = ""
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
if a.Nil {
return a, nil
} else if b.Nil {
return b, nil
}
a.Value = itr.fn(a.Value, b.Value)
return a, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *stringExprIterator) next() (a, b *StringPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// stringExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type stringExprFunc func(a, b string) string
// stringReduceBooleanIterator executes a reducer for every interval and buffers the result.
type stringReduceBooleanIterator struct {
input *bufStringIterator
create func() (StringPointAggregator, BooleanPointEmitter)
dims []string
opt IteratorOptions
points []BooleanPoint
keepTags bool
}
func newStringReduceBooleanIterator(input StringIterator, opt IteratorOptions, createFn func() (StringPointAggregator, BooleanPointEmitter)) *stringReduceBooleanIterator {
return &stringReduceBooleanIterator{
input: newBufStringIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *stringReduceBooleanIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *stringReduceBooleanIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *stringReduceBooleanIterator) Next() (*BooleanPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// stringReduceBooleanPoint stores the reduced data for a name/tag combination.
type stringReduceBooleanPoint struct {
Name string
Tags Tags
Aggregator StringPointAggregator
Emitter BooleanPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *stringReduceBooleanIterator) reduce() ([]BooleanPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*stringReduceBooleanPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &stringReduceBooleanPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateString(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]BooleanPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(booleanPointsByTime(a)))
}
return a, nil
}
// stringStreamBooleanIterator streams inputs into the iterator and emits points gradually.
type stringStreamBooleanIterator struct {
input *bufStringIterator
create func() (StringPointAggregator, BooleanPointEmitter)
dims []string
opt IteratorOptions
m map[string]*stringReduceBooleanPoint
points []BooleanPoint
}
// newStringStreamBooleanIterator returns a new instance of stringStreamBooleanIterator.
func newStringStreamBooleanIterator(input StringIterator, createFn func() (StringPointAggregator, BooleanPointEmitter), opt IteratorOptions) *stringStreamBooleanIterator {
return &stringStreamBooleanIterator{
input: newBufStringIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*stringReduceBooleanPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *stringStreamBooleanIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *stringStreamBooleanIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *stringStreamBooleanIterator) Next() (*BooleanPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *stringStreamBooleanIterator) reduce() ([]BooleanPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []BooleanPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &stringReduceBooleanPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateString(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// stringBooleanExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type stringBooleanExprIterator struct {
left *bufStringIterator
right *bufStringIterator
fn stringBooleanExprFunc
points []StringPoint // must be size 2
storePrev bool
}
func newStringBooleanExprIterator(left, right StringIterator, opt IteratorOptions, fn func(a, b string) bool) *stringBooleanExprIterator {
var points []StringPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []StringPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToString(opt.FillValue)
points = []StringPoint{{Value: value}, {Value: value}}
}
return &stringBooleanExprIterator{
left: newBufStringIterator(left),
right: newBufStringIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *stringBooleanExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *stringBooleanExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *stringBooleanExprIterator) Next() (*BooleanPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = ""
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = ""
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
p := &BooleanPoint{
Name: a.Name,
Tags: a.Tags,
Time: a.Time,
Nil: a.Nil || b.Nil,
Aggregated: a.Aggregated,
}
if !p.Nil {
p.Value = itr.fn(a.Value, b.Value)
}
return p, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *stringBooleanExprIterator) next() (a, b *StringPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// stringBooleanExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type stringBooleanExprFunc func(a, b string) bool
// stringTransformIterator executes a function to modify an existing point for every
// output of the input iterator.
type stringTransformIterator struct {
input StringIterator
fn stringTransformFunc
}
// Stats returns stats from the input iterator.
func (itr *stringTransformIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *stringTransformIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *stringTransformIterator) Next() (*StringPoint, error) {
p, err := itr.input.Next()
if err != nil {
return nil, err
} else if p != nil {
p = itr.fn(p)
}
return p, nil
}
// stringTransformFunc creates or modifies a point.
// The point passed in may be modified and returned rather than allocating a
// new point if possible.
type stringTransformFunc func(p *StringPoint) *StringPoint
// stringBoolTransformIterator executes a function to modify an existing point for every
// output of the input iterator.
type stringBoolTransformIterator struct {
input StringIterator
fn stringBoolTransformFunc
}
// Stats returns stats from the input iterator.
func (itr *stringBoolTransformIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *stringBoolTransformIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *stringBoolTransformIterator) Next() (*BooleanPoint, error) {
p, err := itr.input.Next()
if err != nil {
return nil, err
} else if p != nil {
return itr.fn(p), nil
}
return nil, nil
}
// stringBoolTransformFunc creates or modifies a point.
// The point passed in may be modified and returned rather than allocating a
// new point if possible.
type stringBoolTransformFunc func(p *StringPoint) *BooleanPoint
// stringDedupeIterator only outputs unique points.
// This differs from the DistinctIterator in that it compares all aux fields too.
// This iterator is relatively inefficient and should only be used on small
// datasets such as meta query results.
type stringDedupeIterator struct {
input StringIterator
m map[string]struct{} // lookup of points already sent
}
type stringIteratorMapper struct {
e *Emitter
buf []interface{}
driver IteratorMap // which iterator to use for the primary value, can be nil
fields []IteratorMap // which iterator to use for an aux field
point StringPoint
}
func newStringIteratorMapper(itrs []Iterator, driver IteratorMap, fields []IteratorMap, opt IteratorOptions) *stringIteratorMapper {
e := NewEmitter(itrs, opt.Ascending, 0)
e.OmitTime = true
return &stringIteratorMapper{
e: e,
buf: make([]interface{}, len(itrs)),
driver: driver,
fields: fields,
point: StringPoint{
Aux: make([]interface{}, len(fields)),
},
}
}
func (itr *stringIteratorMapper) Next() (*StringPoint, error) {
t, name, tags, err := itr.e.loadBuf()
if err != nil || t == ZeroTime {
return nil, err
}
itr.point.Time = t
itr.point.Name = name
itr.point.Tags = tags
itr.e.readInto(t, name, tags, itr.buf)
if itr.driver != nil {
if v := itr.driver.Value(tags, itr.buf); v != nil {
if v, ok := v.(string); ok {
itr.point.Value = v
itr.point.Nil = false
} else {
itr.point.Value = ""
itr.point.Nil = true
}
} else {
itr.point.Value = ""
itr.point.Nil = true
}
}
for i, f := range itr.fields {
itr.point.Aux[i] = f.Value(tags, itr.buf)
}
return &itr.point, nil
}
func (itr *stringIteratorMapper) Stats() IteratorStats {
stats := IteratorStats{}
for _, itr := range itr.e.itrs {
stats.Add(itr.Stats())
}
return stats
}
func (itr *stringIteratorMapper) Close() error {
return itr.e.Close()
}
type stringFilterIterator struct {
input StringIterator
cond Expr
opt IteratorOptions
m map[string]interface{}
}
func newStringFilterIterator(input StringIterator, cond Expr, opt IteratorOptions) StringIterator {
// Strip out time conditions from the WHERE clause.
// TODO(jsternberg): This should really be done for us when creating the IteratorOptions struct.
n := RewriteFunc(CloneExpr(cond), func(n Node) Node {
switch n := n.(type) {
case *BinaryExpr:
if n.LHS.String() == "time" {
return &BooleanLiteral{Val: true}
}
}
return n
})
cond, _ = n.(Expr)
if cond == nil {
return input
} else if n, ok := cond.(*BooleanLiteral); ok && n.Val {
return input
}
return &stringFilterIterator{
input: input,
cond: cond,
opt: opt,
m: make(map[string]interface{}),
}
}
func (itr *stringFilterIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *stringFilterIterator) Close() error { return itr.input.Close() }
func (itr *stringFilterIterator) Next() (*StringPoint, error) {
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
}
for i, ref := range itr.opt.Aux {
itr.m[ref.Val] = p.Aux[i]
}
for k, v := range p.Tags.KeyValues() {
itr.m[k] = v
}
if !EvalBool(itr.cond, itr.m) {
continue
}
return p, nil
}
}
// newStringDedupeIterator returns a new instance of stringDedupeIterator.
func newStringDedupeIterator(input StringIterator) *stringDedupeIterator {
return &stringDedupeIterator{
input: input,
m: make(map[string]struct{}),
}
}
// Stats returns stats from the input iterator.
func (itr *stringDedupeIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *stringDedupeIterator) Close() error { return itr.input.Close() }
// Next returns the next unique point from the input iterator.
func (itr *stringDedupeIterator) Next() (*StringPoint, error) {
for {
// Read next point.
p, err := itr.input.Next()
if p == nil || err != nil {
return nil, err
}
// Serialize to bytes to store in lookup.
buf, err := proto.Marshal(encodeStringPoint(p))
if err != nil {
return nil, err
}
// If the point has already been output then move to the next point.
if _, ok := itr.m[string(buf)]; ok {
continue
}
// Otherwise mark it as emitted and return point.
itr.m[string(buf)] = struct{}{}
return p, nil
}
}
// stringReaderIterator represents an iterator that streams from a reader.
type stringReaderIterator struct {
r io.Reader
dec *StringPointDecoder
}
// newStringReaderIterator returns a new instance of stringReaderIterator.
func newStringReaderIterator(r io.Reader, stats IteratorStats) *stringReaderIterator {
dec := NewStringPointDecoder(r)
dec.stats = stats
return &stringReaderIterator{
r: r,
dec: dec,
}
}
// Stats returns stats about points processed.
func (itr *stringReaderIterator) Stats() IteratorStats { return itr.dec.stats }
// Close closes the underlying reader, if applicable.
func (itr *stringReaderIterator) Close() error {
if r, ok := itr.r.(io.ReadCloser); ok {
return r.Close()
}
return nil
}
// Next returns the next point from the iterator.
func (itr *stringReaderIterator) Next() (*StringPoint, error) {
// OPTIMIZE(benbjohnson): Reuse point on iterator.
// Unmarshal next point.
p := &StringPoint{}
if err := itr.dec.DecodeStringPoint(p); err == io.EOF {
return nil, nil
} else if err != nil {
return nil, err
}
return p, nil
}
// BooleanIterator represents a stream of boolean points.
type BooleanIterator interface {
Iterator
Next() (*BooleanPoint, error)
}
// newBooleanIterators converts a slice of Iterator to a slice of BooleanIterator.
// Drop and closes any iterator in itrs that is not a BooleanIterator and cannot
// be cast to a BooleanIterator.
func newBooleanIterators(itrs []Iterator) []BooleanIterator {
a := make([]BooleanIterator, 0, len(itrs))
for _, itr := range itrs {
switch itr := itr.(type) {
case BooleanIterator:
a = append(a, itr)
default:
itr.Close()
}
}
return a
}
// bufBooleanIterator represents a buffered BooleanIterator.
type bufBooleanIterator struct {
itr BooleanIterator
buf *BooleanPoint
}
// newBufBooleanIterator returns a buffered BooleanIterator.
func newBufBooleanIterator(itr BooleanIterator) *bufBooleanIterator {
return &bufBooleanIterator{itr: itr}
}
// Stats returns statistics from the input iterator.
func (itr *bufBooleanIterator) Stats() IteratorStats { return itr.itr.Stats() }
// Close closes the underlying iterator.
func (itr *bufBooleanIterator) Close() error { return itr.itr.Close() }
// peek returns the next point without removing it from the iterator.
func (itr *bufBooleanIterator) peek() (*BooleanPoint, error) {
p, err := itr.Next()
if err != nil {
return nil, err
}
itr.unread(p)
return p, nil
}
// peekTime returns the time of the next point.
// Returns zero time if no more points available.
func (itr *bufBooleanIterator) peekTime() (int64, error) {
p, err := itr.peek()
if p == nil || err != nil {
return ZeroTime, err
}
return p.Time, nil
}
// Next returns the current buffer, if exists, or calls the underlying iterator.
func (itr *bufBooleanIterator) Next() (*BooleanPoint, error) {
buf := itr.buf
if buf != nil {
itr.buf = nil
return buf, nil
}
return itr.itr.Next()
}
// NextInWindow returns the next value if it is between [startTime, endTime).
// If the next value is outside the range then it is moved to the buffer.
func (itr *bufBooleanIterator) NextInWindow(startTime, endTime int64) (*BooleanPoint, error) {
v, err := itr.Next()
if v == nil || err != nil {
return nil, err
} else if t := v.Time; t >= endTime || t < startTime {
itr.unread(v)
return nil, nil
}
return v, nil
}
// unread sets v to the buffer. It is read on the next call to Next().
func (itr *bufBooleanIterator) unread(v *BooleanPoint) { itr.buf = v }
// booleanMergeIterator represents an iterator that combines multiple boolean iterators.
type booleanMergeIterator struct {
inputs []BooleanIterator
heap *booleanMergeHeap
init bool
// Current iterator and window.
curr *booleanMergeHeapItem
window struct {
name string
tags string
startTime int64
endTime int64
}
}
// newBooleanMergeIterator returns a new instance of booleanMergeIterator.
func newBooleanMergeIterator(inputs []BooleanIterator, opt IteratorOptions) *booleanMergeIterator {
itr := &booleanMergeIterator{
inputs: inputs,
heap: &booleanMergeHeap{
items: make([]*booleanMergeHeapItem, 0, len(inputs)),
opt: opt,
},
}
// Initialize heap items.
for _, input := range inputs {
// Wrap in buffer, ignore any inputs without anymore points.
bufInput := newBufBooleanIterator(input)
// Append to the heap.
itr.heap.items = append(itr.heap.items, &booleanMergeHeapItem{itr: bufInput})
}
return itr
}
// Stats returns an aggregation of stats from the underlying iterators.
func (itr *booleanMergeIterator) Stats() IteratorStats {
var stats IteratorStats
for _, input := range itr.inputs {
stats.Add(input.Stats())
}
return stats
}
// Close closes the underlying iterators.
func (itr *booleanMergeIterator) Close() error {
for _, input := range itr.inputs {
input.Close()
}
itr.curr = nil
itr.inputs = nil
itr.heap.items = nil
return nil
}
// Next returns the next point from the iterator.
func (itr *booleanMergeIterator) Next() (*BooleanPoint, error) {
// Initialize the heap. This needs to be done lazily on the first call to this iterator
// so that iterator initialization done through the Select() call returns quickly.
// Queries can only be interrupted after the Select() call completes so any operations
// done during iterator creation cannot be interrupted, which is why we do it here
// instead so an interrupt can happen while initializing the heap.
if !itr.init {
items := itr.heap.items
itr.heap.items = make([]*booleanMergeHeapItem, 0, len(items))
for _, item := range items {
if p, err := item.itr.peek(); err != nil {
return nil, err
} else if p == nil {
continue
}
itr.heap.items = append(itr.heap.items, item)
}
heap.Init(itr.heap)
itr.init = true
}
for {
// Retrieve the next iterator if we don't have one.
if itr.curr == nil {
if len(itr.heap.items) == 0 {
return nil, nil
}
itr.curr = heap.Pop(itr.heap).(*booleanMergeHeapItem)
// Read point and set current window.
p, err := itr.curr.itr.Next()
if err != nil {
return nil, err
}
tags := p.Tags.Subset(itr.heap.opt.Dimensions)
itr.window.name, itr.window.tags = p.Name, tags.ID()
itr.window.startTime, itr.window.endTime = itr.heap.opt.Window(p.Time)
return p, nil
}
// Read the next point from the current iterator.
p, err := itr.curr.itr.Next()
if err != nil {
return nil, err
}
// If there are no more points then remove iterator from heap and find next.
if p == nil {
itr.curr = nil
continue
}
// Check if the point is inside of our current window.
inWindow := true
if window := itr.window; window.name != p.Name {
inWindow = false
} else if tags := p.Tags.Subset(itr.heap.opt.Dimensions); window.tags != tags.ID() {
inWindow = false
} else if opt := itr.heap.opt; opt.Ascending && p.Time >= window.endTime {
inWindow = false
} else if !opt.Ascending && p.Time < window.startTime {
inWindow = false
}
// If it's outside our window then push iterator back on the heap and find new iterator.
if !inWindow {
itr.curr.itr.unread(p)
heap.Push(itr.heap, itr.curr)
itr.curr = nil
continue
}
return p, nil
}
}
// booleanMergeHeap represents a heap of booleanMergeHeapItems.
// Items are sorted by their next window and then by name/tags.
type booleanMergeHeap struct {
opt IteratorOptions
items []*booleanMergeHeapItem
}
func (h *booleanMergeHeap) Len() int { return len(h.items) }
func (h *booleanMergeHeap) Swap(i, j int) { h.items[i], h.items[j] = h.items[j], h.items[i] }
func (h *booleanMergeHeap) Less(i, j int) bool {
x, err := h.items[i].itr.peek()
if err != nil {
return true
}
y, err := h.items[j].itr.peek()
if err != nil {
return false
}
if h.opt.Ascending {
if x.Name != y.Name {
return x.Name < y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); xTags.ID() != yTags.ID() {
return xTags.ID() < yTags.ID()
}
} else {
if x.Name != y.Name {
return x.Name > y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); xTags.ID() != yTags.ID() {
return xTags.ID() > yTags.ID()
}
}
xt, _ := h.opt.Window(x.Time)
yt, _ := h.opt.Window(y.Time)
if h.opt.Ascending {
return xt < yt
}
return xt > yt
}
func (h *booleanMergeHeap) Push(x interface{}) {
h.items = append(h.items, x.(*booleanMergeHeapItem))
}
func (h *booleanMergeHeap) Pop() interface{} {
old := h.items
n := len(old)
item := old[n-1]
h.items = old[0 : n-1]
return item
}
type booleanMergeHeapItem struct {
itr *bufBooleanIterator
}
// booleanSortedMergeIterator is an iterator that sorts and merges multiple iterators into one.
type booleanSortedMergeIterator struct {
inputs []BooleanIterator
heap *booleanSortedMergeHeap
init bool
}
// newBooleanSortedMergeIterator returns an instance of booleanSortedMergeIterator.
func newBooleanSortedMergeIterator(inputs []BooleanIterator, opt IteratorOptions) Iterator {
itr := &booleanSortedMergeIterator{
inputs: inputs,
heap: &booleanSortedMergeHeap{
items: make([]*booleanSortedMergeHeapItem, 0, len(inputs)),
opt: opt,
},
}
// Initialize heap items.
for _, input := range inputs {
// Append to the heap.
itr.heap.items = append(itr.heap.items, &booleanSortedMergeHeapItem{itr: input})
}
return itr
}
// Stats returns an aggregation of stats from the underlying iterators.
func (itr *booleanSortedMergeIterator) Stats() IteratorStats {
var stats IteratorStats
for _, input := range itr.inputs {
stats.Add(input.Stats())
}
return stats
}
// Close closes the underlying iterators.
func (itr *booleanSortedMergeIterator) Close() error {
for _, input := range itr.inputs {
input.Close()
}
return nil
}
// Next returns the next points from the iterator.
func (itr *booleanSortedMergeIterator) Next() (*BooleanPoint, error) { return itr.pop() }
// pop returns the next point from the heap.
// Reads the next point from item's cursor and puts it back on the heap.
func (itr *booleanSortedMergeIterator) pop() (*BooleanPoint, error) {
// Initialize the heap. See the MergeIterator to see why this has to be done lazily.
if !itr.init {
items := itr.heap.items
itr.heap.items = make([]*booleanSortedMergeHeapItem, 0, len(items))
for _, item := range items {
var err error
if item.point, err = item.itr.Next(); err != nil {
return nil, err
} else if item.point == nil {
continue
}
itr.heap.items = append(itr.heap.items, item)
}
heap.Init(itr.heap)
itr.init = true
}
if len(itr.heap.items) == 0 {
return nil, nil
}
// Read the next item from the heap.
item := heap.Pop(itr.heap).(*booleanSortedMergeHeapItem)
if item.err != nil {
return nil, item.err
} else if item.point == nil {
return nil, nil
}
// Copy the point for return.
p := item.point.Clone()
// Read the next item from the cursor. Push back to heap if one exists.
if item.point, item.err = item.itr.Next(); item.point != nil {
heap.Push(itr.heap, item)
}
return p, nil
}
// booleanSortedMergeHeap represents a heap of booleanSortedMergeHeapItems.
type booleanSortedMergeHeap struct {
opt IteratorOptions
items []*booleanSortedMergeHeapItem
}
func (h *booleanSortedMergeHeap) Len() int { return len(h.items) }
func (h *booleanSortedMergeHeap) Swap(i, j int) { h.items[i], h.items[j] = h.items[j], h.items[i] }
func (h *booleanSortedMergeHeap) Less(i, j int) bool {
x, y := h.items[i].point, h.items[j].point
if h.opt.Ascending {
if x.Name != y.Name {
return x.Name < y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); !xTags.Equals(&yTags) {
return xTags.ID() < yTags.ID()
}
return x.Time < y.Time
}
if x.Name != y.Name {
return x.Name > y.Name
} else if xTags, yTags := x.Tags.Subset(h.opt.Dimensions), y.Tags.Subset(h.opt.Dimensions); !xTags.Equals(&yTags) {
return xTags.ID() > yTags.ID()
}
return x.Time > y.Time
}
func (h *booleanSortedMergeHeap) Push(x interface{}) {
h.items = append(h.items, x.(*booleanSortedMergeHeapItem))
}
func (h *booleanSortedMergeHeap) Pop() interface{} {
old := h.items
n := len(old)
item := old[n-1]
h.items = old[0 : n-1]
return item
}
type booleanSortedMergeHeapItem struct {
point *BooleanPoint
err error
itr BooleanIterator
}
// booleanParallelIterator represents an iterator that pulls data in a separate goroutine.
type booleanParallelIterator struct {
input BooleanIterator
ch chan booleanPointError
once sync.Once
closing chan struct{}
wg sync.WaitGroup
}
// newBooleanParallelIterator returns a new instance of booleanParallelIterator.
func newBooleanParallelIterator(input BooleanIterator) *booleanParallelIterator {
itr := &booleanParallelIterator{
input: input,
ch: make(chan booleanPointError, 256),
closing: make(chan struct{}),
}
itr.wg.Add(1)
go itr.monitor()
return itr
}
// Stats returns stats from the underlying iterator.
func (itr *booleanParallelIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the underlying iterators.
func (itr *booleanParallelIterator) Close() error {
itr.once.Do(func() { close(itr.closing) })
itr.wg.Wait()
return itr.input.Close()
}
// Next returns the next point from the iterator.
func (itr *booleanParallelIterator) Next() (*BooleanPoint, error) {
v, ok := <-itr.ch
if !ok {
return nil, io.EOF
}
return v.point, v.err
}
// monitor runs in a separate goroutine and actively pulls the next point.
func (itr *booleanParallelIterator) monitor() {
defer close(itr.ch)
defer itr.wg.Done()
for {
// Read next point.
p, err := itr.input.Next()
if p != nil {
p = p.Clone()
}
select {
case <-itr.closing:
return
case itr.ch <- booleanPointError{point: p, err: err}:
}
}
}
type booleanPointError struct {
point *BooleanPoint
err error
}
// booleanLimitIterator represents an iterator that limits points per group.
type booleanLimitIterator struct {
input BooleanIterator
opt IteratorOptions
n int
prev struct {
name string
tags Tags
}
}
// newBooleanLimitIterator returns a new instance of booleanLimitIterator.
func newBooleanLimitIterator(input BooleanIterator, opt IteratorOptions) *booleanLimitIterator {
return &booleanLimitIterator{
input: input,
opt: opt,
}
}
// Stats returns stats from the underlying iterator.
func (itr *booleanLimitIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the underlying iterators.
func (itr *booleanLimitIterator) Close() error { return itr.input.Close() }
// Next returns the next point from the iterator.
func (itr *booleanLimitIterator) Next() (*BooleanPoint, error) {
for {
p, err := itr.input.Next()
if p == nil || err != nil {
return nil, err
}
// Reset window and counter if a new window is encountered.
if p.Name != itr.prev.name || !p.Tags.Equals(&itr.prev.tags) {
itr.prev.name = p.Name
itr.prev.tags = p.Tags
itr.n = 0
}
// Increment counter.
itr.n++
// Read next point if not beyond the offset.
if itr.n <= itr.opt.Offset {
continue
}
// Read next point if we're beyond the limit.
if itr.opt.Limit > 0 && (itr.n-itr.opt.Offset) > itr.opt.Limit {
continue
}
return p, nil
}
}
type booleanFillIterator struct {
input *bufBooleanIterator
prev BooleanPoint
startTime int64
endTime int64
auxFields []interface{}
init bool
opt IteratorOptions
window struct {
name string
tags Tags
time int64
offset int64
}
}
func newBooleanFillIterator(input BooleanIterator, expr Expr, opt IteratorOptions) *booleanFillIterator {
if opt.Fill == NullFill {
if expr, ok := expr.(*Call); ok && expr.Name == "count" {
opt.Fill = NumberFill
opt.FillValue = false
}
}
var startTime, endTime int64
if opt.Ascending {
startTime, _ = opt.Window(opt.StartTime)
endTime, _ = opt.Window(opt.EndTime)
} else {
startTime, _ = opt.Window(opt.EndTime)
endTime, _ = opt.Window(opt.StartTime)
}
var auxFields []interface{}
if len(opt.Aux) > 0 {
auxFields = make([]interface{}, len(opt.Aux))
}
return &booleanFillIterator{
input: newBufBooleanIterator(input),
prev: BooleanPoint{Nil: true},
startTime: startTime,
endTime: endTime,
auxFields: auxFields,
opt: opt,
}
}
func (itr *booleanFillIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *booleanFillIterator) Close() error { return itr.input.Close() }
func (itr *booleanFillIterator) Next() (*BooleanPoint, error) {
if !itr.init {
p, err := itr.input.peek()
if p == nil || err != nil {
return nil, err
}
itr.window.name, itr.window.tags = p.Name, p.Tags
itr.window.time = itr.startTime
if itr.opt.Location != nil {
_, itr.window.offset = itr.opt.Zone(itr.window.time)
}
itr.init = true
}
p, err := itr.input.Next()
if err != nil {
return nil, err
}
// Check if the next point is outside of our window or is nil.
for p == nil || p.Name != itr.window.name || p.Tags.ID() != itr.window.tags.ID() {
// If we are inside of an interval, unread the point and continue below to
// constructing a new point.
if itr.opt.Ascending {
if itr.window.time <= itr.endTime {
itr.input.unread(p)
p = nil
break
}
} else {
if itr.window.time >= itr.endTime {
itr.input.unread(p)
p = nil
break
}
}
// We are *not* in a current interval. If there is no next point,
// we are at the end of all intervals.
if p == nil {
return nil, nil
}
// Set the new interval.
itr.window.name, itr.window.tags = p.Name, p.Tags
itr.window.time = itr.startTime
if itr.opt.Location != nil {
_, itr.window.offset = itr.opt.Zone(itr.window.time)
}
itr.prev = BooleanPoint{Nil: true}
break
}
// Check if the point is our next expected point.
if p == nil || (itr.opt.Ascending && p.Time > itr.window.time) || (!itr.opt.Ascending && p.Time < itr.window.time) {
if p != nil {
itr.input.unread(p)
}
p = &BooleanPoint{
Name: itr.window.name,
Tags: itr.window.tags,
Time: itr.window.time,
Aux: itr.auxFields,
}
switch itr.opt.Fill {
case LinearFill:
fallthrough
case NullFill:
p.Nil = true
case NumberFill:
p.Value = castToBoolean(itr.opt.FillValue)
case PreviousFill:
if !itr.prev.Nil {
p.Value = itr.prev.Value
p.Nil = itr.prev.Nil
} else {
p.Nil = true
}
}
} else {
itr.prev = *p
}
// Advance the expected time. Do not advance to a new window here
// as there may be lingering points with the same timestamp in the previous
// window.
if itr.opt.Ascending {
itr.window.time += int64(itr.opt.Interval.Duration)
} else {
itr.window.time -= int64(itr.opt.Interval.Duration)
}
// Check to see if we have passed over an offset change and adjust the time
// to account for this new offset.
if itr.opt.Location != nil {
if _, offset := itr.opt.Zone(itr.window.time - 1); offset != itr.window.offset {
diff := itr.window.offset - offset
if abs(diff) < int64(itr.opt.Interval.Duration) {
itr.window.time += diff
}
itr.window.offset = offset
}
}
return p, nil
}
// booleanIntervalIterator represents a boolean implementation of IntervalIterator.
type booleanIntervalIterator struct {
input BooleanIterator
opt IteratorOptions
}
func newBooleanIntervalIterator(input BooleanIterator, opt IteratorOptions) *booleanIntervalIterator {
return &booleanIntervalIterator{input: input, opt: opt}
}
func (itr *booleanIntervalIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *booleanIntervalIterator) Close() error { return itr.input.Close() }
func (itr *booleanIntervalIterator) Next() (*BooleanPoint, error) {
p, err := itr.input.Next()
if p == nil || err != nil {
return nil, err
}
p.Time, _ = itr.opt.Window(p.Time)
// If we see the minimum allowable time, set the time to zero so we don't
// break the default returned time for aggregate queries without times.
if p.Time == MinTime {
p.Time = 0
}
return p, nil
}
// booleanInterruptIterator represents a boolean implementation of InterruptIterator.
type booleanInterruptIterator struct {
input BooleanIterator
closing <-chan struct{}
count int
}
func newBooleanInterruptIterator(input BooleanIterator, closing <-chan struct{}) *booleanInterruptIterator {
return &booleanInterruptIterator{input: input, closing: closing}
}
func (itr *booleanInterruptIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *booleanInterruptIterator) Close() error { return itr.input.Close() }
func (itr *booleanInterruptIterator) Next() (*BooleanPoint, error) {
// Only check if the channel is closed every N points. This
// intentionally checks on both 0 and N so that if the iterator
// has been interrupted before the first point is emitted it will
// not emit any points.
if itr.count&0xFF == 0xFF {
select {
case <-itr.closing:
return nil, itr.Close()
default:
// Reset iterator count to zero and fall through to emit the next point.
itr.count = 0
}
}
// Increment the counter for every point read.
itr.count++
return itr.input.Next()
}
// booleanCloseInterruptIterator represents a boolean implementation of CloseInterruptIterator.
type booleanCloseInterruptIterator struct {
input BooleanIterator
closing <-chan struct{}
done chan struct{}
once sync.Once
}
func newBooleanCloseInterruptIterator(input BooleanIterator, closing <-chan struct{}) *booleanCloseInterruptIterator {
itr := &booleanCloseInterruptIterator{
input: input,
closing: closing,
done: make(chan struct{}),
}
go itr.monitor()
return itr
}
func (itr *booleanCloseInterruptIterator) monitor() {
select {
case <-itr.closing:
itr.Close()
case <-itr.done:
}
}
func (itr *booleanCloseInterruptIterator) Stats() IteratorStats {
return itr.input.Stats()
}
func (itr *booleanCloseInterruptIterator) Close() error {
itr.once.Do(func() {
close(itr.done)
itr.input.Close()
})
return nil
}
func (itr *booleanCloseInterruptIterator) Next() (*BooleanPoint, error) {
p, err := itr.input.Next()
if err != nil {
// Check if the iterator was closed.
select {
case <-itr.done:
return nil, nil
default:
return nil, err
}
}
return p, nil
}
// auxBooleanPoint represents a combination of a point and an error for the AuxIterator.
type auxBooleanPoint struct {
point *BooleanPoint
err error
}
// booleanAuxIterator represents a boolean implementation of AuxIterator.
type booleanAuxIterator struct {
input *bufBooleanIterator
output chan auxBooleanPoint
fields *auxIteratorFields
background bool
}
func newBooleanAuxIterator(input BooleanIterator, opt IteratorOptions) *booleanAuxIterator {
return &booleanAuxIterator{
input: newBufBooleanIterator(input),
output: make(chan auxBooleanPoint, 1),
fields: newAuxIteratorFields(opt),
}
}
func (itr *booleanAuxIterator) Background() {
itr.background = true
itr.Start()
go DrainIterator(itr)
}
func (itr *booleanAuxIterator) Start() { go itr.stream() }
func (itr *booleanAuxIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *booleanAuxIterator) Close() error { return itr.input.Close() }
func (itr *booleanAuxIterator) Next() (*BooleanPoint, error) {
p := <-itr.output
return p.point, p.err
}
func (itr *booleanAuxIterator) Iterator(name string, typ DataType) Iterator {
return itr.fields.iterator(name, typ)
}
func (itr *booleanAuxIterator) stream() {
for {
// Read next point.
p, err := itr.input.Next()
if err != nil {
itr.output <- auxBooleanPoint{err: err}
itr.fields.sendError(err)
break
} else if p == nil {
break
}
// Send point to output and to each field iterator.
itr.output <- auxBooleanPoint{point: p}
if ok := itr.fields.send(p); !ok && itr.background {
break
}
}
close(itr.output)
itr.fields.close()
}
// booleanChanIterator represents a new instance of booleanChanIterator.
type booleanChanIterator struct {
buf struct {
i int
filled bool
points [2]BooleanPoint
}
err error
cond *sync.Cond
done bool
}
func (itr *booleanChanIterator) Stats() IteratorStats { return IteratorStats{} }
func (itr *booleanChanIterator) Close() error {
itr.cond.L.Lock()
// Mark the channel iterator as done and signal all waiting goroutines to start again.
itr.done = true
itr.cond.Broadcast()
// Do not defer the unlock so we don't create an unnecessary allocation.
itr.cond.L.Unlock()
return nil
}
func (itr *booleanChanIterator) setBuf(name string, tags Tags, time int64, value interface{}) bool {
itr.cond.L.Lock()
defer itr.cond.L.Unlock()
// Wait for either the iterator to be done (so we don't have to set the value)
// or for the buffer to have been read and ready for another write.
for !itr.done && itr.buf.filled {
itr.cond.Wait()
}
// Do not set the value and return false to signal that the iterator is closed.
// Do this after the above wait as the above for loop may have exited because
// the iterator was closed.
if itr.done {
return false
}
switch v := value.(type) {
case bool:
itr.buf.points[itr.buf.i] = BooleanPoint{Name: name, Tags: tags, Time: time, Value: v}
default:
itr.buf.points[itr.buf.i] = BooleanPoint{Name: name, Tags: tags, Time: time, Nil: true}
}
itr.buf.filled = true
// Signal to all waiting goroutines that a new value is ready to read.
itr.cond.Signal()
return true
}
func (itr *booleanChanIterator) setErr(err error) {
itr.cond.L.Lock()
defer itr.cond.L.Unlock()
itr.err = err
// Signal to all waiting goroutines that a new value is ready to read.
itr.cond.Signal()
}
func (itr *booleanChanIterator) Next() (*BooleanPoint, error) {
itr.cond.L.Lock()
defer itr.cond.L.Unlock()
// Check for an error and return one if there.
if itr.err != nil {
return nil, itr.err
}
// Wait until either a value is available in the buffer or
// the iterator is closed.
for !itr.done && !itr.buf.filled {
itr.cond.Wait()
}
// Return nil once the channel is done and the buffer is empty.
if itr.done && !itr.buf.filled {
return nil, nil
}
// Always read from the buffer if it exists, even if the iterator
// is closed. This prevents the last value from being truncated by
// the parent iterator.
p := &itr.buf.points[itr.buf.i]
itr.buf.i = (itr.buf.i + 1) % len(itr.buf.points)
itr.buf.filled = false
itr.cond.Signal()
return p, nil
}
// booleanReduceFloatIterator executes a reducer for every interval and buffers the result.
type booleanReduceFloatIterator struct {
input *bufBooleanIterator
create func() (BooleanPointAggregator, FloatPointEmitter)
dims []string
opt IteratorOptions
points []FloatPoint
keepTags bool
}
func newBooleanReduceFloatIterator(input BooleanIterator, opt IteratorOptions, createFn func() (BooleanPointAggregator, FloatPointEmitter)) *booleanReduceFloatIterator {
return &booleanReduceFloatIterator{
input: newBufBooleanIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *booleanReduceFloatIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *booleanReduceFloatIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *booleanReduceFloatIterator) Next() (*FloatPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// booleanReduceFloatPoint stores the reduced data for a name/tag combination.
type booleanReduceFloatPoint struct {
Name string
Tags Tags
Aggregator BooleanPointAggregator
Emitter FloatPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *booleanReduceFloatIterator) reduce() ([]FloatPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*booleanReduceFloatPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &booleanReduceFloatPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateBoolean(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]FloatPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(floatPointsByTime(a)))
}
return a, nil
}
// booleanStreamFloatIterator streams inputs into the iterator and emits points gradually.
type booleanStreamFloatIterator struct {
input *bufBooleanIterator
create func() (BooleanPointAggregator, FloatPointEmitter)
dims []string
opt IteratorOptions
m map[string]*booleanReduceFloatPoint
points []FloatPoint
}
// newBooleanStreamFloatIterator returns a new instance of booleanStreamFloatIterator.
func newBooleanStreamFloatIterator(input BooleanIterator, createFn func() (BooleanPointAggregator, FloatPointEmitter), opt IteratorOptions) *booleanStreamFloatIterator {
return &booleanStreamFloatIterator{
input: newBufBooleanIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*booleanReduceFloatPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *booleanStreamFloatIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *booleanStreamFloatIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *booleanStreamFloatIterator) Next() (*FloatPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *booleanStreamFloatIterator) reduce() ([]FloatPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []FloatPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &booleanReduceFloatPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateBoolean(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// booleanFloatExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type booleanFloatExprIterator struct {
left *bufBooleanIterator
right *bufBooleanIterator
fn booleanFloatExprFunc
points []BooleanPoint // must be size 2
storePrev bool
}
func newBooleanFloatExprIterator(left, right BooleanIterator, opt IteratorOptions, fn func(a, b bool) float64) *booleanFloatExprIterator {
var points []BooleanPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []BooleanPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToBoolean(opt.FillValue)
points = []BooleanPoint{{Value: value}, {Value: value}}
}
return &booleanFloatExprIterator{
left: newBufBooleanIterator(left),
right: newBufBooleanIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *booleanFloatExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *booleanFloatExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *booleanFloatExprIterator) Next() (*FloatPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = false
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = false
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
p := &FloatPoint{
Name: a.Name,
Tags: a.Tags,
Time: a.Time,
Nil: a.Nil || b.Nil,
Aggregated: a.Aggregated,
}
if !p.Nil {
p.Value = itr.fn(a.Value, b.Value)
}
return p, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *booleanFloatExprIterator) next() (a, b *BooleanPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// booleanFloatExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type booleanFloatExprFunc func(a, b bool) float64
// booleanReduceIntegerIterator executes a reducer for every interval and buffers the result.
type booleanReduceIntegerIterator struct {
input *bufBooleanIterator
create func() (BooleanPointAggregator, IntegerPointEmitter)
dims []string
opt IteratorOptions
points []IntegerPoint
keepTags bool
}
func newBooleanReduceIntegerIterator(input BooleanIterator, opt IteratorOptions, createFn func() (BooleanPointAggregator, IntegerPointEmitter)) *booleanReduceIntegerIterator {
return &booleanReduceIntegerIterator{
input: newBufBooleanIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *booleanReduceIntegerIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *booleanReduceIntegerIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *booleanReduceIntegerIterator) Next() (*IntegerPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// booleanReduceIntegerPoint stores the reduced data for a name/tag combination.
type booleanReduceIntegerPoint struct {
Name string
Tags Tags
Aggregator BooleanPointAggregator
Emitter IntegerPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *booleanReduceIntegerIterator) reduce() ([]IntegerPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*booleanReduceIntegerPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &booleanReduceIntegerPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateBoolean(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]IntegerPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(integerPointsByTime(a)))
}
return a, nil
}
// booleanStreamIntegerIterator streams inputs into the iterator and emits points gradually.
type booleanStreamIntegerIterator struct {
input *bufBooleanIterator
create func() (BooleanPointAggregator, IntegerPointEmitter)
dims []string
opt IteratorOptions
m map[string]*booleanReduceIntegerPoint
points []IntegerPoint
}
// newBooleanStreamIntegerIterator returns a new instance of booleanStreamIntegerIterator.
func newBooleanStreamIntegerIterator(input BooleanIterator, createFn func() (BooleanPointAggregator, IntegerPointEmitter), opt IteratorOptions) *booleanStreamIntegerIterator {
return &booleanStreamIntegerIterator{
input: newBufBooleanIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*booleanReduceIntegerPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *booleanStreamIntegerIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *booleanStreamIntegerIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *booleanStreamIntegerIterator) Next() (*IntegerPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *booleanStreamIntegerIterator) reduce() ([]IntegerPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []IntegerPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &booleanReduceIntegerPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateBoolean(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// booleanIntegerExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type booleanIntegerExprIterator struct {
left *bufBooleanIterator
right *bufBooleanIterator
fn booleanIntegerExprFunc
points []BooleanPoint // must be size 2
storePrev bool
}
func newBooleanIntegerExprIterator(left, right BooleanIterator, opt IteratorOptions, fn func(a, b bool) int64) *booleanIntegerExprIterator {
var points []BooleanPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []BooleanPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToBoolean(opt.FillValue)
points = []BooleanPoint{{Value: value}, {Value: value}}
}
return &booleanIntegerExprIterator{
left: newBufBooleanIterator(left),
right: newBufBooleanIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *booleanIntegerExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *booleanIntegerExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *booleanIntegerExprIterator) Next() (*IntegerPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = false
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = false
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
p := &IntegerPoint{
Name: a.Name,
Tags: a.Tags,
Time: a.Time,
Nil: a.Nil || b.Nil,
Aggregated: a.Aggregated,
}
if !p.Nil {
p.Value = itr.fn(a.Value, b.Value)
}
return p, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *booleanIntegerExprIterator) next() (a, b *BooleanPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// booleanIntegerExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type booleanIntegerExprFunc func(a, b bool) int64
// booleanReduceStringIterator executes a reducer for every interval and buffers the result.
type booleanReduceStringIterator struct {
input *bufBooleanIterator
create func() (BooleanPointAggregator, StringPointEmitter)
dims []string
opt IteratorOptions
points []StringPoint
keepTags bool
}
func newBooleanReduceStringIterator(input BooleanIterator, opt IteratorOptions, createFn func() (BooleanPointAggregator, StringPointEmitter)) *booleanReduceStringIterator {
return &booleanReduceStringIterator{
input: newBufBooleanIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *booleanReduceStringIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *booleanReduceStringIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *booleanReduceStringIterator) Next() (*StringPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// booleanReduceStringPoint stores the reduced data for a name/tag combination.
type booleanReduceStringPoint struct {
Name string
Tags Tags
Aggregator BooleanPointAggregator
Emitter StringPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *booleanReduceStringIterator) reduce() ([]StringPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*booleanReduceStringPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &booleanReduceStringPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateBoolean(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]StringPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(stringPointsByTime(a)))
}
return a, nil
}
// booleanStreamStringIterator streams inputs into the iterator and emits points gradually.
type booleanStreamStringIterator struct {
input *bufBooleanIterator
create func() (BooleanPointAggregator, StringPointEmitter)
dims []string
opt IteratorOptions
m map[string]*booleanReduceStringPoint
points []StringPoint
}
// newBooleanStreamStringIterator returns a new instance of booleanStreamStringIterator.
func newBooleanStreamStringIterator(input BooleanIterator, createFn func() (BooleanPointAggregator, StringPointEmitter), opt IteratorOptions) *booleanStreamStringIterator {
return &booleanStreamStringIterator{
input: newBufBooleanIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*booleanReduceStringPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *booleanStreamStringIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *booleanStreamStringIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *booleanStreamStringIterator) Next() (*StringPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *booleanStreamStringIterator) reduce() ([]StringPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []StringPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &booleanReduceStringPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateBoolean(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// booleanStringExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type booleanStringExprIterator struct {
left *bufBooleanIterator
right *bufBooleanIterator
fn booleanStringExprFunc
points []BooleanPoint // must be size 2
storePrev bool
}
func newBooleanStringExprIterator(left, right BooleanIterator, opt IteratorOptions, fn func(a, b bool) string) *booleanStringExprIterator {
var points []BooleanPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []BooleanPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToBoolean(opt.FillValue)
points = []BooleanPoint{{Value: value}, {Value: value}}
}
return &booleanStringExprIterator{
left: newBufBooleanIterator(left),
right: newBufBooleanIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *booleanStringExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *booleanStringExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *booleanStringExprIterator) Next() (*StringPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = false
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = false
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
p := &StringPoint{
Name: a.Name,
Tags: a.Tags,
Time: a.Time,
Nil: a.Nil || b.Nil,
Aggregated: a.Aggregated,
}
if !p.Nil {
p.Value = itr.fn(a.Value, b.Value)
}
return p, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *booleanStringExprIterator) next() (a, b *BooleanPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// booleanStringExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type booleanStringExprFunc func(a, b bool) string
// booleanReduceBooleanIterator executes a reducer for every interval and buffers the result.
type booleanReduceBooleanIterator struct {
input *bufBooleanIterator
create func() (BooleanPointAggregator, BooleanPointEmitter)
dims []string
opt IteratorOptions
points []BooleanPoint
keepTags bool
}
func newBooleanReduceBooleanIterator(input BooleanIterator, opt IteratorOptions, createFn func() (BooleanPointAggregator, BooleanPointEmitter)) *booleanReduceBooleanIterator {
return &booleanReduceBooleanIterator{
input: newBufBooleanIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
}
}
// Stats returns stats from the input iterator.
func (itr *booleanReduceBooleanIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *booleanReduceBooleanIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *booleanReduceBooleanIterator) Next() (*BooleanPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// booleanReduceBooleanPoint stores the reduced data for a name/tag combination.
type booleanReduceBooleanPoint struct {
Name string
Tags Tags
Aggregator BooleanPointAggregator
Emitter BooleanPointEmitter
}
// reduce executes fn once for every point in the next window.
// The previous value for the dimension is passed to fn.
func (itr *booleanReduceBooleanIterator) reduce() ([]BooleanPoint, error) {
// Calculate next window.
var (
startTime, endTime int64
window struct {
name string
tags string
}
)
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
} else if p.Nil {
continue
}
// Unread the point so it can be processed.
itr.input.unread(p)
startTime, endTime = itr.opt.Window(p.Time)
window.name, window.tags = p.Name, p.Tags.Subset(itr.opt.Dimensions).ID()
break
}
// Create points by tags.
m := make(map[string]*booleanReduceBooleanPoint)
for {
// Read next point.
curr, err := itr.input.NextInWindow(startTime, endTime)
if err != nil {
return nil, err
} else if curr == nil {
break
} else if curr.Nil {
continue
} else if curr.Name != window.name {
itr.input.unread(curr)
break
}
// Ensure this point is within the same final window.
if curr.Name != window.name {
itr.input.unread(curr)
break
} else if tags := curr.Tags.Subset(itr.opt.Dimensions); tags.ID() != window.tags {
itr.input.unread(curr)
break
}
// Retrieve the tags on this point for this level of the query.
// This may be different than the bucket dimensions.
tags := curr.Tags.Subset(itr.dims)
id := tags.ID()
// Retrieve the aggregator for this name/tag combination or create one.
rp := m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &booleanReduceBooleanPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
m[id] = rp
}
rp.Aggregator.AggregateBoolean(curr)
}
// Reverse sort points by name & tag if our output is supposed to be ordered.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
if len(keys) > 1 && itr.opt.Ordered {
sort.Sort(reverseStringSlice(keys))
}
// Assume the points are already sorted until proven otherwise.
sortedByTime := true
// Emit the points for each name & tag combination.
a := make([]BooleanPoint, 0, len(m))
for _, k := range keys {
rp := m[k]
points := rp.Emitter.Emit()
for i := len(points) - 1; i >= 0; i-- {
points[i].Name = rp.Name
if !itr.keepTags {
points[i].Tags = rp.Tags
}
// Set the points time to the interval time if the reducer didn't provide one.
if points[i].Time == ZeroTime {
points[i].Time = startTime
} else {
sortedByTime = false
}
a = append(a, points[i])
}
}
// Points may be out of order. Perform a stable sort by time if requested.
if !sortedByTime && itr.opt.Ordered {
sort.Stable(sort.Reverse(booleanPointsByTime(a)))
}
return a, nil
}
// booleanStreamBooleanIterator streams inputs into the iterator and emits points gradually.
type booleanStreamBooleanIterator struct {
input *bufBooleanIterator
create func() (BooleanPointAggregator, BooleanPointEmitter)
dims []string
opt IteratorOptions
m map[string]*booleanReduceBooleanPoint
points []BooleanPoint
}
// newBooleanStreamBooleanIterator returns a new instance of booleanStreamBooleanIterator.
func newBooleanStreamBooleanIterator(input BooleanIterator, createFn func() (BooleanPointAggregator, BooleanPointEmitter), opt IteratorOptions) *booleanStreamBooleanIterator {
return &booleanStreamBooleanIterator{
input: newBufBooleanIterator(input),
create: createFn,
dims: opt.GetDimensions(),
opt: opt,
m: make(map[string]*booleanReduceBooleanPoint),
}
}
// Stats returns stats from the input iterator.
func (itr *booleanStreamBooleanIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *booleanStreamBooleanIterator) Close() error { return itr.input.Close() }
// Next returns the next value for the stream iterator.
func (itr *booleanStreamBooleanIterator) Next() (*BooleanPoint, error) {
// Calculate next window if we have no more points.
if len(itr.points) == 0 {
var err error
itr.points, err = itr.reduce()
if len(itr.points) == 0 {
return nil, err
}
}
// Pop next point off the stack.
p := &itr.points[len(itr.points)-1]
itr.points = itr.points[:len(itr.points)-1]
return p, nil
}
// reduce creates and manages aggregators for every point from the input.
// After aggregating a point, it always tries to emit a value using the emitter.
func (itr *booleanStreamBooleanIterator) reduce() ([]BooleanPoint, error) {
for {
// Read next point.
curr, err := itr.input.Next()
if curr == nil {
// Close all of the aggregators to flush any remaining points to emit.
var points []BooleanPoint
for _, rp := range itr.m {
if aggregator, ok := rp.Aggregator.(io.Closer); ok {
if err := aggregator.Close(); err != nil {
return nil, err
}
pts := rp.Emitter.Emit()
if len(pts) == 0 {
continue
}
for i := range pts {
pts[i].Name = rp.Name
pts[i].Tags = rp.Tags
}
points = append(points, pts...)
}
}
// Eliminate the aggregators and emitters.
itr.m = nil
return points, nil
} else if err != nil {
return nil, err
} else if curr.Nil {
continue
}
tags := curr.Tags.Subset(itr.dims)
id := curr.Name
if len(tags.m) > 0 {
id += "\x00" + tags.ID()
}
// Retrieve the aggregator for this name/tag combination or create one.
rp := itr.m[id]
if rp == nil {
aggregator, emitter := itr.create()
rp = &booleanReduceBooleanPoint{
Name: curr.Name,
Tags: tags,
Aggregator: aggregator,
Emitter: emitter,
}
itr.m[id] = rp
}
rp.Aggregator.AggregateBoolean(curr)
// Attempt to emit points from the aggregator.
points := rp.Emitter.Emit()
if len(points) == 0 {
continue
}
for i := range points {
points[i].Name = rp.Name
points[i].Tags = rp.Tags
}
return points, nil
}
}
// booleanExprIterator executes a function to modify an existing point
// for every output of the input iterator.
type booleanExprIterator struct {
left *bufBooleanIterator
right *bufBooleanIterator
fn booleanExprFunc
points []BooleanPoint // must be size 2
storePrev bool
}
func newBooleanExprIterator(left, right BooleanIterator, opt IteratorOptions, fn func(a, b bool) bool) *booleanExprIterator {
var points []BooleanPoint
switch opt.Fill {
case NullFill, PreviousFill:
points = []BooleanPoint{{Nil: true}, {Nil: true}}
case NumberFill:
value := castToBoolean(opt.FillValue)
points = []BooleanPoint{{Value: value}, {Value: value}}
}
return &booleanExprIterator{
left: newBufBooleanIterator(left),
right: newBufBooleanIterator(right),
points: points,
fn: fn,
storePrev: opt.Fill == PreviousFill,
}
}
func (itr *booleanExprIterator) Stats() IteratorStats {
stats := itr.left.Stats()
stats.Add(itr.right.Stats())
return stats
}
func (itr *booleanExprIterator) Close() error {
itr.left.Close()
itr.right.Close()
return nil
}
func (itr *booleanExprIterator) Next() (*BooleanPoint, error) {
for {
a, b, err := itr.next()
if err != nil || (a == nil && b == nil) {
return nil, err
}
// If any of these are nil and we are using fill(none), skip these points.
if (a == nil || a.Nil || b == nil || b.Nil) && itr.points == nil {
continue
}
// If one of the two points is nil, we need to fill it with a fake nil
// point that has the same name, tags, and time as the other point.
// There should never be a time when both of these are nil.
if a == nil {
p := *b
a = &p
a.Value = false
a.Nil = true
} else if b == nil {
p := *a
b = &p
b.Value = false
b.Nil = true
}
// If a value is nil, use the fill values if the fill value is non-nil.
if a.Nil && !itr.points[0].Nil {
a.Value = itr.points[0].Value
a.Nil = false
}
if b.Nil && !itr.points[1].Nil {
b.Value = itr.points[1].Value
b.Nil = false
}
if itr.storePrev {
itr.points[0], itr.points[1] = *a, *b
}
if a.Nil {
return a, nil
} else if b.Nil {
return b, nil
}
a.Value = itr.fn(a.Value, b.Value)
return a, nil
}
}
// next returns the next points within each iterator. If the iterators are
// uneven, it organizes them so only matching points are returned.
func (itr *booleanExprIterator) next() (a, b *BooleanPoint, err error) {
// Retrieve the next value for both the left and right.
a, err = itr.left.Next()
if err != nil {
return nil, nil, err
}
b, err = itr.right.Next()
if err != nil {
return nil, nil, err
}
// If we have a point from both, make sure that they match each other.
if a != nil && b != nil {
if a.Name > b.Name {
itr.left.unread(a)
return nil, b, nil
} else if a.Name < b.Name {
itr.right.unread(b)
return a, nil, nil
}
if ltags, rtags := a.Tags.ID(), b.Tags.ID(); ltags > rtags {
itr.left.unread(a)
return nil, b, nil
} else if ltags < rtags {
itr.right.unread(b)
return a, nil, nil
}
if a.Time > b.Time {
itr.left.unread(a)
return nil, b, nil
} else if a.Time < b.Time {
itr.right.unread(b)
return a, nil, nil
}
}
return a, b, nil
}
// booleanExprFunc creates or modifies a point by combining two
// points. The point passed in may be modified and returned rather than
// allocating a new point if possible. One of the points may be nil, but at
// least one of the points will be non-nil.
type booleanExprFunc func(a, b bool) bool
// booleanTransformIterator executes a function to modify an existing point for every
// output of the input iterator.
type booleanTransformIterator struct {
input BooleanIterator
fn booleanTransformFunc
}
// Stats returns stats from the input iterator.
func (itr *booleanTransformIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *booleanTransformIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *booleanTransformIterator) Next() (*BooleanPoint, error) {
p, err := itr.input.Next()
if err != nil {
return nil, err
} else if p != nil {
p = itr.fn(p)
}
return p, nil
}
// booleanTransformFunc creates or modifies a point.
// The point passed in may be modified and returned rather than allocating a
// new point if possible.
type booleanTransformFunc func(p *BooleanPoint) *BooleanPoint
// booleanBoolTransformIterator executes a function to modify an existing point for every
// output of the input iterator.
type booleanBoolTransformIterator struct {
input BooleanIterator
fn booleanBoolTransformFunc
}
// Stats returns stats from the input iterator.
func (itr *booleanBoolTransformIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *booleanBoolTransformIterator) Close() error { return itr.input.Close() }
// Next returns the minimum value for the next available interval.
func (itr *booleanBoolTransformIterator) Next() (*BooleanPoint, error) {
p, err := itr.input.Next()
if err != nil {
return nil, err
} else if p != nil {
return itr.fn(p), nil
}
return nil, nil
}
// booleanBoolTransformFunc creates or modifies a point.
// The point passed in may be modified and returned rather than allocating a
// new point if possible.
type booleanBoolTransformFunc func(p *BooleanPoint) *BooleanPoint
// booleanDedupeIterator only outputs unique points.
// This differs from the DistinctIterator in that it compares all aux fields too.
// This iterator is relatively inefficient and should only be used on small
// datasets such as meta query results.
type booleanDedupeIterator struct {
input BooleanIterator
m map[string]struct{} // lookup of points already sent
}
type booleanIteratorMapper struct {
e *Emitter
buf []interface{}
driver IteratorMap // which iterator to use for the primary value, can be nil
fields []IteratorMap // which iterator to use for an aux field
point BooleanPoint
}
func newBooleanIteratorMapper(itrs []Iterator, driver IteratorMap, fields []IteratorMap, opt IteratorOptions) *booleanIteratorMapper {
e := NewEmitter(itrs, opt.Ascending, 0)
e.OmitTime = true
return &booleanIteratorMapper{
e: e,
buf: make([]interface{}, len(itrs)),
driver: driver,
fields: fields,
point: BooleanPoint{
Aux: make([]interface{}, len(fields)),
},
}
}
func (itr *booleanIteratorMapper) Next() (*BooleanPoint, error) {
t, name, tags, err := itr.e.loadBuf()
if err != nil || t == ZeroTime {
return nil, err
}
itr.point.Time = t
itr.point.Name = name
itr.point.Tags = tags
itr.e.readInto(t, name, tags, itr.buf)
if itr.driver != nil {
if v := itr.driver.Value(tags, itr.buf); v != nil {
if v, ok := v.(bool); ok {
itr.point.Value = v
itr.point.Nil = false
} else {
itr.point.Value = false
itr.point.Nil = true
}
} else {
itr.point.Value = false
itr.point.Nil = true
}
}
for i, f := range itr.fields {
itr.point.Aux[i] = f.Value(tags, itr.buf)
}
return &itr.point, nil
}
func (itr *booleanIteratorMapper) Stats() IteratorStats {
stats := IteratorStats{}
for _, itr := range itr.e.itrs {
stats.Add(itr.Stats())
}
return stats
}
func (itr *booleanIteratorMapper) Close() error {
return itr.e.Close()
}
type booleanFilterIterator struct {
input BooleanIterator
cond Expr
opt IteratorOptions
m map[string]interface{}
}
func newBooleanFilterIterator(input BooleanIterator, cond Expr, opt IteratorOptions) BooleanIterator {
// Strip out time conditions from the WHERE clause.
// TODO(jsternberg): This should really be done for us when creating the IteratorOptions struct.
n := RewriteFunc(CloneExpr(cond), func(n Node) Node {
switch n := n.(type) {
case *BinaryExpr:
if n.LHS.String() == "time" {
return &BooleanLiteral{Val: true}
}
}
return n
})
cond, _ = n.(Expr)
if cond == nil {
return input
} else if n, ok := cond.(*BooleanLiteral); ok && n.Val {
return input
}
return &booleanFilterIterator{
input: input,
cond: cond,
opt: opt,
m: make(map[string]interface{}),
}
}
func (itr *booleanFilterIterator) Stats() IteratorStats { return itr.input.Stats() }
func (itr *booleanFilterIterator) Close() error { return itr.input.Close() }
func (itr *booleanFilterIterator) Next() (*BooleanPoint, error) {
for {
p, err := itr.input.Next()
if err != nil || p == nil {
return nil, err
}
for i, ref := range itr.opt.Aux {
itr.m[ref.Val] = p.Aux[i]
}
for k, v := range p.Tags.KeyValues() {
itr.m[k] = v
}
if !EvalBool(itr.cond, itr.m) {
continue
}
return p, nil
}
}
// newBooleanDedupeIterator returns a new instance of booleanDedupeIterator.
func newBooleanDedupeIterator(input BooleanIterator) *booleanDedupeIterator {
return &booleanDedupeIterator{
input: input,
m: make(map[string]struct{}),
}
}
// Stats returns stats from the input iterator.
func (itr *booleanDedupeIterator) Stats() IteratorStats { return itr.input.Stats() }
// Close closes the iterator and all child iterators.
func (itr *booleanDedupeIterator) Close() error { return itr.input.Close() }
// Next returns the next unique point from the input iterator.
func (itr *booleanDedupeIterator) Next() (*BooleanPoint, error) {
for {
// Read next point.
p, err := itr.input.Next()
if p == nil || err != nil {
return nil, err
}
// Serialize to bytes to store in lookup.
buf, err := proto.Marshal(encodeBooleanPoint(p))
if err != nil {
return nil, err
}
// If the point has already been output then move to the next point.
if _, ok := itr.m[string(buf)]; ok {
continue
}
// Otherwise mark it as emitted and return point.
itr.m[string(buf)] = struct{}{}
return p, nil
}
}
// booleanReaderIterator represents an iterator that streams from a reader.
type booleanReaderIterator struct {
r io.Reader
dec *BooleanPointDecoder
}
// newBooleanReaderIterator returns a new instance of booleanReaderIterator.
func newBooleanReaderIterator(r io.Reader, stats IteratorStats) *booleanReaderIterator {
dec := NewBooleanPointDecoder(r)
dec.stats = stats
return &booleanReaderIterator{
r: r,
dec: dec,
}
}
// Stats returns stats about points processed.
func (itr *booleanReaderIterator) Stats() IteratorStats { return itr.dec.stats }
// Close closes the underlying reader, if applicable.
func (itr *booleanReaderIterator) Close() error {
if r, ok := itr.r.(io.ReadCloser); ok {
return r.Close()
}
return nil
}
// Next returns the next point from the iterator.
func (itr *booleanReaderIterator) Next() (*BooleanPoint, error) {
// OPTIMIZE(benbjohnson): Reuse point on iterator.
// Unmarshal next point.
p := &BooleanPoint{}
if err := itr.dec.DecodeBooleanPoint(p); err == io.EOF {
return nil, nil
} else if err != nil {
return nil, err
}
return p, nil
}
// IteratorEncoder is an encoder for encoding an iterator's points to w.
type IteratorEncoder struct {
w io.Writer
// Frequency with which stats are emitted.
StatsInterval time.Duration
}
// NewIteratorEncoder encodes an iterator's points to w.
func NewIteratorEncoder(w io.Writer) *IteratorEncoder {
return &IteratorEncoder{
w: w,
StatsInterval: DefaultStatsInterval,
}
}
// EncodeIterator encodes and writes all of itr's points to the underlying writer.
func (enc *IteratorEncoder) EncodeIterator(itr Iterator) error {
switch itr := itr.(type) {
case FloatIterator:
return enc.encodeFloatIterator(itr)
case IntegerIterator:
return enc.encodeIntegerIterator(itr)
case StringIterator:
return enc.encodeStringIterator(itr)
case BooleanIterator:
return enc.encodeBooleanIterator(itr)
default:
panic(fmt.Sprintf("unsupported iterator for encoder: %T", itr))
}
}
// encodeFloatIterator encodes all points from itr to the underlying writer.
func (enc *IteratorEncoder) encodeFloatIterator(itr FloatIterator) error {
ticker := time.NewTicker(enc.StatsInterval)
defer ticker.Stop()
// Emit initial stats.
if err := enc.encodeStats(itr.Stats()); err != nil {
return err
}
// Continually stream points from the iterator into the encoder.
penc := NewFloatPointEncoder(enc.w)
for {
// Emit stats periodically.
select {
case <-ticker.C:
if err := enc.encodeStats(itr.Stats()); err != nil {
return err
}
default:
}
// Retrieve the next point from the iterator.
p, err := itr.Next()
if err != nil {
return err
} else if p == nil {
break
}
// Write the point to the point encoder.
if err := penc.EncodeFloatPoint(p); err != nil {
return err
}
}
// Emit final stats.
if err := enc.encodeStats(itr.Stats()); err != nil {
return err
}
return nil
}
// encodeIntegerIterator encodes all points from itr to the underlying writer.
func (enc *IteratorEncoder) encodeIntegerIterator(itr IntegerIterator) error {
ticker := time.NewTicker(enc.StatsInterval)
defer ticker.Stop()
// Emit initial stats.
if err := enc.encodeStats(itr.Stats()); err != nil {
return err
}
// Continually stream points from the iterator into the encoder.
penc := NewIntegerPointEncoder(enc.w)
for {
// Emit stats periodically.
select {
case <-ticker.C:
if err := enc.encodeStats(itr.Stats()); err != nil {
return err
}
default:
}
// Retrieve the next point from the iterator.
p, err := itr.Next()
if err != nil {
return err
} else if p == nil {
break
}
// Write the point to the point encoder.
if err := penc.EncodeIntegerPoint(p); err != nil {
return err
}
}
// Emit final stats.
if err := enc.encodeStats(itr.Stats()); err != nil {
return err
}
return nil
}
// encodeStringIterator encodes all points from itr to the underlying writer.
func (enc *IteratorEncoder) encodeStringIterator(itr StringIterator) error {
ticker := time.NewTicker(enc.StatsInterval)
defer ticker.Stop()
// Emit initial stats.
if err := enc.encodeStats(itr.Stats()); err != nil {
return err
}
// Continually stream points from the iterator into the encoder.
penc := NewStringPointEncoder(enc.w)
for {
// Emit stats periodically.
select {
case <-ticker.C:
if err := enc.encodeStats(itr.Stats()); err != nil {
return err
}
default:
}
// Retrieve the next point from the iterator.
p, err := itr.Next()
if err != nil {
return err
} else if p == nil {
break
}
// Write the point to the point encoder.
if err := penc.EncodeStringPoint(p); err != nil {
return err
}
}
// Emit final stats.
if err := enc.encodeStats(itr.Stats()); err != nil {
return err
}
return nil
}
// encodeBooleanIterator encodes all points from itr to the underlying writer.
func (enc *IteratorEncoder) encodeBooleanIterator(itr BooleanIterator) error {
ticker := time.NewTicker(enc.StatsInterval)
defer ticker.Stop()
// Emit initial stats.
if err := enc.encodeStats(itr.Stats()); err != nil {
return err
}
// Continually stream points from the iterator into the encoder.
penc := NewBooleanPointEncoder(enc.w)
for {
// Emit stats periodically.
select {
case <-ticker.C:
if err := enc.encodeStats(itr.Stats()); err != nil {
return err
}
default:
}
// Retrieve the next point from the iterator.
p, err := itr.Next()
if err != nil {
return err
} else if p == nil {
break
}
// Write the point to the point encoder.
if err := penc.EncodeBooleanPoint(p); err != nil {
return err
}
}
// Emit final stats.
if err := enc.encodeStats(itr.Stats()); err != nil {
return err
}
return nil
}
// encode a stats object in the point stream.
func (enc *IteratorEncoder) encodeStats(stats IteratorStats) error {
buf, err := proto.Marshal(&internal.Point{
Name: proto.String(""),
Tags: proto.String(""),
Time: proto.Int64(0),
Nil: proto.Bool(false),
Stats: encodeIteratorStats(&stats),
})
if err != nil {
return err
}
if err := binary.Write(enc.w, binary.BigEndian, uint32(len(buf))); err != nil {
return err
}
if _, err := enc.w.Write(buf); err != nil {
return err
}
return nil
}