add vendoring with go dep

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/codec.go

@@ -0,0 +1,119 @@
+package tsdb
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"math"
+)
+
+const (
+	fieldFloat   = 1
+	fieldInteger = 2
+	fieldBoolean = 3
+	fieldString  = 4
+)
+
+var (
+	// ErrFieldNotFound is returned when a field cannot be found.
+	ErrFieldNotFound = errors.New("field not found")
+
+	// ErrFieldUnmappedID is returned when the system is presented, during decode, with a field ID
+	// there is no mapping for.
+	ErrFieldUnmappedID = errors.New("field ID not mapped")
+)
+
+// FieldCodec provides encoding and decoding functionality for the fields of a given
+// Measurement.
+type FieldCodec struct {
+	fieldsByID   map[uint8]*Field
+	fieldsByName map[string]*Field
+}
+
+// NewFieldCodec returns a FieldCodec for the given Measurement. Must be called with
+// a RLock that protects the Measurement.
+func NewFieldCodec(fields map[string]*Field) *FieldCodec {
+	fieldsByID := make(map[uint8]*Field, len(fields))
+	fieldsByName := make(map[string]*Field, len(fields))
+	for _, f := range fields {
+		fieldsByID[f.ID] = f
+		fieldsByName[f.Name] = f
+	}
+	return &FieldCodec{fieldsByID: fieldsByID, fieldsByName: fieldsByName}
+}
+
+// FieldIDByName returns the ID for the given field.
+func (f *FieldCodec) FieldIDByName(s string) (uint8, error) {
+	fi := f.fieldsByName[s]
+	if fi == nil {
+		return 0, ErrFieldNotFound
+	}
+	return fi.ID, nil
+}
+
+// DecodeByID scans a byte slice for a field with the given ID, converts it to its
+// expected type, and return that value.
+func (f *FieldCodec) DecodeByID(targetID uint8, b []byte) (interface{}, error) {
+	var value interface{}
+	for {
+		if len(b) == 0 {
+			// No more bytes.
+			return nil, ErrFieldNotFound
+		}
+
+		field := f.fieldsByID[b[0]]
+		if field == nil {
+			// This can happen, though is very unlikely. If this node receives encoded data, to be written
+			// to disk, and is queried for that data before its metastore is updated, there will be no field
+			// mapping for the data during decode. All this can happen because data is encoded by the node
+			// that first received the write request, not the node that actually writes the data to disk.
+			// So if this happens, the read must be aborted.
+			return nil, ErrFieldUnmappedID
+		}
+
+		switch field.Type {
+		case fieldFloat:
+			if field.ID == targetID {
+				value = math.Float64frombits(binary.BigEndian.Uint64(b[1:9]))
+			}
+			b = b[9:]
+		case fieldInteger:
+			if field.ID == targetID {
+				value = int64(binary.BigEndian.Uint64(b[1:9]))
+			}
+			b = b[9:]
+		case fieldBoolean:
+			if field.ID == targetID {
+				value = b[1] == 1
+			}
+			b = b[2:]
+		case fieldString:
+			length := binary.BigEndian.Uint16(b[1:3])
+			if field.ID == targetID {
+				value = string(b[3 : 3+length])
+			}
+			b = b[3+length:]
+		default:
+			panic(fmt.Sprintf("unsupported value type during decode by id: %T", field.Type))
+		}
+
+		if value != nil {
+			return value, nil
+		}
+	}
+}
+
+// DecodeByName scans a byte slice for a field with the given name, converts it to its
+// expected type, and return that value.
+func (f *FieldCodec) DecodeByName(name string, b []byte) (interface{}, error) {
+	fi := f.FieldByName(name)
+	if fi == nil {
+		return 0, ErrFieldNotFound
+	}
+	return f.DecodeByID(fi.ID, b)
+}
+
+// FieldByName returns the field by its name. It will return a nil if not found
+func (f *FieldCodec) FieldByName(name string) *Field {
+	return f.fieldsByName[name]
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/database.go

@@ -0,0 +1,244 @@
+// Pacage tsdb abstracts the various shard types supported by the influx_tsm command.
+package tsdb // import "github.com/influxdata/influxdb/cmd/influx_tsm/tsdb"
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/influxdata/influxdb/pkg/slices"
+)
+
+// Flags for differentiating between engines
+const (
+	B1 = iota
+	BZ1
+	TSM1
+)
+
+// EngineFormat holds the flag for the engine
+type EngineFormat int
+
+// String returns the string format of the engine.
+func (e EngineFormat) String() string {
+	switch e {
+	case TSM1:
+		return "tsm1"
+	case B1:
+		return "b1"
+	case BZ1:
+		return "bz1"
+	default:
+		panic("unrecognized shard engine format")
+	}
+}
+
+// ShardInfo is the description of a shard on disk.
+type ShardInfo struct {
+	Database        string
+	RetentionPolicy string
+	Path            string
+	Format          EngineFormat
+	Size            int64
+}
+
+// FormatAsString returns the format of the shard as a string.
+func (s *ShardInfo) FormatAsString() string {
+	return s.Format.String()
+}
+
+// FullPath returns the full path to the shard, given the data directory root.
+func (s *ShardInfo) FullPath(dataPath string) string {
+	return filepath.Join(dataPath, s.Database, s.RetentionPolicy, s.Path)
+}
+
+// ShardInfos is an array of ShardInfo
+type ShardInfos []*ShardInfo
+
+func (s ShardInfos) Len() int      { return len(s) }
+func (s ShardInfos) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s ShardInfos) Less(i, j int) bool {
+	if s[i].Database == s[j].Database {
+		if s[i].RetentionPolicy == s[j].RetentionPolicy {
+			return s[i].Path < s[j].Path
+		}
+
+		return s[i].RetentionPolicy < s[j].RetentionPolicy
+	}
+
+	return s[i].Database < s[j].Database
+}
+
+// Databases returns the sorted unique set of databases for the shards.
+func (s ShardInfos) Databases() []string {
+	dbm := make(map[string]bool)
+	for _, ss := range s {
+		dbm[ss.Database] = true
+	}
+
+	var dbs []string
+	for k := range dbm {
+		dbs = append(dbs, k)
+	}
+	sort.Strings(dbs)
+	return dbs
+}
+
+// FilterFormat returns a copy of the ShardInfos, with shards of the given
+// format removed.
+func (s ShardInfos) FilterFormat(fmt EngineFormat) ShardInfos {
+	var a ShardInfos
+	for _, si := range s {
+		if si.Format != fmt {
+			a = append(a, si)
+		}
+	}
+	return a
+}
+
+// Size returns the space on disk consumed by the shards.
+func (s ShardInfos) Size() int64 {
+	var sz int64
+	for _, si := range s {
+		sz += si.Size
+	}
+	return sz
+}
+
+// ExclusiveDatabases returns a copy of the ShardInfo, with shards associated
+// with the given databases present. If the given set is empty, all databases
+// are returned.
+func (s ShardInfos) ExclusiveDatabases(exc []string) ShardInfos {
+	var a ShardInfos
+
+	// Empty set? Return everything.
+	if len(exc) == 0 {
+		a = make(ShardInfos, len(s))
+		copy(a, s)
+		return a
+	}
+
+	for _, si := range s {
+		if slices.Exists(exc, si.Database) {
+			a = append(a, si)
+		}
+	}
+	return a
+}
+
+// Database represents an entire database on disk.
+type Database struct {
+	path string
+}
+
+// NewDatabase creates a database instance using data at path.
+func NewDatabase(path string) *Database {
+	return &Database{path: path}
+}
+
+// Name returns the name of the database.
+func (d *Database) Name() string {
+	return path.Base(d.path)
+}
+
+// Path returns the path to the database.
+func (d *Database) Path() string {
+	return d.path
+}
+
+// Shards returns information for every shard in the database.
+func (d *Database) Shards() ([]*ShardInfo, error) {
+	fd, err := os.Open(d.path)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get each retention policy.
+	rps, err := fd.Readdirnames(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	// Process each retention policy.
+	var shardInfos []*ShardInfo
+	for _, rp := range rps {
+		rpfd, err := os.Open(filepath.Join(d.path, rp))
+		if err != nil {
+			return nil, err
+		}
+
+		// Process each shard
+		shards, err := rpfd.Readdirnames(-1)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, sh := range shards {
+			fmt, sz, err := shardFormat(filepath.Join(d.path, rp, sh))
+			if err != nil {
+				return nil, err
+			}
+
+			si := &ShardInfo{
+				Database:        d.Name(),
+				RetentionPolicy: path.Base(rp),
+				Path:            sh,
+				Format:          fmt,
+				Size:            sz,
+			}
+			shardInfos = append(shardInfos, si)
+		}
+	}
+
+	sort.Sort(ShardInfos(shardInfos))
+	return shardInfos, nil
+}
+
+// shardFormat returns the format and size on disk of the shard at path.
+func shardFormat(path string) (EngineFormat, int64, error) {
+	// If it's a directory then it's a tsm1 engine
+	fi, err := os.Stat(path)
+	if err != nil {
+		return 0, 0, err
+	}
+	if fi.Mode().IsDir() {
+		return TSM1, fi.Size(), nil
+	}
+
+	// It must be a BoltDB-based engine.
+	db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return 0, 0, err
+	}
+	defer db.Close()
+
+	var format EngineFormat
+	err = db.View(func(tx *bolt.Tx) error {
+		// Retrieve the meta bucket.
+		b := tx.Bucket([]byte("meta"))
+
+		// If no format is specified then it must be an original b1 database.
+		if b == nil {
+			format = B1
+			return nil
+		}
+
+		// There is an actual format indicator.
+		switch f := string(b.Get([]byte("format"))); f {
+		case "b1", "v1":
+			format = B1
+		case "bz1":
+			format = BZ1
+		default:
+			return fmt.Errorf("unrecognized engine format: %s", f)
+		}
+
+		return nil
+	})
+
+	return format, fi.Size(), err
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/internal/meta.pb.go

@@ -0,0 +1,122 @@
+// Code generated by protoc-gen-gogo.
+// source: internal/meta.proto
+// DO NOT EDIT!
+
+/*
+Package internal is a generated protocol buffer package.
+
+It is generated from these files:
+	internal/meta.proto
+
+It has these top-level messages:
+	Series
+	Tag
+	MeasurementFields
+	Field
+*/
+package internal
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+type Series struct {
+	Key              *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"`
+	Tags             []*Tag  `protobuf:"bytes,2,rep,name=Tags" json:"Tags,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *Series) Reset()         { *m = Series{} }
+func (m *Series) String() string { return proto.CompactTextString(m) }
+func (*Series) ProtoMessage()    {}
+
+func (m *Series) GetKey() string {
+	if m != nil && m.Key != nil {
+		return *m.Key
+	}
+	return ""
+}
+
+func (m *Series) GetTags() []*Tag {
+	if m != nil {
+		return m.Tags
+	}
+	return nil
+}
+
+type Tag struct {
+	Key              *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"`
+	Value            *string `protobuf:"bytes,2,req,name=Value" json:"Value,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *Tag) Reset()         { *m = Tag{} }
+func (m *Tag) String() string { return proto.CompactTextString(m) }
+func (*Tag) ProtoMessage()    {}
+
+func (m *Tag) GetKey() string {
+	if m != nil && m.Key != nil {
+		return *m.Key
+	}
+	return ""
+}
+
+func (m *Tag) GetValue() string {
+	if m != nil && m.Value != nil {
+		return *m.Value
+	}
+	return ""
+}
+
+type MeasurementFields struct {
+	Fields           []*Field `protobuf:"bytes,1,rep,name=Fields" json:"Fields,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *MeasurementFields) Reset()         { *m = MeasurementFields{} }
+func (m *MeasurementFields) String() string { return proto.CompactTextString(m) }
+func (*MeasurementFields) ProtoMessage()    {}
+
+func (m *MeasurementFields) GetFields() []*Field {
+	if m != nil {
+		return m.Fields
+	}
+	return nil
+}
+
+type Field struct {
+	ID               *int32  `protobuf:"varint,1,req,name=ID" json:"ID,omitempty"`
+	Name             *string `protobuf:"bytes,2,req,name=Name" json:"Name,omitempty"`
+	Type             *int32  `protobuf:"varint,3,req,name=Type" json:"Type,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *Field) Reset()         { *m = Field{} }
+func (m *Field) String() string { return proto.CompactTextString(m) }
+func (*Field) ProtoMessage()    {}
+
+func (m *Field) GetID() int32 {
+	if m != nil && m.ID != nil {
+		return *m.ID
+	}
+	return 0
+}
+
+func (m *Field) GetName() string {
+	if m != nil && m.Name != nil {
+		return *m.Name
+	}
+	return ""
+}
+
+func (m *Field) GetType() int32 {
+	if m != nil && m.Type != nil {
+		return *m.Type
+	}
+	return 0
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/types.go

@@ -0,0 +1,60 @@
+package tsdb
+
+import (
+	"encoding/binary"
+	"strings"
+
+	"github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/internal"
+	"github.com/influxdata/influxdb/influxql"
+
+	"github.com/gogo/protobuf/proto"
+)
+
+// Field represents an encoded field.
+type Field struct {
+	ID   uint8             `json:"id,omitempty"`
+	Name string            `json:"name,omitempty"`
+	Type influxql.DataType `json:"type,omitempty"`
+}
+
+// MeasurementFields is a mapping from measurements to its fields.
+type MeasurementFields struct {
+	Fields map[string]*Field `json:"fields"`
+	Codec  *FieldCodec
+}
+
+// UnmarshalBinary decodes the object from a binary format.
+func (m *MeasurementFields) UnmarshalBinary(buf []byte) error {
+	var pb internal.MeasurementFields
+	if err := proto.Unmarshal(buf, &pb); err != nil {
+		return err
+	}
+	m.Fields = make(map[string]*Field)
+	for _, f := range pb.Fields {
+		m.Fields[f.GetName()] = &Field{ID: uint8(f.GetID()), Name: f.GetName(), Type: influxql.DataType(f.GetType())}
+	}
+	return nil
+}
+
+// Series represents a series in the shard.
+type Series struct {
+	Key  string
+	Tags map[string]string
+}
+
+// MeasurementFromSeriesKey returns the Measurement name for a given series.
+func MeasurementFromSeriesKey(key string) string {
+	return strings.SplitN(key, ",", 2)[0]
+}
+
+// DecodeKeyValue decodes the key and value from bytes.
+func DecodeKeyValue(field string, dec *FieldCodec, k, v []byte) (int64, interface{}) {
+	// Convert key to a timestamp.
+	key := int64(binary.BigEndian.Uint64(k[0:8]))
+
+	decValue, err := dec.DecodeByName(field, v)
+	if err != nil {
+		return key, nil
+	}
+	return key, decValue
+}