add vendoring with go dep

vendor/github.com/influxdata/influxdb/cmd/influx/cli/cli_internal_test.go

@@ -0,0 +1,58 @@
+package cli
+
+import "testing"
+
+func TestParseCommand_InsertInto(t *testing.T) {
+	t.Parallel()
+
+	c := CommandLine{}
+
+	tests := []struct {
+		cmd, db, rp string
+	}{
+		{
+			cmd: `INSERT INTO test cpu,host=serverA,region=us-west value=1.0`,
+			db:  "",
+			rp:  "test",
+		},
+		{
+			cmd: ` INSERT INTO .test cpu,host=serverA,region=us-west value=1.0`,
+			db:  "",
+			rp:  "test",
+		},
+		{
+			cmd: `INSERT INTO   "test test" cpu,host=serverA,region=us-west value=1.0`,
+			db:  "",
+			rp:  "test test",
+		},
+		{
+			cmd: `Insert iNTO test.test cpu,host=serverA,region=us-west value=1.0`,
+			db:  "test",
+			rp:  "test",
+		},
+		{
+			cmd: `insert into "test test" cpu,host=serverA,region=us-west value=1.0`,
+			db:  "",
+			rp:  "test test",
+		},
+		{
+			cmd: `insert into "d b"."test test" cpu,host=serverA,region=us-west value=1.0`,
+			db:  "d b",
+			rp:  "test test",
+		},
+	}
+
+	for _, test := range tests {
+		t.Logf("command: %s", test.cmd)
+		bp, err := c.parseInsert(test.cmd)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if bp.Database != test.db {
+			t.Fatalf(`Command "insert into" db parsing failed, expected: %q, actual: %q`, test.db, bp.Database)
+		}
+		if bp.RetentionPolicy != test.rp {
+			t.Fatalf(`Command "insert into" rp parsing failed, expected: %q, actual: %q`, test.rp, bp.RetentionPolicy)
+		}
+	}
+}

vendor/github.com/influxdata/influxdb/cmd/influx/cli/cli_test.go

@@ -0,0 +1,594 @@
+package cli_test
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/influxdata/influxdb/client"
+	"github.com/influxdata/influxdb/cmd/influx/cli"
+	"github.com/influxdata/influxdb/influxql"
+	"github.com/peterh/liner"
+)
+
+const (
+	CLIENT_VERSION = "y.y"
+	SERVER_VERSION = "x.x"
+)
+
+func TestNewCLI(t *testing.T) {
+	t.Parallel()
+	c := cli.New(CLIENT_VERSION)
+
+	if c == nil {
+		t.Fatal("CommandLine shouldn't be nil.")
+	}
+
+	if c.ClientVersion != CLIENT_VERSION {
+		t.Fatalf("CommandLine version is %s but should be %s", c.ClientVersion, CLIENT_VERSION)
+	}
+}
+
+func TestRunCLI(t *testing.T) {
+	t.Parallel()
+	ts := emptyTestServer()
+	defer ts.Close()
+
+	u, _ := url.Parse(ts.URL)
+	h, p, _ := net.SplitHostPort(u.Host)
+	c := cli.New(CLIENT_VERSION)
+	c.Host = h
+	c.Port, _ = strconv.Atoi(p)
+	c.IgnoreSignals = true
+	c.ForceTTY = true
+	go func() {
+		close(c.Quit)
+	}()
+	if err := c.Run(); err != nil {
+		t.Fatalf("Run failed with error: %s", err)
+	}
+}
+
+func TestRunCLI_ExecuteInsert(t *testing.T) {
+	t.Parallel()
+	ts := emptyTestServer()
+	defer ts.Close()
+
+	u, _ := url.Parse(ts.URL)
+	h, p, _ := net.SplitHostPort(u.Host)
+	c := cli.New(CLIENT_VERSION)
+	c.Host = h
+	c.Port, _ = strconv.Atoi(p)
+	c.ClientConfig.Precision = "ms"
+	c.Execute = "INSERT sensor,floor=1 value=2"
+	c.IgnoreSignals = true
+	c.ForceTTY = true
+	if err := c.Run(); err != nil {
+		t.Fatalf("Run failed with error: %s", err)
+	}
+}
+
+func TestSetAuth(t *testing.T) {
+	t.Parallel()
+	c := cli.New(CLIENT_VERSION)
+	config := client.NewConfig()
+	client, _ := client.NewClient(config)
+	c.Client = client
+	u := "userx"
+	p := "pwdy"
+	c.SetAuth("auth " + u + " " + p)
+
+	// validate CLI configuration
+	if c.ClientConfig.Username != u {
+		t.Fatalf("Username is %s but should be %s", c.ClientConfig.Username, u)
+	}
+	if c.ClientConfig.Password != p {
+		t.Fatalf("Password is %s but should be %s", c.ClientConfig.Password, p)
+	}
+}
+
+func TestSetPrecision(t *testing.T) {
+	t.Parallel()
+	c := cli.New(CLIENT_VERSION)
+	config := client.NewConfig()
+	client, _ := client.NewClient(config)
+	c.Client = client
+
+	// validate set non-default precision
+	p := "ns"
+	c.SetPrecision("precision " + p)
+	if c.ClientConfig.Precision != p {
+		t.Fatalf("Precision is %s but should be %s", c.ClientConfig.Precision, p)
+	}
+
+	// validate set default precision which equals empty string
+	p = "rfc3339"
+	c.SetPrecision("precision " + p)
+	if c.ClientConfig.Precision != "" {
+		t.Fatalf("Precision is %s but should be empty", c.ClientConfig.Precision)
+	}
+}
+
+func TestSetFormat(t *testing.T) {
+	t.Parallel()
+	c := cli.New(CLIENT_VERSION)
+	config := client.NewConfig()
+	client, _ := client.NewClient(config)
+	c.Client = client
+
+	// validate set non-default format
+	f := "json"
+	c.SetFormat("format " + f)
+	if c.Format != f {
+		t.Fatalf("Format is %s but should be %s", c.Format, f)
+	}
+}
+
+func Test_SetChunked(t *testing.T) {
+	t.Parallel()
+	c := cli.New(CLIENT_VERSION)
+	config := client.NewConfig()
+	client, _ := client.NewClient(config)
+	c.Client = client
+
+	// make sure chunked is on by default
+	if got, exp := c.Chunked, true; got != exp {
+		t.Fatalf("chunked should be on by default.  got %v, exp %v", got, exp)
+	}
+
+	// turn chunked off
+	if err := c.ParseCommand("Chunked"); err != nil {
+		t.Fatalf("setting chunked failed: err: %s", err)
+	}
+
+	if got, exp := c.Chunked, false; got != exp {
+		t.Fatalf("setting chunked failed.  got %v, exp %v", got, exp)
+	}
+
+	// turn chunked back on
+	if err := c.ParseCommand("Chunked"); err != nil {
+		t.Fatalf("setting chunked failed: err: %s", err)
+	}
+
+	if got, exp := c.Chunked, true; got != exp {
+		t.Fatalf("setting chunked failed.  got %v, exp %v", got, exp)
+	}
+}
+
+func Test_SetChunkSize(t *testing.T) {
+	t.Parallel()
+	c := cli.New(CLIENT_VERSION)
+	config := client.NewConfig()
+	client, _ := client.NewClient(config)
+	c.Client = client
+
+	// check default chunk size
+	if got, exp := c.ChunkSize, 0; got != exp {
+		t.Fatalf("unexpected chunk size.  got %d, exp %d", got, exp)
+	}
+
+	tests := []struct {
+		command string
+		exp     int
+	}{
+		{"chunk size 20", 20},
+		{"   CHunk     siZE  55    ", 55},
+		{"chunk 10", 10},
+		{"     chuNK     15", 15},
+		{"chunk size -60", 0},
+		{"chunk size 10", 10},
+		{"chunk size 0", 0},
+		{"chunk size 10", 10},
+		{"chunk size junk", 10},
+	}
+
+	for _, test := range tests {
+		if err := c.ParseCommand(test.command); err != nil {
+			t.Logf("command: %q", test.command)
+			t.Fatalf("setting chunked failed: err: %s", err)
+		}
+
+		if got, exp := c.ChunkSize, test.exp; got != exp {
+			t.Logf("command: %q", test.command)
+			t.Fatalf("unexpected chunk size.  got %d, exp %d", got, exp)
+		}
+	}
+}
+
+func TestSetWriteConsistency(t *testing.T) {
+	t.Parallel()
+	c := cli.New(CLIENT_VERSION)
+	config := client.NewConfig()
+	client, _ := client.NewClient(config)
+	c.Client = client
+
+	// set valid write consistency
+	consistency := "all"
+	c.SetWriteConsistency("consistency " + consistency)
+	if c.ClientConfig.WriteConsistency != consistency {
+		t.Fatalf("WriteConsistency is %s but should be %s", c.ClientConfig.WriteConsistency, consistency)
+	}
+
+	// set different valid write consistency and validate change
+	consistency = "quorum"
+	c.SetWriteConsistency("consistency " + consistency)
+	if c.ClientConfig.WriteConsistency != consistency {
+		t.Fatalf("WriteConsistency is %s but should be %s", c.ClientConfig.WriteConsistency, consistency)
+	}
+
+	// set invalid write consistency and verify there was no change
+	invalidConsistency := "invalid_consistency"
+	c.SetWriteConsistency("consistency " + invalidConsistency)
+	if c.ClientConfig.WriteConsistency == invalidConsistency {
+		t.Fatalf("WriteConsistency is %s but should be %s", c.ClientConfig.WriteConsistency, consistency)
+	}
+}
+
+func TestParseCommand_CommandsExist(t *testing.T) {
+	t.Parallel()
+	c, err := client.NewClient(client.Config{})
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	m := cli.CommandLine{Client: c, Line: liner.NewLiner()}
+	tests := []struct {
+		cmd string
+	}{
+		{cmd: "gopher"},
+		{cmd: "auth"},
+		{cmd: "help"},
+		{cmd: "format"},
+		{cmd: "precision"},
+		{cmd: "settings"},
+	}
+	for _, test := range tests {
+		if err := m.ParseCommand(test.cmd); err != nil {
+			t.Fatalf(`Got error %v for command %q, expected nil`, err, test.cmd)
+		}
+	}
+}
+
+func TestParseCommand_Connect(t *testing.T) {
+	t.Parallel()
+	ts := emptyTestServer()
+	defer ts.Close()
+
+	u, _ := url.Parse(ts.URL)
+	cmd := "connect " + u.Host
+	c := cli.CommandLine{}
+
+	// assert connection is established
+	if err := c.ParseCommand(cmd); err != nil {
+		t.Fatalf("There was an error while connecting to %v: %v", u.Path, err)
+	}
+
+	// assert server version is populated
+	if c.ServerVersion != SERVER_VERSION {
+		t.Fatalf("Server version is %s but should be %s.", c.ServerVersion, SERVER_VERSION)
+	}
+}
+
+func TestParseCommand_TogglePretty(t *testing.T) {
+	t.Parallel()
+	c := cli.CommandLine{}
+	if c.Pretty {
+		t.Fatalf(`Pretty should be false.`)
+	}
+	c.ParseCommand("pretty")
+	if !c.Pretty {
+		t.Fatalf(`Pretty should be true.`)
+	}
+	c.ParseCommand("pretty")
+	if c.Pretty {
+		t.Fatalf(`Pretty should be false.`)
+	}
+}
+
+func TestParseCommand_Exit(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		cmd string
+	}{
+		{cmd: "exit"},
+		{cmd: " exit"},
+		{cmd: "exit "},
+		{cmd: "Exit "},
+	}
+
+	for _, test := range tests {
+		c := cli.CommandLine{Quit: make(chan struct{}, 1)}
+		c.ParseCommand(test.cmd)
+		// channel should be closed
+		if _, ok := <-c.Quit; ok {
+			t.Fatalf(`Command "exit" failed for %q.`, test.cmd)
+		}
+	}
+}
+
+func TestParseCommand_Quit(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		cmd string
+	}{
+		{cmd: "quit"},
+		{cmd: " quit"},
+		{cmd: "quit "},
+		{cmd: "Quit "},
+	}
+
+	for _, test := range tests {
+		c := cli.CommandLine{Quit: make(chan struct{}, 1)}
+		c.ParseCommand(test.cmd)
+		// channel should be closed
+		if _, ok := <-c.Quit; ok {
+			t.Fatalf(`Command "quit" failed for %q.`, test.cmd)
+		}
+	}
+}
+
+func TestParseCommand_Use(t *testing.T) {
+	t.Parallel()
+	ts := emptyTestServer()
+	defer ts.Close()
+
+	u, _ := url.Parse(ts.URL)
+	config := client.Config{URL: *u}
+	c, err := client.NewClient(config)
+	if err != nil {
+		t.Fatalf("unexpected error.  expected %v, actual %v", nil, err)
+	}
+
+	tests := []struct {
+		cmd string
+	}{
+		{cmd: "use db"},
+		{cmd: " use db"},
+		{cmd: "use db "},
+		{cmd: "use db;"},
+		{cmd: "use db; "},
+		{cmd: "Use db"},
+	}
+
+	for _, test := range tests {
+		m := cli.CommandLine{Client: c}
+		if err := m.ParseCommand(test.cmd); err != nil {
+			t.Fatalf(`Got error %v for command %q, expected nil.`, err, test.cmd)
+		}
+
+		if m.Database != "db" {
+			t.Fatalf(`Command "use" changed database to %q. Expected db`, m.Database)
+		}
+	}
+}
+
+func TestParseCommand_UseAuth(t *testing.T) {
+	t.Parallel()
+	ts := emptyTestServer()
+	defer ts.Close()
+
+	u, _ := url.Parse(ts.URL)
+	tests := []struct {
+		cmd      string
+		user     string
+		database string
+	}{
+		{
+			cmd:      "use db",
+			user:     "admin",
+			database: "db",
+		},
+		{
+			cmd:      "use blank",
+			user:     "admin",
+			database: "",
+		},
+		{
+			cmd:      "use db",
+			user:     "anonymous",
+			database: "db",
+		},
+		{
+			cmd:      "use blank",
+			user:     "anonymous",
+			database: "blank",
+		},
+	}
+
+	for i, tt := range tests {
+		config := client.Config{URL: *u, Username: tt.user}
+		fmt.Println("using auth:", tt.user)
+		c, err := client.NewClient(config)
+		if err != nil {
+			t.Errorf("%d. unexpected error.  expected %v, actual %v", i, nil, err)
+			continue
+		}
+		m := cli.CommandLine{Client: c}
+		m.ClientConfig.Username = tt.user
+
+		if err := m.ParseCommand(tt.cmd); err != nil {
+			t.Fatalf(`%d. Got error %v for command %q, expected nil.`, i, err, tt.cmd)
+		}
+
+		if m.Database != tt.database {
+			t.Fatalf(`%d. Command "use" changed database to %q. Expected %q`, i, m.Database, tt.database)
+		}
+	}
+}
+
+func TestParseCommand_Consistency(t *testing.T) {
+	t.Parallel()
+	c := cli.CommandLine{}
+	tests := []struct {
+		cmd string
+	}{
+		{cmd: "consistency one"},
+		{cmd: " consistency one"},
+		{cmd: "consistency one "},
+		{cmd: "consistency one;"},
+		{cmd: "consistency one; "},
+		{cmd: "Consistency one"},
+	}
+
+	for _, test := range tests {
+		if err := c.ParseCommand(test.cmd); err != nil {
+			t.Fatalf(`Got error %v for command %q, expected nil.`, err, test.cmd)
+		}
+
+		if c.ClientConfig.WriteConsistency != "one" {
+			t.Fatalf(`Command "consistency" changed consistency to %q. Expected one`, c.ClientConfig.WriteConsistency)
+		}
+	}
+}
+
+func TestParseCommand_Insert(t *testing.T) {
+	t.Parallel()
+	ts := emptyTestServer()
+	defer ts.Close()
+
+	u, _ := url.Parse(ts.URL)
+	config := client.Config{URL: *u}
+	c, err := client.NewClient(config)
+	if err != nil {
+		t.Fatalf("unexpected error.  expected %v, actual %v", nil, err)
+	}
+	m := cli.CommandLine{Client: c}
+
+	tests := []struct {
+		cmd string
+	}{
+		{cmd: "INSERT cpu,host=serverA,region=us-west value=1.0"},
+		{cmd: " INSERT cpu,host=serverA,region=us-west value=1.0"},
+		{cmd: "INSERT   cpu,host=serverA,region=us-west value=1.0"},
+		{cmd: "insert cpu,host=serverA,region=us-west    value=1.0    "},
+		{cmd: "insert"},
+		{cmd: "Insert "},
+		{cmd: "insert c"},
+		{cmd: "insert int"},
+	}
+
+	for _, test := range tests {
+		if err := m.ParseCommand(test.cmd); err != nil {
+			t.Fatalf(`Got error %v for command %q, expected nil.`, err, test.cmd)
+		}
+	}
+}
+
+func TestParseCommand_History(t *testing.T) {
+	t.Parallel()
+	c := cli.CommandLine{Line: liner.NewLiner()}
+	defer c.Line.Close()
+
+	// append one entry to history
+	c.Line.AppendHistory("abc")
+
+	tests := []struct {
+		cmd string
+	}{
+		{cmd: "history"},
+		{cmd: " history"},
+		{cmd: "history "},
+		{cmd: "History "},
+	}
+
+	for _, test := range tests {
+		if err := c.ParseCommand(test.cmd); err != nil {
+			t.Fatalf(`Got error %v for command %q, expected nil.`, err, test.cmd)
+		}
+	}
+
+	// buf size should be at least 1
+	var buf bytes.Buffer
+	c.Line.WriteHistory(&buf)
+	if buf.Len() < 1 {
+		t.Fatal("History is borked")
+	}
+}
+
+func TestParseCommand_HistoryWithBlankCommand(t *testing.T) {
+	t.Parallel()
+	c := cli.CommandLine{Line: liner.NewLiner()}
+	defer c.Line.Close()
+
+	// append one entry to history
+	c.Line.AppendHistory("x")
+
+	tests := []struct {
+		cmd string
+		err error
+	}{
+		{cmd: "history"},
+		{cmd: " history"},
+		{cmd: "history "},
+		{cmd: "", err: cli.ErrBlankCommand},      // shouldn't be persisted in history
+		{cmd: " ", err: cli.ErrBlankCommand},     // shouldn't be persisted in history
+		{cmd: "     ", err: cli.ErrBlankCommand}, // shouldn't be persisted in history
+	}
+
+	// a blank command will return cli.ErrBlankCommand.
+	for _, test := range tests {
+		if err := c.ParseCommand(test.cmd); err != test.err {
+			t.Errorf(`Got error %v for command %q, expected %v`, err, test.cmd, test.err)
+		}
+	}
+
+	// buf shall not contain empty commands
+	var buf bytes.Buffer
+	c.Line.WriteHistory(&buf)
+	scanner := bufio.NewScanner(&buf)
+	for scanner.Scan() {
+		if strings.TrimSpace(scanner.Text()) == "" {
+			t.Fatal("Empty commands should not be persisted in history.")
+		}
+	}
+}
+
+// helper methods
+
+func emptyTestServer() *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Influxdb-Version", SERVER_VERSION)
+
+		// Fake authorization entirely based on the username.
+		authorized := false
+		user, _, _ := r.BasicAuth()
+		switch user {
+		case "", "admin":
+			authorized = true
+		}
+
+		switch r.URL.Path {
+		case "/query":
+			values := r.URL.Query()
+			parser := influxql.NewParser(bytes.NewBufferString(values.Get("q")))
+			q, err := parser.ParseQuery()
+			if err != nil {
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+			stmt := q.Statements[0]
+
+			switch stmt.(type) {
+			case *influxql.ShowDatabasesStatement:
+				if authorized {
+					io.WriteString(w, `{"results":[{"series":[{"name":"databases","columns":["name"],"values":[["db"]]}]}]}`)
+				} else {
+					w.WriteHeader(http.StatusUnauthorized)
+					io.WriteString(w, fmt.Sprintf(`{"error":"error authorizing query: %s not authorized to execute statement 'SHOW DATABASES', requires admin privilege"}`, user))
+				}
+			case *influxql.ShowDiagnosticsStatement:
+				io.WriteString(w, `{"results":[{}]}`)
+			}
+		case "/write":
+			w.WriteHeader(http.StatusOK)
+		}
+	}))
+}

vendor/github.com/influxdata/influxdb/cmd/influx/cli/parser.go

@@ -0,0 +1,34 @@
+package cli
+
+import (
+	"bytes"
+	"fmt"
+)
+
+func parseDatabaseAndRetentionPolicy(stmt []byte) (string, string, error) {
+	var db, rp []byte
+	var quoted bool
+	var seperatorCount int
+
+	stmt = bytes.TrimSpace(stmt)
+
+	for _, b := range stmt {
+		if b == '"' {
+			quoted = !quoted
+			continue
+		}
+		if b == '.' && !quoted {
+			seperatorCount++
+			if seperatorCount > 1 {
+				return "", "", fmt.Errorf("unable to parse database and retention policy from %s", string(stmt))
+			}
+			continue
+		}
+		if seperatorCount == 1 {
+			rp = append(rp, b)
+			continue
+		}
+		db = append(db, b)
+	}
+	return string(db), string(rp), nil
+}

vendor/github.com/influxdata/influxdb/cmd/influx/cli/parser_internal_test.go

@@ -0,0 +1,90 @@
+package cli
+
+import (
+	"errors"
+	"testing"
+)
+
+func Test_parseDatabaseAndretentionPolicy(t *testing.T) {
+	tests := []struct {
+		stmt string
+		db   string
+		rp   string
+		err  error
+	}{
+		{
+			stmt: `foo`,
+			db:   "foo",
+		},
+		{
+			stmt: `"foo.bar"`,
+			db:   "foo.bar",
+		},
+		{
+			stmt: `"foo.bar".`,
+			db:   "foo.bar",
+		},
+		{
+			stmt: `."foo.bar"`,
+			rp:   "foo.bar",
+		},
+		{
+			stmt: `foo.bar`,
+			db:   "foo",
+			rp:   "bar",
+		},
+		{
+			stmt: `"foo".bar`,
+			db:   "foo",
+			rp:   "bar",
+		},
+		{
+			stmt: `"foo"."bar"`,
+			db:   "foo",
+			rp:   "bar",
+		},
+		{
+			stmt: `"foo.bin"."bar"`,
+			db:   "foo.bin",
+			rp:   "bar",
+		},
+		{
+			stmt: `"foo.bin"."bar.baz...."`,
+			db:   "foo.bin",
+			rp:   "bar.baz....",
+		},
+		{
+			stmt: `  "foo.bin"."bar.baz...."  `,
+			db:   "foo.bin",
+			rp:   "bar.baz....",
+		},
+
+		{
+			stmt: `"foo.bin"."bar".boom`,
+			err:  errors.New("foo"),
+		},
+		{
+			stmt: "foo.bar.",
+			err:  errors.New("foo"),
+		},
+	}
+
+	for _, test := range tests {
+		db, rp, err := parseDatabaseAndRetentionPolicy([]byte(test.stmt))
+		if err != nil && test.err == nil {
+			t.Errorf("unexpected error: got %s", err)
+			continue
+		}
+		if test.err != nil && err == nil {
+			t.Errorf("expected err: got: nil, exp: %s", test.err)
+			continue
+		}
+		if db != test.db {
+			t.Errorf("unexpected database: got: %s, exp: %s", db, test.db)
+		}
+		if rp != test.rp {
+			t.Errorf("unexpected retention policy: got: %s, exp: %s", rp, test.rp)
+		}
+	}
+
+}

vendor/github.com/influxdata/influxdb/cmd/influx/main.go

@@ -0,0 +1,120 @@
+// The influx command is a CLI client to InfluxDB.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/influxdata/influxdb/client"
+	"github.com/influxdata/influxdb/cmd/influx/cli"
+)
+
+// These variables are populated via the Go linker.
+var (
+	version string
+)
+
+const (
+	// defaultFormat is the default format of the results when issuing queries
+	defaultFormat = "column"
+
+	// defaultPrecision is the default timestamp format of the results when issuing queries
+	defaultPrecision = "ns"
+
+	// defaultPPS is the default points per second that the import will throttle at
+	// by default it's 0, which means it will not throttle
+	defaultPPS = 0
+)
+
+func init() {
+	// If version is not set, make that clear.
+	if version == "" {
+		version = "unknown"
+	}
+}
+
+func main() {
+	c := cli.New(version)
+
+	fs := flag.NewFlagSet("InfluxDB shell version "+version, flag.ExitOnError)
+	fs.StringVar(&c.Host, "host", client.DefaultHost, "Influxdb host to connect to.")
+	fs.IntVar(&c.Port, "port", client.DefaultPort, "Influxdb port to connect to.")
+	fs.StringVar(&c.ClientConfig.UnixSocket, "socket", "", "Influxdb unix socket to connect to.")
+	fs.StringVar(&c.ClientConfig.Username, "username", "", "Username to connect to the server.")
+	fs.StringVar(&c.ClientConfig.Password, "password", "", `Password to connect to the server.  Leaving blank will prompt for password (--password="").`)
+	fs.StringVar(&c.Database, "database", c.Database, "Database to connect to the server.")
+	fs.BoolVar(&c.Ssl, "ssl", false, "Use https for connecting to cluster.")
+	fs.BoolVar(&c.ClientConfig.UnsafeSsl, "unsafeSsl", false, "Set this when connecting to the cluster using https and not use SSL verification.")
+	fs.StringVar(&c.Format, "format", defaultFormat, "Format specifies the format of the server responses:  json, csv, or column.")
+	fs.StringVar(&c.ClientConfig.Precision, "precision", defaultPrecision, "Precision specifies the format of the timestamp:  rfc3339,h,m,s,ms,u or ns.")
+	fs.StringVar(&c.ClientConfig.WriteConsistency, "consistency", "all", "Set write consistency level: any, one, quorum, or all.")
+	fs.BoolVar(&c.Pretty, "pretty", false, "Turns on pretty print for the json format.")
+	fs.StringVar(&c.Execute, "execute", c.Execute, "Execute command and quit.")
+	fs.BoolVar(&c.ShowVersion, "version", false, "Displays the InfluxDB version.")
+	fs.BoolVar(&c.Import, "import", false, "Import a previous database.")
+	fs.IntVar(&c.ImporterConfig.PPS, "pps", defaultPPS, "How many points per second the import will allow.  By default it is zero and will not throttle importing.")
+	fs.StringVar(&c.ImporterConfig.Path, "path", "", "path to the file to import")
+	fs.BoolVar(&c.ImporterConfig.Compressed, "compressed", false, "set to true if the import file is compressed")
+
+	// Define our own custom usage to print
+	fs.Usage = func() {
+		fmt.Println(`Usage of influx:
+  -version
+       Display the version and exit.
+  -host 'host name'
+       Host to connect to.
+  -port 'port #'
+       Port to connect to.
+  -socket 'unix domain socket'
+       Unix socket to connect to.
+  -database 'database name'
+       Database to connect to the server.
+  -password 'password'
+      Password to connect to the server.  Leaving blank will prompt for password (--password '').
+  -username 'username'
+       Username to connect to the server.
+  -ssl
+        Use https for requests.
+  -unsafeSsl
+        Set this when connecting to the cluster using https and not use SSL verification.
+  -execute 'command'
+       Execute command and quit.
+  -format 'json|csv|column'
+       Format specifies the format of the server responses:  json, csv, or column.
+  -precision 'rfc3339|h|m|s|ms|u|ns'
+       Precision specifies the format of the timestamp:  rfc3339, h, m, s, ms, u or ns.
+  -consistency 'any|one|quorum|all'
+       Set write consistency level: any, one, quorum, or all
+  -pretty
+       Turns on pretty print for the json format.
+  -import
+       Import a previous database export from file
+  -pps
+       How many points per second the import will allow.  By default it is zero and will not throttle importing.
+  -path
+       Path to file to import
+  -compressed
+       Set to true if the import file is compressed
+
+Examples:
+
+    # Use influx in a non-interactive mode to query the database "metrics" and pretty print json:
+    $ influx -database 'metrics' -execute 'select * from cpu' -format 'json' -pretty
+
+    # Connect to a specific database on startup and set database context:
+    $ influx -database 'metrics' -host 'localhost' -port '8086'
+`)
+	}
+	fs.Parse(os.Args[1:])
+
+	if c.ShowVersion {
+		c.Version()
+		os.Exit(0)
+	}
+
+	if err := c.Run(); err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		os.Exit(1)
+	}
+}

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/README.md

@@ -0,0 +1,107 @@
+# `influx_inspect`
+
+## Ways to run
+
+### `influx_inspect`
+Will print usage for the tool.
+
+### `influx_inspect report`
+Displays series meta-data for all shards.  Default location [$HOME/.influxdb]
+
+### `influx_inspect dumptsm`
+Dumps low-level details about tsm1 files
+
+#### Flags
+
+##### `-index` bool
+Dump raw index data.
+
+`default` = false
+
+#### `-blocks` bool
+Dump raw block data.
+
+`default` = false
+
+#### `-all`
+Dump all data. Caution: This may print a lot of information.
+
+`default` = false
+
+#### `-filter-key`
+Only display index and block data match this key substring.
+
+`default` = ""
+
+
+### `influx_inspect export`
+Exports all tsm files to line protocol.  This output file can be imported via the [influx](https://github.com/influxdata/influxdb/tree/master/importer#running-the-import-command) command.
+
+
+#### `-datadir` string
+Data storage path.
+
+`default` = "$HOME/.influxdb/data"
+
+#### `-waldir` string
+WAL storage path.
+
+`default` = "$HOME/.influxdb/wal"
+
+#### `-out` string
+Destination file to export to
+
+`default` = "$HOME/.influxdb/export"
+
+#### `-database` string (optional)
+Database to export.
+
+`default` = ""
+
+#### `-retention` string (optional)
+Retention policy to export.
+
+`default` = ""
+
+#### `-start` string (optional)
+Optional. The time range to start at.
+
+#### `-end` string (optional)
+Optional. The time range to end at.
+
+#### `-compress` bool (optional)
+Compress the output.
+
+`default` = false
+
+#### Sample Commands
+
+Export entire database and compress output:
+```
+influx_inspect export --compress
+```
+
+Export specific retention policy:
+```
+influx_inspect export --database mydb --retention autogen
+```
+
+##### Sample Data
+This is a sample of what the output will look like.
+
+```
+# DDL
+CREATE DATABASE MY_DB_NAME
+CREATE RETENTION POLICY autogen ON MY_DB_NAME DURATION inf REPLICATION 1
+
+# DML
+# CONTEXT-DATABASE:MY_DB_NAME
+# CONTEXT-RETENTION-POLICY:autogen
+randset value=97.9296104805 1439856000000000000
+randset value=25.3849066842 1439856100000000000
+```
+
+# Caveats
+
+The system does not have access to the meta store when exporting TSM shards.  As such, it always creates the retention policy with infinite duration and replication factor of 1.
+End users may want to change this prior to re-importing if they are importing to a cluster or want a different duration for retention.

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/dumptsi/dumptsi.go

@@ -0,0 +1,474 @@
+// Package dumptsi inspects low-level details about tsi1 files.
+package dumptsi
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"text/tabwriter"
+
+	"github.com/influxdata/influxdb/models"
+	"github.com/influxdata/influxdb/tsdb/index/tsi1"
+)
+
+// Command represents the program execution for "influxd dumptsi".
+type Command struct {
+	// Standard input/output, overridden for testing.
+	Stderr io.Writer
+	Stdout io.Writer
+
+	paths []string
+
+	showSeries         bool
+	showMeasurements   bool
+	showTagKeys        bool
+	showTagValues      bool
+	showTagValueSeries bool
+
+	measurementFilter *regexp.Regexp
+	tagKeyFilter      *regexp.Regexp
+	tagValueFilter    *regexp.Regexp
+}
+
+// NewCommand returns a new instance of Command.
+func NewCommand() *Command {
+	return &Command{
+		Stderr: os.Stderr,
+		Stdout: os.Stdout,
+	}
+}
+
+// Run executes the command.
+func (cmd *Command) Run(args ...string) error {
+	var measurementFilter, tagKeyFilter, tagValueFilter string
+	fs := flag.NewFlagSet("dumptsi", flag.ExitOnError)
+	fs.BoolVar(&cmd.showSeries, "series", false, "Show raw series data")
+	fs.BoolVar(&cmd.showMeasurements, "measurements", false, "Show raw measurement data")
+	fs.BoolVar(&cmd.showTagKeys, "tag-keys", false, "Show raw tag key data")
+	fs.BoolVar(&cmd.showTagValues, "tag-values", false, "Show raw tag value data")
+	fs.BoolVar(&cmd.showTagValueSeries, "tag-value-series", false, "Show raw series data for each value")
+	fs.StringVar(&measurementFilter, "measurement-filter", "", "Regex measurement filter")
+	fs.StringVar(&tagKeyFilter, "tag-key-filter", "", "Regex tag key filter")
+	fs.StringVar(&tagValueFilter, "tag-value-filter", "", "Regex tag value filter")
+	fs.SetOutput(cmd.Stdout)
+	fs.Usage = cmd.printUsage
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// Parse filters.
+	if measurementFilter != "" {
+		re, err := regexp.Compile(measurementFilter)
+		if err != nil {
+			return err
+		}
+		cmd.measurementFilter = re
+	}
+	if tagKeyFilter != "" {
+		re, err := regexp.Compile(tagKeyFilter)
+		if err != nil {
+			return err
+		}
+		cmd.tagKeyFilter = re
+	}
+	if tagValueFilter != "" {
+		re, err := regexp.Compile(tagValueFilter)
+		if err != nil {
+			return err
+		}
+		cmd.tagValueFilter = re
+	}
+
+	cmd.paths = fs.Args()
+	if len(cmd.paths) == 0 {
+		fmt.Printf("at least one path required\n\n")
+		fs.Usage()
+		return nil
+	}
+
+	// Some flags imply other flags.
+	if cmd.showTagValueSeries {
+		cmd.showTagValues = true
+	}
+	if cmd.showTagValues {
+		cmd.showTagKeys = true
+	}
+	if cmd.showTagKeys {
+		cmd.showMeasurements = true
+	}
+
+	return cmd.run()
+}
+
+func (cmd *Command) run() error {
+	// Build a file set from the paths on the command line.
+	idx, fs, err := cmd.readFileSet()
+	if err != nil {
+		return err
+	}
+
+	if idx != nil {
+		defer idx.Close()
+	} else {
+		defer fs.Close()
+	}
+	defer fs.Release()
+
+	// Show either raw data or summary stats.
+	if cmd.showSeries || cmd.showMeasurements {
+		if err := cmd.printMerged(fs); err != nil {
+			return err
+		}
+	} else {
+		if err := cmd.printFileSummaries(fs); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (cmd *Command) readFileSet() (*tsi1.Index, *tsi1.FileSet, error) {
+	// If only one path exists and it's a directory then open as an index.
+	if len(cmd.paths) == 1 {
+		fi, err := os.Stat(cmd.paths[0])
+		if err != nil {
+			return nil, nil, err
+		} else if fi.IsDir() {
+			idx := tsi1.NewIndex()
+			idx.Path = cmd.paths[0]
+			idx.CompactionEnabled = false
+			if err := idx.Open(); err != nil {
+				return nil, nil, err
+			}
+			return idx, idx.RetainFileSet(), nil
+		}
+	}
+
+	// Open each file and group into a fileset.
+	var files []tsi1.File
+	for _, path := range cmd.paths {
+		switch ext := filepath.Ext(path); ext {
+		case tsi1.LogFileExt:
+			f := tsi1.NewLogFile(path)
+			if err := f.Open(); err != nil {
+				return nil, nil, err
+			}
+			files = append(files, f)
+
+		case tsi1.IndexFileExt:
+			f := tsi1.NewIndexFile()
+			f.SetPath(path)
+			if err := f.Open(); err != nil {
+				return nil, nil, err
+			}
+			files = append(files, f)
+
+		default:
+			return nil, nil, fmt.Errorf("unexpected file extension: %s", ext)
+		}
+	}
+
+	fs, err := tsi1.NewFileSet(nil, files)
+	if err != nil {
+		return nil, nil, err
+	}
+	fs.Retain()
+
+	return nil, fs, nil
+}
+
+func (cmd *Command) printMerged(fs *tsi1.FileSet) error {
+	if err := cmd.printSeries(fs); err != nil {
+		return err
+	} else if err := cmd.printMeasurements(fs); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (cmd *Command) printSeries(fs *tsi1.FileSet) error {
+	if !cmd.showSeries {
+		return nil
+	}
+
+	// Print header.
+	tw := tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
+	fmt.Fprintln(tw, "Series\t")
+
+	// Iterate over each series.
+	itr := fs.SeriesIterator()
+	for e := itr.Next(); e != nil; e = itr.Next() {
+		name, tags := e.Name(), e.Tags()
+
+		if !cmd.matchSeries(e.Name(), e.Tags()) {
+			continue
+		}
+
+		fmt.Fprintf(tw, "%s%s\t%v\n", name, tags.HashKey(), deletedString(e.Deleted()))
+	}
+
+	// Flush & write footer spacing.
+	if err := tw.Flush(); err != nil {
+		return err
+	}
+	fmt.Fprint(cmd.Stdout, "\n\n")
+
+	return nil
+}
+
+func (cmd *Command) printMeasurements(fs *tsi1.FileSet) error {
+	if !cmd.showMeasurements {
+		return nil
+	}
+
+	tw := tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
+	fmt.Fprintln(tw, "Measurement\t")
+
+	// Iterate over each series.
+	itr := fs.MeasurementIterator()
+	for e := itr.Next(); e != nil; e = itr.Next() {
+		if cmd.measurementFilter != nil && !cmd.measurementFilter.Match(e.Name()) {
+			continue
+		}
+
+		fmt.Fprintf(tw, "%s\t%v\n", e.Name(), deletedString(e.Deleted()))
+		if err := tw.Flush(); err != nil {
+			return err
+		}
+
+		if err := cmd.printTagKeys(fs, e.Name()); err != nil {
+			return err
+		}
+	}
+
+	fmt.Fprint(cmd.Stdout, "\n\n")
+
+	return nil
+}
+
+func (cmd *Command) printTagKeys(fs *tsi1.FileSet, name []byte) error {
+	if !cmd.showTagKeys {
+		return nil
+	}
+
+	// Iterate over each key.
+	tw := tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
+	itr := fs.TagKeyIterator(name)
+	for e := itr.Next(); e != nil; e = itr.Next() {
+		if cmd.tagKeyFilter != nil && !cmd.tagKeyFilter.Match(e.Key()) {
+			continue
+		}
+
+		fmt.Fprintf(tw, "    %s\t%v\n", e.Key(), deletedString(e.Deleted()))
+		if err := tw.Flush(); err != nil {
+			return err
+		}
+
+		if err := cmd.printTagValues(fs, name, e.Key()); err != nil {
+			return err
+		}
+	}
+	fmt.Fprint(cmd.Stdout, "\n")
+
+	return nil
+}
+
+func (cmd *Command) printTagValues(fs *tsi1.FileSet, name, key []byte) error {
+	if !cmd.showTagValues {
+		return nil
+	}
+
+	// Iterate over each value.
+	tw := tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
+	itr := fs.TagValueIterator(name, key)
+	for e := itr.Next(); e != nil; e = itr.Next() {
+		if cmd.tagValueFilter != nil && !cmd.tagValueFilter.Match(e.Value()) {
+			continue
+		}
+
+		fmt.Fprintf(tw, "        %s\t%v\n", e.Value(), deletedString(e.Deleted()))
+		if err := tw.Flush(); err != nil {
+			return err
+		}
+
+		if err := cmd.printTagValueSeries(fs, name, key, e.Value()); err != nil {
+			return err
+		}
+	}
+	fmt.Fprint(cmd.Stdout, "\n")
+
+	return nil
+}
+
+func (cmd *Command) printTagValueSeries(fs *tsi1.FileSet, name, key, value []byte) error {
+	if !cmd.showTagValueSeries {
+		return nil
+	}
+
+	// Iterate over each series.
+	tw := tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
+	itr := fs.TagValueSeriesIterator(name, key, value)
+	for e := itr.Next(); e != nil; e = itr.Next() {
+		if !cmd.matchSeries(e.Name(), e.Tags()) {
+			continue
+		}
+
+		fmt.Fprintf(tw, "            %s%s\n", e.Name(), e.Tags().HashKey())
+		if err := tw.Flush(); err != nil {
+			return err
+		}
+	}
+	fmt.Fprint(cmd.Stdout, "\n")
+
+	return nil
+}
+
+func (cmd *Command) printFileSummaries(fs *tsi1.FileSet) error {
+	for _, f := range fs.Files() {
+		switch f := f.(type) {
+		case *tsi1.LogFile:
+			if err := cmd.printLogFileSummary(f); err != nil {
+				return err
+			}
+		case *tsi1.IndexFile:
+			if err := cmd.printIndexFileSummary(f); err != nil {
+				return err
+			}
+		default:
+			panic("unreachable")
+		}
+		fmt.Fprintln(cmd.Stdout, "")
+	}
+	return nil
+}
+
+func (cmd *Command) printLogFileSummary(f *tsi1.LogFile) error {
+	fmt.Fprintf(cmd.Stdout, "[LOG FILE] %s\n", filepath.Base(f.Path()))
+	tw := tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
+	fmt.Fprintf(tw, "Series:\t%d\n", f.SeriesN())
+	fmt.Fprintf(tw, "Measurements:\t%d\n", f.MeasurementN())
+	fmt.Fprintf(tw, "Tag Keys:\t%d\n", f.TagKeyN())
+	fmt.Fprintf(tw, "Tag Values:\t%d\n", f.TagValueN())
+	return tw.Flush()
+}
+
+func (cmd *Command) printIndexFileSummary(f *tsi1.IndexFile) error {
+	fmt.Fprintf(cmd.Stdout, "[INDEX FILE] %s\n", filepath.Base(f.Path()))
+
+	// Calculate summary stats.
+	seriesN := f.SeriesN()
+	var measurementN, measurementSeriesN, measurementSeriesSize uint64
+	var keyN uint64
+	var valueN, valueSeriesN, valueSeriesSize uint64
+	mitr := f.MeasurementIterator()
+	for me, _ := mitr.Next().(*tsi1.MeasurementBlockElem); me != nil; me, _ = mitr.Next().(*tsi1.MeasurementBlockElem) {
+		kitr := f.TagKeyIterator(me.Name())
+		for ke, _ := kitr.Next().(*tsi1.TagBlockKeyElem); ke != nil; ke, _ = kitr.Next().(*tsi1.TagBlockKeyElem) {
+			vitr := f.TagValueIterator(me.Name(), ke.Key())
+			for ve, _ := vitr.Next().(*tsi1.TagBlockValueElem); ve != nil; ve, _ = vitr.Next().(*tsi1.TagBlockValueElem) {
+				valueN++
+				valueSeriesN += uint64(ve.SeriesN())
+				valueSeriesSize += uint64(len(ve.SeriesData()))
+			}
+			keyN++
+		}
+		measurementN++
+		measurementSeriesN += uint64(me.SeriesN())
+		measurementSeriesSize += uint64(len(me.SeriesData()))
+	}
+
+	// Write stats.
+	tw := tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
+	fmt.Fprintf(tw, "Series:\t%d\n", seriesN)
+	fmt.Fprintf(tw, "Measurements:\t%d\n", measurementN)
+	fmt.Fprintf(tw, "  Series data size:\t%d (%s)\n", measurementSeriesSize, formatSize(measurementSeriesSize))
+	fmt.Fprintf(tw, "  Bytes per series:\t%.01fb\n", float64(measurementSeriesSize)/float64(measurementSeriesN))
+	fmt.Fprintf(tw, "Tag Keys:\t%d\n", keyN)
+	fmt.Fprintf(tw, "Tag Values:\t%d\n", valueN)
+	fmt.Fprintf(tw, "  Series:\t%d\n", valueSeriesN)
+	fmt.Fprintf(tw, "  Series data size:\t%d (%s)\n", valueSeriesSize, formatSize(valueSeriesSize))
+	fmt.Fprintf(tw, "  Bytes per series:\t%.01fb\n", float64(valueSeriesSize)/float64(valueSeriesN))
+	fmt.Fprintf(tw, "Avg tags per series:\t%.01f\n", float64(valueSeriesN)/float64(seriesN))
+	if err := tw.Flush(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// matchSeries returns true if the command filters matches the series.
+func (cmd *Command) matchSeries(name []byte, tags models.Tags) bool {
+	// Filter by measurement.
+	if cmd.measurementFilter != nil && !cmd.measurementFilter.Match(name) {
+		return false
+	}
+
+	// Filter by tag key/value.
+	if cmd.tagKeyFilter != nil || cmd.tagValueFilter != nil {
+		var matched bool
+		for _, tag := range tags {
+			if (cmd.tagKeyFilter == nil || cmd.tagKeyFilter.Match(tag.Key)) && (cmd.tagValueFilter == nil || cmd.tagValueFilter.Match(tag.Value)) {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			return false
+		}
+	}
+
+	return true
+}
+
+// printUsage prints the usage message to STDERR.
+func (cmd *Command) printUsage() {
+	usage := `Dumps low-level details about tsi1 files.
+
+Usage: influx_inspect dumptsi [flags] path...
+
+    -series
+            Dump raw series data
+    -measurements
+            Dump raw measurement data
+    -tag-keys
+            Dump raw tag keys
+    -tag-values
+            Dump raw tag values
+    -tag-value-series
+            Dump raw series for each tag value
+    -measurement-filter REGEXP
+            Filters data by measurement regular expression
+    -tag-key-filter REGEXP
+            Filters data by tag key regular expression
+    -tag-value-filter REGEXP
+            Filters data by tag value regular expression
+
+If no flags are specified then summary stats are provided for each file.
+`
+
+	fmt.Fprintf(cmd.Stdout, usage)
+}
+
+// deletedString returns "(deleted)" if v is true.
+func deletedString(v bool) string {
+	if v {
+		return "(deleted)"
+	}
+	return ""
+}
+
+func formatSize(v uint64) string {
+	denom := uint64(1)
+	var uom string
+	for _, uom = range []string{"b", "kb", "mb", "gb", "tb"} {
+		if denom*1024 > v {
+			break
+		}
+		denom *= 1024
+	}
+	return fmt.Sprintf("%0.01f%s", float64(v)/float64(denom), uom)
+}

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/dumptsm/dumptsm.go

@@ -0,0 +1,332 @@
+// Package dumptsm inspects low-level details about tsm1 files.
+package dumptsm
+
+import (
+	"encoding/binary"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+)
+
+// Command represents the program execution for "influxd dumptsm".
+type Command struct {
+	// Standard input/output, overridden for testing.
+	Stderr io.Writer
+	Stdout io.Writer
+
+	dumpIndex  bool
+	dumpBlocks bool
+	dumpAll    bool
+	filterKey  string
+	path       string
+}
+
+// NewCommand returns a new instance of Command.
+func NewCommand() *Command {
+	return &Command{
+		Stderr: os.Stderr,
+		Stdout: os.Stdout,
+	}
+}
+
+// Run executes the command.
+func (cmd *Command) Run(args ...string) error {
+	fs := flag.NewFlagSet("file", flag.ExitOnError)
+	fs.BoolVar(&cmd.dumpIndex, "index", false, "Dump raw index data")
+	fs.BoolVar(&cmd.dumpBlocks, "blocks", false, "Dump raw block data")
+	fs.BoolVar(&cmd.dumpAll, "all", false, "Dump all data. Caution: This may print a lot of information")
+	fs.StringVar(&cmd.filterKey, "filter-key", "", "Only display index and block data match this key substring")
+
+	fs.SetOutput(cmd.Stdout)
+	fs.Usage = cmd.printUsage
+
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	if fs.Arg(0) == "" {
+		fmt.Printf("TSM file not specified\n\n")
+		fs.Usage()
+		return nil
+	}
+	cmd.path = fs.Args()[0]
+	cmd.dumpBlocks = cmd.dumpBlocks || cmd.dumpAll || cmd.filterKey != ""
+	cmd.dumpIndex = cmd.dumpIndex || cmd.dumpAll || cmd.filterKey != ""
+	return cmd.dump()
+}
+
+func (cmd *Command) dump() error {
+	var errors []error
+
+	f, err := os.Open(cmd.path)
+	if err != nil {
+		return err
+	}
+
+	// Get the file size
+	stat, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	b := make([]byte, 8)
+
+	r, err := tsm1.NewTSMReader(f)
+	if err != nil {
+		return fmt.Errorf("Error opening TSM files: %s", err.Error())
+	}
+	defer r.Close()
+
+	minTime, maxTime := r.TimeRange()
+	keyCount := r.KeyCount()
+
+	blockStats := &blockStats{}
+
+	println("Summary:")
+	fmt.Printf("  File: %s\n", cmd.path)
+	fmt.Printf("  Time Range: %s - %s\n",
+		time.Unix(0, minTime).UTC().Format(time.RFC3339Nano),
+		time.Unix(0, maxTime).UTC().Format(time.RFC3339Nano),
+	)
+	fmt.Printf("  Duration: %s ", time.Unix(0, maxTime).Sub(time.Unix(0, minTime)))
+	fmt.Printf("  Series: %d ", keyCount)
+	fmt.Printf("  File Size: %d\n", stat.Size())
+	println()
+
+	tw := tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
+
+	if cmd.dumpIndex {
+		println("Index:")
+		tw.Flush()
+		println()
+
+		fmt.Fprintln(tw, "  "+strings.Join([]string{"Pos", "Min Time", "Max Time", "Ofs", "Size", "Key", "Field"}, "\t"))
+		var pos int
+		for i := 0; i < keyCount; i++ {
+			key, _ := r.KeyAt(i)
+			for _, e := range r.Entries(string(key)) {
+				pos++
+				split := strings.Split(string(key), "#!~#")
+
+				// Possible corruption? Try to read as much as we can and point to the problem.
+				measurement := split[0]
+				field := split[1]
+
+				if cmd.filterKey != "" && !strings.Contains(string(key), cmd.filterKey) {
+					continue
+				}
+				fmt.Fprintln(tw, "  "+strings.Join([]string{
+					strconv.FormatInt(int64(pos), 10),
+					time.Unix(0, e.MinTime).UTC().Format(time.RFC3339Nano),
+					time.Unix(0, e.MaxTime).UTC().Format(time.RFC3339Nano),
+					strconv.FormatInt(int64(e.Offset), 10),
+					strconv.FormatInt(int64(e.Size), 10),
+					measurement,
+					field,
+				}, "\t"))
+				tw.Flush()
+			}
+		}
+	}
+
+	tw = tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
+	fmt.Fprintln(tw, "  "+strings.Join([]string{"Blk", "Chk", "Ofs", "Len", "Type", "Min Time", "Points", "Enc [T/V]", "Len [T/V]"}, "\t"))
+
+	// Starting at 5 because the magic number is 4 bytes + 1 byte version
+	i := int64(5)
+	var blockCount, pointCount, blockSize int64
+	indexSize := r.IndexSize()
+
+	// Start at the beginning and read every block
+	for j := 0; j < keyCount; j++ {
+		key, _ := r.KeyAt(j)
+		for _, e := range r.Entries(string(key)) {
+
+			f.Seek(int64(e.Offset), 0)
+			f.Read(b[:4])
+
+			chksum := binary.BigEndian.Uint32(b[:4])
+
+			buf := make([]byte, e.Size-4)
+			f.Read(buf)
+
+			blockSize += int64(e.Size)
+
+			if cmd.filterKey != "" && !strings.Contains(string(key), cmd.filterKey) {
+				i += blockSize
+				blockCount++
+				continue
+			}
+
+			blockType := buf[0]
+
+			encoded := buf[1:]
+
+			var v []tsm1.Value
+			v, err := tsm1.DecodeBlock(buf, v)
+			if err != nil {
+				return err
+			}
+			startTime := time.Unix(0, v[0].UnixNano())
+
+			pointCount += int64(len(v))
+
+			// Length of the timestamp block
+			tsLen, j := binary.Uvarint(encoded)
+
+			// Unpack the timestamp bytes
+			ts := encoded[int(j) : int(j)+int(tsLen)]
+
+			// Unpack the value bytes
+			values := encoded[int(j)+int(tsLen):]
+
+			tsEncoding := timeEnc[int(ts[0]>>4)]
+			vEncoding := encDescs[int(blockType+1)][values[0]>>4]
+
+			typeDesc := blockTypes[blockType]
+
+			blockStats.inc(0, ts[0]>>4)
+			blockStats.inc(int(blockType+1), values[0]>>4)
+			blockStats.size(len(buf))
+
+			if cmd.dumpBlocks {
+				fmt.Fprintln(tw, "  "+strings.Join([]string{
+					strconv.FormatInt(blockCount, 10),
+					strconv.FormatUint(uint64(chksum), 10),
+					strconv.FormatInt(i, 10),
+					strconv.FormatInt(int64(len(buf)), 10),
+					typeDesc,
+					startTime.UTC().Format(time.RFC3339Nano),
+					strconv.FormatInt(int64(len(v)), 10),
+					fmt.Sprintf("%s/%s", tsEncoding, vEncoding),
+					fmt.Sprintf("%d/%d", len(ts), len(values)),
+				}, "\t"))
+			}
+
+			i += blockSize
+			blockCount++
+		}
+	}
+
+	if cmd.dumpBlocks {
+		println("Blocks:")
+		tw.Flush()
+		println()
+	}
+
+	var blockSizeAvg int64
+	if blockCount > 0 {
+		blockSizeAvg = blockSize / blockCount
+	}
+	fmt.Printf("Statistics\n")
+	fmt.Printf("  Blocks:\n")
+	fmt.Printf("    Total: %d Size: %d Min: %d Max: %d Avg: %d\n",
+		blockCount, blockSize, blockStats.min, blockStats.max, blockSizeAvg)
+	fmt.Printf("  Index:\n")
+	fmt.Printf("    Total: %d Size: %d\n", blockCount, indexSize)
+	fmt.Printf("  Points:\n")
+	fmt.Printf("    Total: %d", pointCount)
+	println()
+
+	println("  Encoding:")
+	for i, counts := range blockStats.counts {
+		if len(counts) == 0 {
+			continue
+		}
+		fmt.Printf("    %s: ", strings.Title(fieldType[i]))
+		for j, v := range counts {
+			fmt.Printf("\t%s: %d (%d%%) ", encDescs[i][j], v, int(float64(v)/float64(blockCount)*100))
+		}
+		println()
+	}
+	fmt.Printf("  Compression:\n")
+	fmt.Printf("    Per block: %0.2f bytes/point\n", float64(blockSize)/float64(pointCount))
+	fmt.Printf("    Total: %0.2f bytes/point\n", float64(stat.Size())/float64(pointCount))
+
+	if len(errors) > 0 {
+		println()
+		fmt.Printf("Errors (%d):\n", len(errors))
+		for _, err := range errors {
+			fmt.Printf("  * %v\n", err)
+		}
+		println()
+		return fmt.Errorf("error count %d", len(errors))
+	}
+	return nil
+}
+
+// printUsage prints the usage message to STDERR.
+func (cmd *Command) printUsage() {
+	usage := `Dumps low-level details about tsm1 files.
+
+Usage: influx_inspect dumptsm [flags] <path
+
+    -index
+            Dump raw index data
+    -blocks
+            Dump raw block data
+    -all
+            Dump all data. Caution: This may print a lot of information
+    -filter-key <name>
+            Only display index and block data match this key substring
+`
+
+	fmt.Fprintf(cmd.Stdout, usage)
+}
+
+var (
+	fieldType = []string{
+		"timestamp", "float", "int", "bool", "string",
+	}
+	blockTypes = []string{
+		"float64", "int64", "bool", "string",
+	}
+	timeEnc = []string{
+		"none", "s8b", "rle",
+	}
+	floatEnc = []string{
+		"none", "gor",
+	}
+	intEnc = []string{
+		"none", "s8b", "rle",
+	}
+	boolEnc = []string{
+		"none", "bp",
+	}
+	stringEnc = []string{
+		"none", "snpy",
+	}
+	encDescs = [][]string{
+		timeEnc, floatEnc, intEnc, boolEnc, stringEnc,
+	}
+)
+
+type blockStats struct {
+	min, max int
+	counts   [][]int
+}
+
+func (b *blockStats) inc(typ int, enc byte) {
+	for len(b.counts) <= typ {
+		b.counts = append(b.counts, []int{})
+	}
+	for len(b.counts[typ]) <= int(enc) {
+		b.counts[typ] = append(b.counts[typ], 0)
+	}
+	b.counts[typ][enc]++
+}
+
+func (b *blockStats) size(sz int) {
+	if b.min == 0 || sz < b.min {
+		b.min = sz
+	}
+	if b.min == 0 || sz > b.max {
+		b.max = sz
+	}
+}

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/dumptsm/dumptsm_test.go

@@ -0,0 +1,3 @@
+package dumptsm_test
+
+// TODO: write some tests

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/export/export.go

@@ -0,0 +1,408 @@
+// Package export exports TSM files into InfluxDB line protocol format.
+package export
+
+import (
+	"bufio"
+	"compress/gzip"
+	"flag"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/influxdata/influxdb/influxql"
+	"github.com/influxdata/influxdb/models"
+	"github.com/influxdata/influxdb/pkg/escape"
+	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+)
+
+// Command represents the program execution for "influx_inspect export".
+type Command struct {
+	// Standard input/output, overridden for testing.
+	Stderr io.Writer
+	Stdout io.Writer
+
+	dataDir         string
+	walDir          string
+	out             string
+	database        string
+	retentionPolicy string
+	startTime       int64
+	endTime         int64
+	compress        bool
+
+	manifest map[string]struct{}
+	tsmFiles map[string][]string
+	walFiles map[string][]string
+}
+
+// NewCommand returns a new instance of Command.
+func NewCommand() *Command {
+	return &Command{
+		Stderr: os.Stderr,
+		Stdout: os.Stdout,
+
+		manifest: make(map[string]struct{}),
+		tsmFiles: make(map[string][]string),
+		walFiles: make(map[string][]string),
+	}
+}
+
+// Run executes the command.
+func (cmd *Command) Run(args ...string) error {
+	var start, end string
+	fs := flag.NewFlagSet("export", flag.ExitOnError)
+	fs.StringVar(&cmd.dataDir, "datadir", os.Getenv("HOME")+"/.influxdb/data", "Data storage path")
+	fs.StringVar(&cmd.walDir, "waldir", os.Getenv("HOME")+"/.influxdb/wal", "WAL storage path")
+	fs.StringVar(&cmd.out, "out", os.Getenv("HOME")+"/.influxdb/export", "Destination file to export to")
+	fs.StringVar(&cmd.database, "database", "", "Optional: the database to export")
+	fs.StringVar(&cmd.retentionPolicy, "retention", "", "Optional: the retention policy to export (requires -database)")
+	fs.StringVar(&start, "start", "", "Optional: the start time to export (RFC3339 format)")
+	fs.StringVar(&end, "end", "", "Optional: the end time to export (RFC3339 format)")
+	fs.BoolVar(&cmd.compress, "compress", false, "Compress the output")
+
+	fs.SetOutput(cmd.Stdout)
+	fs.Usage = func() {
+		fmt.Fprintf(cmd.Stdout, "Exports TSM files into InfluxDB line protocol format.\n\n")
+		fmt.Fprintf(cmd.Stdout, "Usage: %s export [flags]\n\n", filepath.Base(os.Args[0]))
+		fs.PrintDefaults()
+	}
+
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// set defaults
+	if start != "" {
+		s, err := time.Parse(time.RFC3339, start)
+		if err != nil {
+			return err
+		}
+		cmd.startTime = s.UnixNano()
+	} else {
+		cmd.startTime = math.MinInt64
+	}
+	if end != "" {
+		e, err := time.Parse(time.RFC3339, end)
+		if err != nil {
+			return err
+		}
+		cmd.endTime = e.UnixNano()
+	} else {
+		// set end time to max if it is not set.
+		cmd.endTime = math.MaxInt64
+	}
+
+	if err := cmd.validate(); err != nil {
+		return err
+	}
+
+	return cmd.export()
+}
+
+func (cmd *Command) validate() error {
+	if cmd.retentionPolicy != "" && cmd.database == "" {
+		return fmt.Errorf("must specify a db")
+	}
+	if cmd.startTime != 0 && cmd.endTime != 0 && cmd.endTime < cmd.startTime {
+		return fmt.Errorf("end time before start time")
+	}
+	return nil
+}
+
+func (cmd *Command) export() error {
+	if err := cmd.walkTSMFiles(); err != nil {
+		return err
+	}
+	if err := cmd.walkWALFiles(); err != nil {
+		return err
+	}
+	return cmd.write()
+}
+
+func (cmd *Command) walkTSMFiles() error {
+	return filepath.Walk(cmd.dataDir, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// check to see if this is a tsm file
+		if filepath.Ext(path) != "."+tsm1.TSMFileExtension {
+			return nil
+		}
+
+		relPath, err := filepath.Rel(cmd.dataDir, path)
+		if err != nil {
+			return err
+		}
+		dirs := strings.Split(relPath, string(byte(os.PathSeparator)))
+		if len(dirs) < 2 {
+			return fmt.Errorf("invalid directory structure for %s", path)
+		}
+		if dirs[0] == cmd.database || cmd.database == "" {
+			if dirs[1] == cmd.retentionPolicy || cmd.retentionPolicy == "" {
+				key := filepath.Join(dirs[0], dirs[1])
+				cmd.manifest[key] = struct{}{}
+				cmd.tsmFiles[key] = append(cmd.tsmFiles[key], path)
+			}
+		}
+		return nil
+	})
+}
+
+func (cmd *Command) walkWALFiles() error {
+	return filepath.Walk(cmd.walDir, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// check to see if this is a wal file
+		fileName := filepath.Base(path)
+		if filepath.Ext(path) != "."+tsm1.WALFileExtension || !strings.HasPrefix(fileName, tsm1.WALFilePrefix) {
+			return nil
+		}
+
+		relPath, err := filepath.Rel(cmd.walDir, path)
+		if err != nil {
+			return err
+		}
+		dirs := strings.Split(relPath, string(byte(os.PathSeparator)))
+		if len(dirs) < 2 {
+			return fmt.Errorf("invalid directory structure for %s", path)
+		}
+		if dirs[0] == cmd.database || cmd.database == "" {
+			if dirs[1] == cmd.retentionPolicy || cmd.retentionPolicy == "" {
+				key := filepath.Join(dirs[0], dirs[1])
+				cmd.manifest[key] = struct{}{}
+				cmd.walFiles[key] = append(cmd.walFiles[key], path)
+			}
+		}
+		return nil
+	})
+}
+
+func (cmd *Command) write() error {
+	// open our output file and create an output buffer
+	f, err := os.Create(cmd.out)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	// Because calling (*os.File).Write is relatively expensive,
+	// and we don't *need* to sync to disk on every written line of export,
+	// use a sized buffered writer so that we only sync the file every megabyte.
+	bw := bufio.NewWriterSize(f, 1024*1024)
+	defer bw.Flush()
+
+	var w io.Writer = bw
+
+	if cmd.compress {
+		gzw := gzip.NewWriter(w)
+		defer gzw.Close()
+		w = gzw
+	}
+
+	s, e := time.Unix(0, cmd.startTime).Format(time.RFC3339), time.Unix(0, cmd.endTime).Format(time.RFC3339)
+	fmt.Fprintf(w, "# INFLUXDB EXPORT: %s - %s\n", s, e)
+
+	// Write out all the DDL
+	fmt.Fprintln(w, "# DDL")
+	for key := range cmd.manifest {
+		keys := strings.Split(key, string(os.PathSeparator))
+		db, rp := influxql.QuoteIdent(keys[0]), influxql.QuoteIdent(keys[1])
+		fmt.Fprintf(w, "CREATE DATABASE %s WITH NAME %s\n", db, rp)
+	}
+
+	fmt.Fprintln(w, "# DML")
+	for key := range cmd.manifest {
+		keys := strings.Split(key, string(os.PathSeparator))
+		fmt.Fprintf(w, "# CONTEXT-DATABASE:%s\n", keys[0])
+		fmt.Fprintf(w, "# CONTEXT-RETENTION-POLICY:%s\n", keys[1])
+		if files, ok := cmd.tsmFiles[key]; ok {
+			fmt.Fprintf(cmd.Stdout, "writing out tsm file data for %s...", key)
+			if err := cmd.writeTsmFiles(w, files); err != nil {
+				return err
+			}
+			fmt.Fprintln(cmd.Stdout, "complete.")
+		}
+		if _, ok := cmd.walFiles[key]; ok {
+			fmt.Fprintf(cmd.Stdout, "writing out wal file data for %s...", key)
+			if err := cmd.writeWALFiles(w, cmd.walFiles[key], key); err != nil {
+				return err
+			}
+			fmt.Fprintln(cmd.Stdout, "complete.")
+		}
+	}
+	return nil
+}
+
+func (cmd *Command) writeTsmFiles(w io.Writer, files []string) error {
+	fmt.Fprintln(w, "# writing tsm data")
+
+	// we need to make sure we write the same order that the files were written
+	sort.Strings(files)
+
+	for _, f := range files {
+		if err := cmd.exportTSMFile(f, w); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (cmd *Command) exportTSMFile(tsmFilePath string, w io.Writer) error {
+	f, err := os.Open(tsmFilePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	r, err := tsm1.NewTSMReader(f)
+	if err != nil {
+		fmt.Fprintf(cmd.Stderr, "unable to read %s, skipping: %s\n", tsmFilePath, err.Error())
+		return nil
+	}
+	defer r.Close()
+
+	if sgStart, sgEnd := r.TimeRange(); sgStart > cmd.endTime || sgEnd < cmd.startTime {
+		return nil
+	}
+
+	for i := 0; i < r.KeyCount(); i++ {
+		key, _ := r.KeyAt(i)
+		values, err := r.ReadAll(string(key))
+		if err != nil {
+			fmt.Fprintf(cmd.Stderr, "unable to read key %q in %s, skipping: %s\n", string(key), tsmFilePath, err.Error())
+			continue
+		}
+		measurement, field := tsm1.SeriesAndFieldFromCompositeKey(key)
+		field = escape.Bytes(field)
+
+		if err := cmd.writeValues(w, measurement, string(field), values); err != nil {
+			// An error from writeValues indicates an IO error, which should be returned.
+			return err
+		}
+	}
+	return nil
+}
+
+func (cmd *Command) writeWALFiles(w io.Writer, files []string, key string) error {
+	fmt.Fprintln(w, "# writing wal data")
+
+	// we need to make sure we write the same order that the wal received the data
+	sort.Strings(files)
+
+	var once sync.Once
+	warnDelete := func() {
+		once.Do(func() {
+			msg := fmt.Sprintf(`WARNING: detected deletes in wal file.
+Some series for %q may be brought back by replaying this data.
+To resolve, you can either let the shard snapshot prior to exporting the data
+or manually editing the exported file.
+			`, key)
+			fmt.Fprintln(cmd.Stderr, msg)
+		})
+	}
+
+	for _, f := range files {
+		if err := cmd.exportWALFile(f, w, warnDelete); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// exportWAL reads every WAL entry from r and exports it to w.
+func (cmd *Command) exportWALFile(walFilePath string, w io.Writer, warnDelete func()) error {
+	f, err := os.Open(walFilePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	r := tsm1.NewWALSegmentReader(f)
+	defer r.Close()
+
+	for r.Next() {
+		entry, err := r.Read()
+		if err != nil {
+			n := r.Count()
+			fmt.Fprintf(cmd.Stderr, "file %s corrupt at position %d", walFilePath, n)
+			break
+		}
+
+		switch t := entry.(type) {
+		case *tsm1.DeleteWALEntry, *tsm1.DeleteRangeWALEntry:
+			warnDelete()
+			continue
+		case *tsm1.WriteWALEntry:
+			for key, values := range t.Values {
+				measurement, field := tsm1.SeriesAndFieldFromCompositeKey([]byte(key))
+				// measurements are stored escaped, field names are not
+				field = escape.Bytes(field)
+
+				if err := cmd.writeValues(w, measurement, string(field), values); err != nil {
+					// An error from writeValues indicates an IO error, which should be returned.
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// writeValues writes every value in values to w, using the given series key and field name.
+// If any call to w.Write fails, that error is returned.
+func (cmd *Command) writeValues(w io.Writer, seriesKey []byte, field string, values []tsm1.Value) error {
+	buf := []byte(string(seriesKey) + " " + field + "=")
+	prefixLen := len(buf)
+
+	for _, value := range values {
+		ts := value.UnixNano()
+		if (ts < cmd.startTime) || (ts > cmd.endTime) {
+			continue
+		}
+
+		// Re-slice buf to be "<series_key> <field>=".
+		buf = buf[:prefixLen]
+
+		// Append the correct representation of the value.
+		switch v := value.Value().(type) {
+		case float64:
+			buf = strconv.AppendFloat(buf, v, 'g', -1, 64)
+		case int64:
+			buf = strconv.AppendInt(buf, v, 10)
+			buf = append(buf, 'i')
+		case bool:
+			buf = strconv.AppendBool(buf, v)
+		case string:
+			buf = append(buf, '"')
+			buf = append(buf, models.EscapeStringField(v)...)
+			buf = append(buf, '"')
+		default:
+			// This shouldn't be possible, but we'll format it anyway.
+			buf = append(buf, fmt.Sprintf("%v", v)...)
+		}
+
+		// Now buf has "<series_key> <field>=<value>".
+		// Append the timestamp and a newline, then write it.
+		buf = append(buf, ' ')
+		buf = strconv.AppendInt(buf, ts, 10)
+		buf = append(buf, '\n')
+		if _, err := w.Write(buf); err != nil {
+			// Underlying IO error needs to be returned.
+			return err
+		}
+	}
+
+	return nil
+}

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/export/export_test.go

@@ -0,0 +1,340 @@
+package export
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"math"
+	"math/rand"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/golang/snappy"
+	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+)
+
+type corpus map[string][]tsm1.Value
+
+var (
+	basicCorpus = corpus{
+		tsm1.SeriesFieldKey("floats,k=f", "f"): []tsm1.Value{
+			tsm1.NewValue(1, float64(1.5)),
+			tsm1.NewValue(2, float64(3)),
+		},
+		tsm1.SeriesFieldKey("ints,k=i", "i"): []tsm1.Value{
+			tsm1.NewValue(10, int64(15)),
+			tsm1.NewValue(20, int64(30)),
+		},
+		tsm1.SeriesFieldKey("bools,k=b", "b"): []tsm1.Value{
+			tsm1.NewValue(100, true),
+			tsm1.NewValue(200, false),
+		},
+		tsm1.SeriesFieldKey("strings,k=s", "s"): []tsm1.Value{
+			tsm1.NewValue(1000, "1k"),
+			tsm1.NewValue(2000, "2k"),
+		},
+	}
+
+	basicCorpusExpLines = []string{
+		"floats,k=f f=1.5 1",
+		"floats,k=f f=3 2",
+		"ints,k=i i=15i 10",
+		"ints,k=i i=30i 20",
+		"bools,k=b b=true 100",
+		"bools,k=b b=false 200",
+		`strings,k=s s="1k" 1000`,
+		`strings,k=s s="2k" 2000`,
+	}
+
+	escapeStringCorpus = corpus{
+		tsm1.SeriesFieldKey("t", "s"): []tsm1.Value{
+			tsm1.NewValue(1, `1. "quotes"`),
+			tsm1.NewValue(2, `2. back\slash`),
+			tsm1.NewValue(3, `3. bs\q"`),
+		},
+	}
+
+	escCorpusExpLines = []string{
+		`t s="1. \"quotes\"" 1`,
+		`t s="2. back\\slash" 2`,
+		`t s="3. bs\\q\"" 3`,
+	}
+)
+
+func Test_exportWALFile(t *testing.T) {
+	for _, c := range []struct {
+		corpus corpus
+		lines  []string
+	}{
+		{corpus: basicCorpus, lines: basicCorpusExpLines},
+		{corpus: escapeStringCorpus, lines: escCorpusExpLines},
+	} {
+		walFile := writeCorpusToWALFile(c.corpus)
+		defer os.Remove(walFile.Name())
+
+		var out bytes.Buffer
+		if err := newCommand().exportWALFile(walFile.Name(), &out, func() {}); err != nil {
+			t.Fatal(err)
+		}
+
+		lines := strings.Split(out.String(), "\n")
+		for _, exp := range c.lines {
+			found := false
+			for _, l := range lines {
+				if exp == l {
+					found = true
+					break
+				}
+			}
+
+			if !found {
+				t.Fatalf("expected line %q to be in exported output:\n%s", exp, out.String())
+			}
+		}
+	}
+}
+
+func Test_exportTSMFile(t *testing.T) {
+	for _, c := range []struct {
+		corpus corpus
+		lines  []string
+	}{
+		{corpus: basicCorpus, lines: basicCorpusExpLines},
+		{corpus: escapeStringCorpus, lines: escCorpusExpLines},
+	} {
+		tsmFile := writeCorpusToTSMFile(c.corpus)
+		defer os.Remove(tsmFile.Name())
+
+		var out bytes.Buffer
+		if err := newCommand().exportTSMFile(tsmFile.Name(), &out); err != nil {
+			t.Fatal(err)
+		}
+
+		lines := strings.Split(out.String(), "\n")
+		for _, exp := range c.lines {
+			found := false
+			for _, l := range lines {
+				if exp == l {
+					found = true
+					break
+				}
+			}
+
+			if !found {
+				t.Fatalf("expected line %q to be in exported output:\n%s", exp, out.String())
+			}
+		}
+	}
+}
+
+var sink interface{}
+
+func benchmarkExportTSM(c corpus, b *testing.B) {
+	// Garbage collection is relatively likely to happen during export, so track allocations.
+	b.ReportAllocs()
+
+	f := writeCorpusToTSMFile(c)
+	defer os.Remove(f.Name())
+
+	cmd := newCommand()
+	var out bytes.Buffer
+	b.ResetTimer()
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		if err := cmd.exportTSMFile(f.Name(), &out); err != nil {
+			b.Fatal(err)
+		}
+
+		sink = out.Bytes()
+		out.Reset()
+	}
+}
+
+func BenchmarkExportTSMFloats_100s_250vps(b *testing.B) {
+	benchmarkExportTSM(makeFloatsCorpus(100, 250), b)
+}
+
+func BenchmarkExportTSMInts_100s_250vps(b *testing.B) {
+	benchmarkExportTSM(makeIntsCorpus(100, 250), b)
+}
+
+func BenchmarkExportTSMBools_100s_250vps(b *testing.B) {
+	benchmarkExportTSM(makeBoolsCorpus(100, 250), b)
+}
+
+func BenchmarkExportTSMStrings_100s_250vps(b *testing.B) {
+	benchmarkExportTSM(makeStringsCorpus(100, 250), b)
+}
+
+func benchmarkExportWAL(c corpus, b *testing.B) {
+	// Garbage collection is relatively likely to happen during export, so track allocations.
+	b.ReportAllocs()
+
+	f := writeCorpusToWALFile(c)
+	defer os.Remove(f.Name())
+
+	cmd := newCommand()
+	var out bytes.Buffer
+	b.ResetTimer()
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		if err := cmd.exportWALFile(f.Name(), &out, func() {}); err != nil {
+			b.Fatal(err)
+		}
+
+		sink = out.Bytes()
+		out.Reset()
+	}
+}
+
+func BenchmarkExportWALFloats_100s_250vps(b *testing.B) {
+	benchmarkExportWAL(makeFloatsCorpus(100, 250), b)
+}
+
+func BenchmarkExportWALInts_100s_250vps(b *testing.B) {
+	benchmarkExportWAL(makeIntsCorpus(100, 250), b)
+}
+
+func BenchmarkExportWALBools_100s_250vps(b *testing.B) {
+	benchmarkExportWAL(makeBoolsCorpus(100, 250), b)
+}
+
+func BenchmarkExportWALStrings_100s_250vps(b *testing.B) {
+	benchmarkExportWAL(makeStringsCorpus(100, 250), b)
+}
+
+// newCommand returns a command that discards its output and that accepts all timestamps.
+func newCommand() *Command {
+	return &Command{
+		Stderr:    ioutil.Discard,
+		Stdout:    ioutil.Discard,
+		startTime: math.MinInt64,
+		endTime:   math.MaxInt64,
+	}
+}
+
+// makeCorpus returns a new corpus filled with values generated by fn.
+// The RNG passed to fn is seeded with numSeries * numValuesPerSeries, for predictable output.
+func makeCorpus(numSeries, numValuesPerSeries int, fn func(*rand.Rand) interface{}) corpus {
+	rng := rand.New(rand.NewSource(int64(numSeries) * int64(numValuesPerSeries)))
+	var unixNano int64
+	corpus := make(corpus, numSeries)
+	for i := 0; i < numSeries; i++ {
+		vals := make([]tsm1.Value, numValuesPerSeries)
+		for j := 0; j < numValuesPerSeries; j++ {
+			vals[j] = tsm1.NewValue(unixNano, fn(rng))
+			unixNano++
+		}
+
+		k := fmt.Sprintf("m,t=%d", i)
+		corpus[tsm1.SeriesFieldKey(k, "x")] = vals
+	}
+
+	return corpus
+}
+
+func makeFloatsCorpus(numSeries, numFloatsPerSeries int) corpus {
+	return makeCorpus(numSeries, numFloatsPerSeries, func(rng *rand.Rand) interface{} {
+		return rng.Float64()
+	})
+}
+
+func makeIntsCorpus(numSeries, numIntsPerSeries int) corpus {
+	return makeCorpus(numSeries, numIntsPerSeries, func(rng *rand.Rand) interface{} {
+		// This will only return positive integers. That's probably okay.
+		return rng.Int63()
+	})
+}
+
+func makeBoolsCorpus(numSeries, numBoolsPerSeries int) corpus {
+	return makeCorpus(numSeries, numBoolsPerSeries, func(rng *rand.Rand) interface{} {
+		return rand.Int63n(2) == 1
+	})
+}
+
+func makeStringsCorpus(numSeries, numStringsPerSeries int) corpus {
+	return makeCorpus(numSeries, numStringsPerSeries, func(rng *rand.Rand) interface{} {
+		// The string will randomly have 2-6 parts
+		parts := make([]string, rand.Intn(4)+2)
+
+		for i := range parts {
+			// Each part is a random base36-encoded number
+			parts[i] = strconv.FormatInt(rand.Int63(), 36)
+		}
+
+		// Join the individual parts with underscores.
+		return strings.Join(parts, "_")
+	})
+}
+
+// writeCorpusToWALFile writes the given corpus as a WAL file, and returns a handle to that file.
+// It is the caller's responsibility to remove the returned temp file.
+// writeCorpusToWALFile will panic on any error that occurs.
+func writeCorpusToWALFile(c corpus) *os.File {
+	walFile, err := ioutil.TempFile("", "export_test_corpus_wal")
+	if err != nil {
+		panic(err)
+	}
+
+	e := &tsm1.WriteWALEntry{Values: c}
+	b, err := e.Encode(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	w := tsm1.NewWALSegmentWriter(walFile)
+	if err := w.Write(e.Type(), snappy.Encode(nil, b)); err != nil {
+		panic(err)
+	}
+
+	if err := w.Flush(); err != nil {
+		panic(err)
+	}
+	// (*tsm1.WALSegmentWriter).sync isn't exported, but it only Syncs the file anyway.
+	if err := walFile.Sync(); err != nil {
+		panic(err)
+	}
+
+	return walFile
+}
+
+// writeCorpusToTSMFile writes the given corpus as a TSM file, and returns a handle to that file.
+// It is the caller's responsibility to remove the returned temp file.
+// writeCorpusToTSMFile will panic on any error that occurs.
+func writeCorpusToTSMFile(c corpus) *os.File {
+	tsmFile, err := ioutil.TempFile("", "export_test_corpus_tsm")
+	if err != nil {
+		panic(err)
+	}
+
+	w, err := tsm1.NewTSMWriter(tsmFile)
+	if err != nil {
+		panic(err)
+	}
+
+	// Write the series in alphabetical order so that each test run is comparable,
+	// given an identical corpus.
+	keys := make([]string, 0, len(c))
+	for k := range c {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		if err := w.Write(k, c[k]); err != nil {
+			panic(err)
+		}
+	}
+
+	if err := w.WriteIndex(); err != nil {
+		panic(err)
+	}
+
+	if err := w.Close(); err != nil {
+		panic(err)
+	}
+
+	return tsmFile
+}

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/help/help.go

@@ -0,0 +1,43 @@
+// Package help contains the help for the influx_inspect command.
+package help
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+// Command displays help for command-line sub-commands.
+type Command struct {
+	Stdout io.Writer
+}
+
+// NewCommand returns a new instance of Command.
+func NewCommand() *Command {
+	return &Command{
+		Stdout: os.Stdout,
+	}
+}
+
+// Run executes the command.
+func (cmd *Command) Run(args ...string) error {
+	fmt.Fprintln(cmd.Stdout, strings.TrimSpace(usage))
+	return nil
+}
+
+const usage = `
+Usage: influx_inspect [[command] [arguments]]
+
+The commands are:
+
+    dumptsm              dumps low-level details about tsm1 files.
+    export               exports raw data from a shard to line protocol
+    help                 display this help message
+    report               displays a shard level report
+    verify               verifies integrity of TSM files
+
+"help" is the default command.
+
+Use "influx_inspect [command] -help" for more information about a command.
+`

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/help/help_test.go

@@ -0,0 +1,3 @@
+package help_test
+
+// TODO: write some tests

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/main.go

@@ -0,0 +1,90 @@
+// The influx_inspect command displays detailed information about InfluxDB data files.
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/influxdata/influxdb/cmd"
+	"github.com/influxdata/influxdb/cmd/influx_inspect/dumptsi"
+	"github.com/influxdata/influxdb/cmd/influx_inspect/dumptsm"
+	"github.com/influxdata/influxdb/cmd/influx_inspect/export"
+	"github.com/influxdata/influxdb/cmd/influx_inspect/help"
+	"github.com/influxdata/influxdb/cmd/influx_inspect/report"
+	"github.com/influxdata/influxdb/cmd/influx_inspect/verify"
+	_ "github.com/influxdata/influxdb/tsdb/engine"
+)
+
+func main() {
+	m := NewMain()
+	if err := m.Run(os.Args[1:]...); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+// Main represents the program execution.
+type Main struct {
+	Logger *log.Logger
+
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewMain returns a new instance of Main.
+func NewMain() *Main {
+	return &Main{
+		Logger: log.New(os.Stderr, "[influx_inspect] ", log.LstdFlags),
+		Stdin:  os.Stdin,
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+}
+
+// Run determines and runs the command specified by the CLI args.
+func (m *Main) Run(args ...string) error {
+	name, args := cmd.ParseCommandName(args)
+
+	// Extract name from args.
+	switch name {
+	case "", "help":
+		if err := help.NewCommand().Run(args...); err != nil {
+			return fmt.Errorf("help: %s", err)
+		}
+	case "dumptsi":
+		name := dumptsi.NewCommand()
+		if err := name.Run(args...); err != nil {
+			return fmt.Errorf("dumptsi: %s", err)
+		}
+	case "dumptsmdev":
+		fmt.Fprintf(m.Stderr, "warning: dumptsmdev is deprecated, use dumptsm instead.\n")
+		fallthrough
+	case "dumptsm":
+		name := dumptsm.NewCommand()
+		if err := name.Run(args...); err != nil {
+			return fmt.Errorf("dumptsm: %s", err)
+		}
+	case "export":
+		name := export.NewCommand()
+		if err := name.Run(args...); err != nil {
+			return fmt.Errorf("export: %s", err)
+		}
+	case "report":
+		name := report.NewCommand()
+		if err := name.Run(args...); err != nil {
+			return fmt.Errorf("report: %s", err)
+		}
+	case "verify":
+		name := verify.NewCommand()
+		if err := name.Run(args...); err != nil {
+			return fmt.Errorf("verify: %s", err)
+		}
+	default:
+		return fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'influx_inspect help' for usage`+"\n\n", name)
+	}
+
+	return nil
+}

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/report/report.go

@@ -0,0 +1,192 @@
+// Package report reports statistics about TSM files.
+package report
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/influxdata/influxdb/models"
+	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+	"github.com/retailnext/hllpp"
+)
+
+// Command represents the program execution for "influxd report".
+type Command struct {
+	Stderr io.Writer
+	Stdout io.Writer
+
+	dir      string
+	pattern  string
+	detailed bool
+}
+
+// NewCommand returns a new instance of Command.
+func NewCommand() *Command {
+	return &Command{
+		Stderr: os.Stderr,
+		Stdout: os.Stdout,
+	}
+}
+
+// Run executes the command.
+func (cmd *Command) Run(args ...string) error {
+	fs := flag.NewFlagSet("report", flag.ExitOnError)
+	fs.StringVar(&cmd.pattern, "pattern", "", "Include only files matching a pattern")
+	fs.BoolVar(&cmd.detailed, "detailed", false, "Report detailed cardinality estimates")
+
+	fs.SetOutput(cmd.Stdout)
+	fs.Usage = cmd.printUsage
+
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	cmd.dir = fs.Arg(0)
+
+	start := time.Now()
+
+	files, err := filepath.Glob(filepath.Join(cmd.dir, fmt.Sprintf("*.%s", tsm1.TSMFileExtension)))
+	if err != nil {
+		return err
+	}
+
+	var filtered []string
+	if cmd.pattern != "" {
+		for _, f := range files {
+			if strings.Contains(f, cmd.pattern) {
+				filtered = append(filtered, f)
+			}
+		}
+		files = filtered
+	}
+
+	if len(files) == 0 {
+		return fmt.Errorf("no tsm files at %v", cmd.dir)
+	}
+
+	tw := tabwriter.NewWriter(cmd.Stdout, 8, 8, 1, '\t', 0)
+	fmt.Fprintln(tw, strings.Join([]string{"File", "Series", "Load Time"}, "\t"))
+
+	totalSeries := hllpp.New()
+	tagCardinalities := map[string]*hllpp.HLLPP{}
+	measCardinalities := map[string]*hllpp.HLLPP{}
+	fieldCardinalities := map[string]*hllpp.HLLPP{}
+
+	for _, f := range files {
+		file, err := os.OpenFile(f, os.O_RDONLY, 0600)
+		if err != nil {
+			fmt.Fprintf(cmd.Stderr, "error: %s: %v. Skipping.\n", f, err)
+			continue
+		}
+
+		loadStart := time.Now()
+		reader, err := tsm1.NewTSMReader(file)
+		if err != nil {
+			fmt.Fprintf(cmd.Stderr, "error: %s: %v. Skipping.\n", file.Name(), err)
+			continue
+		}
+		loadTime := time.Since(loadStart)
+
+		seriesCount := reader.KeyCount()
+		for i := 0; i < seriesCount; i++ {
+			key, _ := reader.KeyAt(i)
+			totalSeries.Add([]byte(key))
+
+			if cmd.detailed {
+				sep := strings.Index(string(key), "#!~#")
+				seriesKey, field := key[:sep], key[sep+4:]
+				measurement, tags := models.ParseKey(seriesKey)
+
+				measCount, ok := measCardinalities[measurement]
+				if !ok {
+					measCount = hllpp.New()
+					measCardinalities[measurement] = measCount
+				}
+				measCount.Add([]byte(key))
+
+				fieldCount, ok := fieldCardinalities[measurement]
+				if !ok {
+					fieldCount = hllpp.New()
+					fieldCardinalities[measurement] = fieldCount
+				}
+				fieldCount.Add([]byte(field))
+
+				for _, t := range tags {
+					tagCount, ok := tagCardinalities[string(t.Key)]
+					if !ok {
+						tagCount = hllpp.New()
+						tagCardinalities[string(t.Key)] = tagCount
+					}
+					tagCount.Add(t.Value)
+				}
+			}
+		}
+		reader.Close()
+
+		fmt.Fprintln(tw, strings.Join([]string{
+			filepath.Base(file.Name()),
+			strconv.FormatInt(int64(seriesCount), 10),
+			loadTime.String(),
+		}, "\t"))
+		tw.Flush()
+	}
+
+	tw.Flush()
+	println()
+	fmt.Printf("Statistics\n")
+	fmt.Printf("\tSeries:\n")
+	fmt.Printf("\t\tTotal (est): %d\n", totalSeries.Count())
+
+	if cmd.detailed {
+		fmt.Printf("\tMeasurements (est):\n")
+		for _, t := range sortKeys(measCardinalities) {
+			fmt.Printf("\t\t%v: %d (%d%%)\n", t, measCardinalities[t].Count(), int((float64(measCardinalities[t].Count())/float64(totalSeries.Count()))*100))
+		}
+
+		fmt.Printf("\tFields (est):\n")
+		for _, t := range sortKeys(fieldCardinalities) {
+			fmt.Printf("\t\t%v: %d\n", t, fieldCardinalities[t].Count())
+		}
+
+		fmt.Printf("\tTags (est):\n")
+		for _, t := range sortKeys(tagCardinalities) {
+			fmt.Printf("\t\t%v: %d\n", t, tagCardinalities[t].Count())
+		}
+	}
+
+	fmt.Printf("Completed in %s\n", time.Since(start))
+	return nil
+}
+
+// sortKeys is a quick helper to return the sorted set of a map's keys
+func sortKeys(vals map[string]*hllpp.HLLPP) (keys []string) {
+	for k := range vals {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	return keys
+}
+
+// printUsage prints the usage message to STDERR.
+func (cmd *Command) printUsage() {
+	usage := `Displays shard level report.
+
+Usage: influx_inspect report [flags]
+
+    -pattern <pattern>
+            Include only files matching a pattern.
+    -detailed
+            Report detailed cardinality estimates.
+            Defaults to "false".
+`
+
+	fmt.Fprintf(cmd.Stdout, usage)
+}

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/report/report_test.go

@@ -0,0 +1,3 @@
+package report_test
+
+// TODO: write some tests

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/verify/verify.go

@@ -0,0 +1,120 @@
+// Package verify verifies integrity of TSM files.
+package verify
+
+import (
+	"flag"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"os"
+	"path/filepath"
+	"text/tabwriter"
+	"time"
+
+	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+)
+
+// Command represents the program execution for "influx_inspect verify".
+type Command struct {
+	Stderr io.Writer
+	Stdout io.Writer
+}
+
+// NewCommand returns a new instance of Command.
+func NewCommand() *Command {
+	return &Command{
+		Stderr: os.Stderr,
+		Stdout: os.Stdout,
+	}
+}
+
+// Run executes the command.
+func (cmd *Command) Run(args ...string) error {
+	var path string
+	fs := flag.NewFlagSet("verify", flag.ExitOnError)
+	fs.StringVar(&path, "dir", os.Getenv("HOME")+"/.influxdb", "Root storage path. [$HOME/.influxdb]")
+
+	fs.SetOutput(cmd.Stdout)
+	fs.Usage = cmd.printUsage
+
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	start := time.Now()
+	dataPath := filepath.Join(path, "data")
+
+	brokenBlocks := 0
+	totalBlocks := 0
+
+	// No need to do this in a loop
+	ext := fmt.Sprintf(".%s", tsm1.TSMFileExtension)
+
+	// Get all TSM files by walking through the data dir
+	files := []string{}
+	err := filepath.Walk(dataPath, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if filepath.Ext(path) == ext {
+			files = append(files, path)
+		}
+		return nil
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	tw := tabwriter.NewWriter(cmd.Stdout, 16, 8, 0, '\t', 0)
+
+	// Verify the checksums of every block in every file
+	for _, f := range files {
+		file, err := os.OpenFile(f, os.O_RDONLY, 0600)
+		if err != nil {
+			return err
+		}
+
+		reader, err := tsm1.NewTSMReader(file)
+		if err != nil {
+			return err
+		}
+
+		blockItr := reader.BlockIterator()
+		brokenFileBlocks := 0
+		count := 0
+		for blockItr.Next() {
+			totalBlocks++
+			key, _, _, _, checksum, buf, err := blockItr.Read()
+			if err != nil {
+				brokenBlocks++
+				fmt.Fprintf(tw, "%s: could not get checksum for key %v block %d due to error: %q\n", f, key, count, err)
+			} else if expected := crc32.ChecksumIEEE(buf); checksum != expected {
+				brokenBlocks++
+				fmt.Fprintf(tw, "%s: got %d but expected %d for key %v, block %d\n", f, checksum, expected, key, count)
+			}
+			count++
+		}
+		if brokenFileBlocks == 0 {
+			fmt.Fprintf(tw, "%s: healthy\n", f)
+		}
+		reader.Close()
+	}
+
+	fmt.Fprintf(tw, "Broken Blocks: %d / %d, in %vs\n", brokenBlocks, totalBlocks, time.Since(start).Seconds())
+	tw.Flush()
+	return nil
+}
+
+// printUsage prints the usage message to STDERR.
+func (cmd *Command) printUsage() {
+	usage := fmt.Sprintf(`Verifies the integrity of TSM files.
+
+Usage: influx_inspect verify [flags]
+
+    -dir <path>
+            Root storage path
+            Defaults to "%[1]s/.influxdb".
+ `, os.Getenv("HOME"))
+
+	fmt.Fprintf(cmd.Stdout, usage)
+}

vendor/github.com/influxdata/influxdb/cmd/influx_inspect/verify/verify_test.go

@@ -0,0 +1,3 @@
+package verify_test
+
+// TODO: write some tests

vendor/github.com/influxdata/influxdb/cmd/influx_stress/README.md

@@ -0,0 +1,43 @@
+# `influx_stress`
+
+If you run into any issues with this tool please mention @jackzampolin when you create an issue.
+
+## Ways to run
+
+### `influx_stress`
+This runs a basic stress test with the [default config](https://github.com/influxdata/influxdb/blob/master/stress/stress.toml) For more information on the configuration file please see the default.
+
+### `influx_stress -config someConfig.toml`
+This runs the stress test with a valid configuration file located at `someConfig.tom`
+
+### `influx_stress -v2 -config someConfig.iql`
+This runs the stress test with a valid `v2` configuration file. For more information about the `v2` stress test see the [v2 stress README](https://github.com/influxdata/influxdb/blob/master/stress/v2/README.md).
+
+## Flags
+
+If flags are defined they overwrite the config from any file passed in.
+
+### `-addr` string
+IP address and port of database where response times will persist (e.g., localhost:8086)
+
+`default` = "http://localhost:8086"
+
+### `-config` string
+The relative path to the stress test configuration file.
+
+`default` = [config](https://github.com/influxdata/influxdb/blob/master/stress/stress.toml)
+
+### `-cpuprofile` filename
+Writes the result of Go's cpu profile to filename
+
+`default` = no profiling
+
+### `-database` string
+Name of database on `-addr` that `influx_stress` will persist write and query response times
+
+`default` = "stress"
+
+### `-tags` value
+A comma separated list of tags to add to write and query response times.
+
+`default` = ""

vendor/github.com/influxdata/influxdb/cmd/influx_stress/examples/template.toml

@@ -0,0 +1,92 @@
+# This section can be removed
+[provision]
+  # The basic provisioner simply deletes and creates database.
+  # If `reset_database` is false, it will not attempt to delete the database
+  [provision.basic]
+    # If enabled the provisioner will actually run
+    enabled = true
+    # Address of the instance that is to be provisioned
+    address = "localhost:8086"
+    # Database the will be created/deleted
+    database = "stress"
+    # Attempt to delete database
+    reset_database = true
+
+# This section cannot be commented out
+# To prevent writes set `enabled=false`
+# in [write.influx_client.basic]
+[write]
+  [write.point_generator]
+    # The basic point generator will generate points of the form
+    # `cpu,host=server-%v,location=us-west value=234 123456`
+    [write.point_generator.basic]
+      # number of points that will be written for each of the series
+      point_count = 100
+      # number of series
+      series_count = 100000
+      # How much time between each timestamp
+      tick = "10s"
+      # Randomize timestamp a bit (not functional)
+      jitter = true
+      # Precision of points that are being written
+      precision = "s"
+      # name of the measurement that will be written
+      measurement = "cpu"
+      # The date for the first point that is written into influx
+      start_date = "2006-Jan-02"
+      # Defines a tag for a series
+      [[write.point_generator.basic.tag]]
+        key = "host"
+        value = "server"
+      [[write.point_generator.basic.tag]]
+        key = "location"
+        value = "us-west"
+      # Defines a field for a series
+      [[write.point_generator.basic.field]]
+        key = "value"
+        value = "float64" # supported types: float64, int, bool
+
+
+  [write.influx_client]
+    [write.influx_client.basic]
+      # If enabled the writer will actually write
+      enabled = true
+      # Addresses is an array of the Influxdb instances
+      addresses = ["localhost:8086"] # stress_test_server runs on port 1234
+      # Database that is being written to
+      database = "stress"
+      # Precision of points that are being written
+      precision = "s"
+      # Size of batches that are sent to db
+      batch_size = 10000
+      # Interval between each batch
+      batch_interval = "0s"
+      # How many concurrent writers to the db
+      concurrency = 10
+      # ssl enabled?
+      ssl = false
+      # format of points that are written to influxdb
+      format = "line_http" # line_udp (not supported yet), graphite_tcp (not supported yet), graphite_udp (not supported yet)
+
+# This section can be removed
+[read]
+  [read.query_generator]
+    [read.query_generator.basic]
+      # Template of the query that will be ran against the instance
+      template = "SELECT count(value) FROM cpu where host='server-%v'"
+      # How many times the templated query will be ran
+      query_count = 250
+
+  [read.query_client]
+    [read.query_client.basic]
+      # if enabled the reader will actually read
+      enabled = true
+      # Address of the instance that will be queried
+      addresses = ["localhost:8086"]
+      # Database that will be queried
+      database = "stress"
+      # Interval bewteen queries
+      query_interval = "100ms"
+      # Number of concurrent queriers
+      concurrency = 1
+

vendor/github.com/influxdata/influxdb/cmd/influx_stress/influx_stress.go

@@ -0,0 +1,71 @@
+// Command influx_stress is deprecated; use github.com/influxdata/influx-stress instead.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"runtime/pprof"
+
+	"github.com/influxdata/influxdb/stress"
+	v2 "github.com/influxdata/influxdb/stress/v2"
+)
+
+var (
+	useV2      = flag.Bool("v2", false, "Use version 2 of stress tool")
+	config     = flag.String("config", "", "The stress test file")
+	cpuprofile = flag.String("cpuprofile", "", "Write the cpu profile to `filename`")
+	db         = flag.String("db", "", "target database within test system for write and query load")
+)
+
+func main() {
+	o := stress.NewOutputConfig()
+	flag.Parse()
+
+	if *cpuprofile != "" {
+		f, err := os.Create(*cpuprofile)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+
+	if *useV2 {
+		if *config != "" {
+			v2.RunStress(*config)
+		} else {
+			v2.RunStress("stress/v2/iql/file.iql")
+		}
+	} else {
+
+		c, err := stress.NewConfig(*config)
+		if err != nil {
+			log.Fatal(err)
+			return
+		}
+
+		if *db != "" {
+			c.Provision.Basic.Database = *db
+			c.Write.InfluxClients.Basic.Database = *db
+			c.Read.QueryClients.Basic.Database = *db
+		}
+
+		w := stress.NewWriter(c.Write.PointGenerators.Basic, &c.Write.InfluxClients.Basic)
+		r := stress.NewQuerier(&c.Read.QueryGenerators.Basic, &c.Read.QueryClients.Basic)
+		s := stress.NewStressTest(&c.Provision.Basic, w, r)
+
+		bw := stress.NewBroadcastChannel()
+		bw.Register(c.Write.InfluxClients.Basic.BasicWriteHandler)
+		bw.Register(o.HTTPHandler("write"))
+
+		br := stress.NewBroadcastChannel()
+		br.Register(c.Read.QueryClients.Basic.BasicReadHandler)
+		br.Register(o.HTTPHandler("read"))
+
+		s.Start(bw.Handle, br.Handle)
+
+	}
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/README.md

@@ -0,0 +1,152 @@
+# Converting b1 and bz1 shards to tsm1
+
+`influx_tsm` is a tool for converting b1 and bz1 shards to tsm1
+format. Converting shards to tsm1 format results in a very significant
+reduction in disk usage, and significantly improved write-throughput,
+when writing data into those shards.
+
+Conversion can be controlled on a database-by-database basis. By
+default a database is backed up before it is converted, allowing you
+to roll back any changes. Because of the backup process, ensure the
+host system has at least as much free disk space as the disk space
+consumed by the _data_ directory of your InfluxDB system.
+
+The tool automatically ignores tsm1 shards, and can be run
+idempotently on any database.
+
+Conversion is an offline process, and the InfluxDB system must be
+stopped during conversion. However the conversion process reads and
+writes shards directly on disk and should be fast.
+
+## Steps
+
+Follow these steps to perform a conversion.
+
+* Identify the databases you wish to convert. You can convert one or more databases at a time. By default all databases are converted.
+* Decide on parallel operation. By default the conversion operation peforms each operation in a serial manner. This minimizes load on the host system performing the conversion, but also takes the most time. If you wish to minimize the time conversion takes, enable parallel mode. Conversion will then perform as many operations as possible in parallel, but the process may place significant load on the host system (CPU, disk, and RAM, usage will all increase).
+* Stop all write-traffic to your InfluxDB system.
+* Restart the InfluxDB service and wait until all WAL data is flushed to disk -- this has completed when the system responds to queries. This is to ensure all data is present in shards.
+* Stop the InfluxDB service. It should not be restarted until conversion is complete.
+* Run conversion tool. Depending on the size of the data directory, this might be a lengthy operation. Consider running the conversion tool under a "screen" session to avoid any interruptions.
+* Unless you ran the conversion tool as the same user as that which runs InfluxDB, then you may need to set the correct read-and-write permissions on the new tsm1 directories.
+* Restart node and ensure data looks correct.
+* If everything looks OK, you may then wish to remove or archive the backed-up databases.
+* Restart write traffic.
+
+## Example session
+
+Below is an example session, showing a database being converted.
+
+```
+$ # Create a backup location that the `influxdb` user has full access to
+$ mkdir -m 0777 /path/to/influxdb_backup
+$ sudo -u influxdb influx_tsm -backup /path/to/influxdb_backup -parallel /var/lib/influxdb/data
+
+b1 and bz1 shard conversion.
+-----------------------------------
+Data directory is:                  /var/lib/influxdb/data
+Backup directory is:                /path/to/influxdb_backup
+Databases specified:                all
+Database backups enabled:           yes
+Parallel mode enabled (GOMAXPROCS): yes (8)
+
+
+Found 1 shards that will be converted.
+
+Database        Retention       Path                                                    Engine  Size
+_internal       monitor         /var/lib/influxdb/data/_internal/monitor/1           bz1     65536
+
+These shards will be converted. Proceed? y/N: y
+Conversion starting....
+Backing up 1 databases...
+2016/01/28 12:23:43.699266 Backup of databse '_internal' started
+2016/01/28 12:23:43.699883 Backing up file /var/lib/influxdb/data/_internal/monitor/1
+2016/01/28 12:23:43.700052 Database _internal backed up (851.776µs)
+2016/01/28 12:23:43.700320 Starting conversion of shard: /var/lib/influxdb/data/_internal/monitor/1
+2016/01/28 12:23:43.706276 Conversion of /var/lib/influxdb/data/_internal/monitor/1 successful (6.040148ms)
+
+Summary statistics
+========================================
+Databases converted:                 1
+Shards converted:                    1
+TSM files created:                   1
+Points read:                         369
+Points written:                      369
+NaN filtered:                        0
+Inf filtered:                        0
+Points without fields filtered:      0
+Disk usage pre-conversion (bytes):   65536
+Disk usage post-conversion (bytes):  11000
+Reduction factor:                    83%
+Bytes per TSM point:                 29.81
+Total conversion time:               7.330443ms
+
+$ # restart node, verify data
+$ sudo rm -r /path/to/influxdb_backup
+```
+
+Note that the tool first lists the shards that will be converted,
+before asking for confirmation. You can abort the conversion process
+at this step if you just wish to see what would be converted, or if
+the list of shards does not look correct.
+
+__WARNING:__ If you run the `influx_tsm` tool as a user other than the
+`influxdb` user (or the user that the InfluxDB process runs under),
+please make sure to verify the shard permissions are correct prior to
+starting InfluxDB. If needed, shard permissions can be corrected with
+the `chown` command. For example:
+
+```
+sudo chown -R influxdb:influxdb /var/lib/influxdb
+```
+
+## Rolling back a conversion
+
+After a successful backup (the message `Database XYZ backed up` was
+logged), you have a duplicate of that database in the _backup_
+directory you provided on the command line. If, when checking your
+data after a successful conversion, you notice things missing or
+something just isn't right, you can "undo" the conversion:
+
+- Shut down your node (this is very important)
+- Remove the database's directory from the influxdb `data` directory (default: `~/.influxdb/data/XYZ` for binary installations or `/var/lib/influxdb/data/XYZ` for packaged installations)
+- Copy (to really make sure the shard is preserved) the database's directory from the backup directory you created into the `data` directory.
+
+Using the same directories as above, and assuming a database named `stats`:
+
+```
+$ sudo rm -r /var/lib/influxdb/data/stats
+$ sudo cp -r /path/to/influxdb_backup/stats /var/lib/influxdb/data/
+$ # restart influxd node
+```
+
+#### How to avoid downtime when upgrading shards
+
+*Identify non-`tsm1` shards*
+
+Non-`tsm1` shards are files of the form: `data/<database>/<retention_policy>/<shard_id>`.
+
+`tsm1` shards are files of the form: `data/<database>/<retention_policy>/<shard_id>/<file>.tsm`.
+
+*Determine which `bz`/`bz1` shards are cold for writes*
+
+Run the `SHOW SHARDS` query to see the start and end dates for shards.
+If the date range for a shard does not span the current time then the shard is said to be cold for writes.
+This means that no new points are expected to be added to the shard.
+The shard whose date range spans now is said to be hot for writes.
+You can only safely convert cold shards without stopping the InfluxDB process.
+
+*Convert cold shards*
+
+1. Copy each of the cold shards you'd like to convert to a new directory with the structure `/tmp/data/<database>/<retention_policy>/<shard_id>`.
+2. Run the `influx_tsm` tool on the copied files:
+```
+influx_tsm -parallel /tmp/data/
+```
+3. Remove the existing cold `b1`/`bz1` shards from the production data directory.
+4. Move the new `tsm1` shards into the original directory, overwriting the existing `b1`/`bz1` shards of the same name. Do this simultaneously with step 3 to avoid any query errors.
+5. Wait an hour, a day, or a week (depending on your retention period) for any hot `b1`/`bz1` shards to become cold and repeat steps 1 through 4 on the newly cold shards.
+
+> **Note:** Any points written to the cold shards after making a copy will be lost when the `tsm1` shard overwrites the existing cold shard.
+Nothing in InfluxDB will prevent writes to cold shards, they are merely unexpected, not impossible.
+It is your responsibility to prevent writes to cold shards to prevent data loss.

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/b1/reader.go

@@ -0,0 +1,270 @@
+// Package b1 reads data from b1 shards.
+package b1 // import "github.com/influxdata/influxdb/cmd/influx_tsm/b1"
+
+import (
+	"encoding/binary"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/stats"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/tsdb"
+	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+)
+
+// DefaultChunkSize is the size of chunks read from the b1 shard
+const DefaultChunkSize int = 1000
+
+var excludedBuckets = map[string]bool{
+	"fields": true,
+	"meta":   true,
+	"series": true,
+	"wal":    true,
+}
+
+// Reader is used to read all data from a b1 shard.
+type Reader struct {
+	path string
+	db   *bolt.DB
+	tx   *bolt.Tx
+
+	cursors    []*cursor
+	currCursor int
+
+	keyBuf   string
+	values   []tsm1.Value
+	valuePos int
+
+	fields map[string]*tsdb.MeasurementFields
+	codecs map[string]*tsdb.FieldCodec
+
+	stats *stats.Stats
+}
+
+// NewReader returns a reader for the b1 shard at path.
+func NewReader(path string, stats *stats.Stats, chunkSize int) *Reader {
+	r := &Reader{
+		path:   path,
+		fields: make(map[string]*tsdb.MeasurementFields),
+		codecs: make(map[string]*tsdb.FieldCodec),
+		stats:  stats,
+	}
+
+	if chunkSize <= 0 {
+		chunkSize = DefaultChunkSize
+	}
+
+	r.values = make([]tsm1.Value, chunkSize)
+
+	return r
+}
+
+// Open opens the reader.
+func (r *Reader) Open() error {
+	// Open underlying storage.
+	db, err := bolt.Open(r.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return err
+	}
+	r.db = db
+
+	// Load fields.
+	if err := r.db.View(func(tx *bolt.Tx) error {
+		meta := tx.Bucket([]byte("fields"))
+		c := meta.Cursor()
+
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			mf := &tsdb.MeasurementFields{}
+			if err := mf.UnmarshalBinary(v); err != nil {
+				return err
+			}
+			r.fields[string(k)] = mf
+			r.codecs[string(k)] = tsdb.NewFieldCodec(mf.Fields)
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	seriesSet := make(map[string]bool)
+
+	// ignore series index and find all series in this shard
+	if err := r.db.View(func(tx *bolt.Tx) error {
+		tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
+			key := string(name)
+			if !excludedBuckets[key] {
+				seriesSet[key] = true
+			}
+			return nil
+		})
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	r.tx, err = r.db.Begin(false)
+	if err != nil {
+		return err
+	}
+
+	// Create cursor for each field of each series.
+	for s := range seriesSet {
+		measurement := tsdb.MeasurementFromSeriesKey(s)
+		fields := r.fields[measurement]
+		if fields == nil {
+			r.stats.IncrFiltered()
+			continue
+		}
+		for _, f := range fields.Fields {
+			c := newCursor(r.tx, s, f.Name, r.codecs[measurement])
+			c.SeekTo(0)
+			r.cursors = append(r.cursors, c)
+		}
+	}
+	sort.Sort(cursors(r.cursors))
+
+	return nil
+}
+
+// Next returns whether any data remains to be read. It must be called before
+// the next call to Read().
+func (r *Reader) Next() bool {
+	r.valuePos = 0
+OUTER:
+	for {
+		if r.currCursor >= len(r.cursors) {
+			// All cursors drained. No more data remains.
+			return false
+		}
+
+		cc := r.cursors[r.currCursor]
+		r.keyBuf = tsm1.SeriesFieldKey(cc.series, cc.field)
+
+		for {
+			k, v := cc.Next()
+			if k == -1 {
+				// Go to next cursor and try again.
+				r.currCursor++
+				if r.valuePos == 0 {
+					// The previous cursor had no data. Instead of returning
+					// just go immediately to the next cursor.
+					continue OUTER
+				}
+				// There is some data available. Indicate that it should be read.
+				return true
+			}
+
+			if f, ok := v.(float64); ok {
+				if math.IsInf(f, 0) {
+					r.stats.AddPointsRead(1)
+					r.stats.IncrInf()
+					continue
+				}
+
+				if math.IsNaN(f) {
+					r.stats.AddPointsRead(1)
+					r.stats.IncrNaN()
+					continue
+				}
+			}
+
+			r.values[r.valuePos] = tsm1.NewValue(k, v)
+			r.valuePos++
+
+			if r.valuePos >= len(r.values) {
+				return true
+			}
+		}
+	}
+}
+
+// Read returns the next chunk of data in the shard, converted to tsm1 values. Data is
+// emitted completely for every field, in every series, before the next field is processed.
+// Data from Read() adheres to the requirements for writing to tsm1 shards
+func (r *Reader) Read() (string, []tsm1.Value, error) {
+	return r.keyBuf, r.values[:r.valuePos], nil
+}
+
+// Close closes the reader.
+func (r *Reader) Close() error {
+	r.tx.Rollback()
+	return r.db.Close()
+}
+
+// cursor provides ordered iteration across a series.
+type cursor struct {
+	// Bolt cursor and readahead buffer.
+	cursor *bolt.Cursor
+	keyBuf int64
+	valBuf interface{}
+
+	series string
+	field  string
+	dec    *tsdb.FieldCodec
+}
+
+// Cursor returns an iterator for a key over a single field.
+func newCursor(tx *bolt.Tx, series string, field string, dec *tsdb.FieldCodec) *cursor {
+	cur := &cursor{
+		keyBuf: -2,
+		series: series,
+		field:  field,
+		dec:    dec,
+	}
+
+	// Retrieve series bucket.
+	b := tx.Bucket([]byte(series))
+	if b != nil {
+		cur.cursor = b.Cursor()
+	}
+
+	return cur
+}
+
+// Seek moves the cursor to a position.
+func (c *cursor) SeekTo(seek int64) {
+	var seekBytes [8]byte
+	binary.BigEndian.PutUint64(seekBytes[:], uint64(seek))
+	k, v := c.cursor.Seek(seekBytes[:])
+	c.keyBuf, c.valBuf = tsdb.DecodeKeyValue(c.field, c.dec, k, v)
+}
+
+// Next returns the next key/value pair from the cursor.
+func (c *cursor) Next() (key int64, value interface{}) {
+	for {
+		k, v := func() (int64, interface{}) {
+			if c.keyBuf != -2 {
+				k, v := c.keyBuf, c.valBuf
+				c.keyBuf = -2
+				return k, v
+			}
+
+			k, v := c.cursor.Next()
+			if k == nil {
+				return -1, nil
+			}
+			return tsdb.DecodeKeyValue(c.field, c.dec, k, v)
+		}()
+
+		if k != -1 && v == nil {
+			// There is a point in the series at the next timestamp,
+			// but not for this cursor's field. Go to the next point.
+			continue
+		}
+		return k, v
+	}
+}
+
+// Sort b1 cursors in correct order for writing to TSM files.
+
+type cursors []*cursor
+
+func (a cursors) Len() int      { return len(a) }
+func (a cursors) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a cursors) Less(i, j int) bool {
+	if a[i].series == a[j].series {
+		return a[i].field < a[j].field
+	}
+	return a[i].series < a[j].series
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/bz1/reader.go

@@ -0,0 +1,371 @@
+// Package bz1 reads data from bz1 shards.
+package bz1 // import "github.com/influxdata/influxdb/cmd/influx_tsm/bz1"
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/golang/snappy"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/stats"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/tsdb"
+	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+)
+
+// DefaultChunkSize is the size of chunks read from the bz1 shard
+const DefaultChunkSize = 1000
+
+// Reader is used to read all data from a bz1 shard.
+type Reader struct {
+	path string
+	db   *bolt.DB
+	tx   *bolt.Tx
+
+	cursors    []*cursor
+	currCursor int
+
+	keyBuf   string
+	values   []tsm1.Value
+	valuePos int
+
+	fields map[string]*tsdb.MeasurementFields
+	codecs map[string]*tsdb.FieldCodec
+
+	stats *stats.Stats
+}
+
+// NewReader returns a reader for the bz1 shard at path.
+func NewReader(path string, stats *stats.Stats, chunkSize int) *Reader {
+	r := &Reader{
+		path:   path,
+		fields: make(map[string]*tsdb.MeasurementFields),
+		codecs: make(map[string]*tsdb.FieldCodec),
+		stats:  stats,
+	}
+
+	if chunkSize <= 0 {
+		chunkSize = DefaultChunkSize
+	}
+
+	r.values = make([]tsm1.Value, chunkSize)
+
+	return r
+}
+
+// Open opens the reader.
+func (r *Reader) Open() error {
+	// Open underlying storage.
+	db, err := bolt.Open(r.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return err
+	}
+	r.db = db
+
+	seriesSet := make(map[string]bool)
+
+	if err := r.db.View(func(tx *bolt.Tx) error {
+		var data []byte
+
+		meta := tx.Bucket([]byte("meta"))
+		if meta == nil {
+			// No data in this shard.
+			return nil
+		}
+
+		pointsBucket := tx.Bucket([]byte("points"))
+		if pointsBucket == nil {
+			return nil
+		}
+
+		if err := pointsBucket.ForEach(func(key, _ []byte) error {
+			seriesSet[string(key)] = true
+			return nil
+		}); err != nil {
+			return err
+		}
+
+		buf := meta.Get([]byte("fields"))
+		if buf == nil {
+			// No data in this shard.
+			return nil
+		}
+
+		data, err = snappy.Decode(nil, buf)
+		if err != nil {
+			return err
+		}
+		if err := json.Unmarshal(data, &r.fields); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	// Build the codec for each measurement.
+	for k, v := range r.fields {
+		r.codecs[k] = tsdb.NewFieldCodec(v.Fields)
+	}
+
+	r.tx, err = r.db.Begin(false)
+	if err != nil {
+		return err
+	}
+
+	// Create cursor for each field of each series.
+	for s := range seriesSet {
+		measurement := tsdb.MeasurementFromSeriesKey(s)
+		fields := r.fields[measurement]
+		if fields == nil {
+			r.stats.IncrFiltered()
+			continue
+		}
+		for _, f := range fields.Fields {
+			c := newCursor(r.tx, s, f.Name, r.codecs[measurement])
+			if c == nil {
+				continue
+			}
+			c.SeekTo(0)
+			r.cursors = append(r.cursors, c)
+		}
+	}
+	sort.Sort(cursors(r.cursors))
+
+	return nil
+}
+
+// Next returns whether there is any more data to be read.
+func (r *Reader) Next() bool {
+	r.valuePos = 0
+OUTER:
+	for {
+		if r.currCursor >= len(r.cursors) {
+			// All cursors drained. No more data remains.
+			return false
+		}
+
+		cc := r.cursors[r.currCursor]
+		r.keyBuf = tsm1.SeriesFieldKey(cc.series, cc.field)
+
+		for {
+			k, v := cc.Next()
+			if k == -1 {
+				// Go to next cursor and try again.
+				r.currCursor++
+				if r.valuePos == 0 {
+					// The previous cursor had no data. Instead of returning
+					// just go immediately to the next cursor.
+					continue OUTER
+				}
+				// There is some data available. Indicate that it should be read.
+				return true
+			}
+
+			if f, ok := v.(float64); ok {
+				if math.IsInf(f, 0) {
+					r.stats.AddPointsRead(1)
+					r.stats.IncrInf()
+					continue
+				}
+
+				if math.IsNaN(f) {
+					r.stats.AddPointsRead(1)
+					r.stats.IncrNaN()
+					continue
+				}
+			}
+
+			r.values[r.valuePos] = tsm1.NewValue(k, v)
+			r.valuePos++
+
+			if r.valuePos >= len(r.values) {
+				return true
+			}
+		}
+	}
+}
+
+// Read returns the next chunk of data in the shard, converted to tsm1 values. Data is
+// emitted completely for every field, in every series, before the next field is processed.
+// Data from Read() adheres to the requirements for writing to tsm1 shards
+func (r *Reader) Read() (string, []tsm1.Value, error) {
+	return r.keyBuf, r.values[:r.valuePos], nil
+}
+
+// Close closes the reader.
+func (r *Reader) Close() error {
+	r.tx.Rollback()
+	return r.db.Close()
+}
+
+// cursor provides ordered iteration across a series.
+type cursor struct {
+	cursor       *bolt.Cursor
+	buf          []byte // uncompressed buffer
+	off          int    // buffer offset
+	fieldIndices []int
+	index        int
+
+	series string
+	field  string
+	dec    *tsdb.FieldCodec
+
+	keyBuf int64
+	valBuf interface{}
+}
+
+// newCursor returns an instance of a bz1 cursor.
+func newCursor(tx *bolt.Tx, series string, field string, dec *tsdb.FieldCodec) *cursor {
+	// Retrieve points bucket. Ignore if there is no bucket.
+	b := tx.Bucket([]byte("points")).Bucket([]byte(series))
+	if b == nil {
+		return nil
+	}
+
+	return &cursor{
+		cursor: b.Cursor(),
+		series: series,
+		field:  field,
+		dec:    dec,
+		keyBuf: -2,
+	}
+}
+
+// Seek moves the cursor to a position.
+func (c *cursor) SeekTo(seek int64) {
+	var seekBytes [8]byte
+	binary.BigEndian.PutUint64(seekBytes[:], uint64(seek))
+
+	// Move cursor to appropriate block and set to buffer.
+	k, v := c.cursor.Seek(seekBytes[:])
+	if v == nil { // get the last block, it might have this time
+		_, v = c.cursor.Last()
+	} else if seek < int64(binary.BigEndian.Uint64(k)) { // the seek key is less than this block, go back one and check
+		_, v = c.cursor.Prev()
+
+		// if the previous block max time is less than the seek value, reset to where we were originally
+		if v == nil || seek > int64(binary.BigEndian.Uint64(v[0:8])) {
+			_, v = c.cursor.Seek(seekBytes[:])
+		}
+	}
+	c.setBuf(v)
+
+	// Read current block up to seek position.
+	c.seekBuf(seekBytes[:])
+
+	// Return current entry.
+	c.keyBuf, c.valBuf = c.read()
+}
+
+// seekBuf moves the cursor to a position within the current buffer.
+func (c *cursor) seekBuf(seek []byte) (key, value []byte) {
+	for {
+		// Slice off the current entry.
+		buf := c.buf[c.off:]
+
+		// Exit if current entry's timestamp is on or after the seek.
+		if len(buf) == 0 {
+			return
+		}
+
+		if bytes.Compare(buf[0:8], seek) != -1 {
+			return
+		}
+
+		c.off += entryHeaderSize + entryDataSize(buf)
+	}
+}
+
+// Next returns the next key/value pair from the cursor. If there are no values
+// remaining, -1 is returned.
+func (c *cursor) Next() (int64, interface{}) {
+	for {
+		k, v := func() (int64, interface{}) {
+			if c.keyBuf != -2 {
+				k, v := c.keyBuf, c.valBuf
+				c.keyBuf = -2
+				return k, v
+			}
+
+			// Ignore if there is no buffer.
+			if len(c.buf) == 0 {
+				return -1, nil
+			}
+
+			// Move forward to next entry.
+			c.off += entryHeaderSize + entryDataSize(c.buf[c.off:])
+
+			// If no items left then read first item from next block.
+			if c.off >= len(c.buf) {
+				_, v := c.cursor.Next()
+				c.setBuf(v)
+			}
+
+			return c.read()
+		}()
+
+		if k != -1 && v == nil {
+			// There is a point in the series at the next timestamp,
+			// but not for this cursor's field. Go to the next point.
+			continue
+		}
+		return k, v
+	}
+}
+
+// setBuf saves a compressed block to the buffer.
+func (c *cursor) setBuf(block []byte) {
+	// Clear if the block is empty.
+	if len(block) == 0 {
+		c.buf, c.off, c.fieldIndices, c.index = c.buf[0:0], 0, c.fieldIndices[0:0], 0
+		return
+	}
+
+	// Otherwise decode block into buffer.
+	// Skip over the first 8 bytes since they are the max timestamp.
+	buf, err := snappy.Decode(nil, block[8:])
+	if err != nil {
+		c.buf = c.buf[0:0]
+		fmt.Printf("block decode error: %s\n", err)
+	}
+
+	c.buf, c.off = buf, 0
+}
+
+// read reads the current key and value from the current block.
+func (c *cursor) read() (key int64, value interface{}) {
+	// Return nil if the offset is at the end of the buffer.
+	if c.off >= len(c.buf) {
+		return -1, nil
+	}
+
+	// Otherwise read the current entry.
+	buf := c.buf[c.off:]
+	dataSize := entryDataSize(buf)
+
+	return tsdb.DecodeKeyValue(c.field, c.dec, buf[0:8], buf[entryHeaderSize:entryHeaderSize+dataSize])
+}
+
+// Sort bz1 cursors in correct order for writing to TSM files.
+
+type cursors []*cursor
+
+func (a cursors) Len() int      { return len(a) }
+func (a cursors) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a cursors) Less(i, j int) bool {
+	if a[i].series == a[j].series {
+		return a[i].field < a[j].field
+	}
+	return a[i].series < a[j].series
+}
+
+// entryHeaderSize is the number of bytes required for the header.
+const entryHeaderSize = 8 + 4
+
+// entryDataSize returns the size of an entry's data field, in bytes.
+func entryDataSize(v []byte) int { return int(binary.BigEndian.Uint32(v[8:12])) }

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/converter.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/influxdata/influxdb/cmd/influx_tsm/stats"
+	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+)
+
+const (
+	maxBlocksPerKey = 65535
+)
+
+// KeyIterator is used to iterate over b* keys for conversion to tsm keys
+type KeyIterator interface {
+	Next() bool
+	Read() (string, []tsm1.Value, error)
+}
+
+// Converter encapsulates the logic for converting b*1 shards to tsm1 shards.
+type Converter struct {
+	path           string
+	maxTSMFileSize uint32
+	sequence       int
+	stats          *stats.Stats
+}
+
+// NewConverter returns a new instance of the Converter.
+func NewConverter(path string, sz uint32, stats *stats.Stats) *Converter {
+	return &Converter{
+		path:           path,
+		maxTSMFileSize: sz,
+		stats:          stats,
+	}
+}
+
+// Process writes the data provided by iter to a tsm1 shard.
+func (c *Converter) Process(iter KeyIterator) error {
+	// Ensure the tsm1 directory exists.
+	if err := os.MkdirAll(c.path, 0777); err != nil {
+		return err
+	}
+
+	// Iterate until no more data remains.
+	var w tsm1.TSMWriter
+	var keyCount map[string]int
+
+	for iter.Next() {
+		k, v, err := iter.Read()
+		if err != nil {
+			return err
+		}
+
+		if w == nil {
+			w, err = c.nextTSMWriter()
+			if err != nil {
+				return err
+			}
+			keyCount = map[string]int{}
+		}
+		if err := w.Write(k, v); err != nil {
+			return err
+		}
+		keyCount[k]++
+
+		c.stats.AddPointsRead(len(v))
+		c.stats.AddPointsWritten(len(v))
+
+		// If we have a max file size configured and we're over it, start a new TSM file.
+		if w.Size() > c.maxTSMFileSize || keyCount[k] == maxBlocksPerKey {
+			if err := w.WriteIndex(); err != nil && err != tsm1.ErrNoValues {
+				return err
+			}
+
+			c.stats.AddTSMBytes(w.Size())
+
+			if err := w.Close(); err != nil {
+				return err
+			}
+			w = nil
+		}
+	}
+
+	if w != nil {
+		if err := w.WriteIndex(); err != nil && err != tsm1.ErrNoValues {
+			return err
+		}
+		c.stats.AddTSMBytes(w.Size())
+
+		if err := w.Close(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// nextTSMWriter returns the next TSMWriter for the Converter.
+func (c *Converter) nextTSMWriter() (tsm1.TSMWriter, error) {
+	c.sequence++
+	fileName := filepath.Join(c.path, fmt.Sprintf("%09d-%09d.%s", 1, c.sequence, tsm1.TSMFileExtension))
+
+	fd, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0666)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the writer for the new TSM file.
+	w, err := tsm1.NewTSMWriter(fd)
+	if err != nil {
+		return nil, err
+	}
+
+	c.stats.IncrTSMFileCount()
+	return w, nil
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/main.go

@@ -0,0 +1,415 @@
+// Command influx_tsm converts b1 or bz1 shards (from InfluxDB releases earlier than v0.11)
+// to the current tsm1 format.
+package main
+
+import (
+	"bufio"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"sort"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"net/http"
+	_ "net/http/pprof"
+
+	"github.com/influxdata/influxdb/cmd/influx_tsm/b1"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/bz1"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/tsdb"
+)
+
+// ShardReader reads b* shards and converts to tsm shards
+type ShardReader interface {
+	KeyIterator
+	Open() error
+	Close() error
+}
+
+const (
+	tsmExt = "tsm"
+)
+
+var description = `
+Convert a database from b1 or bz1 format to tsm1 format.
+
+This tool will backup the directories before conversion (if not disabled).
+The backed-up files must be removed manually, generally after starting up the
+node again to make sure all of data has been converted correctly.
+
+To restore a backup:
+  Shut down the node, remove the converted directory, and
+  copy the backed-up directory to the original location.`
+
+type options struct {
+	DataPath       string
+	BackupPath     string
+	DBs            []string
+	DebugAddr      string
+	TSMSize        uint64
+	Parallel       bool
+	SkipBackup     bool
+	UpdateInterval time.Duration
+	Yes            bool
+	CPUFile        string
+}
+
+func (o *options) Parse() error {
+	fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	var dbs string
+
+	fs.StringVar(&dbs, "dbs", "", "Comma-delimited list of databases to convert. Default is to convert all databases.")
+	fs.Uint64Var(&opts.TSMSize, "sz", maxTSMSz, "Maximum size of individual TSM files.")
+	fs.BoolVar(&opts.Parallel, "parallel", false, "Perform parallel conversion. (up to GOMAXPROCS shards at once)")
+	fs.BoolVar(&opts.SkipBackup, "nobackup", false, "Disable database backups. Not recommended.")
+	fs.StringVar(&opts.BackupPath, "backup", "", "The location to backup up the current databases. Must not be within the data directory.")
+	fs.StringVar(&opts.DebugAddr, "debug", "", "If set, http debugging endpoints will be enabled on the given address")
+	fs.DurationVar(&opts.UpdateInterval, "interval", 5*time.Second, "How often status updates are printed.")
+	fs.BoolVar(&opts.Yes, "y", false, "Don't ask, just convert")
+	fs.StringVar(&opts.CPUFile, "profile", "", "CPU Profile location")
+	fs.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage: %v [options] <data-path> \n", os.Args[0])
+		fmt.Fprintf(os.Stderr, "%v\n\nOptions:\n", description)
+		fs.PrintDefaults()
+		fmt.Fprintf(os.Stderr, "\n")
+	}
+
+	if err := fs.Parse(os.Args[1:]); err != nil {
+		return err
+	}
+
+	if len(fs.Args()) < 1 {
+		return errors.New("no data directory specified")
+	}
+	var err error
+	if o.DataPath, err = filepath.Abs(fs.Args()[0]); err != nil {
+		return err
+	}
+	if o.DataPath, err = filepath.EvalSymlinks(filepath.Clean(o.DataPath)); err != nil {
+		return err
+	}
+
+	if o.TSMSize > maxTSMSz {
+		return fmt.Errorf("bad TSM file size, maximum TSM file size is %d", maxTSMSz)
+	}
+
+	// Check if specific databases were requested.
+	o.DBs = strings.Split(dbs, ",")
+	if len(o.DBs) == 1 && o.DBs[0] == "" {
+		o.DBs = nil
+	}
+
+	if !o.SkipBackup {
+		if o.BackupPath == "" {
+			return errors.New("either -nobackup or -backup DIR must be set")
+		}
+		if o.BackupPath, err = filepath.Abs(o.BackupPath); err != nil {
+			return err
+		}
+		if o.BackupPath, err = filepath.EvalSymlinks(filepath.Clean(o.BackupPath)); err != nil {
+			if os.IsNotExist(err) {
+				return errors.New("backup directory must already exist")
+			}
+			return err
+		}
+
+		if strings.HasPrefix(o.BackupPath, o.DataPath) {
+			fmt.Println(o.BackupPath, o.DataPath)
+			return errors.New("backup directory cannot be contained within data directory")
+		}
+	}
+
+	if o.DebugAddr != "" {
+		log.Printf("Starting debugging server on http://%v", o.DebugAddr)
+		go func() {
+			log.Fatal(http.ListenAndServe(o.DebugAddr, nil))
+		}()
+	}
+
+	return nil
+}
+
+var opts options
+
+const maxTSMSz uint64 = 2 * 1024 * 1024 * 1024
+
+func init() {
+	log.SetOutput(os.Stderr)
+	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
+}
+
+func main() {
+	if err := opts.Parse(); err != nil {
+		log.Fatal(err)
+	}
+
+	// Determine the list of databases
+	dbs, err := ioutil.ReadDir(opts.DataPath)
+	if err != nil {
+		log.Fatalf("failed to access data directory at %v: %v\n", opts.DataPath, err)
+	}
+	fmt.Println() // Cleanly separate output from start of program.
+
+	if opts.Parallel {
+		if !isEnvSet("GOMAXPROCS") {
+			// Only modify GOMAXPROCS if it wasn't set in the environment
+			// This means 'GOMAXPROCS=1 influx_tsm -parallel' will not actually
+			// run in parallel
+			runtime.GOMAXPROCS(runtime.NumCPU())
+		}
+	}
+
+	var badUser string
+	if opts.SkipBackup {
+		badUser = "(NOT RECOMMENDED)"
+	}
+
+	// Dump summary of what is about to happen.
+	fmt.Println("b1 and bz1 shard conversion.")
+	fmt.Println("-----------------------------------")
+	fmt.Println("Data directory is:                 ", opts.DataPath)
+	if !opts.SkipBackup {
+		fmt.Println("Backup directory is:               ", opts.BackupPath)
+	}
+	fmt.Println("Databases specified:               ", allDBs(opts.DBs))
+	fmt.Println("Database backups enabled:          ", yesno(!opts.SkipBackup), badUser)
+	fmt.Printf("Parallel mode enabled (GOMAXPROCS): %s (%d)\n", yesno(opts.Parallel), runtime.GOMAXPROCS(0))
+	fmt.Println()
+
+	shards := collectShards(dbs)
+
+	// Anything to convert?
+	fmt.Printf("\nFound %d shards that will be converted.\n", len(shards))
+	if len(shards) == 0 {
+		fmt.Println("Nothing to do.")
+		return
+	}
+
+	// Display list of convertible shards.
+	fmt.Println()
+	w := new(tabwriter.Writer)
+	w.Init(os.Stdout, 0, 8, 1, '\t', 0)
+	fmt.Fprintln(w, "Database\tRetention\tPath\tEngine\tSize")
+	for _, si := range shards {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%d\n", si.Database, si.RetentionPolicy, si.FullPath(opts.DataPath), si.FormatAsString(), si.Size)
+	}
+	w.Flush()
+
+	if !opts.Yes {
+		// Get confirmation from user.
+		fmt.Printf("\nThese shards will be converted. Proceed? y/N: ")
+		liner := bufio.NewReader(os.Stdin)
+		yn, err := liner.ReadString('\n')
+		if err != nil {
+			log.Fatalf("failed to read response: %v", err)
+		}
+		yn = strings.TrimRight(strings.ToLower(yn), "\n")
+		if yn != "y" {
+			log.Fatal("Conversion aborted.")
+		}
+	}
+	fmt.Println("Conversion starting....")
+
+	if opts.CPUFile != "" {
+		f, err := os.Create(opts.CPUFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		if err = pprof.StartCPUProfile(f); err != nil {
+			log.Fatal(err)
+		}
+		defer pprof.StopCPUProfile()
+	}
+
+	tr := newTracker(shards, opts)
+
+	if err := tr.Run(); err != nil {
+		log.Fatalf("Error occurred preventing completion: %v\n", err)
+	}
+
+	tr.PrintStats()
+}
+
+func collectShards(dbs []os.FileInfo) tsdb.ShardInfos {
+	// Get the list of shards for conversion.
+	var shards tsdb.ShardInfos
+	for _, db := range dbs {
+		d := tsdb.NewDatabase(filepath.Join(opts.DataPath, db.Name()))
+		shs, err := d.Shards()
+		if err != nil {
+			log.Fatalf("Failed to access shards for database %v: %v\n", d.Name(), err)
+		}
+		shards = append(shards, shs...)
+	}
+
+	sort.Sort(shards)
+	shards = shards.FilterFormat(tsdb.TSM1)
+	if len(dbs) > 0 {
+		shards = shards.ExclusiveDatabases(opts.DBs)
+	}
+
+	return shards
+}
+
+// backupDatabase backs up the database named db
+func backupDatabase(db string) error {
+	copyFile := func(path string, info os.FileInfo, err error) error {
+		// Strip the DataPath from the path and replace with BackupPath.
+		toPath := strings.Replace(path, opts.DataPath, opts.BackupPath, 1)
+
+		if info.IsDir() {
+			return os.MkdirAll(toPath, info.Mode())
+		}
+
+		in, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer in.Close()
+
+		srcInfo, err := os.Stat(path)
+		if err != nil {
+			return err
+		}
+
+		out, err := os.OpenFile(toPath, os.O_CREATE|os.O_WRONLY, info.Mode())
+		if err != nil {
+			return err
+		}
+		defer out.Close()
+
+		dstInfo, err := os.Stat(toPath)
+		if err != nil {
+			return err
+		}
+
+		if dstInfo.Size() == srcInfo.Size() {
+			log.Printf("Backup file already found for %v with correct size, skipping.", path)
+			return nil
+		}
+
+		if dstInfo.Size() > srcInfo.Size() {
+			log.Printf("Invalid backup file found for %v, replacing with good copy.", path)
+			if err := out.Truncate(0); err != nil {
+				return err
+			}
+			if _, err := out.Seek(0, io.SeekStart); err != nil {
+				return err
+			}
+		}
+
+		if dstInfo.Size() > 0 {
+			log.Printf("Resuming backup of file %v, starting at %v bytes", path, dstInfo.Size())
+		}
+
+		off, err := out.Seek(0, io.SeekEnd)
+		if err != nil {
+			return err
+		}
+		if _, err := in.Seek(off, io.SeekStart); err != nil {
+			return err
+		}
+
+		log.Printf("Backing up file %v", path)
+
+		_, err = io.Copy(out, in)
+
+		return err
+	}
+
+	return filepath.Walk(filepath.Join(opts.DataPath, db), copyFile)
+}
+
+// convertShard converts the shard in-place.
+func convertShard(si *tsdb.ShardInfo, tr *tracker) error {
+	src := si.FullPath(opts.DataPath)
+	dst := fmt.Sprintf("%v.%v", src, tsmExt)
+
+	var reader ShardReader
+	switch si.Format {
+	case tsdb.BZ1:
+		reader = bz1.NewReader(src, &tr.Stats, 0)
+	case tsdb.B1:
+		reader = b1.NewReader(src, &tr.Stats, 0)
+	default:
+		return fmt.Errorf("Unsupported shard format: %v", si.FormatAsString())
+	}
+
+	// Open the shard, and create a converter.
+	if err := reader.Open(); err != nil {
+		return fmt.Errorf("Failed to open %v for conversion: %v", src, err)
+	}
+	defer reader.Close()
+	converter := NewConverter(dst, uint32(opts.TSMSize), &tr.Stats)
+
+	// Perform the conversion.
+	if err := converter.Process(reader); err != nil {
+		return fmt.Errorf("Conversion of %v failed: %v", src, err)
+	}
+
+	// Delete source shard, and rename new tsm1 shard.
+	if err := reader.Close(); err != nil {
+		return fmt.Errorf("Conversion of %v failed due to close: %v", src, err)
+	}
+
+	if err := os.RemoveAll(si.FullPath(opts.DataPath)); err != nil {
+		return fmt.Errorf("Deletion of %v failed: %v", src, err)
+	}
+	if err := os.Rename(dst, src); err != nil {
+		return fmt.Errorf("Rename of %v to %v failed: %v", dst, src, err)
+	}
+
+	return nil
+}
+
+// ParallelGroup allows the maximum parrallelism of a set of operations to be controlled.
+type ParallelGroup chan struct{}
+
+// NewParallelGroup returns a group which allows n operations to run in parallel. A value of 0
+// means no operations will ever run.
+func NewParallelGroup(n int) ParallelGroup {
+	return make(chan struct{}, n)
+}
+
+// Do executes one operation of the ParallelGroup
+func (p ParallelGroup) Do(f func()) {
+	p <- struct{}{} // acquire working slot
+	defer func() { <-p }()
+
+	f()
+}
+
+// yesno returns "yes" for true, "no" for false.
+func yesno(b bool) string {
+	if b {
+		return "yes"
+	}
+	return "no"
+}
+
+// allDBs returns "all" if all databases are requested for conversion.
+func allDBs(dbs []string) string {
+	if dbs == nil {
+		return "all"
+	}
+	return fmt.Sprintf("%v", dbs)
+}
+
+// isEnvSet checks to see if a variable was set in the environment
+func isEnvSet(name string) bool {
+	for _, s := range os.Environ() {
+		if strings.SplitN(s, "=", 2)[0] == name {
+			return true
+		}
+	}
+	return false
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/stats/stats.go

@@ -0,0 +1,55 @@
+// Package stats contains statistics for converting non-TSM shards to TSM.
+package stats
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+// Stats are the statistics captured while converting non-TSM shards to TSM
+type Stats struct {
+	NanFiltered     uint64
+	InfFiltered     uint64
+	FieldsFiltered  uint64
+	PointsWritten   uint64
+	PointsRead      uint64
+	TsmFilesCreated uint64
+	TsmBytesWritten uint64
+	CompletedShards uint64
+	TotalTime       time.Duration
+}
+
+// AddPointsRead increments the number of read points.
+func (s *Stats) AddPointsRead(n int) {
+	atomic.AddUint64(&s.PointsRead, uint64(n))
+}
+
+// AddPointsWritten increments the number of written points.
+func (s *Stats) AddPointsWritten(n int) {
+	atomic.AddUint64(&s.PointsWritten, uint64(n))
+}
+
+// AddTSMBytes increments the number of TSM Bytes.
+func (s *Stats) AddTSMBytes(n uint32) {
+	atomic.AddUint64(&s.TsmBytesWritten, uint64(n))
+}
+
+// IncrTSMFileCount increments the number of TSM files created.
+func (s *Stats) IncrTSMFileCount() {
+	atomic.AddUint64(&s.TsmFilesCreated, 1)
+}
+
+// IncrNaN increments the number of NaNs filtered.
+func (s *Stats) IncrNaN() {
+	atomic.AddUint64(&s.NanFiltered, 1)
+}
+
+// IncrInf increments the number of Infs filtered.
+func (s *Stats) IncrInf() {
+	atomic.AddUint64(&s.InfFiltered, 1)
+}
+
+// IncrFiltered increments the number of fields filtered.
+func (s *Stats) IncrFiltered() {
+	atomic.AddUint64(&s.FieldsFiltered, 1)
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tracker.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/influxdata/influxdb/cmd/influx_tsm/stats"
+	"github.com/influxdata/influxdb/cmd/influx_tsm/tsdb"
+)
+
+// tracker will orchestrate and track the conversions of non-TSM shards to TSM
+type tracker struct {
+	Stats stats.Stats
+
+	shards tsdb.ShardInfos
+	opts   options
+
+	pg ParallelGroup
+	wg sync.WaitGroup
+}
+
+// newTracker will setup and return a clean tracker instance
+func newTracker(shards tsdb.ShardInfos, opts options) *tracker {
+	t := &tracker{
+		shards: shards,
+		opts:   opts,
+		pg:     NewParallelGroup(runtime.GOMAXPROCS(0)),
+	}
+
+	return t
+}
+
+func (t *tracker) Run() error {
+	conversionStart := time.Now()
+
+	// Backup each directory.
+	if !opts.SkipBackup {
+		databases := t.shards.Databases()
+		fmt.Printf("Backing up %d databases...\n", len(databases))
+		t.wg.Add(len(databases))
+		for i := range databases {
+			db := databases[i]
+			go t.pg.Do(func() {
+				defer t.wg.Done()
+
+				start := time.Now()
+				log.Printf("Backup of database '%v' started", db)
+				err := backupDatabase(db)
+				if err != nil {
+					log.Fatalf("Backup of database %v failed: %v\n", db, err)
+				}
+				log.Printf("Database %v backed up (%v)\n", db, time.Since(start))
+			})
+		}
+		t.wg.Wait()
+	} else {
+		fmt.Println("Database backup disabled.")
+	}
+
+	t.wg.Add(len(t.shards))
+	for i := range t.shards {
+		si := t.shards[i]
+		go t.pg.Do(func() {
+			defer func() {
+				atomic.AddUint64(&t.Stats.CompletedShards, 1)
+				t.wg.Done()
+			}()
+
+			start := time.Now()
+			log.Printf("Starting conversion of shard: %v", si.FullPath(opts.DataPath))
+			if err := convertShard(si, t); err != nil {
+				log.Fatalf("Failed to convert %v: %v\n", si.FullPath(opts.DataPath), err)
+			}
+			log.Printf("Conversion of %v successful (%v)\n", si.FullPath(opts.DataPath), time.Since(start))
+		})
+	}
+
+	done := make(chan struct{})
+	go func() {
+		t.wg.Wait()
+		close(done)
+	}()
+
+WAIT_LOOP:
+	for {
+		select {
+		case <-done:
+			break WAIT_LOOP
+		case <-time.After(opts.UpdateInterval):
+			t.StatusUpdate()
+		}
+	}
+
+	t.Stats.TotalTime = time.Since(conversionStart)
+
+	return nil
+}
+
+func (t *tracker) StatusUpdate() {
+	shardCount := atomic.LoadUint64(&t.Stats.CompletedShards)
+	pointCount := atomic.LoadUint64(&t.Stats.PointsRead)
+	pointWritten := atomic.LoadUint64(&t.Stats.PointsWritten)
+
+	log.Printf("Still Working: Completed Shards: %d/%d Points read/written: %d/%d", shardCount, len(t.shards), pointCount, pointWritten)
+}
+
+func (t *tracker) PrintStats() {
+	preSize := t.shards.Size()
+	postSize := int64(t.Stats.TsmBytesWritten)
+
+	fmt.Printf("\nSummary statistics\n========================================\n")
+	fmt.Printf("Databases converted:                 %d\n", len(t.shards.Databases()))
+	fmt.Printf("Shards converted:                    %d\n", len(t.shards))
+	fmt.Printf("TSM files created:                   %d\n", t.Stats.TsmFilesCreated)
+	fmt.Printf("Points read:                         %d\n", t.Stats.PointsRead)
+	fmt.Printf("Points written:                      %d\n", t.Stats.PointsWritten)
+	fmt.Printf("NaN filtered:                        %d\n", t.Stats.NanFiltered)
+	fmt.Printf("Inf filtered:                        %d\n", t.Stats.InfFiltered)
+	fmt.Printf("Points without fields filtered:      %d\n", t.Stats.FieldsFiltered)
+	fmt.Printf("Disk usage pre-conversion (bytes):   %d\n", preSize)
+	fmt.Printf("Disk usage post-conversion (bytes):  %d\n", postSize)
+	fmt.Printf("Reduction factor:                    %d%%\n", 100*(preSize-postSize)/preSize)
+	fmt.Printf("Bytes per TSM point:                 %.2f\n", float64(postSize)/float64(t.Stats.PointsWritten))
+	fmt.Printf("Total conversion time:               %v\n", t.Stats.TotalTime)
+	fmt.Println()
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/codec.go

@@ -0,0 +1,119 @@
+package tsdb
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"math"
+)
+
+const (
+	fieldFloat   = 1
+	fieldInteger = 2
+	fieldBoolean = 3
+	fieldString  = 4
+)
+
+var (
+	// ErrFieldNotFound is returned when a field cannot be found.
+	ErrFieldNotFound = errors.New("field not found")
+
+	// ErrFieldUnmappedID is returned when the system is presented, during decode, with a field ID
+	// there is no mapping for.
+	ErrFieldUnmappedID = errors.New("field ID not mapped")
+)
+
+// FieldCodec provides encoding and decoding functionality for the fields of a given
+// Measurement.
+type FieldCodec struct {
+	fieldsByID   map[uint8]*Field
+	fieldsByName map[string]*Field
+}
+
+// NewFieldCodec returns a FieldCodec for the given Measurement. Must be called with
+// a RLock that protects the Measurement.
+func NewFieldCodec(fields map[string]*Field) *FieldCodec {
+	fieldsByID := make(map[uint8]*Field, len(fields))
+	fieldsByName := make(map[string]*Field, len(fields))
+	for _, f := range fields {
+		fieldsByID[f.ID] = f
+		fieldsByName[f.Name] = f
+	}
+	return &FieldCodec{fieldsByID: fieldsByID, fieldsByName: fieldsByName}
+}
+
+// FieldIDByName returns the ID for the given field.
+func (f *FieldCodec) FieldIDByName(s string) (uint8, error) {
+	fi := f.fieldsByName[s]
+	if fi == nil {
+		return 0, ErrFieldNotFound
+	}
+	return fi.ID, nil
+}
+
+// DecodeByID scans a byte slice for a field with the given ID, converts it to its
+// expected type, and return that value.
+func (f *FieldCodec) DecodeByID(targetID uint8, b []byte) (interface{}, error) {
+	var value interface{}
+	for {
+		if len(b) == 0 {
+			// No more bytes.
+			return nil, ErrFieldNotFound
+		}
+
+		field := f.fieldsByID[b[0]]
+		if field == nil {
+			// This can happen, though is very unlikely. If this node receives encoded data, to be written
+			// to disk, and is queried for that data before its metastore is updated, there will be no field
+			// mapping for the data during decode. All this can happen because data is encoded by the node
+			// that first received the write request, not the node that actually writes the data to disk.
+			// So if this happens, the read must be aborted.
+			return nil, ErrFieldUnmappedID
+		}
+
+		switch field.Type {
+		case fieldFloat:
+			if field.ID == targetID {
+				value = math.Float64frombits(binary.BigEndian.Uint64(b[1:9]))
+			}
+			b = b[9:]
+		case fieldInteger:
+			if field.ID == targetID {
+				value = int64(binary.BigEndian.Uint64(b[1:9]))
+			}
+			b = b[9:]
+		case fieldBoolean:
+			if field.ID == targetID {
+				value = b[1] == 1
+			}
+			b = b[2:]
+		case fieldString:
+			length := binary.BigEndian.Uint16(b[1:3])
+			if field.ID == targetID {
+				value = string(b[3 : 3+length])
+			}
+			b = b[3+length:]
+		default:
+			panic(fmt.Sprintf("unsupported value type during decode by id: %T", field.Type))
+		}
+
+		if value != nil {
+			return value, nil
+		}
+	}
+}
+
+// DecodeByName scans a byte slice for a field with the given name, converts it to its
+// expected type, and return that value.
+func (f *FieldCodec) DecodeByName(name string, b []byte) (interface{}, error) {
+	fi := f.FieldByName(name)
+	if fi == nil {
+		return 0, ErrFieldNotFound
+	}
+	return f.DecodeByID(fi.ID, b)
+}
+
+// FieldByName returns the field by its name. It will return a nil if not found
+func (f *FieldCodec) FieldByName(name string) *Field {
+	return f.fieldsByName[name]
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/database.go

@@ -0,0 +1,244 @@
+// Pacage tsdb abstracts the various shard types supported by the influx_tsm command.
+package tsdb // import "github.com/influxdata/influxdb/cmd/influx_tsm/tsdb"
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/influxdata/influxdb/pkg/slices"
+)
+
+// Flags for differentiating between engines
+const (
+	B1 = iota
+	BZ1
+	TSM1
+)
+
+// EngineFormat holds the flag for the engine
+type EngineFormat int
+
+// String returns the string format of the engine.
+func (e EngineFormat) String() string {
+	switch e {
+	case TSM1:
+		return "tsm1"
+	case B1:
+		return "b1"
+	case BZ1:
+		return "bz1"
+	default:
+		panic("unrecognized shard engine format")
+	}
+}
+
+// ShardInfo is the description of a shard on disk.
+type ShardInfo struct {
+	Database        string
+	RetentionPolicy string
+	Path            string
+	Format          EngineFormat
+	Size            int64
+}
+
+// FormatAsString returns the format of the shard as a string.
+func (s *ShardInfo) FormatAsString() string {
+	return s.Format.String()
+}
+
+// FullPath returns the full path to the shard, given the data directory root.
+func (s *ShardInfo) FullPath(dataPath string) string {
+	return filepath.Join(dataPath, s.Database, s.RetentionPolicy, s.Path)
+}
+
+// ShardInfos is an array of ShardInfo
+type ShardInfos []*ShardInfo
+
+func (s ShardInfos) Len() int      { return len(s) }
+func (s ShardInfos) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s ShardInfos) Less(i, j int) bool {
+	if s[i].Database == s[j].Database {
+		if s[i].RetentionPolicy == s[j].RetentionPolicy {
+			return s[i].Path < s[j].Path
+		}
+
+		return s[i].RetentionPolicy < s[j].RetentionPolicy
+	}
+
+	return s[i].Database < s[j].Database
+}
+
+// Databases returns the sorted unique set of databases for the shards.
+func (s ShardInfos) Databases() []string {
+	dbm := make(map[string]bool)
+	for _, ss := range s {
+		dbm[ss.Database] = true
+	}
+
+	var dbs []string
+	for k := range dbm {
+		dbs = append(dbs, k)
+	}
+	sort.Strings(dbs)
+	return dbs
+}
+
+// FilterFormat returns a copy of the ShardInfos, with shards of the given
+// format removed.
+func (s ShardInfos) FilterFormat(fmt EngineFormat) ShardInfos {
+	var a ShardInfos
+	for _, si := range s {
+		if si.Format != fmt {
+			a = append(a, si)
+		}
+	}
+	return a
+}
+
+// Size returns the space on disk consumed by the shards.
+func (s ShardInfos) Size() int64 {
+	var sz int64
+	for _, si := range s {
+		sz += si.Size
+	}
+	return sz
+}
+
+// ExclusiveDatabases returns a copy of the ShardInfo, with shards associated
+// with the given databases present. If the given set is empty, all databases
+// are returned.
+func (s ShardInfos) ExclusiveDatabases(exc []string) ShardInfos {
+	var a ShardInfos
+
+	// Empty set? Return everything.
+	if len(exc) == 0 {
+		a = make(ShardInfos, len(s))
+		copy(a, s)
+		return a
+	}
+
+	for _, si := range s {
+		if slices.Exists(exc, si.Database) {
+			a = append(a, si)
+		}
+	}
+	return a
+}
+
+// Database represents an entire database on disk.
+type Database struct {
+	path string
+}
+
+// NewDatabase creates a database instance using data at path.
+func NewDatabase(path string) *Database {
+	return &Database{path: path}
+}
+
+// Name returns the name of the database.
+func (d *Database) Name() string {
+	return path.Base(d.path)
+}
+
+// Path returns the path to the database.
+func (d *Database) Path() string {
+	return d.path
+}
+
+// Shards returns information for every shard in the database.
+func (d *Database) Shards() ([]*ShardInfo, error) {
+	fd, err := os.Open(d.path)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get each retention policy.
+	rps, err := fd.Readdirnames(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	// Process each retention policy.
+	var shardInfos []*ShardInfo
+	for _, rp := range rps {
+		rpfd, err := os.Open(filepath.Join(d.path, rp))
+		if err != nil {
+			return nil, err
+		}
+
+		// Process each shard
+		shards, err := rpfd.Readdirnames(-1)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, sh := range shards {
+			fmt, sz, err := shardFormat(filepath.Join(d.path, rp, sh))
+			if err != nil {
+				return nil, err
+			}
+
+			si := &ShardInfo{
+				Database:        d.Name(),
+				RetentionPolicy: path.Base(rp),
+				Path:            sh,
+				Format:          fmt,
+				Size:            sz,
+			}
+			shardInfos = append(shardInfos, si)
+		}
+	}
+
+	sort.Sort(ShardInfos(shardInfos))
+	return shardInfos, nil
+}
+
+// shardFormat returns the format and size on disk of the shard at path.
+func shardFormat(path string) (EngineFormat, int64, error) {
+	// If it's a directory then it's a tsm1 engine
+	fi, err := os.Stat(path)
+	if err != nil {
+		return 0, 0, err
+	}
+	if fi.Mode().IsDir() {
+		return TSM1, fi.Size(), nil
+	}
+
+	// It must be a BoltDB-based engine.
+	db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return 0, 0, err
+	}
+	defer db.Close()
+
+	var format EngineFormat
+	err = db.View(func(tx *bolt.Tx) error {
+		// Retrieve the meta bucket.
+		b := tx.Bucket([]byte("meta"))
+
+		// If no format is specified then it must be an original b1 database.
+		if b == nil {
+			format = B1
+			return nil
+		}
+
+		// There is an actual format indicator.
+		switch f := string(b.Get([]byte("format"))); f {
+		case "b1", "v1":
+			format = B1
+		case "bz1":
+			format = BZ1
+		default:
+			return fmt.Errorf("unrecognized engine format: %s", f)
+		}
+
+		return nil
+	})
+
+	return format, fi.Size(), err
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/internal/meta.pb.go

@@ -0,0 +1,122 @@
+// Code generated by protoc-gen-gogo.
+// source: internal/meta.proto
+// DO NOT EDIT!
+
+/*
+Package internal is a generated protocol buffer package.
+
+It is generated from these files:
+	internal/meta.proto
+
+It has these top-level messages:
+	Series
+	Tag
+	MeasurementFields
+	Field
+*/
+package internal
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+type Series struct {
+	Key              *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"`
+	Tags             []*Tag  `protobuf:"bytes,2,rep,name=Tags" json:"Tags,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *Series) Reset()         { *m = Series{} }
+func (m *Series) String() string { return proto.CompactTextString(m) }
+func (*Series) ProtoMessage()    {}
+
+func (m *Series) GetKey() string {
+	if m != nil && m.Key != nil {
+		return *m.Key
+	}
+	return ""
+}
+
+func (m *Series) GetTags() []*Tag {
+	if m != nil {
+		return m.Tags
+	}
+	return nil
+}
+
+type Tag struct {
+	Key              *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"`
+	Value            *string `protobuf:"bytes,2,req,name=Value" json:"Value,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *Tag) Reset()         { *m = Tag{} }
+func (m *Tag) String() string { return proto.CompactTextString(m) }
+func (*Tag) ProtoMessage()    {}
+
+func (m *Tag) GetKey() string {
+	if m != nil && m.Key != nil {
+		return *m.Key
+	}
+	return ""
+}
+
+func (m *Tag) GetValue() string {
+	if m != nil && m.Value != nil {
+		return *m.Value
+	}
+	return ""
+}
+
+type MeasurementFields struct {
+	Fields           []*Field `protobuf:"bytes,1,rep,name=Fields" json:"Fields,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *MeasurementFields) Reset()         { *m = MeasurementFields{} }
+func (m *MeasurementFields) String() string { return proto.CompactTextString(m) }
+func (*MeasurementFields) ProtoMessage()    {}
+
+func (m *MeasurementFields) GetFields() []*Field {
+	if m != nil {
+		return m.Fields
+	}
+	return nil
+}
+
+type Field struct {
+	ID               *int32  `protobuf:"varint,1,req,name=ID" json:"ID,omitempty"`
+	Name             *string `protobuf:"bytes,2,req,name=Name" json:"Name,omitempty"`
+	Type             *int32  `protobuf:"varint,3,req,name=Type" json:"Type,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *Field) Reset()         { *m = Field{} }
+func (m *Field) String() string { return proto.CompactTextString(m) }
+func (*Field) ProtoMessage()    {}
+
+func (m *Field) GetID() int32 {
+	if m != nil && m.ID != nil {
+		return *m.ID
+	}
+	return 0
+}
+
+func (m *Field) GetName() string {
+	if m != nil && m.Name != nil {
+		return *m.Name
+	}
+	return ""
+}
+
+func (m *Field) GetType() int32 {
+	if m != nil && m.Type != nil {
+		return *m.Type
+	}
+	return 0
+}

vendor/github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/types.go

@@ -0,0 +1,60 @@
+package tsdb
+
+import (
+	"encoding/binary"
+	"strings"
+
+	"github.com/influxdata/influxdb/cmd/influx_tsm/tsdb/internal"
+	"github.com/influxdata/influxdb/influxql"
+
+	"github.com/gogo/protobuf/proto"
+)
+
+// Field represents an encoded field.
+type Field struct {
+	ID   uint8             `json:"id,omitempty"`
+	Name string            `json:"name,omitempty"`
+	Type influxql.DataType `json:"type,omitempty"`
+}
+
+// MeasurementFields is a mapping from measurements to its fields.
+type MeasurementFields struct {
+	Fields map[string]*Field `json:"fields"`
+	Codec  *FieldCodec
+}
+
+// UnmarshalBinary decodes the object from a binary format.
+func (m *MeasurementFields) UnmarshalBinary(buf []byte) error {
+	var pb internal.MeasurementFields
+	if err := proto.Unmarshal(buf, &pb); err != nil {
+		return err
+	}
+	m.Fields = make(map[string]*Field)
+	for _, f := range pb.Fields {
+		m.Fields[f.GetName()] = &Field{ID: uint8(f.GetID()), Name: f.GetName(), Type: influxql.DataType(f.GetType())}
+	}
+	return nil
+}
+
+// Series represents a series in the shard.
+type Series struct {
+	Key  string
+	Tags map[string]string
+}
+
+// MeasurementFromSeriesKey returns the Measurement name for a given series.
+func MeasurementFromSeriesKey(key string) string {
+	return strings.SplitN(key, ",", 2)[0]
+}
+
+// DecodeKeyValue decodes the key and value from bytes.
+func DecodeKeyValue(field string, dec *FieldCodec, k, v []byte) (int64, interface{}) {
+	// Convert key to a timestamp.
+	key := int64(binary.BigEndian.Uint64(k[0:8]))
+
+	decValue, err := dec.DecodeByName(field, v)
+	if err != nil {
+		return key, nil
+	}
+	return key, decValue
+}

vendor/github.com/influxdata/influxdb/cmd/influxd/backup/backup.go

@@ -0,0 +1,387 @@
+// Package backup is the backup subcommand for the influxd command.
+package backup
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/influxdata/influxdb/services/snapshotter"
+	"github.com/influxdata/influxdb/tcp"
+)
+
+const (
+	// Suffix is a suffix added to the backup while it's in-process.
+	Suffix = ".pending"
+
+	// Metafile is the base name given to the metastore backups.
+	Metafile = "meta"
+
+	// BackupFilePattern is the beginning of the pattern for a backup
+	// file. They follow the scheme <database>.<retention>.<shardID>.<increment>
+	BackupFilePattern = "%s.%s.%05d"
+)
+
+// Command represents the program execution for "influxd backup".
+type Command struct {
+	// The logger passed to the ticker during execution.
+	Logger *log.Logger
+
+	// Standard input/output, overridden for testing.
+	Stderr io.Writer
+	Stdout io.Writer
+
+	host     string
+	path     string
+	database string
+}
+
+// NewCommand returns a new instance of Command with default settings.
+func NewCommand() *Command {
+	return &Command{
+		Stderr: os.Stderr,
+		Stdout: os.Stdout,
+	}
+}
+
+// Run executes the program.
+func (cmd *Command) Run(args ...string) error {
+	// Set up logger.
+	cmd.Logger = log.New(cmd.Stderr, "", log.LstdFlags)
+
+	// Parse command line arguments.
+	retentionPolicy, shardID, since, err := cmd.parseFlags(args)
+	if err != nil {
+		return err
+	}
+
+	// based on the arguments passed in we only backup the minimum
+	if shardID != "" {
+		// always backup the metastore
+		if err := cmd.backupMetastore(); err != nil {
+			return err
+		}
+		err = cmd.backupShard(retentionPolicy, shardID, since)
+	} else if retentionPolicy != "" {
+		err = cmd.backupRetentionPolicy(retentionPolicy, since)
+	} else if cmd.database != "" {
+		err = cmd.backupDatabase(since)
+	} else {
+		err = cmd.backupMetastore()
+	}
+
+	if err != nil {
+		cmd.Logger.Printf("backup failed: %v", err)
+		return err
+	}
+
+	cmd.Logger.Println("backup complete")
+
+	return nil
+}
+
+// parseFlags parses and validates the command line arguments into a request object.
+func (cmd *Command) parseFlags(args []string) (retentionPolicy, shardID string, since time.Time, err error) {
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+
+	fs.StringVar(&cmd.host, "host", "localhost:8088", "")
+	fs.StringVar(&cmd.database, "database", "", "")
+	fs.StringVar(&retentionPolicy, "retention", "", "")
+	fs.StringVar(&shardID, "shard", "", "")
+	var sinceArg string
+	fs.StringVar(&sinceArg, "since", "", "")
+
+	fs.SetOutput(cmd.Stderr)
+	fs.Usage = cmd.printUsage
+
+	err = fs.Parse(args)
+	if err != nil {
+		return
+	}
+	if sinceArg != "" {
+		since, err = time.Parse(time.RFC3339, sinceArg)
+		if err != nil {
+			return
+		}
+	}
+
+	// Ensure that only one arg is specified.
+	if fs.NArg() == 0 {
+		return "", "", time.Unix(0, 0), errors.New("backup destination path required")
+	} else if fs.NArg() != 1 {
+		return "", "", time.Unix(0, 0), errors.New("only one backup path allowed")
+	}
+	cmd.path = fs.Arg(0)
+
+	err = os.MkdirAll(cmd.path, 0700)
+
+	return
+}
+
+// backupShard will write a tar archive of the passed in shard with any TSM files that have been
+// created since the time passed in
+func (cmd *Command) backupShard(retentionPolicy string, shardID string, since time.Time) error {
+	id, err := strconv.ParseUint(shardID, 10, 64)
+	if err != nil {
+		return err
+	}
+
+	shardArchivePath, err := cmd.nextPath(filepath.Join(cmd.path, fmt.Sprintf(BackupFilePattern, cmd.database, retentionPolicy, id)))
+	if err != nil {
+		return err
+	}
+
+	cmd.Logger.Printf("backing up db=%v rp=%v shard=%v to %s since %s",
+		cmd.database, retentionPolicy, shardID, shardArchivePath, since)
+
+	req := &snapshotter.Request{
+		Type:            snapshotter.RequestShardBackup,
+		Database:        cmd.database,
+		RetentionPolicy: retentionPolicy,
+		ShardID:         id,
+		Since:           since,
+	}
+
+	// TODO: verify shard backup data
+	return cmd.downloadAndVerify(req, shardArchivePath, nil)
+}
+
+// backupDatabase will request the database information from the server and then backup the metastore and
+// every shard in every retention policy in the database. Each shard will be written to a separate tar.
+func (cmd *Command) backupDatabase(since time.Time) error {
+	cmd.Logger.Printf("backing up db=%s since %s", cmd.database, since)
+
+	req := &snapshotter.Request{
+		Type:     snapshotter.RequestDatabaseInfo,
+		Database: cmd.database,
+	}
+
+	response, err := cmd.requestInfo(req)
+	if err != nil {
+		return err
+	}
+
+	return cmd.backupResponsePaths(response, since)
+}
+
+// backupRetentionPolicy will request the retention policy information from the server and then backup
+// the metastore and every shard in the retention policy. Each shard will be written to a separate tar.
+func (cmd *Command) backupRetentionPolicy(retentionPolicy string, since time.Time) error {
+	cmd.Logger.Printf("backing up rp=%s since %s", retentionPolicy, since)
+
+	req := &snapshotter.Request{
+		Type:            snapshotter.RequestRetentionPolicyInfo,
+		Database:        cmd.database,
+		RetentionPolicy: retentionPolicy,
+	}
+
+	response, err := cmd.requestInfo(req)
+	if err != nil {
+		return err
+	}
+
+	return cmd.backupResponsePaths(response, since)
+}
+
+// backupResponsePaths will backup the metastore and all shard paths in the response struct
+func (cmd *Command) backupResponsePaths(response *snapshotter.Response, since time.Time) error {
+	if err := cmd.backupMetastore(); err != nil {
+		return err
+	}
+
+	// loop through the returned paths and back up each shard
+	for _, path := range response.Paths {
+		rp, id, err := retentionAndShardFromPath(path)
+		if err != nil {
+			return err
+		}
+
+		if err := cmd.backupShard(rp, id, since); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// backupMetastore will backup the metastore on the host to the passed in path. Database and retention policy backups
+// will force a backup of the metastore as well as requesting a specific shard backup from the command line
+func (cmd *Command) backupMetastore() error {
+	metastoreArchivePath, err := cmd.nextPath(filepath.Join(cmd.path, Metafile))
+	if err != nil {
+		return err
+	}
+
+	cmd.Logger.Printf("backing up metastore to %s", metastoreArchivePath)
+
+	req := &snapshotter.Request{
+		Type: snapshotter.RequestMetastoreBackup,
+	}
+
+	return cmd.downloadAndVerify(req, metastoreArchivePath, func(file string) error {
+		binData, err := ioutil.ReadFile(file)
+		if err != nil {
+			return err
+		}
+
+		magic := binary.BigEndian.Uint64(binData[:8])
+		if magic != snapshotter.BackupMagicHeader {
+			cmd.Logger.Println("Invalid metadata blob, ensure the metadata service is running (default port 8088)")
+			return errors.New("invalid metadata received")
+		}
+
+		return nil
+	})
+}
+
+// nextPath returns the next file to write to.
+func (cmd *Command) nextPath(path string) (string, error) {
+	// Iterate through incremental files until one is available.
+	for i := 0; ; i++ {
+		s := fmt.Sprintf(path+".%02d", i)
+		if _, err := os.Stat(s); os.IsNotExist(err) {
+			return s, nil
+		} else if err != nil {
+			return "", err
+		}
+	}
+}
+
+// downloadAndVerify will download either the metastore or shard to a temp file and then
+// rename it to a good backup file name after complete
+func (cmd *Command) downloadAndVerify(req *snapshotter.Request, path string, validator func(string) error) error {
+	tmppath := path + Suffix
+	if err := cmd.download(req, tmppath); err != nil {
+		return err
+	}
+
+	if validator != nil {
+		if err := validator(tmppath); err != nil {
+			if rmErr := os.Remove(tmppath); rmErr != nil {
+				cmd.Logger.Printf("Error cleaning up temporary file: %v", rmErr)
+			}
+			return err
+		}
+	}
+
+	f, err := os.Stat(tmppath)
+	if err != nil {
+		return err
+	}
+
+	// There was nothing downloaded, don't create an empty backup file.
+	if f.Size() == 0 {
+		return os.Remove(tmppath)
+	}
+
+	// Rename temporary file to final path.
+	if err := os.Rename(tmppath, path); err != nil {
+		return fmt.Errorf("rename: %s", err)
+	}
+
+	return nil
+}
+
+// download downloads a snapshot of either the metastore or a shard from a host to a given path.
+func (cmd *Command) download(req *snapshotter.Request, path string) error {
+	// Create local file to write to.
+	f, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("open temp file: %s", err)
+	}
+	defer f.Close()
+
+	for i := 0; i < 10; i++ {
+		if err = func() error {
+			// Connect to snapshotter service.
+			conn, err := tcp.Dial("tcp", cmd.host, snapshotter.MuxHeader)
+			if err != nil {
+				return err
+			}
+			defer conn.Close()
+
+			// Write the request
+			if err := json.NewEncoder(conn).Encode(req); err != nil {
+				return fmt.Errorf("encode snapshot request: %s", err)
+			}
+
+			// Read snapshot from the connection
+			if n, err := io.Copy(f, conn); err != nil || n == 0 {
+				return fmt.Errorf("copy backup to file: err=%v, n=%d", err, n)
+			}
+			return nil
+		}(); err == nil {
+			break
+		} else if err != nil {
+			cmd.Logger.Printf("Download shard %v failed %s.  Retrying (%d)...\n", req.ShardID, err, i)
+			time.Sleep(time.Second)
+		}
+	}
+
+	return err
+}
+
+// requestInfo will request the database or retention policy information from the host
+func (cmd *Command) requestInfo(request *snapshotter.Request) (*snapshotter.Response, error) {
+	// Connect to snapshotter service.
+	conn, err := tcp.Dial("tcp", cmd.host, snapshotter.MuxHeader)
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	// Write the request
+	if err := json.NewEncoder(conn).Encode(request); err != nil {
+		return nil, fmt.Errorf("encode snapshot request: %s", err)
+	}
+
+	// Read the response
+	var r snapshotter.Response
+	if err := json.NewDecoder(conn).Decode(&r); err != nil {
+		return nil, err
+	}
+
+	return &r, nil
+}
+
+// printUsage prints the usage message to STDERR.
+func (cmd *Command) printUsage() {
+	fmt.Fprintf(cmd.Stdout, `Downloads a snapshot of a data node and saves it to disk.
+
+Usage: influxd backup [flags] PATH
+
+    -host <host:port>
+            The host to connect to snapshot. Defaults to 127.0.0.1:8088.
+    -database <name>
+            The database to backup.
+    -retention <name>
+            Optional. The retention policy to backup.
+    -shard <id>
+            Optional. The shard id to backup. If specified, retention is required.
+    -since <2015-12-24T08:12:23>
+            Optional. Do an incremental backup since the passed in RFC3339
+            formatted time.
+
+`)
+}
+
+// retentionAndShardFromPath will take the shard relative path and split it into the
+// retention policy name and shard ID. The first part of the path should be the database name.
+func retentionAndShardFromPath(path string) (retention, shard string, err error) {
+	a := strings.Split(path, string(filepath.Separator))
+	if len(a) != 3 {
+		return "", "", fmt.Errorf("expected database, retention policy, and shard id in path: %s", path)
+	}
+
+	return a[1], a[2], nil
+}

vendor/github.com/influxdata/influxdb/cmd/influxd/help/help.go

@@ -0,0 +1,46 @@
+// Package help is the help subcommand of the influxd command.
+package help
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+// Command displays help for command-line sub-commands.
+type Command struct {
+	Stdout io.Writer
+}
+
+// NewCommand returns a new instance of Command.
+func NewCommand() *Command {
+	return &Command{
+		Stdout: os.Stdout,
+	}
+}
+
+// Run executes the command.
+func (cmd *Command) Run(args ...string) error {
+	fmt.Fprintln(cmd.Stdout, strings.TrimSpace(usage))
+	return nil
+}
+
+const usage = `
+Configure and start an InfluxDB server.
+
+Usage: influxd [[command] [arguments]]
+
+The commands are:
+
+    backup               downloads a snapshot of a data node and saves it to disk
+    config               display the default configuration
+    help                 display this help message
+    restore              uses a snapshot of a data node to rebuild a cluster
+    run                  run node with existing configuration
+    version              displays the InfluxDB version
+
+"run" is the default command.
+
+Use "influxd [command] -help" for more information about a command.
+`

vendor/github.com/influxdata/influxdb/cmd/influxd/main.go

@@ -0,0 +1,177 @@
+// Command influxd is the InfluxDB server.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/influxdata/influxdb/cmd"
+	"github.com/influxdata/influxdb/cmd/influxd/backup"
+	"github.com/influxdata/influxdb/cmd/influxd/help"
+	"github.com/influxdata/influxdb/cmd/influxd/restore"
+	"github.com/influxdata/influxdb/cmd/influxd/run"
+	"github.com/uber-go/zap"
+)
+
+// These variables are populated via the Go linker.
+var (
+	version string
+	commit  string
+	branch  string
+)
+
+func init() {
+	// If commit, branch, or build time are not set, make that clear.
+	if version == "" {
+		version = "unknown"
+	}
+	if commit == "" {
+		commit = "unknown"
+	}
+	if branch == "" {
+		branch = "unknown"
+	}
+}
+
+func main() {
+	rand.Seed(time.Now().UnixNano())
+
+	m := NewMain()
+	if err := m.Run(os.Args[1:]...); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+// Main represents the program execution.
+type Main struct {
+	Logger zap.Logger
+
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewMain return a new instance of Main.
+func NewMain() *Main {
+	return &Main{
+		Logger: zap.New(
+			zap.NewTextEncoder(),
+			zap.Output(os.Stderr),
+		),
+		Stdin:  os.Stdin,
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+}
+
+// Run determines and runs the command specified by the CLI args.
+func (m *Main) Run(args ...string) error {
+	name, args := cmd.ParseCommandName(args)
+
+	// Extract name from args.
+	switch name {
+	case "", "run":
+		cmd := run.NewCommand()
+
+		// Tell the server the build details.
+		cmd.Version = version
+		cmd.Commit = commit
+		cmd.Branch = branch
+		cmd.Logger = m.Logger
+
+		if err := cmd.Run(args...); err != nil {
+			return fmt.Errorf("run: %s", err)
+		}
+
+		signalCh := make(chan os.Signal, 1)
+		signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
+		m.Logger.Info("Listening for signals")
+
+		// Block until one of the signals above is received
+		<-signalCh
+		m.Logger.Info("Signal received, initializing clean shutdown...")
+		go cmd.Close()
+
+		// Block again until another signal is received, a shutdown timeout elapses,
+		// or the Command is gracefully closed
+		m.Logger.Info("Waiting for clean shutdown...")
+		select {
+		case <-signalCh:
+			m.Logger.Info("second signal received, initializing hard shutdown")
+		case <-time.After(time.Second * 30):
+			m.Logger.Info("time limit reached, initializing hard shutdown")
+		case <-cmd.Closed:
+			m.Logger.Info("server shutdown completed")
+		}
+
+		// goodbye.
+
+	case "backup":
+		name := backup.NewCommand()
+		if err := name.Run(args...); err != nil {
+			return fmt.Errorf("backup: %s", err)
+		}
+	case "restore":
+		name := restore.NewCommand()
+		if err := name.Run(args...); err != nil {
+			return fmt.Errorf("restore: %s", err)
+		}
+	case "config":
+		if err := run.NewPrintConfigCommand().Run(args...); err != nil {
+			return fmt.Errorf("config: %s", err)
+		}
+	case "version":
+		if err := NewVersionCommand().Run(args...); err != nil {
+			return fmt.Errorf("version: %s", err)
+		}
+	case "help":
+		if err := help.NewCommand().Run(args...); err != nil {
+			return fmt.Errorf("help: %s", err)
+		}
+	default:
+		return fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'influxd help' for usage`+"\n\n", name)
+	}
+
+	return nil
+}
+
+// VersionCommand represents the command executed by "influxd version".
+type VersionCommand struct {
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewVersionCommand return a new instance of VersionCommand.
+func NewVersionCommand() *VersionCommand {
+	return &VersionCommand{
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+}
+
+// Run prints the current version and commit info.
+func (cmd *VersionCommand) Run(args ...string) error {
+	// Parse flags in case -h is specified.
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	fs.Usage = func() { fmt.Fprintln(cmd.Stderr, versionUsage) }
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// Print version info.
+	fmt.Fprintf(cmd.Stdout, "InfluxDB v%s (git: %s %s)\n", version, branch, commit)
+
+	return nil
+}
+
+var versionUsage = `Displays the InfluxDB version, build branch and git commit hash.
+
+Usage: influxd version
+`

vendor/github.com/influxdata/influxdb/cmd/influxd/restore/restore.go

@@ -0,0 +1,355 @@
+// Package restore is the restore subcommand for the influxd command,
+// for restoring from a backup.
+package restore
+
+import (
+	"archive/tar"
+	"bytes"
+	"encoding/binary"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/influxdata/influxdb/cmd/influxd/backup"
+	"github.com/influxdata/influxdb/services/meta"
+	"github.com/influxdata/influxdb/services/snapshotter"
+)
+
+// Command represents the program execution for "influxd restore".
+type Command struct {
+	Stdout io.Writer
+	Stderr io.Writer
+
+	backupFilesPath string
+	metadir         string
+	datadir         string
+	database        string
+	retention       string
+	shard           string
+
+	// TODO: when the new meta stuff is done this should not be exported or be gone
+	MetaConfig *meta.Config
+}
+
+// NewCommand returns a new instance of Command with default settings.
+func NewCommand() *Command {
+	return &Command{
+		Stdout:     os.Stdout,
+		Stderr:     os.Stderr,
+		MetaConfig: meta.NewConfig(),
+	}
+}
+
+// Run executes the program.
+func (cmd *Command) Run(args ...string) error {
+	if err := cmd.parseFlags(args); err != nil {
+		return err
+	}
+
+	if cmd.metadir != "" {
+		if err := cmd.unpackMeta(); err != nil {
+			return err
+		}
+	}
+
+	if cmd.shard != "" {
+		return cmd.unpackShard(cmd.shard)
+	} else if cmd.retention != "" {
+		return cmd.unpackRetention()
+	} else if cmd.datadir != "" {
+		return cmd.unpackDatabase()
+	}
+	return nil
+}
+
+// parseFlags parses and validates the command line arguments.
+func (cmd *Command) parseFlags(args []string) error {
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	fs.StringVar(&cmd.metadir, "metadir", "", "")
+	fs.StringVar(&cmd.datadir, "datadir", "", "")
+	fs.StringVar(&cmd.database, "database", "", "")
+	fs.StringVar(&cmd.retention, "retention", "", "")
+	fs.StringVar(&cmd.shard, "shard", "", "")
+	fs.SetOutput(cmd.Stdout)
+	fs.Usage = cmd.printUsage
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	cmd.MetaConfig = meta.NewConfig()
+	cmd.MetaConfig.Dir = cmd.metadir
+
+	// Require output path.
+	cmd.backupFilesPath = fs.Arg(0)
+	if cmd.backupFilesPath == "" {
+		return fmt.Errorf("path with backup files required")
+	}
+
+	// validate the arguments
+	if cmd.metadir == "" && cmd.database == "" {
+		return fmt.Errorf("-metadir or -database are required to restore")
+	}
+
+	if cmd.database != "" && cmd.datadir == "" {
+		return fmt.Errorf("-datadir is required to restore")
+	}
+
+	if cmd.shard != "" {
+		if cmd.database == "" {
+			return fmt.Errorf("-database is required to restore shard")
+		}
+		if cmd.retention == "" {
+			return fmt.Errorf("-retention is required to restore shard")
+		}
+	} else if cmd.retention != "" && cmd.database == "" {
+		return fmt.Errorf("-database is required to restore retention policy")
+	}
+
+	return nil
+}
+
+// unpackMeta reads the metadata from the backup directory and initializes a raft
+// cluster and replaces the root metadata.
+func (cmd *Command) unpackMeta() error {
+	// find the meta file
+	metaFiles, err := filepath.Glob(filepath.Join(cmd.backupFilesPath, backup.Metafile+".*"))
+	if err != nil {
+		return err
+	}
+
+	if len(metaFiles) == 0 {
+		return fmt.Errorf("no metastore backups in %s", cmd.backupFilesPath)
+	}
+
+	latest := metaFiles[len(metaFiles)-1]
+
+	fmt.Fprintf(cmd.Stdout, "Using metastore snapshot: %v\n", latest)
+	// Read the metastore backup
+	f, err := os.Open(latest)
+	if err != nil {
+		return err
+	}
+
+	var buf bytes.Buffer
+	if _, err := io.Copy(&buf, f); err != nil {
+		return fmt.Errorf("copy: %s", err)
+	}
+
+	b := buf.Bytes()
+	var i int
+
+	// Make sure the file is actually a meta store backup file
+	magic := binary.BigEndian.Uint64(b[:8])
+	if magic != snapshotter.BackupMagicHeader {
+		return fmt.Errorf("invalid metadata file")
+	}
+	i += 8
+
+	// Size of the meta store bytes
+	length := int(binary.BigEndian.Uint64(b[i : i+8]))
+	i += 8
+	metaBytes := b[i : i+length]
+	i += int(length)
+
+	// Size of the node.json bytes
+	length = int(binary.BigEndian.Uint64(b[i : i+8]))
+	i += 8
+	nodeBytes := b[i : i+length]
+
+	// Unpack into metadata.
+	var data meta.Data
+	if err := data.UnmarshalBinary(metaBytes); err != nil {
+		return fmt.Errorf("unmarshal: %s", err)
+	}
+
+	// Copy meta config and remove peers so it starts in single mode.
+	c := cmd.MetaConfig
+	c.Dir = cmd.metadir
+
+	// Create the meta dir
+	if os.MkdirAll(c.Dir, 0700); err != nil {
+		return err
+	}
+
+	// Write node.json back to meta dir
+	if err := ioutil.WriteFile(filepath.Join(c.Dir, "node.json"), nodeBytes, 0655); err != nil {
+		return err
+	}
+
+	client := meta.NewClient(c)
+	if err := client.Open(); err != nil {
+		return err
+	}
+	defer client.Close()
+
+	// Force set the full metadata.
+	if err := client.SetData(&data); err != nil {
+		return fmt.Errorf("set data: %s", err)
+	}
+
+	// remove the raft.db file if it exists
+	err = os.Remove(filepath.Join(cmd.metadir, "raft.db"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+
+	// remove the node.json file if it exists
+	err = os.Remove(filepath.Join(cmd.metadir, "node.json"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+
+	return nil
+}
+
+// unpackShard will look for all backup files in the path matching this shard ID
+// and restore them to the data dir
+func (cmd *Command) unpackShard(shardID string) error {
+	// make sure the shard isn't already there so we don't clobber anything
+	restorePath := filepath.Join(cmd.datadir, cmd.database, cmd.retention, shardID)
+	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("shard already present: %s", restorePath)
+	}
+
+	id, err := strconv.ParseUint(shardID, 10, 64)
+	if err != nil {
+		return err
+	}
+
+	// find the shard backup files
+	pat := filepath.Join(cmd.backupFilesPath, fmt.Sprintf(backup.BackupFilePattern, cmd.database, cmd.retention, id))
+	return cmd.unpackFiles(pat + ".*")
+}
+
+// unpackDatabase will look for all backup files in the path matching this database
+// and restore them to the data dir
+func (cmd *Command) unpackDatabase() error {
+	// make sure the shard isn't already there so we don't clobber anything
+	restorePath := filepath.Join(cmd.datadir, cmd.database)
+	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("database already present: %s", restorePath)
+	}
+
+	// find the database backup files
+	pat := filepath.Join(cmd.backupFilesPath, cmd.database)
+	return cmd.unpackFiles(pat + ".*")
+}
+
+// unpackRetention will look for all backup files in the path matching this retention
+// and restore them to the data dir
+func (cmd *Command) unpackRetention() error {
+	// make sure the shard isn't already there so we don't clobber anything
+	restorePath := filepath.Join(cmd.datadir, cmd.database, cmd.retention)
+	if _, err := os.Stat(restorePath); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("retention already present: %s", restorePath)
+	}
+
+	// find the retention backup files
+	pat := filepath.Join(cmd.backupFilesPath, cmd.database)
+	return cmd.unpackFiles(fmt.Sprintf("%s.%s.*", pat, cmd.retention))
+}
+
+// unpackFiles will look for backup files matching the pattern and restore them to the data dir
+func (cmd *Command) unpackFiles(pat string) error {
+	fmt.Printf("Restoring from backup %s\n", pat)
+
+	backupFiles, err := filepath.Glob(pat)
+	if err != nil {
+		return err
+	}
+
+	if len(backupFiles) == 0 {
+		return fmt.Errorf("no backup files for %s in %s", pat, cmd.backupFilesPath)
+	}
+
+	for _, fn := range backupFiles {
+		if err := cmd.unpackTar(fn); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// unpackTar will restore a single tar archive to the data dir
+func (cmd *Command) unpackTar(tarFile string) error {
+	f, err := os.Open(tarFile)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	tr := tar.NewReader(f)
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+
+		if err := cmd.unpackFile(tr, hdr.Name); err != nil {
+			return err
+		}
+	}
+}
+
+// unpackFile will copy the current file from the tar archive to the data dir
+func (cmd *Command) unpackFile(tr *tar.Reader, fileName string) error {
+	nativeFileName := filepath.FromSlash(fileName)
+	fn := filepath.Join(cmd.datadir, nativeFileName)
+	fmt.Printf("unpacking %s\n", fn)
+
+	if err := os.MkdirAll(filepath.Dir(fn), 0777); err != nil {
+		return fmt.Errorf("error making restore dir: %s", err.Error())
+	}
+
+	ff, err := os.Create(fn)
+	if err != nil {
+		return err
+	}
+	defer ff.Close()
+
+	if _, err := io.Copy(ff, tr); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// printUsage prints the usage message to STDERR.
+func (cmd *Command) printUsage() {
+	fmt.Fprintf(cmd.Stdout, `Uses backups from the PATH to restore the metastore, databases,
+retention policies, or specific shards. The InfluxDB process must not be
+running during a restore.
+
+Usage: influxd restore [flags] PATH
+
+    -metadir <path>
+            Optional. If set the metastore will be recovered to the given path.
+    -datadir <path>
+            Optional. If set the restore process will recover the specified
+            database, retention policy or shard to the given directory.
+    -database <name>
+            Optional. Required if no metadir given. Will restore the database
+            TSM files.
+    -retention <name>
+            Optional. If given, database is required. Will restore the retention policy's
+            TSM files.
+    -shard <id>
+            Optional. If given, database and retention are required. Will restore the shard's
+            TSM files.
+
+`)
+}

vendor/github.com/influxdata/influxdb/cmd/influxd/run/command.go

@@ -0,0 +1,261 @@
+// Package run is the run (default) subcommand for the influxd command.
+package run
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"time"
+
+	"github.com/uber-go/zap"
+)
+
+const logo = `
+ 8888888           .d888 888                   8888888b.  888888b.
+   888            d88P"  888                   888  "Y88b 888  "88b
+   888            888    888                   888    888 888  .88P
+   888   88888b.  888888 888 888  888 888  888 888    888 8888888K.
+   888   888 "88b 888    888 888  888  Y8bd8P' 888    888 888  "Y88b
+   888   888  888 888    888 888  888   X88K   888    888 888    888
+   888   888  888 888    888 Y88b 888 .d8""8b. 888  .d88P 888   d88P
+ 8888888 888  888 888    888  "Y88888 888  888 8888888P"  8888888P"
+
+`
+
+// Command represents the command executed by "influxd run".
+type Command struct {
+	Version   string
+	Branch    string
+	Commit    string
+	BuildTime string
+
+	closing chan struct{}
+	Closed  chan struct{}
+
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+	Logger zap.Logger
+
+	Server *Server
+}
+
+// NewCommand return a new instance of Command.
+func NewCommand() *Command {
+	return &Command{
+		closing: make(chan struct{}),
+		Closed:  make(chan struct{}),
+		Stdin:   os.Stdin,
+		Stdout:  os.Stdout,
+		Stderr:  os.Stderr,
+		Logger:  zap.New(zap.NullEncoder()),
+	}
+}
+
+// Run parses the config from args and runs the server.
+func (cmd *Command) Run(args ...string) error {
+	// Parse the command line flags.
+	options, err := cmd.ParseFlags(args...)
+	if err != nil {
+		return err
+	}
+
+	// Print sweet InfluxDB logo.
+	fmt.Print(logo)
+
+	// Mark start-up in log.
+	cmd.Logger.Info(fmt.Sprintf("InfluxDB starting, version %s, branch %s, commit %s",
+		cmd.Version, cmd.Branch, cmd.Commit))
+	cmd.Logger.Info(fmt.Sprintf("Go version %s, GOMAXPROCS set to %d", runtime.Version(), runtime.GOMAXPROCS(0)))
+
+	// Write the PID file.
+	if err := cmd.writePIDFile(options.PIDFile); err != nil {
+		return fmt.Errorf("write pid file: %s", err)
+	}
+
+	// Parse config
+	config, err := cmd.ParseConfig(options.GetConfigPath())
+	if err != nil {
+		return fmt.Errorf("parse config: %s", err)
+	}
+
+	// Apply any environment variables on top of the parsed config
+	if err := config.ApplyEnvOverrides(); err != nil {
+		return fmt.Errorf("apply env config: %v", err)
+	}
+
+	// Validate the configuration.
+	if err := config.Validate(); err != nil {
+		return fmt.Errorf("%s. To generate a valid configuration file run `influxd config > influxdb.generated.conf`", err)
+	}
+
+	if config.HTTPD.PprofEnabled {
+		// Turn on block profiling to debug stuck databases
+		runtime.SetBlockProfileRate(int(1 * time.Second))
+	}
+
+	// Create server from config and start it.
+	buildInfo := &BuildInfo{
+		Version: cmd.Version,
+		Commit:  cmd.Commit,
+		Branch:  cmd.Branch,
+		Time:    cmd.BuildTime,
+	}
+	s, err := NewServer(config, buildInfo)
+	if err != nil {
+		return fmt.Errorf("create server: %s", err)
+	}
+	s.Logger = cmd.Logger
+	s.CPUProfile = options.CPUProfile
+	s.MemProfile = options.MemProfile
+	if err := s.Open(); err != nil {
+		return fmt.Errorf("open server: %s", err)
+	}
+	cmd.Server = s
+
+	// Begin monitoring the server's error channel.
+	go cmd.monitorServerErrors()
+
+	return nil
+}
+
+// Close shuts down the server.
+func (cmd *Command) Close() error {
+	defer close(cmd.Closed)
+	close(cmd.closing)
+	if cmd.Server != nil {
+		return cmd.Server.Close()
+	}
+	return nil
+}
+
+func (cmd *Command) monitorServerErrors() {
+	logger := log.New(cmd.Stderr, "", log.LstdFlags)
+	for {
+		select {
+		case err := <-cmd.Server.Err():
+			logger.Println(err)
+		case <-cmd.closing:
+			return
+		}
+	}
+}
+
+// ParseFlags parses the command line flags from args and returns an options set.
+func (cmd *Command) ParseFlags(args ...string) (Options, error) {
+	var options Options
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	fs.StringVar(&options.ConfigPath, "config", "", "")
+	fs.StringVar(&options.PIDFile, "pidfile", "", "")
+	// Ignore hostname option.
+	_ = fs.String("hostname", "", "")
+	fs.StringVar(&options.CPUProfile, "cpuprofile", "", "")
+	fs.StringVar(&options.MemProfile, "memprofile", "", "")
+	fs.Usage = func() { fmt.Fprintln(cmd.Stderr, usage) }
+	if err := fs.Parse(args); err != nil {
+		return Options{}, err
+	}
+	return options, nil
+}
+
+// writePIDFile writes the process ID to path.
+func (cmd *Command) writePIDFile(path string) error {
+	// Ignore if path is not set.
+	if path == "" {
+		return nil
+	}
+
+	// Ensure the required directory structure exists.
+	err := os.MkdirAll(filepath.Dir(path), 0777)
+	if err != nil {
+		return fmt.Errorf("mkdir: %s", err)
+	}
+
+	// Retrieve the PID and write it.
+	pid := strconv.Itoa(os.Getpid())
+	if err := ioutil.WriteFile(path, []byte(pid), 0666); err != nil {
+		return fmt.Errorf("write file: %s", err)
+	}
+
+	return nil
+}
+
+// ParseConfig parses the config at path.
+// It returns a demo configuration if path is blank.
+func (cmd *Command) ParseConfig(path string) (*Config, error) {
+	// Use demo configuration if no config path is specified.
+	if path == "" {
+		cmd.Logger.Info("no configuration provided, using default settings")
+		return NewDemoConfig()
+	}
+
+	cmd.Logger.Info(fmt.Sprintf("Using configuration at: %s", path))
+
+	config := NewConfig()
+	if err := config.FromTomlFile(path); err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}
+
+const usage = `Runs the InfluxDB server.
+
+Usage: influxd run [flags]
+
+    -config <path>
+            Set the path to the configuration file.
+            This defaults to the environment variable INFLUXDB_CONFIG_PATH,
+            ~/.influxdb/influxdb.conf, or /etc/influxdb/influxdb.conf if a file
+            is present at any of these locations.
+            Disable the automatic loading of a configuration file using
+            the null device (such as /dev/null).
+    -pidfile <path>
+            Write process ID to a file.
+    -cpuprofile <path>
+            Write CPU profiling information to a file.
+    -memprofile <path>
+            Write memory usage information to a file.
+`
+
+// Options represents the command line options that can be parsed.
+type Options struct {
+	ConfigPath string
+	PIDFile    string
+	CPUProfile string
+	MemProfile string
+}
+
+// GetConfigPath returns the config path from the options.
+// It will return a path by searching in this order:
+//   1. The CLI option in ConfigPath
+//   2. The environment variable INFLUXDB_CONFIG_PATH
+//   3. The first influxdb.conf file on the path:
+//        - ~/.influxdb
+//        - /etc/influxdb
+func (opt *Options) GetConfigPath() string {
+	if opt.ConfigPath != "" {
+		if opt.ConfigPath == os.DevNull {
+			return ""
+		}
+		return opt.ConfigPath
+	} else if envVar := os.Getenv("INFLUXDB_CONFIG_PATH"); envVar != "" {
+		return envVar
+	}
+
+	for _, path := range []string{
+		os.ExpandEnv("${HOME}/.influxdb/influxdb.conf"),
+		"/etc/influxdb/influxdb.conf",
+	} {
+		if _, err := os.Stat(path); err == nil {
+			return path
+		}
+	}
+	return ""
+}

vendor/github.com/influxdata/influxdb/cmd/influxd/run/config.go

@@ -0,0 +1,363 @@
+package run
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/user"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/influxdata/influxdb/coordinator"
+	"github.com/influxdata/influxdb/monitor"
+	"github.com/influxdata/influxdb/monitor/diagnostics"
+	"github.com/influxdata/influxdb/services/collectd"
+	"github.com/influxdata/influxdb/services/continuous_querier"
+	"github.com/influxdata/influxdb/services/graphite"
+	"github.com/influxdata/influxdb/services/httpd"
+	"github.com/influxdata/influxdb/services/meta"
+	"github.com/influxdata/influxdb/services/opentsdb"
+	"github.com/influxdata/influxdb/services/precreator"
+	"github.com/influxdata/influxdb/services/retention"
+	"github.com/influxdata/influxdb/services/subscriber"
+	"github.com/influxdata/influxdb/services/udp"
+	"github.com/influxdata/influxdb/tsdb"
+)
+
+const (
+	// DefaultBindAddress is the default address for various RPC services.
+	DefaultBindAddress = "127.0.0.1:8088"
+)
+
+// Config represents the configuration format for the influxd binary.
+type Config struct {
+	Meta        *meta.Config       `toml:"meta"`
+	Data        tsdb.Config        `toml:"data"`
+	Coordinator coordinator.Config `toml:"coordinator"`
+	Retention   retention.Config   `toml:"retention"`
+	Precreator  precreator.Config  `toml:"shard-precreation"`
+
+	Monitor        monitor.Config    `toml:"monitor"`
+	Subscriber     subscriber.Config `toml:"subscriber"`
+	HTTPD          httpd.Config      `toml:"http"`
+	GraphiteInputs []graphite.Config `toml:"graphite"`
+	CollectdInputs []collectd.Config `toml:"collectd"`
+	OpenTSDBInputs []opentsdb.Config `toml:"opentsdb"`
+	UDPInputs      []udp.Config      `toml:"udp"`
+
+	ContinuousQuery continuous_querier.Config `toml:"continuous_queries"`
+
+	// Server reporting
+	ReportingDisabled bool `toml:"reporting-disabled"`
+
+	// BindAddress is the address that all TCP services use (Raft, Snapshot, Cluster, etc.)
+	BindAddress string `toml:"bind-address"`
+}
+
+// NewConfig returns an instance of Config with reasonable defaults.
+func NewConfig() *Config {
+	c := &Config{}
+	c.Meta = meta.NewConfig()
+	c.Data = tsdb.NewConfig()
+	c.Coordinator = coordinator.NewConfig()
+	c.Precreator = precreator.NewConfig()
+
+	c.Monitor = monitor.NewConfig()
+	c.Subscriber = subscriber.NewConfig()
+	c.HTTPD = httpd.NewConfig()
+
+	c.GraphiteInputs = []graphite.Config{graphite.NewConfig()}
+	c.CollectdInputs = []collectd.Config{collectd.NewConfig()}
+	c.OpenTSDBInputs = []opentsdb.Config{opentsdb.NewConfig()}
+	c.UDPInputs = []udp.Config{udp.NewConfig()}
+
+	c.ContinuousQuery = continuous_querier.NewConfig()
+	c.Retention = retention.NewConfig()
+	c.BindAddress = DefaultBindAddress
+
+	return c
+}
+
+// NewDemoConfig returns the config that runs when no config is specified.
+func NewDemoConfig() (*Config, error) {
+	c := NewConfig()
+
+	var homeDir string
+	// By default, store meta and data files in current users home directory
+	u, err := user.Current()
+	if err == nil {
+		homeDir = u.HomeDir
+	} else if os.Getenv("HOME") != "" {
+		homeDir = os.Getenv("HOME")
+	} else {
+		return nil, fmt.Errorf("failed to determine current user for storage")
+	}
+
+	c.Meta.Dir = filepath.Join(homeDir, ".influxdb/meta")
+	c.Data.Dir = filepath.Join(homeDir, ".influxdb/data")
+	c.Data.WALDir = filepath.Join(homeDir, ".influxdb/wal")
+
+	return c, nil
+}
+
+// trimBOM trims the Byte-Order-Marks from the beginning of the file.
+// This is for Windows compatability only.
+// See https://github.com/influxdata/telegraf/issues/1378.
+func trimBOM(f []byte) []byte {
+	return bytes.TrimPrefix(f, []byte("\xef\xbb\xbf"))
+}
+
+// FromTomlFile loads the config from a TOML file.
+func (c *Config) FromTomlFile(fpath string) error {
+	bs, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return err
+	}
+	bs = trimBOM(bs)
+	return c.FromToml(string(bs))
+}
+
+// FromToml loads the config from TOML.
+func (c *Config) FromToml(input string) error {
+	// Replace deprecated [cluster] with [coordinator]
+	re := regexp.MustCompile(`(?m)^\s*\[cluster\]`)
+	input = re.ReplaceAllStringFunc(input, func(in string) string {
+		in = strings.TrimSpace(in)
+		out := "[coordinator]"
+		log.Printf("deprecated config option %s replaced with %s; %s will not be supported in a future release\n", in, out, in)
+		return out
+	})
+
+	_, err := toml.Decode(input, c)
+	return err
+}
+
+// Validate returns an error if the config is invalid.
+func (c *Config) Validate() error {
+	if err := c.Meta.Validate(); err != nil {
+		return err
+	}
+
+	if err := c.Data.Validate(); err != nil {
+		return err
+	}
+
+	if err := c.Monitor.Validate(); err != nil {
+		return err
+	}
+
+	if err := c.ContinuousQuery.Validate(); err != nil {
+		return err
+	}
+
+	if err := c.Retention.Validate(); err != nil {
+		return err
+	}
+
+	if err := c.Precreator.Validate(); err != nil {
+		return err
+	}
+
+	if err := c.Subscriber.Validate(); err != nil {
+		return err
+	}
+
+	for _, graphite := range c.GraphiteInputs {
+		if err := graphite.Validate(); err != nil {
+			return fmt.Errorf("invalid graphite config: %v", err)
+		}
+	}
+
+	for _, collectd := range c.CollectdInputs {
+		if err := collectd.Validate(); err != nil {
+			return fmt.Errorf("invalid collectd config: %v", err)
+		}
+	}
+
+	return nil
+}
+
+// ApplyEnvOverrides apply the environment configuration on top of the config.
+func (c *Config) ApplyEnvOverrides() error {
+	return c.applyEnvOverrides("INFLUXDB", reflect.ValueOf(c), "")
+}
+
+func (c *Config) applyEnvOverrides(prefix string, spec reflect.Value, structKey string) error {
+	// If we have a pointer, dereference it
+	element := spec
+	if spec.Kind() == reflect.Ptr {
+		element = spec.Elem()
+	}
+
+	value := os.Getenv(prefix)
+
+	switch element.Kind() {
+	case reflect.String:
+		if len(value) == 0 {
+			return nil
+		}
+		element.SetString(value)
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		var intValue int64
+
+		// Handle toml.Duration
+		if element.Type().Name() == "Duration" {
+			dur, err := time.ParseDuration(value)
+			if err != nil {
+				return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", prefix, structKey, element.Type().String(), value)
+			}
+			intValue = dur.Nanoseconds()
+		} else {
+			var err error
+			intValue, err = strconv.ParseInt(value, 0, element.Type().Bits())
+			if err != nil {
+				return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", prefix, structKey, element.Type().String(), value)
+			}
+		}
+		element.SetInt(intValue)
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		intValue, err := strconv.ParseUint(value, 0, element.Type().Bits())
+		if err != nil {
+			return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", prefix, structKey, element.Type().String(), value)
+		}
+		element.SetUint(intValue)
+	case reflect.Bool:
+		boolValue, err := strconv.ParseBool(value)
+		if err != nil {
+			return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", prefix, structKey, element.Type().String(), value)
+		}
+		element.SetBool(boolValue)
+	case reflect.Float32, reflect.Float64:
+		floatValue, err := strconv.ParseFloat(value, element.Type().Bits())
+		if err != nil {
+			return fmt.Errorf("failed to apply %v to %v using type %v and value '%v'", prefix, structKey, element.Type().String(), value)
+		}
+		element.SetFloat(floatValue)
+	case reflect.Slice:
+		// If the type is s slice, apply to each using the index as a suffix, e.g. GRAPHITE_0, GRAPHITE_0_TEMPLATES_0 or GRAPHITE_0_TEMPLATES="item1,item2"
+		for j := 0; j < element.Len(); j++ {
+			f := element.Index(j)
+			if err := c.applyEnvOverrides(prefix, f, structKey); err != nil {
+				return err
+			}
+
+			if err := c.applyEnvOverrides(fmt.Sprintf("%s_%d", prefix, j), f, structKey); err != nil {
+				return err
+			}
+		}
+
+		// If the type is s slice but have value not parsed as slice e.g. GRAPHITE_0_TEMPLATES="item1,item2"
+		if element.Len() == 0 && len(value) > 0 {
+			rules := strings.Split(value, ",")
+
+			for _, rule := range rules {
+				element.Set(reflect.Append(element, reflect.ValueOf(rule)))
+			}
+		}
+	case reflect.Struct:
+		typeOfSpec := element.Type()
+		for i := 0; i < element.NumField(); i++ {
+			field := element.Field(i)
+
+			// Skip any fields that we cannot set
+			if !field.CanSet() && field.Kind() != reflect.Slice {
+				continue
+			}
+
+			fieldName := typeOfSpec.Field(i).Name
+
+			configName := typeOfSpec.Field(i).Tag.Get("toml")
+			// Replace hyphens with underscores to avoid issues with shells
+			configName = strings.Replace(configName, "-", "_", -1)
+
+			envKey := strings.ToUpper(configName)
+			if prefix != "" {
+				envKey = strings.ToUpper(fmt.Sprintf("%s_%s", prefix, configName))
+			}
+
+			// If it's a sub-config, recursively apply
+			if field.Kind() == reflect.Struct || field.Kind() == reflect.Ptr ||
+				field.Kind() == reflect.Slice || field.Kind() == reflect.Array {
+				if err := c.applyEnvOverrides(envKey, field, fieldName); err != nil {
+					return err
+				}
+				continue
+			}
+
+			value := os.Getenv(envKey)
+			// Skip any fields we don't have a value to set
+			if len(value) == 0 {
+				continue
+			}
+
+			if err := c.applyEnvOverrides(envKey, field, fieldName); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// Diagnostics returns a diagnostics representation of Config.
+func (c *Config) Diagnostics() (*diagnostics.Diagnostics, error) {
+	return diagnostics.RowFromMap(map[string]interface{}{
+		"reporting-disabled": c.ReportingDisabled,
+		"bind-address":       c.BindAddress,
+	}), nil
+}
+
+func (c *Config) diagnosticsClients() map[string]diagnostics.Client {
+	// Config settings that are always present.
+	m := map[string]diagnostics.Client{
+		"config": c,
+
+		"config-data":        c.Data,
+		"config-meta":        c.Meta,
+		"config-coordinator": c.Coordinator,
+		"config-retention":   c.Retention,
+		"config-precreator":  c.Precreator,
+
+		"config-monitor":    c.Monitor,
+		"config-subscriber": c.Subscriber,
+		"config-httpd":      c.HTTPD,
+
+		"config-cqs": c.ContinuousQuery,
+	}
+
+	// Config settings that can be repeated and can be disabled.
+	if g := graphite.Configs(c.GraphiteInputs); g.Enabled() {
+		m["config-graphite"] = g
+	}
+	if cc := collectd.Configs(c.CollectdInputs); cc.Enabled() {
+		m["config-collectd"] = cc
+	}
+	if t := opentsdb.Configs(c.OpenTSDBInputs); t.Enabled() {
+		m["config-opentsdb"] = t
+	}
+	if u := udp.Configs(c.UDPInputs); u.Enabled() {
+		m["config-udp"] = u
+	}
+
+	return m
+}
+
+// registerDiagnostics registers the config settings with the Monitor.
+func (c *Config) registerDiagnostics(m *monitor.Monitor) {
+	for name, dc := range c.diagnosticsClients() {
+		m.RegisterDiagnosticsClient(name, dc)
+	}
+}
+
+// registerDiagnostics deregisters the config settings from the Monitor.
+func (c *Config) deregisterDiagnostics(m *monitor.Monitor) {
+	for name := range c.diagnosticsClients() {
+		m.DeregisterDiagnosticsClient(name)
+	}
+}

vendor/github.com/influxdata/influxdb/cmd/influxd/run/config_command.go

@@ -0,0 +1,92 @@
+package run
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/BurntSushi/toml"
+)
+
+// PrintConfigCommand represents the command executed by "influxd config".
+type PrintConfigCommand struct {
+	Stdin  io.Reader
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// NewPrintConfigCommand return a new instance of PrintConfigCommand.
+func NewPrintConfigCommand() *PrintConfigCommand {
+	return &PrintConfigCommand{
+		Stdin:  os.Stdin,
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+}
+
+// Run parses and prints the current config loaded.
+func (cmd *PrintConfigCommand) Run(args ...string) error {
+	// Parse command flags.
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	configPath := fs.String("config", "", "")
+	fs.Usage = func() { fmt.Fprintln(cmd.Stderr, printConfigUsage) }
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	// Parse config from path.
+	opt := Options{ConfigPath: *configPath}
+	config, err := cmd.parseConfig(opt.GetConfigPath())
+	if err != nil {
+		return fmt.Errorf("parse config: %s", err)
+	}
+
+	// Apply any environment variables on top of the parsed config
+	if err := config.ApplyEnvOverrides(); err != nil {
+		return fmt.Errorf("apply env config: %v", err)
+	}
+
+	// Validate the configuration.
+	if err := config.Validate(); err != nil {
+		return fmt.Errorf("%s. To generate a valid configuration file run `influxd config > influxdb.generated.conf`", err)
+	}
+
+	toml.NewEncoder(cmd.Stdout).Encode(config)
+	fmt.Fprint(cmd.Stdout, "\n")
+
+	return nil
+}
+
+// ParseConfig parses the config at path.
+// Returns a demo configuration if path is blank.
+func (cmd *PrintConfigCommand) parseConfig(path string) (*Config, error) {
+	config, err := NewDemoConfig()
+	if err != nil {
+		config = NewConfig()
+	}
+
+	if path == "" {
+		return config, nil
+	}
+
+	fmt.Fprintf(os.Stderr, "Merging with configuration at: %s\n", path)
+
+	if err := config.FromTomlFile(path); err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
+var printConfigUsage = `Displays the default configuration.
+
+Usage: influxd config [flags]
+
+    -config <path>
+            Set the path to the initial configuration file.
+            This defaults to the environment variable INFLUXDB_CONFIG_PATH,
+            ~/.influxdb/influxdb.conf, or /etc/influxdb/influxdb.conf if a file
+            is present at any of these locations.
+            Disable the automatic loading of a configuration file using
+            the null device (such as /dev/null).
+`

vendor/github.com/influxdata/influxdb/cmd/influxd/run/config_test.go

@@ -0,0 +1,312 @@
+package run_test
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/BurntSushi/toml"
+	"github.com/influxdata/influxdb/cmd/influxd/run"
+)
+
+// Ensure the configuration can be parsed.
+func TestConfig_Parse(t *testing.T) {
+	// Parse configuration.
+	var c run.Config
+	if err := c.FromToml(`
+[meta]
+dir = "/tmp/meta"
+
+[data]
+dir = "/tmp/data"
+
+[coordinator]
+
+[http]
+bind-address = ":8087"
+
+[[graphite]]
+protocol = "udp"
+
+[[graphite]]
+protocol = "tcp"
+
+[[collectd]]
+bind-address = ":1000"
+
+[[collectd]]
+bind-address = ":1010"
+
+[[opentsdb]]
+bind-address = ":2000"
+
+[[opentsdb]]
+bind-address = ":2010"
+
+[[opentsdb]]
+bind-address = ":2020"
+
+[[udp]]
+bind-address = ":4444"
+
+[monitoring]
+enabled = true
+
+[subscriber]
+enabled = true
+
+[continuous_queries]
+enabled = true
+`); err != nil {
+		t.Fatal(err)
+	}
+
+	// Validate configuration.
+	if c.Meta.Dir != "/tmp/meta" {
+		t.Fatalf("unexpected meta dir: %s", c.Meta.Dir)
+	} else if c.Data.Dir != "/tmp/data" {
+		t.Fatalf("unexpected data dir: %s", c.Data.Dir)
+	} else if c.HTTPD.BindAddress != ":8087" {
+		t.Fatalf("unexpected api bind address: %s", c.HTTPD.BindAddress)
+	} else if len(c.GraphiteInputs) != 2 {
+		t.Fatalf("unexpected graphiteInputs count: %d", len(c.GraphiteInputs))
+	} else if c.GraphiteInputs[0].Protocol != "udp" {
+		t.Fatalf("unexpected graphite protocol(0): %s", c.GraphiteInputs[0].Protocol)
+	} else if c.GraphiteInputs[1].Protocol != "tcp" {
+		t.Fatalf("unexpected graphite protocol(1): %s", c.GraphiteInputs[1].Protocol)
+	} else if c.CollectdInputs[0].BindAddress != ":1000" {
+		t.Fatalf("unexpected collectd bind address: %s", c.CollectdInputs[0].BindAddress)
+	} else if c.CollectdInputs[1].BindAddress != ":1010" {
+		t.Fatalf("unexpected collectd bind address: %s", c.CollectdInputs[1].BindAddress)
+	} else if c.OpenTSDBInputs[0].BindAddress != ":2000" {
+		t.Fatalf("unexpected opentsdb bind address: %s", c.OpenTSDBInputs[0].BindAddress)
+	} else if c.OpenTSDBInputs[1].BindAddress != ":2010" {
+		t.Fatalf("unexpected opentsdb bind address: %s", c.OpenTSDBInputs[1].BindAddress)
+	} else if c.OpenTSDBInputs[2].BindAddress != ":2020" {
+		t.Fatalf("unexpected opentsdb bind address: %s", c.OpenTSDBInputs[2].BindAddress)
+	} else if c.UDPInputs[0].BindAddress != ":4444" {
+		t.Fatalf("unexpected udp bind address: %s", c.UDPInputs[0].BindAddress)
+	} else if c.Subscriber.Enabled != true {
+		t.Fatalf("unexpected subscriber enabled: %v", c.Subscriber.Enabled)
+	} else if c.ContinuousQuery.Enabled != true {
+		t.Fatalf("unexpected continuous query enabled: %v", c.ContinuousQuery.Enabled)
+	}
+}
+
+// Ensure the configuration can be parsed.
+func TestConfig_Parse_EnvOverride(t *testing.T) {
+	// Parse configuration.
+	var c run.Config
+	if _, err := toml.Decode(`
+[meta]
+dir = "/tmp/meta"
+
+[data]
+dir = "/tmp/data"
+
+[coordinator]
+
+[admin]
+bind-address = ":8083"
+
+[http]
+bind-address = ":8087"
+
+[[graphite]]
+protocol = "udp"
+templates = [
+  "default.* .template.in.config"
+]
+
+[[graphite]]
+protocol = "tcp"
+
+[[collectd]]
+bind-address = ":1000"
+
+[[collectd]]
+bind-address = ":1010"
+
+[[opentsdb]]
+bind-address = ":2000"
+
+[[opentsdb]]
+bind-address = ":2010"
+
+[[udp]]
+bind-address = ":4444"
+
+[[udp]]
+
+[monitoring]
+enabled = true
+
+[continuous_queries]
+enabled = true
+`, &c); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := os.Setenv("INFLUXDB_UDP_BIND_ADDRESS", ":1234"); err != nil {
+		t.Fatalf("failed to set env var: %v", err)
+	}
+
+	if err := os.Setenv("INFLUXDB_UDP_0_BIND_ADDRESS", ":5555"); err != nil {
+		t.Fatalf("failed to set env var: %v", err)
+	}
+
+	if err := os.Setenv("INFLUXDB_GRAPHITE_0_TEMPLATES_0", "overide.* .template.0"); err != nil {
+		t.Fatalf("failed to set env var: %v", err)
+	}
+
+	if err := os.Setenv("INFLUXDB_GRAPHITE_1_TEMPLATES", "overide.* .template.1.1,overide.* .template.1.2"); err != nil {
+		t.Fatalf("failed to set env var: %v", err)
+	}
+
+	if err := os.Setenv("INFLUXDB_GRAPHITE_1_PROTOCOL", "udp"); err != nil {
+		t.Fatalf("failed to set env var: %v", err)
+	}
+
+	if err := os.Setenv("INFLUXDB_COLLECTD_1_BIND_ADDRESS", ":1020"); err != nil {
+		t.Fatalf("failed to set env var: %v", err)
+	}
+
+	if err := os.Setenv("INFLUXDB_OPENTSDB_0_BIND_ADDRESS", ":2020"); err != nil {
+		t.Fatalf("failed to set env var: %v", err)
+	}
+
+	// uint64 type
+	if err := os.Setenv("INFLUXDB_DATA_CACHE_MAX_MEMORY_SIZE", "1000"); err != nil {
+		t.Fatalf("failed to set env var: %v", err)
+	}
+
+	if err := c.ApplyEnvOverrides(); err != nil {
+		t.Fatalf("failed to apply env overrides: %v", err)
+	}
+
+	if c.UDPInputs[0].BindAddress != ":5555" {
+		t.Fatalf("unexpected udp bind address: %s", c.UDPInputs[0].BindAddress)
+	}
+
+	if c.UDPInputs[1].BindAddress != ":1234" {
+		t.Fatalf("unexpected udp bind address: %s", c.UDPInputs[1].BindAddress)
+	}
+
+	if len(c.GraphiteInputs[0].Templates) != 1 || c.GraphiteInputs[0].Templates[0] != "overide.* .template.0" {
+		t.Fatalf("unexpected graphite 0 templates: %+v", c.GraphiteInputs[0].Templates)
+	}
+
+	if len(c.GraphiteInputs[1].Templates) != 2 || c.GraphiteInputs[1].Templates[1] != "overide.* .template.1.2" {
+		t.Fatalf("unexpected graphite 1 templates: %+v", c.GraphiteInputs[1].Templates)
+	}
+
+	if c.GraphiteInputs[1].Protocol != "udp" {
+		t.Fatalf("unexpected graphite protocol: %s", c.GraphiteInputs[1].Protocol)
+	}
+
+	if c.CollectdInputs[1].BindAddress != ":1020" {
+		t.Fatalf("unexpected collectd bind address: %s", c.CollectdInputs[1].BindAddress)
+	}
+
+	if c.OpenTSDBInputs[0].BindAddress != ":2020" {
+		t.Fatalf("unexpected opentsdb bind address: %s", c.OpenTSDBInputs[0].BindAddress)
+	}
+
+	if c.Data.CacheMaxMemorySize != 1000 {
+		t.Fatalf("unexpected cache max memory size: %v", c.Data.CacheMaxMemorySize)
+	}
+}
+
+func TestConfig_ValidateNoServiceConfigured(t *testing.T) {
+	var c run.Config
+	if _, err := toml.Decode(`
+[meta]
+enabled = false
+
+[data]
+enabled = false
+`, &c); err != nil {
+		t.Fatal(err)
+	}
+
+	if e := c.Validate(); e == nil {
+		t.Fatalf("got nil, expected error")
+	}
+}
+
+func TestConfig_ValidateMonitorStore_MetaOnly(t *testing.T) {
+	c := run.NewConfig()
+	if _, err := toml.Decode(`
+[monitor]
+store-enabled = true
+
+[meta]
+dir = "foo"
+
+[data]
+enabled = false
+`, &c); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := c.Validate(); err == nil {
+		t.Fatalf("got nil, expected error")
+	}
+}
+
+func TestConfig_DeprecatedOptions(t *testing.T) {
+	// Parse configuration.
+	var c run.Config
+	if err := c.FromToml(`
+[cluster]
+max-select-point = 100
+`); err != nil {
+		t.Fatal(err)
+	}
+
+	// Validate configuration.
+	if c.Coordinator.MaxSelectPointN != 100 {
+		t.Fatalf("unexpected coordinator max select points: %d", c.Coordinator.MaxSelectPointN)
+
+	}
+}
+
+// Ensure that Config.Validate correctly validates the individual subsections.
+func TestConfig_InvalidSubsections(t *testing.T) {
+	// Precondition: NewDemoConfig must validate correctly.
+	c, err := run.NewDemoConfig()
+	if err != nil {
+		t.Fatalf("error creating demo config: %s", err)
+	}
+	if err := c.Validate(); err != nil {
+		t.Fatalf("new demo config failed validation: %s", err)
+	}
+
+	// For each subsection, load a config with a single invalid setting.
+	for _, tc := range []struct {
+		section string
+		kv      string
+	}{
+		{"meta", `dir = ""`},
+		{"data", `dir = ""`},
+		{"monitor", `store-database = ""`},
+		{"continuous_queries", `run-interval = "0s"`},
+		{"subscriber", `http-timeout = "0s"`},
+		{"retention", `check-interval = "0s"`},
+		{"shard-precreation", `advance-period = "0s"`},
+	} {
+		c, err := run.NewDemoConfig()
+		if err != nil {
+			t.Fatalf("error creating demo config: %s", err)
+		}
+
+		s := fmt.Sprintf("\n[%s]\n%s\n", tc.section, tc.kv)
+		if err := c.FromToml(s); err != nil {
+			t.Fatalf("error loading toml %q: %s", s, err)
+		}
+
+		if err := c.Validate(); err == nil {
+			t.Fatalf("expected error but got nil for config: %s", s)
+		}
+	}
+}

vendor/github.com/influxdata/influxdb/cmd/influxd/run/server.go

@@ -0,0 +1,616 @@
+package run
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"time"
+
+	"github.com/influxdata/influxdb"
+	"github.com/influxdata/influxdb/coordinator"
+	"github.com/influxdata/influxdb/influxql"
+	"github.com/influxdata/influxdb/models"
+	"github.com/influxdata/influxdb/monitor"
+	"github.com/influxdata/influxdb/services/collectd"
+	"github.com/influxdata/influxdb/services/continuous_querier"
+	"github.com/influxdata/influxdb/services/graphite"
+	"github.com/influxdata/influxdb/services/httpd"
+	"github.com/influxdata/influxdb/services/meta"
+	"github.com/influxdata/influxdb/services/opentsdb"
+	"github.com/influxdata/influxdb/services/precreator"
+	"github.com/influxdata/influxdb/services/retention"
+	"github.com/influxdata/influxdb/services/snapshotter"
+	"github.com/influxdata/influxdb/services/subscriber"
+	"github.com/influxdata/influxdb/services/udp"
+	"github.com/influxdata/influxdb/tcp"
+	"github.com/influxdata/influxdb/tsdb"
+	client "github.com/influxdata/usage-client/v1"
+	"github.com/uber-go/zap"
+
+	// Initialize the engine & index packages
+	_ "github.com/influxdata/influxdb/tsdb/engine"
+	_ "github.com/influxdata/influxdb/tsdb/index"
+)
+
+var startTime time.Time
+
+func init() {
+	startTime = time.Now().UTC()
+}
+
+// BuildInfo represents the build details for the server code.
+type BuildInfo struct {
+	Version string
+	Commit  string
+	Branch  string
+	Time    string
+}
+
+// Server represents a container for the metadata and storage data and services.
+// It is built using a Config and it manages the startup and shutdown of all
+// services in the proper order.
+type Server struct {
+	buildInfo BuildInfo
+
+	err     chan error
+	closing chan struct{}
+
+	BindAddress string
+	Listener    net.Listener
+
+	Logger zap.Logger
+
+	MetaClient *meta.Client
+
+	TSDBStore     *tsdb.Store
+	QueryExecutor *influxql.QueryExecutor
+	PointsWriter  *coordinator.PointsWriter
+	Subscriber    *subscriber.Service
+
+	Services []Service
+
+	// These references are required for the tcp muxer.
+	SnapshotterService *snapshotter.Service
+
+	Monitor *monitor.Monitor
+
+	// Server reporting and registration
+	reportingDisabled bool
+
+	// Profiling
+	CPUProfile string
+	MemProfile string
+
+	// httpAPIAddr is the host:port combination for the main HTTP API for querying and writing data
+	httpAPIAddr string
+
+	// httpUseTLS specifies if we should use a TLS connection to the http servers
+	httpUseTLS bool
+
+	// tcpAddr is the host:port combination for the TCP listener that services mux onto
+	tcpAddr string
+
+	config *Config
+}
+
+// NewServer returns a new instance of Server built from a config.
+func NewServer(c *Config, buildInfo *BuildInfo) (*Server, error) {
+	// We need to ensure that a meta directory always exists even if
+	// we don't start the meta store.  node.json is always stored under
+	// the meta directory.
+	if err := os.MkdirAll(c.Meta.Dir, 0777); err != nil {
+		return nil, fmt.Errorf("mkdir all: %s", err)
+	}
+
+	// 0.10-rc1 and prior would sometimes put the node.json at the root
+	// dir which breaks backup/restore and restarting nodes.  This moves
+	// the file from the root so it's always under the meta dir.
+	oldPath := filepath.Join(filepath.Dir(c.Meta.Dir), "node.json")
+	newPath := filepath.Join(c.Meta.Dir, "node.json")
+
+	if _, err := os.Stat(oldPath); err == nil {
+		if err := os.Rename(oldPath, newPath); err != nil {
+			return nil, err
+		}
+	}
+
+	_, err := influxdb.LoadNode(c.Meta.Dir)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+	}
+
+	if err := raftDBExists(c.Meta.Dir); err != nil {
+		return nil, err
+	}
+
+	// In 0.10.0 bind-address got moved to the top level. Check
+	// The old location to keep things backwards compatible
+	bind := c.BindAddress
+
+	s := &Server{
+		buildInfo: *buildInfo,
+		err:       make(chan error),
+		closing:   make(chan struct{}),
+
+		BindAddress: bind,
+
+		Logger: zap.New(
+			zap.NewTextEncoder(),
+			zap.Output(os.Stderr),
+		),
+
+		MetaClient: meta.NewClient(c.Meta),
+
+		reportingDisabled: c.ReportingDisabled,
+
+		httpAPIAddr: c.HTTPD.BindAddress,
+		httpUseTLS:  c.HTTPD.HTTPSEnabled,
+		tcpAddr:     bind,
+
+		config: c,
+	}
+	s.Monitor = monitor.New(s, c.Monitor)
+	s.config.registerDiagnostics(s.Monitor)
+
+	if err := s.MetaClient.Open(); err != nil {
+		return nil, err
+	}
+
+	s.TSDBStore = tsdb.NewStore(c.Data.Dir)
+	s.TSDBStore.EngineOptions.Config = c.Data
+
+	// Copy TSDB configuration.
+	s.TSDBStore.EngineOptions.EngineVersion = c.Data.Engine
+	s.TSDBStore.EngineOptions.IndexVersion = c.Data.Index
+
+	// Create the Subscriber service
+	s.Subscriber = subscriber.NewService(c.Subscriber)
+
+	// Initialize points writer.
+	s.PointsWriter = coordinator.NewPointsWriter()
+	s.PointsWriter.WriteTimeout = time.Duration(c.Coordinator.WriteTimeout)
+	s.PointsWriter.TSDBStore = s.TSDBStore
+	s.PointsWriter.Subscriber = s.Subscriber
+
+	// Initialize query executor.
+	s.QueryExecutor = influxql.NewQueryExecutor()
+	s.QueryExecutor.StatementExecutor = &coordinator.StatementExecutor{
+		MetaClient:  s.MetaClient,
+		TaskManager: s.QueryExecutor.TaskManager,
+		TSDBStore:   coordinator.LocalTSDBStore{Store: s.TSDBStore},
+		ShardMapper: &coordinator.LocalShardMapper{
+			MetaClient: s.MetaClient,
+			TSDBStore:  coordinator.LocalTSDBStore{Store: s.TSDBStore},
+		},
+		Monitor:           s.Monitor,
+		PointsWriter:      s.PointsWriter,
+		MaxSelectPointN:   c.Coordinator.MaxSelectPointN,
+		MaxSelectSeriesN:  c.Coordinator.MaxSelectSeriesN,
+		MaxSelectBucketsN: c.Coordinator.MaxSelectBucketsN,
+	}
+	s.QueryExecutor.TaskManager.QueryTimeout = time.Duration(c.Coordinator.QueryTimeout)
+	s.QueryExecutor.TaskManager.LogQueriesAfter = time.Duration(c.Coordinator.LogQueriesAfter)
+	s.QueryExecutor.TaskManager.MaxConcurrentQueries = c.Coordinator.MaxConcurrentQueries
+
+	// Initialize the monitor
+	s.Monitor.Version = s.buildInfo.Version
+	s.Monitor.Commit = s.buildInfo.Commit
+	s.Monitor.Branch = s.buildInfo.Branch
+	s.Monitor.BuildTime = s.buildInfo.Time
+	s.Monitor.PointsWriter = (*monitorPointsWriter)(s.PointsWriter)
+	return s, nil
+}
+
+// Statistics returns statistics for the services running in the Server.
+func (s *Server) Statistics(tags map[string]string) []models.Statistic {
+	var statistics []models.Statistic
+	statistics = append(statistics, s.QueryExecutor.Statistics(tags)...)
+	statistics = append(statistics, s.TSDBStore.Statistics(tags)...)
+	statistics = append(statistics, s.PointsWriter.Statistics(tags)...)
+	statistics = append(statistics, s.Subscriber.Statistics(tags)...)
+	for _, srv := range s.Services {
+		if m, ok := srv.(monitor.Reporter); ok {
+			statistics = append(statistics, m.Statistics(tags)...)
+		}
+	}
+	return statistics
+}
+
+func (s *Server) appendSnapshotterService() {
+	srv := snapshotter.NewService()
+	srv.TSDBStore = s.TSDBStore
+	srv.MetaClient = s.MetaClient
+	s.Services = append(s.Services, srv)
+	s.SnapshotterService = srv
+}
+
+// SetLogOutput sets the logger used for all messages. It must not be called
+// after the Open method has been called.
+func (s *Server) SetLogOutput(w io.Writer) {
+	s.Logger = zap.New(zap.NewTextEncoder(), zap.Output(zap.AddSync(w)))
+}
+
+func (s *Server) appendMonitorService() {
+	s.Services = append(s.Services, s.Monitor)
+}
+
+func (s *Server) appendRetentionPolicyService(c retention.Config) {
+	if !c.Enabled {
+		return
+	}
+	srv := retention.NewService(c)
+	srv.MetaClient = s.MetaClient
+	srv.TSDBStore = s.TSDBStore
+	s.Services = append(s.Services, srv)
+}
+
+func (s *Server) appendHTTPDService(c httpd.Config) {
+	if !c.Enabled {
+		return
+	}
+	srv := httpd.NewService(c)
+	srv.Handler.MetaClient = s.MetaClient
+	srv.Handler.QueryAuthorizer = meta.NewQueryAuthorizer(s.MetaClient)
+	srv.Handler.WriteAuthorizer = meta.NewWriteAuthorizer(s.MetaClient)
+	srv.Handler.QueryExecutor = s.QueryExecutor
+	srv.Handler.Monitor = s.Monitor
+	srv.Handler.PointsWriter = s.PointsWriter
+	srv.Handler.Version = s.buildInfo.Version
+
+	s.Services = append(s.Services, srv)
+}
+
+func (s *Server) appendCollectdService(c collectd.Config) {
+	if !c.Enabled {
+		return
+	}
+	srv := collectd.NewService(c)
+	srv.MetaClient = s.MetaClient
+	srv.PointsWriter = s.PointsWriter
+	s.Services = append(s.Services, srv)
+}
+
+func (s *Server) appendOpenTSDBService(c opentsdb.Config) error {
+	if !c.Enabled {
+		return nil
+	}
+	srv, err := opentsdb.NewService(c)
+	if err != nil {
+		return err
+	}
+	srv.PointsWriter = s.PointsWriter
+	srv.MetaClient = s.MetaClient
+	s.Services = append(s.Services, srv)
+	return nil
+}
+
+func (s *Server) appendGraphiteService(c graphite.Config) error {
+	if !c.Enabled {
+		return nil
+	}
+	srv, err := graphite.NewService(c)
+	if err != nil {
+		return err
+	}
+
+	srv.PointsWriter = s.PointsWriter
+	srv.MetaClient = s.MetaClient
+	srv.Monitor = s.Monitor
+	s.Services = append(s.Services, srv)
+	return nil
+}
+
+func (s *Server) appendPrecreatorService(c precreator.Config) error {
+	if !c.Enabled {
+		return nil
+	}
+	srv, err := precreator.NewService(c)
+	if err != nil {
+		return err
+	}
+
+	srv.MetaClient = s.MetaClient
+	s.Services = append(s.Services, srv)
+	return nil
+}
+
+func (s *Server) appendUDPService(c udp.Config) {
+	if !c.Enabled {
+		return
+	}
+	srv := udp.NewService(c)
+	srv.PointsWriter = s.PointsWriter
+	srv.MetaClient = s.MetaClient
+	s.Services = append(s.Services, srv)
+}
+
+func (s *Server) appendContinuousQueryService(c continuous_querier.Config) {
+	if !c.Enabled {
+		return
+	}
+	srv := continuous_querier.NewService(c)
+	srv.MetaClient = s.MetaClient
+	srv.QueryExecutor = s.QueryExecutor
+	s.Services = append(s.Services, srv)
+}
+
+// Err returns an error channel that multiplexes all out of band errors received from all services.
+func (s *Server) Err() <-chan error { return s.err }
+
+// Open opens the meta and data store and all services.
+func (s *Server) Open() error {
+	// Start profiling, if set.
+	startProfile(s.CPUProfile, s.MemProfile)
+
+	// Open shared TCP connection.
+	ln, err := net.Listen("tcp", s.BindAddress)
+	if err != nil {
+		return fmt.Errorf("listen: %s", err)
+	}
+	s.Listener = ln
+
+	// Multiplex listener.
+	mux := tcp.NewMux()
+	go mux.Serve(ln)
+
+	// Append services.
+	s.appendMonitorService()
+	s.appendPrecreatorService(s.config.Precreator)
+	s.appendSnapshotterService()
+	s.appendContinuousQueryService(s.config.ContinuousQuery)
+	s.appendHTTPDService(s.config.HTTPD)
+	s.appendRetentionPolicyService(s.config.Retention)
+	for _, i := range s.config.GraphiteInputs {
+		if err := s.appendGraphiteService(i); err != nil {
+			return err
+		}
+	}
+	for _, i := range s.config.CollectdInputs {
+		s.appendCollectdService(i)
+	}
+	for _, i := range s.config.OpenTSDBInputs {
+		if err := s.appendOpenTSDBService(i); err != nil {
+			return err
+		}
+	}
+	for _, i := range s.config.UDPInputs {
+		s.appendUDPService(i)
+	}
+
+	s.Subscriber.MetaClient = s.MetaClient
+	s.Subscriber.MetaClient = s.MetaClient
+	s.PointsWriter.MetaClient = s.MetaClient
+	s.Monitor.MetaClient = s.MetaClient
+
+	s.SnapshotterService.Listener = mux.Listen(snapshotter.MuxHeader)
+
+	// Configure logging for all services and clients.
+	if s.config.Meta.LoggingEnabled {
+		s.MetaClient.WithLogger(s.Logger)
+	}
+	s.TSDBStore.WithLogger(s.Logger)
+	if s.config.Data.QueryLogEnabled {
+		s.QueryExecutor.WithLogger(s.Logger)
+	}
+	s.PointsWriter.WithLogger(s.Logger)
+	s.Subscriber.WithLogger(s.Logger)
+	for _, svc := range s.Services {
+		svc.WithLogger(s.Logger)
+	}
+	s.SnapshotterService.WithLogger(s.Logger)
+	s.Monitor.WithLogger(s.Logger)
+
+	// Open TSDB store.
+	if err := s.TSDBStore.Open(); err != nil {
+		return fmt.Errorf("open tsdb store: %s", err)
+	}
+
+	// Open the subcriber service
+	if err := s.Subscriber.Open(); err != nil {
+		return fmt.Errorf("open subscriber: %s", err)
+	}
+
+	// Open the points writer service
+	if err := s.PointsWriter.Open(); err != nil {
+		return fmt.Errorf("open points writer: %s", err)
+	}
+
+	for _, service := range s.Services {
+		if err := service.Open(); err != nil {
+			return fmt.Errorf("open service: %s", err)
+		}
+	}
+
+	// Start the reporting service, if not disabled.
+	if !s.reportingDisabled {
+		go s.startServerReporting()
+	}
+
+	return nil
+}
+
+// Close shuts down the meta and data stores and all services.
+func (s *Server) Close() error {
+	stopProfile()
+
+	// Close the listener first to stop any new connections
+	if s.Listener != nil {
+		s.Listener.Close()
+	}
+
+	// Close services to allow any inflight requests to complete
+	// and prevent new requests from being accepted.
+	for _, service := range s.Services {
+		service.Close()
+	}
+
+	s.config.deregisterDiagnostics(s.Monitor)
+
+	if s.PointsWriter != nil {
+		s.PointsWriter.Close()
+	}
+
+	if s.QueryExecutor != nil {
+		s.QueryExecutor.Close()
+	}
+
+	// Close the TSDBStore, no more reads or writes at this point
+	if s.TSDBStore != nil {
+		s.TSDBStore.Close()
+	}
+
+	if s.Subscriber != nil {
+		s.Subscriber.Close()
+	}
+
+	if s.MetaClient != nil {
+		s.MetaClient.Close()
+	}
+
+	close(s.closing)
+	return nil
+}
+
+// startServerReporting starts periodic server reporting.
+func (s *Server) startServerReporting() {
+	s.reportServer()
+
+	ticker := time.NewTicker(24 * time.Hour)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-s.closing:
+			return
+		case <-ticker.C:
+			s.reportServer()
+		}
+	}
+}
+
+// reportServer reports usage statistics about the system.
+func (s *Server) reportServer() {
+	dbs := s.MetaClient.Databases()
+	numDatabases := len(dbs)
+
+	var (
+		numMeasurements int64
+		numSeries       int64
+	)
+
+	for _, db := range dbs {
+		name := db.Name
+		n, err := s.TSDBStore.SeriesCardinality(name)
+		if err != nil {
+			s.Logger.Error(fmt.Sprintf("Unable to get series cardinality for database %s: %v", name, err))
+		} else {
+			numSeries += n
+		}
+
+		n, err = s.TSDBStore.MeasurementsCardinality(name)
+		if err != nil {
+			s.Logger.Error(fmt.Sprintf("Unable to get measurement cardinality for database %s: %v", name, err))
+		} else {
+			numMeasurements += n
+		}
+	}
+
+	clusterID := s.MetaClient.ClusterID()
+	cl := client.New("")
+	usage := client.Usage{
+		Product: "influxdb",
+		Data: []client.UsageData{
+			{
+				Values: client.Values{
+					"os":               runtime.GOOS,
+					"arch":             runtime.GOARCH,
+					"version":          s.buildInfo.Version,
+					"cluster_id":       fmt.Sprintf("%v", clusterID),
+					"num_series":       numSeries,
+					"num_measurements": numMeasurements,
+					"num_databases":    numDatabases,
+					"uptime":           time.Since(startTime).Seconds(),
+				},
+			},
+		},
+	}
+
+	s.Logger.Info("Sending usage statistics to usage.influxdata.com")
+
+	go cl.Save(usage)
+}
+
+// Service represents a service attached to the server.
+type Service interface {
+	WithLogger(log zap.Logger)
+	Open() error
+	Close() error
+}
+
+// prof stores the file locations of active profiles.
+var prof struct {
+	cpu *os.File
+	mem *os.File
+}
+
+// StartProfile initializes the cpu and memory profile, if specified.
+func startProfile(cpuprofile, memprofile string) {
+	if cpuprofile != "" {
+		f, err := os.Create(cpuprofile)
+		if err != nil {
+			log.Fatalf("cpuprofile: %v", err)
+		}
+		log.Printf("writing CPU profile to: %s\n", cpuprofile)
+		prof.cpu = f
+		pprof.StartCPUProfile(prof.cpu)
+	}
+
+	if memprofile != "" {
+		f, err := os.Create(memprofile)
+		if err != nil {
+			log.Fatalf("memprofile: %v", err)
+		}
+		log.Printf("writing mem profile to: %s\n", memprofile)
+		prof.mem = f
+		runtime.MemProfileRate = 4096
+	}
+
+}
+
+// StopProfile closes the cpu and memory profiles if they are running.
+func stopProfile() {
+	if prof.cpu != nil {
+		pprof.StopCPUProfile()
+		prof.cpu.Close()
+		log.Println("CPU profile stopped")
+	}
+	if prof.mem != nil {
+		pprof.Lookup("heap").WriteTo(prof.mem, 0)
+		prof.mem.Close()
+		log.Println("mem profile stopped")
+	}
+}
+
+// monitorPointsWriter is a wrapper around `coordinator.PointsWriter` that helps
+// to prevent a circular dependency between the `cluster` and `monitor` packages.
+type monitorPointsWriter coordinator.PointsWriter
+
+func (pw *monitorPointsWriter) WritePoints(database, retentionPolicy string, points models.Points) error {
+	return (*coordinator.PointsWriter)(pw).WritePointsPrivileged(database, retentionPolicy, models.ConsistencyLevelAny, points)
+}
+
+func raftDBExists(dir string) error {
+	// Check to see if there is a raft db, if so, error out with a message
+	// to downgrade, export, and then import the meta data
+	raftFile := filepath.Join(dir, "raft.db")
+	if _, err := os.Stat(raftFile); err == nil {
+		return fmt.Errorf("detected %s. To proceed, you'll need to either 1) downgrade to v0.11.x, export your metadata, upgrade to the current version again, and then import the metadata or 2) delete the file, which will effectively reset your database. For more assistance with the upgrade, see: https://docs.influxdata.com/influxdb/v0.12/administration/upgrading/", raftFile)
+	}
+	return nil
+}

vendor/github.com/influxdata/influxdb/cmd/parse.go

@@ -0,0 +1,29 @@
+// Package cmd is the root package of the various command-line utilities for InfluxDB.
+package cmd
+
+import "strings"
+
+// ParseCommandName extracts the command name and args from the args list.
+func ParseCommandName(args []string) (string, []string) {
+	// Retrieve command name as first argument.
+	var name string
+	if len(args) > 0 {
+		if !strings.HasPrefix(args[0], "-") {
+			name = args[0]
+		} else if args[0] == "-h" || args[0] == "-help" || args[0] == "--help" {
+			// Special case -h immediately following binary name
+			name = "help"
+		}
+	}
+
+	// If command is "help" and has an argument then rewrite args to use "-h".
+	if name == "help" && len(args) > 2 && !strings.HasPrefix(args[1], "-") {
+		return args[1], []string{"-h"}
+	}
+
+	// If a named command is specified then return it with its arguments.
+	if name != "" {
+		return name, args[1:]
+	}
+	return "", args
+}