416 lines
11 KiB
Go
416 lines
11 KiB
Go
// Command influx_tsm converts b1 or bz1 shards (from InfluxDB releases earlier than v0.11)
|
|
// to the current tsm1 format.
|
|
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"runtime/pprof"
|
|
"sort"
|
|
"strings"
|
|
"text/tabwriter"
|
|
"time"
|
|
|
|
"net/http"
|
|
_ "net/http/pprof"
|
|
|
|
"github.com/influxdata/influxdb/cmd/influx_tsm/b1"
|
|
"github.com/influxdata/influxdb/cmd/influx_tsm/bz1"
|
|
"github.com/influxdata/influxdb/cmd/influx_tsm/tsdb"
|
|
)
|
|
|
|
// ShardReader reads b* shards and converts to tsm shards
|
|
type ShardReader interface {
|
|
KeyIterator
|
|
Open() error
|
|
Close() error
|
|
}
|
|
|
|
const (
|
|
tsmExt = "tsm"
|
|
)
|
|
|
|
var description = `
|
|
Convert a database from b1 or bz1 format to tsm1 format.
|
|
|
|
This tool will backup the directories before conversion (if not disabled).
|
|
The backed-up files must be removed manually, generally after starting up the
|
|
node again to make sure all of data has been converted correctly.
|
|
|
|
To restore a backup:
|
|
Shut down the node, remove the converted directory, and
|
|
copy the backed-up directory to the original location.`
|
|
|
|
type options struct {
|
|
DataPath string
|
|
BackupPath string
|
|
DBs []string
|
|
DebugAddr string
|
|
TSMSize uint64
|
|
Parallel bool
|
|
SkipBackup bool
|
|
UpdateInterval time.Duration
|
|
Yes bool
|
|
CPUFile string
|
|
}
|
|
|
|
func (o *options) Parse() error {
|
|
fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
|
|
|
|
var dbs string
|
|
|
|
fs.StringVar(&dbs, "dbs", "", "Comma-delimited list of databases to convert. Default is to convert all databases.")
|
|
fs.Uint64Var(&opts.TSMSize, "sz", maxTSMSz, "Maximum size of individual TSM files.")
|
|
fs.BoolVar(&opts.Parallel, "parallel", false, "Perform parallel conversion. (up to GOMAXPROCS shards at once)")
|
|
fs.BoolVar(&opts.SkipBackup, "nobackup", false, "Disable database backups. Not recommended.")
|
|
fs.StringVar(&opts.BackupPath, "backup", "", "The location to backup up the current databases. Must not be within the data directory.")
|
|
fs.StringVar(&opts.DebugAddr, "debug", "", "If set, http debugging endpoints will be enabled on the given address")
|
|
fs.DurationVar(&opts.UpdateInterval, "interval", 5*time.Second, "How often status updates are printed.")
|
|
fs.BoolVar(&opts.Yes, "y", false, "Don't ask, just convert")
|
|
fs.StringVar(&opts.CPUFile, "profile", "", "CPU Profile location")
|
|
fs.Usage = func() {
|
|
fmt.Fprintf(os.Stderr, "Usage: %v [options] <data-path> \n", os.Args[0])
|
|
fmt.Fprintf(os.Stderr, "%v\n\nOptions:\n", description)
|
|
fs.PrintDefaults()
|
|
fmt.Fprintf(os.Stderr, "\n")
|
|
}
|
|
|
|
if err := fs.Parse(os.Args[1:]); err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(fs.Args()) < 1 {
|
|
return errors.New("no data directory specified")
|
|
}
|
|
var err error
|
|
if o.DataPath, err = filepath.Abs(fs.Args()[0]); err != nil {
|
|
return err
|
|
}
|
|
if o.DataPath, err = filepath.EvalSymlinks(filepath.Clean(o.DataPath)); err != nil {
|
|
return err
|
|
}
|
|
|
|
if o.TSMSize > maxTSMSz {
|
|
return fmt.Errorf("bad TSM file size, maximum TSM file size is %d", maxTSMSz)
|
|
}
|
|
|
|
// Check if specific databases were requested.
|
|
o.DBs = strings.Split(dbs, ",")
|
|
if len(o.DBs) == 1 && o.DBs[0] == "" {
|
|
o.DBs = nil
|
|
}
|
|
|
|
if !o.SkipBackup {
|
|
if o.BackupPath == "" {
|
|
return errors.New("either -nobackup or -backup DIR must be set")
|
|
}
|
|
if o.BackupPath, err = filepath.Abs(o.BackupPath); err != nil {
|
|
return err
|
|
}
|
|
if o.BackupPath, err = filepath.EvalSymlinks(filepath.Clean(o.BackupPath)); err != nil {
|
|
if os.IsNotExist(err) {
|
|
return errors.New("backup directory must already exist")
|
|
}
|
|
return err
|
|
}
|
|
|
|
if strings.HasPrefix(o.BackupPath, o.DataPath) {
|
|
fmt.Println(o.BackupPath, o.DataPath)
|
|
return errors.New("backup directory cannot be contained within data directory")
|
|
}
|
|
}
|
|
|
|
if o.DebugAddr != "" {
|
|
log.Printf("Starting debugging server on http://%v", o.DebugAddr)
|
|
go func() {
|
|
log.Fatal(http.ListenAndServe(o.DebugAddr, nil))
|
|
}()
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
var opts options
|
|
|
|
const maxTSMSz uint64 = 2 * 1024 * 1024 * 1024
|
|
|
|
func init() {
|
|
log.SetOutput(os.Stderr)
|
|
log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
|
|
}
|
|
|
|
func main() {
|
|
if err := opts.Parse(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
// Determine the list of databases
|
|
dbs, err := ioutil.ReadDir(opts.DataPath)
|
|
if err != nil {
|
|
log.Fatalf("failed to access data directory at %v: %v\n", opts.DataPath, err)
|
|
}
|
|
fmt.Println() // Cleanly separate output from start of program.
|
|
|
|
if opts.Parallel {
|
|
if !isEnvSet("GOMAXPROCS") {
|
|
// Only modify GOMAXPROCS if it wasn't set in the environment
|
|
// This means 'GOMAXPROCS=1 influx_tsm -parallel' will not actually
|
|
// run in parallel
|
|
runtime.GOMAXPROCS(runtime.NumCPU())
|
|
}
|
|
}
|
|
|
|
var badUser string
|
|
if opts.SkipBackup {
|
|
badUser = "(NOT RECOMMENDED)"
|
|
}
|
|
|
|
// Dump summary of what is about to happen.
|
|
fmt.Println("b1 and bz1 shard conversion.")
|
|
fmt.Println("-----------------------------------")
|
|
fmt.Println("Data directory is: ", opts.DataPath)
|
|
if !opts.SkipBackup {
|
|
fmt.Println("Backup directory is: ", opts.BackupPath)
|
|
}
|
|
fmt.Println("Databases specified: ", allDBs(opts.DBs))
|
|
fmt.Println("Database backups enabled: ", yesno(!opts.SkipBackup), badUser)
|
|
fmt.Printf("Parallel mode enabled (GOMAXPROCS): %s (%d)\n", yesno(opts.Parallel), runtime.GOMAXPROCS(0))
|
|
fmt.Println()
|
|
|
|
shards := collectShards(dbs)
|
|
|
|
// Anything to convert?
|
|
fmt.Printf("\nFound %d shards that will be converted.\n", len(shards))
|
|
if len(shards) == 0 {
|
|
fmt.Println("Nothing to do.")
|
|
return
|
|
}
|
|
|
|
// Display list of convertible shards.
|
|
fmt.Println()
|
|
w := new(tabwriter.Writer)
|
|
w.Init(os.Stdout, 0, 8, 1, '\t', 0)
|
|
fmt.Fprintln(w, "Database\tRetention\tPath\tEngine\tSize")
|
|
for _, si := range shards {
|
|
fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%d\n", si.Database, si.RetentionPolicy, si.FullPath(opts.DataPath), si.FormatAsString(), si.Size)
|
|
}
|
|
w.Flush()
|
|
|
|
if !opts.Yes {
|
|
// Get confirmation from user.
|
|
fmt.Printf("\nThese shards will be converted. Proceed? y/N: ")
|
|
liner := bufio.NewReader(os.Stdin)
|
|
yn, err := liner.ReadString('\n')
|
|
if err != nil {
|
|
log.Fatalf("failed to read response: %v", err)
|
|
}
|
|
yn = strings.TrimRight(strings.ToLower(yn), "\n")
|
|
if yn != "y" {
|
|
log.Fatal("Conversion aborted.")
|
|
}
|
|
}
|
|
fmt.Println("Conversion starting....")
|
|
|
|
if opts.CPUFile != "" {
|
|
f, err := os.Create(opts.CPUFile)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
if err = pprof.StartCPUProfile(f); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer pprof.StopCPUProfile()
|
|
}
|
|
|
|
tr := newTracker(shards, opts)
|
|
|
|
if err := tr.Run(); err != nil {
|
|
log.Fatalf("Error occurred preventing completion: %v\n", err)
|
|
}
|
|
|
|
tr.PrintStats()
|
|
}
|
|
|
|
func collectShards(dbs []os.FileInfo) tsdb.ShardInfos {
|
|
// Get the list of shards for conversion.
|
|
var shards tsdb.ShardInfos
|
|
for _, db := range dbs {
|
|
d := tsdb.NewDatabase(filepath.Join(opts.DataPath, db.Name()))
|
|
shs, err := d.Shards()
|
|
if err != nil {
|
|
log.Fatalf("Failed to access shards for database %v: %v\n", d.Name(), err)
|
|
}
|
|
shards = append(shards, shs...)
|
|
}
|
|
|
|
sort.Sort(shards)
|
|
shards = shards.FilterFormat(tsdb.TSM1)
|
|
if len(dbs) > 0 {
|
|
shards = shards.ExclusiveDatabases(opts.DBs)
|
|
}
|
|
|
|
return shards
|
|
}
|
|
|
|
// backupDatabase backs up the database named db
|
|
func backupDatabase(db string) error {
|
|
copyFile := func(path string, info os.FileInfo, err error) error {
|
|
// Strip the DataPath from the path and replace with BackupPath.
|
|
toPath := strings.Replace(path, opts.DataPath, opts.BackupPath, 1)
|
|
|
|
if info.IsDir() {
|
|
return os.MkdirAll(toPath, info.Mode())
|
|
}
|
|
|
|
in, err := os.Open(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer in.Close()
|
|
|
|
srcInfo, err := os.Stat(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
out, err := os.OpenFile(toPath, os.O_CREATE|os.O_WRONLY, info.Mode())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer out.Close()
|
|
|
|
dstInfo, err := os.Stat(toPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if dstInfo.Size() == srcInfo.Size() {
|
|
log.Printf("Backup file already found for %v with correct size, skipping.", path)
|
|
return nil
|
|
}
|
|
|
|
if dstInfo.Size() > srcInfo.Size() {
|
|
log.Printf("Invalid backup file found for %v, replacing with good copy.", path)
|
|
if err := out.Truncate(0); err != nil {
|
|
return err
|
|
}
|
|
if _, err := out.Seek(0, io.SeekStart); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if dstInfo.Size() > 0 {
|
|
log.Printf("Resuming backup of file %v, starting at %v bytes", path, dstInfo.Size())
|
|
}
|
|
|
|
off, err := out.Seek(0, io.SeekEnd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := in.Seek(off, io.SeekStart); err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Printf("Backing up file %v", path)
|
|
|
|
_, err = io.Copy(out, in)
|
|
|
|
return err
|
|
}
|
|
|
|
return filepath.Walk(filepath.Join(opts.DataPath, db), copyFile)
|
|
}
|
|
|
|
// convertShard converts the shard in-place.
|
|
func convertShard(si *tsdb.ShardInfo, tr *tracker) error {
|
|
src := si.FullPath(opts.DataPath)
|
|
dst := fmt.Sprintf("%v.%v", src, tsmExt)
|
|
|
|
var reader ShardReader
|
|
switch si.Format {
|
|
case tsdb.BZ1:
|
|
reader = bz1.NewReader(src, &tr.Stats, 0)
|
|
case tsdb.B1:
|
|
reader = b1.NewReader(src, &tr.Stats, 0)
|
|
default:
|
|
return fmt.Errorf("Unsupported shard format: %v", si.FormatAsString())
|
|
}
|
|
|
|
// Open the shard, and create a converter.
|
|
if err := reader.Open(); err != nil {
|
|
return fmt.Errorf("Failed to open %v for conversion: %v", src, err)
|
|
}
|
|
defer reader.Close()
|
|
converter := NewConverter(dst, uint32(opts.TSMSize), &tr.Stats)
|
|
|
|
// Perform the conversion.
|
|
if err := converter.Process(reader); err != nil {
|
|
return fmt.Errorf("Conversion of %v failed: %v", src, err)
|
|
}
|
|
|
|
// Delete source shard, and rename new tsm1 shard.
|
|
if err := reader.Close(); err != nil {
|
|
return fmt.Errorf("Conversion of %v failed due to close: %v", src, err)
|
|
}
|
|
|
|
if err := os.RemoveAll(si.FullPath(opts.DataPath)); err != nil {
|
|
return fmt.Errorf("Deletion of %v failed: %v", src, err)
|
|
}
|
|
if err := os.Rename(dst, src); err != nil {
|
|
return fmt.Errorf("Rename of %v to %v failed: %v", dst, src, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// ParallelGroup allows the maximum parrallelism of a set of operations to be controlled.
|
|
type ParallelGroup chan struct{}
|
|
|
|
// NewParallelGroup returns a group which allows n operations to run in parallel. A value of 0
|
|
// means no operations will ever run.
|
|
func NewParallelGroup(n int) ParallelGroup {
|
|
return make(chan struct{}, n)
|
|
}
|
|
|
|
// Do executes one operation of the ParallelGroup
|
|
func (p ParallelGroup) Do(f func()) {
|
|
p <- struct{}{} // acquire working slot
|
|
defer func() { <-p }()
|
|
|
|
f()
|
|
}
|
|
|
|
// yesno returns "yes" for true, "no" for false.
|
|
func yesno(b bool) string {
|
|
if b {
|
|
return "yes"
|
|
}
|
|
return "no"
|
|
}
|
|
|
|
// allDBs returns "all" if all databases are requested for conversion.
|
|
func allDBs(dbs []string) string {
|
|
if dbs == nil {
|
|
return "all"
|
|
}
|
|
return fmt.Sprintf("%v", dbs)
|
|
}
|
|
|
|
// isEnvSet checks to see if a variable was set in the environment
|
|
func isEnvSet(name string) bool {
|
|
for _, s := range os.Environ() {
|
|
if strings.SplitN(s, "=", 2)[0] == name {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|