frodovdr
/
vsphere-influxdb-go
mirror of https://github.com/Oxalide/vsphere-influxdb-go.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #53 from zooplus/master 

Nested Folders and measurement prefix support
This commit is contained in:
Adrian Todorov 2018-01-07 15:07:16 +01:00 committed by GitHub
parent 942b49d61e d00ca9ee6d
commit c252b28988
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 134 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -11,3 +11,6 @@ vsphere-influxdb
# Configuration file # Configuration file
vsphere-influxdb.json vsphere-influxdb.json
# Vim swap files
*.swp

105
vsphere-influxdb.go
View File

@ -1,17 +1,17 @@
/* Copyright 2016 Adrian Todorov, Oxalide ato@oxalide.com /* Copyright 2016 Adrian Todorov, Oxalide ato@oxalide.com
Original project author: https://github.com/cblomart Original project author: https://github.com/cblomart
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
(at your option) any later version. (at your option) any later version.
This program is distributed in the hope that it will be useful, This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details. GNU General Public License for more details.
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
package main package main
@ -60,6 +60,7 @@ type InfluxDB struct {
Username string Username string
Password string Password string
Database string Database string
Prefix string
} }
// VCenter for VMware vCenter connections // VCenter for VMware vCenter connections
@ -106,18 +107,18 @@ func (vcenter *VCenter) Connect() error {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
stdlog.Println("connecting to vcenter: " + vcenter.Hostname) stdlog.Println("Connecting to vcenter:", vcenter.Hostname)
u, err := url.Parse("https://" + vcenter.Username + ":" + vcenter.Password + "@" + vcenter.Hostname + "/sdk") u, err := url.Parse("https://" + vcenter.Username + ":" + vcenter.Password + "@" + vcenter.Hostname + "/sdk")
if err != nil { if err != nil {
errlog.Println("Could not parse vcenter url: ", vcenter.Hostname) errlog.Println("Could not parse vcenter url:", vcenter.Hostname)
errlog.Println("Error: ", err) errlog.Println("Error:", err)
return err return err
} }
client, err := govmomi.NewClient(ctx, u, true) client, err := govmomi.NewClient(ctx, u, true)
if err != nil { if err != nil {
errlog.Println("Could not connect to vcenter: ", vcenter.Hostname) errlog.Println("Could not connect to vcenter:", vcenter.Hostname)
errlog.Println("Error: ", err) errlog.Println("Error:", err)
return err return err
} }
@ -133,7 +134,7 @@ func (vcenter *VCenter) Disconnect() error {
if vcenter.client != nil { if vcenter.client != nil {
if err := vcenter.client.Logout(ctx); err != nil { if err := vcenter.client.Logout(ctx); err != nil {
errlog.Println("Could not disconnect properly from vcenter", vcenter.Hostname, err) errlog.Println("Could not disconnect properly from vcenter:", vcenter.Hostname, err)
return err return err
} }
} }
@ -152,7 +153,7 @@ func (vcenter *VCenter) Init(config Configuration) error {
err := client.RetrieveOne(ctx, *client.ServiceContent.PerfManager, nil, &perfmanager) err := client.RetrieveOne(ctx, *client.ServiceContent.PerfManager, nil, &perfmanager)
if err != nil { if err != nil {
errlog.Println("Could not get performance manager") errlog.Println("Could not get performance manager")
errlog.Println("Error: ", err) errlog.Println("Error:", err)
return err return err
} }
@ -187,7 +188,7 @@ func (vcenter *VCenter) Init(config Configuration) error {
// Query a vcenter // Query a vcenter
func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.Client, nowTime time.Time) { func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.Client, nowTime time.Time) {
stdlog.Println("Setting up query inventory of vcenter: ", vcenter.Hostname) stdlog.Println("Setting up query inventory of vcenter:", vcenter.Hostname)
// Create the contect // Create the contect
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@ -200,7 +201,7 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
var viewManager mo.ViewManager var viewManager mo.ViewManager
err := client.RetrieveOne(ctx, *client.ServiceContent.ViewManager, nil, &viewManager) err := client.RetrieveOne(ctx, *client.ServiceContent.ViewManager, nil, &viewManager)
if err != nil { if err != nil {
errlog.Println("Could not get view manager from vcenter: " + vcenter.Hostname) errlog.Println("Could not get view manager from vcenter:", vcenter.Hostname)
errlog.Println("Error: ", err) errlog.Println("Error: ", err)
return return
} }
@ -209,16 +210,16 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
var rootFolder mo.Folder var rootFolder mo.Folder
err = client.RetrieveOne(ctx, client.ServiceContent.RootFolder, nil, &rootFolder) err = client.RetrieveOne(ctx, client.ServiceContent.RootFolder, nil, &rootFolder)
if err != nil { if err != nil {
errlog.Println("Could not get root folder from vcenter: " + vcenter.Hostname) errlog.Println("Could not get root folder from vcenter:", vcenter.Hostname)
errlog.Println("Error: ", err) errlog.Println("Error:", err)
return return
} }
datacenters := []types.ManagedObjectReference{} datacenters := []types.ManagedObjectReference{}
for _, child := range rootFolder.ChildEntity { for _, child := range rootFolder.ChildEntity {
if child.Type == "Datacenter" { //if child.Type == "Datacenter" {
datacenters = append(datacenters, child) datacenters = append(datacenters, child)
} //}
} }
// Get intresting object types from specified queries // Get intresting object types from specified queries
objectTypes := []string{} objectTypes := []string{}
@ -236,16 +237,16 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
req := types.CreateContainerView{This: viewManager.Reference(), Container: datacenter, Type: objectTypes, Recursive: true} req := types.CreateContainerView{This: viewManager.Reference(), Container: datacenter, Type: objectTypes, Recursive: true}
res, err := methods.CreateContainerView(ctx, client.RoundTripper, &req) res, err := methods.CreateContainerView(ctx, client.RoundTripper, &req)
if err != nil { if err != nil {
errlog.Println("Could not create container view from vcenter: " + vcenter.Hostname) errlog.Println("Could not create container view from vcenter:", vcenter.Hostname)
errlog.Println("Error: ", err) errlog.Println("Error:", err)
continue continue
} }
// Retrieve the created ContentView // Retrieve the created ContentView
var containerView mo.ContainerView var containerView mo.ContainerView
err = client.RetrieveOne(ctx, res.Returnval, nil, &containerView) err = client.RetrieveOne(ctx, res.Returnval, nil, &containerView)
if err != nil { if err != nil {
errlog.Println("Could not get container view from vcenter: " + vcenter.Hostname) errlog.Println("Could not get container view from vcenter:", vcenter.Hostname)
errlog.Println("Error: ", err) errlog.Println("Error:", err)
continue continue
} }
// Add found object to object list // Add found object to object list
@ -344,7 +345,7 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
// Retrieve properties for ResourcePools // Retrieve properties for ResourcePools
if len(respoolRefs) > 0 { if len(respoolRefs) > 0 {
if debug { if debug {
stdlog.Println("going inside ResourcePools") stdlog.Println("Going inside ResourcePools")
} }
err = pc.Retrieve(ctx, respoolRefs, []string{"name", "config", "vm"}, &respool) err = pc.Retrieve(ctx, respoolRefs, []string{"name", "config", "vm"}, &respool)
if err != nil { if err != nil {
@ -352,11 +353,11 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
return return
} }
for _, pool := range respool { for _, pool := range respool {
stdlog.Println(pool.Config.MemoryAllocation.GetResourceAllocationInfo().Limit)
stdlog.Println(pool.Config.CpuAllocation.GetResourceAllocationInfo().Limit)
if debug { if debug {
stdlog.Println("---resourcepool name - you should see every resourcepool here (+VMs inside)----") stdlog.Println("---resourcepool name - you should see every resourcepool here (+VMs inside)----")
stdlog.Println(pool.Name) stdlog.Println(pool.Name)
stdlog.Println(pool.Config.MemoryAllocation.GetResourceAllocationInfo().Limit)
stdlog.Println(pool.Config.CpuAllocation.GetResourceAllocationInfo().Limit)
} }
for _, vm := range pool.Vm { for _, vm := range pool.Vm {
if debug { if debug {
@ -377,7 +378,7 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
// Retrieve properties for clusters, if any // Retrieve properties for clusters, if any
if len(clusterRefs) > 0 { if len(clusterRefs) > 0 {
if debug { if debug {
stdlog.Println("going inside clusters") stdlog.Println("Going inside clusters")
} }
// Step 1 : Get ObjectContents and Host info for VM // Step 1 : Get ObjectContents and Host info for VM
@ -468,16 +469,16 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
propreq := types.RetrieveProperties{SpecSet: []types.PropertyFilterSpec{{ObjectSet: objectSet, PropSet: []types.PropertySpec{*propSpec}}}} propreq := types.RetrieveProperties{SpecSet: []types.PropertyFilterSpec{{ObjectSet: objectSet, PropSet: []types.PropertySpec{*propSpec}}}}
propres, err := client.PropertyCollector().RetrieveProperties(ctx, propreq) propres, err := client.PropertyCollector().RetrieveProperties(ctx, propreq)
if err != nil { if err != nil {
errlog.Println("Could not retrieve object names from vcenter: " + vcenter.Hostname) errlog.Println("Could not retrieve object names from vcenter:", vcenter.Hostname)
errlog.Println("Error: ", err) errlog.Println("Error:", err)
return return
} }
//load retrieved properties //load retrieved properties
err = mo.LoadRetrievePropertiesResponse(propres, &objects) err = mo.LoadRetrievePropertiesResponse(propres, &objects)
if err != nil { if err != nil {
errlog.Println("Could not retrieve object names from vcenter: " + vcenter.Hostname) errlog.Println("Could not retrieve object names from vcenter:", vcenter.Hostname)
errlog.Println("Error: ", err) errlog.Println("Error:", err)
return return
} }
@ -524,8 +525,8 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
perfreq := types.QueryPerf{This: *client.ServiceContent.PerfManager, QuerySpec: queries} perfreq := types.QueryPerf{This: *client.ServiceContent.PerfManager, QuerySpec: queries}
perfres, err := methods.QueryPerf(ctx, client.RoundTripper, &perfreq) perfres, err := methods.QueryPerf(ctx, client.RoundTripper, &perfreq)
if err != nil { if err != nil {
errlog.Println("Could not request perfs from vcenter: " + vcenter.Hostname) errlog.Println("Could not request perfs from vcenter:", vcenter.Hostname)
errlog.Println("Error: ", err) errlog.Println("Error:", err)
return return
} }
@ -635,7 +636,7 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
} }
//create InfluxDB points //create InfluxDB points
pt, err := influxclient.NewPoint(entityName, tags, fields, nowTime) pt, err := influxclient.NewPoint(config.InfluxDB.Prefix+entityName, tags, fields, nowTime)
if err != nil { if err != nil {
errlog.Println(err) errlog.Println(err)
continue continue
@ -645,7 +646,7 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
for measurement, v := range specialFields { for measurement, v := range specialFields {
for name, metric := range v { for name, metric := range v {
for instance, value := range metric { for instance, value := range metric {
pt2, err := influxclient.NewPoint(measurement, specialTags[measurement][name][instance], value, time.Now()) pt2, err := influxclient.NewPoint(config.InfluxDB.Prefix+measurement, specialTags[measurement][name][instance], value, time.Now())
if err != nil { if err != nil {
errlog.Println(err) errlog.Println(err)
continue continue
@ -655,12 +656,12 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
} }
} }
// var respool []mo.ResourcePool // var respool []mo.ResourcePool
// err = pc.Retrieve(ctx, respoolRefs, []string{"name", "config", "vm"}, &respool) // err = pc.Retrieve(ctx, respoolRefs, []string{"name", "config", "vm"}, &respool)
// if err != nil { // if err != nil {
// errlog.Println(err) // errlog.Println(err)
// continue // continue
// } // }
for _, pool := range respool { for _, pool := range respool {
respoolFields := map[string]interface{}{ respoolFields := map[string]interface{}{
@ -668,7 +669,7 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
"memory_limit": pool.Config.MemoryAllocation.GetResourceAllocationInfo().Limit, "memory_limit": pool.Config.MemoryAllocation.GetResourceAllocationInfo().Limit,
} }
respoolTags := map[string]string{"pool_name": pool.Name} respoolTags := map[string]string{"pool_name": pool.Name}
pt3, err := influxclient.NewPoint("resourcepool", respoolTags, respoolFields, time.Now()) pt3, err := influxclient.NewPoint(config.InfluxDB.Prefix+"resourcepool", respoolTags, respoolFields, time.Now())
if err != nil { if err != nil {
errlog.Println(err) errlog.Println(err)
continue continue
@ -682,7 +683,7 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
"free_space": datastore.Summary.FreeSpace, "free_space": datastore.Summary.FreeSpace,
} }
datastoreTags := map[string]string{"ds_name": datastore.Summary.Name, "host": vcName} datastoreTags := map[string]string{"ds_name": datastore.Summary.Name, "host": vcName}
pt4, err := influxclient.NewPoint("datastore", datastoreTags, datastoreFields, time.Now()) pt4, err := influxclient.NewPoint(config.InfluxDB.Prefix+"datastore", datastoreTags, datastoreFields, time.Now())
if err != nil { if err != nil {
errlog.Println(err) errlog.Println(err)
continue continue
@ -699,7 +700,7 @@ func (vcenter *VCenter) Query(config Configuration, InfluxDBClient influxclient.
return return
} }
stdlog.Println("sent data to Influxdb") stdlog.Println("Sent data to Influxdb from:", vcenter.Hostname)
} }
func min(n ...int64) int64 { func min(n ...int64) int64 {
@ -785,7 +786,7 @@ func main() {
flag.BoolVar(&debug, "debug", false, "Debug mode") flag.BoolVar(&debug, "debug", false, "Debug mode")
workerCount := flag.Int("workers", 4, "Number of concurrent workers to query vcenters") workerCount := flag.Int("workers", 4, "Number of concurrent workers to query vcenters")
cfgFile := flag.String("config", "/etc/"+baseName+".json", "Config file to use. Default is /etc/"+baseName+".json") cfgFile := flag.String("config", "/etc/"+baseName+".json", "Config file to use")
flag.Parse() flag.Parse()
stdlog.Println("Starting", baseName, "with config file", *cfgFile) stdlog.Println("Starting", baseName, "with config file", *cfgFile)

151
vsphere-influxdb.json.sample
View File

@ -1,76 +1,81 @@
{ {
"Domain": ".lab", "Domain": ".lab",
"Interval": 60, "Interval": 60,
"VCenters": [ "VCenters": [
{ "Username": "monitoring", "Password": "monixx", "Hostname": "vcenter-01.dc-01.lab" }, { "Username": "monitoring", "Password": "monixx", "Hostname": "vcenter-01.dc-01.lab" },
{ "Username": "monitoring", "Password": "monixx", "Hostname": "vcenter-01.dc-02.lab" }, { "Username": "monitoring", "Password": "monixx", "Hostname": "vcenter-01.dc-02.lab" },
{ "Username": "monitoring", "Password": "monixx", "Hostname": "vcenter-02.dc-02.lab" }, { "Username": "monitoring", "Password": "monixx", "Hostname": "vcenter-02.dc-02.lab" },
{ "Username": "monitoring", "Password": "monixx", "Hostname": "vcenter-01.home.lab" } { "Username": "monitoring", "Password": "monixx", "Hostname": "vcenter-01.home.lab" }
],
], "InfluxDB": {
"InfluxDB": { "Prefix": "vsphere_",
"Hostname": "http://influxdb-01.dc-01.lab:8086", "Hostname": "http://influxdb-01.dc-01.lab:8086",
"Username": "vm", "Username": "vm",
"Password": "vmware334", "Password": "vmware334",
"Database": "vmware_performance" "Database": "vmware_performance"
}, },
"Metrics": [
{ "Metrics": [
"ObjectType": [ "VirtualMachine", "HostSystem" ], {
"Definition": [ "ObjectType": [ "VirtualMachine", "HostSystem" ],
{ "Metric": "cpu.usage.average", "Instances": "*" }, "Definition": [
{ "Metric": "cpu.usage.maximum", "Instances": "*" }, { "Metric": "cpu.usage.average", "Instances": "*" },
{ "Metric": "cpu.usagemhz.average", "Instances": "*" }, { "Metric": "cpu.usage.maximum", "Instances": "*" },
{ "Metric": "cpu.usagemhz.maximum", "Instances": "*" }, { "Metric": "cpu.usagemhz.average", "Instances": "*" },
{ "Metric": "cpu.wait.summation", "Instances": "*" }, { "Metric": "cpu.usagemhz.maximum", "Instances": "*" },
{ "Metric": "cpu.system.summation", "Instances": "*" }, { "Metric": "cpu.wait.summation", "Instances": "*" },
{ "Metric": "cpu.ready.summation", "Instances": "*" }, { "Metric": "cpu.system.summation", "Instances": "*" },
{ "Metric": "mem.usage.average", "Instances": "*" }, { "Metric": "cpu.ready.summation", "Instances": "*" },
{ "Metric": "mem.usage.maximum", "Instances": "*" }, { "Metric": "mem.usage.average", "Instances": "*" },
{ "Metric": "mem.consumed.average", "Instances": "*" }, { "Metric": "mem.usage.maximum", "Instances": "*" },
{ "Metric": "mem.consumed.maximum", "Instances": "*" }, { "Metric": "mem.consumed.average", "Instances": "*" },
{ "Metric": "mem.active.average", "Instances": "*" }, { "Metric": "mem.consumed.maximum", "Instances": "*" },
{ "Metric": "mem.active.maximum", "Instances": "*" }, { "Metric": "mem.active.average", "Instances": "*" },
{ "Metric": "mem.vmmemctl.average", "Instances": "*" }, { "Metric": "mem.active.maximum", "Instances": "*" },
{ "Metric": "mem.vmmemctl.maximum", "Instances": "*" }, { "Metric": "mem.vmmemctl.average", "Instances": "*" },
{ "Metric": "mem.totalCapacity.average", "Instances": "*" }, { "Metric": "mem.vmmemctl.maximum", "Instances": "*" },
{ "Metric": "net.packetsRx.summation", "Instances": "*" }, { "Metric": "mem.totalCapacity.average", "Instances": "*" },
{ "Metric": "net.packetsTx.summation", "Instances": "*" }, { "Metric": "net.packetsRx.summation", "Instances": "*" },
{ "Metric": "net.throughput.usage.average", "Instances": "*" }, { "Metric": "net.packetsTx.summation", "Instances": "*" },
{ "Metric": "net.received.average", "Instances": "*" }, { "Metric": "net.throughput.usage.average", "Instances": "*" },
{ "Metric": "net.transmitted.average", "Instances": "*" }, { "Metric": "net.received.average", "Instances": "*" },
{ "Metric": "net.throughput.usage.nfs.average", "Instances": "*" }, { "Metric": "net.transmitted.average", "Instances": "*" },
{ "Metric": "datastore.numberReadAveraged.average", "Instances": "*" }, { "Metric": "net.throughput.usage.nfs.average", "Instances": "*" },
{ "Metric": "datastore.numberWriteAveraged.average", "Instances": "*" }, { "Metric": "datastore.numberReadAveraged.average", "Instances": "*" },
{ "Metric": "datastore.read.average", "Instances": "*" }, { "Metric": "datastore.numberWriteAveraged.average", "Instances": "*" },
{ "Metric": "datastore.write.average", "Instances": "*" }, { "Metric": "datastore.read.average", "Instances": "*" },
{ "Metric": "datastore.totalReadLatency.average", "Instances": "*" }, { "Metric": "datastore.write.average", "Instances": "*" },
{ "Metric": "datastore.totalWriteLatency.average", "Instances": "*" }, { "Metric": "datastore.totalReadLatency.average", "Instances": "*" },
{ "Metric": "mem.capacity.provisioned.average", "Instances": "*"}, { "Metric": "datastore.totalWriteLatency.average", "Instances": "*" },
{ "Metric": "cpu.corecount.provisioned.average", "Instances": "*" } { "Metric": "mem.capacity.provisioned.average", "Instances": "*"},
] { "Metric": "cpu.corecount.provisioned.average", "Instances": "*" }
}, ]
{ },
"ObjectType": [ "VirtualMachine" ],
"Definition": [ {
{ "Metric": "datastore.datastoreVMObservedLatency.latest", "Instances": "*" } "ObjectType": [ "VirtualMachine" ],
] "Definition": [
}, { "Metric": "datastore.datastoreVMObservedLatency.latest", "Instances": "*" }
{ ]
"ObjectType": [ "HostSystem" ], },
"Definition": [
{ "Metric": "disk.maxTotalLatency.latest", "Instances": "" }, {
{ "Metric": "disk.numberReadAveraged.average", "Instances": "*" }, "ObjectType": [ "HostSystem" ],
{ "Metric": "disk.numberWriteAveraged.average", "Instances": "*" }, "Definition": [
{ "Metric": "net.throughput.contention.summation", "Instances": "*" } { "Metric": "disk.maxTotalLatency.latest", "Instances": "" },
] { "Metric": "disk.numberReadAveraged.average", "Instances": "*" },
}, { "Metric": "disk.numberWriteAveraged.average", "Instances": "*" },
{ { "Metric": "net.throughput.contention.summation", "Instances": "*" }
"ObjectType": [ "Datastore" ], ]
"Definition": [ },
{ "Metric": "disk.capacity.latest", "Instances": "*" },
{ "Metric": "disk.used.latest", "Instances": "*" } {
] "ObjectType": [ "Datastore" ],
} "Definition": [
] { "Metric": "disk.capacity.latest", "Instances": "*" },
{ "Metric": "disk.used.latest", "Instances": "*" }
]
}
]
} }