Adds widget metrics output

Doc cleanup Missed changes
2020-02-18 09:44:29 -06:00 · 2020-02-18 09:44:29 -06:00 · 5ccfc7cbc6
commit 5ccfc7cbc6
parent b61ed8b81b
13 changed files with 273 additions and 28 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -13,7 +13,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 > - **Fixed**: for any bug fixes.
 > - **Security**: in case of vulnerabilities.

-## [3.3.0] - 
+## [3.x.x] - 
+
+- Adds metrics. If run with the `--export :2112` flag (`:2112` is a port),
+  metrics are exposed as Prometheus metrics on that port and can be HTTP
+  GET-ted.  
+
+
+## [3.3.0] - 2020-02-17

 - Added: Logs are now rotated. Settings are currently hard-coded at 4 files of 5MB
  each, so logs shouldn't take up more than 20MB.  I'm going to see how many
--- a/README.md
+++ b/README.md
@ -97,9 +97,9 @@ and these are separated by spaces.
 4. Legal widget names are: cpu, disk, mem, temp, batt, net, procs
 5. Widget names are not case sensitive
 4. The simplest row is a single widget, by name, e.g.
-   ```
-   cpu
-   ```
+    ```
+    cpu
+    ```
 5. **Weights**
    1. Widgets with no weights have a weight of 1.
    2. If multiple widgets are put on a row with no weights, they will all have
@ -117,19 +117,19 @@ and these are separated by spaces.
       second will be 5/7 ~= 67% wide (or, memory will be twice as wide as disk).
 9. If prefixed by a number and colon, the widget will span that number of
   rows downward. E.g.
-   ```
-   mem   2:cpu
-   net
-   ```
+    ```
+    mem   2:cpu
+    net
+    ```
   Here, memory and network will be in the same row as CPU, one over the other,
   and each half as high as CPU; it'll look like this:
-   ```
-    +------+------+
-    | Mem  |      |
-    +------+ CPU  |
-    | Net  |      |
-    +------+------+
-   ```
+    ```
+     +------+------+
+     | Mem  |      |
+     +------+ CPU  |
+     | Net  |      |
+     +------+------+
+    ```
 10. Negative, 0, or non-integer weights will be recorded as "1".  Same for row
    spans. 
 11. Unrecognized widget names will cause the application to abort.                          
@ -146,23 +146,71 @@ and these are separated by spaces.
 Yes, you're clever enough to break the layout algorithm, but if you try to
 build massive edifices, you're in for disappointment.

+### Metrics
+
+gotop can export widget data as Prometheus metrics. This allows users to take
+snapshots of the current state of a machine running gotop, or to query gotop
+remotely.
+
+All metrics are in the `gotop` namespace, and are tagged with
+`goto_<widget>_<enum>`. Metrics are only exported for widgets
+that are enabled, and are updated with the same frequency as the configured
+update interval.  Most widgets are exported as Prometheus gauges.
+
+Metrics are disabled by default, and must be enabled with the `--export` flag.
+The flag takes an interface port in the idiomatic Go format of
+`<addy>:<port>`; a common pattern is `-x :2112`. There is **no security**
+on this feature; I recommend that you run this bound to a localhost interface,
+e.g. `127.0.0.1:7653`, and if you want to access this remotely, run it behind
+a proxy that provides SSL and authentication such as
+[Caddy](https://caddyserver.com).
+
+Once enabled, any widgets that are enabled will appear in the HTTP payload of
+a call to `http://<addy>:<port>/metrics`. For example,
+
+```
+➜  ~ curl -s http://localhost:2112/metrics | egrep -e '^gotop'
+gotop_battery_0 0.6387792286668692
+gotop_cpu_0 12.871287128721228
+gotop_cpu_1 11.000000000001364
+gotop_disk_:dev:nvme0n1p1 0.63
+gotop_memory_main 49.932259713701434
+gotop_memory_swap 0
+gotop_net_recv 129461
+gotop_net_sent 218525
+gotop_temp_coretemp_core0 37
+gotop_temp_coretemp_core1 37
+```
+
+Disk metrics are reformatted to replace `/` with `:` which makes them legal
+Prometheus names:
+
+```
+➜  ~ curl -s http://localhost:2112/metrics | egrep -e '^gotop_disk' | tr ':' '/'
+gotop_disk_/dev/nvme0n1p1 0.63
+```
+
+This feature satisfies a ticket request to provide a "snapshot" for comparison
+with a known state, but it is also foundational for a future feature where
+widgets can be configured with virtual devices fed by data from remote gotop
+instances. The objective for that feature is to allow monitoring of multiple
+remote VMs without having to have a wall of gotops running on a large monitor.
+
 ### CLI Options

 `-c`, `--color=NAME` Set a colorscheme.  
-`-m`, `--minimal` Only show CPU, Mem and Process widgets.  (DEPRECATED for `-l minimal`)
+`-m`, `--minimal` Only show CPU, Mem and Process widgets.  (DEPRECATED for `-l minimal`)  
 `-r`, `--rate=RATE` Number of times per second to update CPU and Mem widgets [default: 1].  
 `-V`, `--version` Print version and exit.  
 `-p`, `--percpu` Show each CPU in the CPU widget.  
 `-a`, `--averagecpu` Show average CPU in the CPU widget.  
 `-f`, `--fahrenheit` Show temperatures in fahrenheit.  
 `-s`, `--statusbar` Show a statusbar with the time.  
-`-b`, `--battery` Show battery level widget (`minimal` turns off). [preview](./assets/screenshots/battery.png)  (DEPRECATED for `-l battery`)
-`-i`, `--interface=NAME` Select network interface [default: all].
-`-l`, `--layout=NAME` Choose a layout. gotop searches for a file by NAME in \$XDG_CONFIG_HOME/gotop, then relative to the current path. "-" reads a layout from stdin, allowing for simple, one-off layouts such as `echo net | gotop -l -`
+`-b`, `--battery` Show battery level widget (`minimal` turns off). [preview](./assets/screenshots/battery.png)  (DEPRECATED for `-l battery`)  
+`-i`, `--interface=NAME` Select network interface. Several interfaces can be defined using comma separated values. Interfaces can also be ignored by prefixing the interface with `!` [default: all].  
+`-l`, `--layout=NAME` Choose a layout. gotop searches for a file by NAME in \$XDG_CONFIG_HOME/gotop, then relative to the current path. "-" reads a layout from stdin, allowing for simple, one-off layouts such as `echo net | gotop -l -`  
+`-x`, `--export=PORT` Enable metrics for export on the specified port. This feature is disabled by default.

-Several interfaces can be defined using comma separated values.
-
-Interfaces can also be ignored using `!`

 ## Built With

@ -172,3 +220,4 @@ Interfaces can also be ignored using `!`
 - [shirou/gopsutil](https://github.com/shirou/gopsutil)
 - [goreleaser/nfpm](https://github.com/goreleaser/nfpm)
 - [distatus/battery](https://github.com/distatus/battery)
+- [prometheus/client_golang](https://github.com/prometheus/client_golang)
--- a/cmd/gotop/main.go
+++ b/cmd/gotop/main.go
@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http"
 	"os"
 	"os/signal"
 	"path/filepath"
@ -14,6 +15,7 @@ import (

 	docopt "github.com/docopt/docopt.go"
 	ui "github.com/gizak/termui/v3"
+	"github.com/prometheus/client_golang/prometheus/promhttp"

 	"github.com/xxxserxxx/gotop"
 	"github.com/xxxserxxx/gotop/colorschemes"
@ -59,6 +61,7 @@ Options:
  -B, --bandwidth=bits	  Specify the number of bits per seconds.
  -l, --layout=NAME       Name of layout spec file for the UI.  Looks first in $XDG_CONFIG_HOME/gotop, then as a path.  Use "-" to pipe.
  -i, --interface=NAME    Select network interface [default: all].
+  -x, --export=PORT       Enable metrics for export on the specified port.

 Several interfaces can be defined using comma separated values.

@ -110,8 +113,11 @@ Colorschemes:
 	if args["--minimal"].(bool) {
 		conf.Layout = "minimal"
 	}
-	if val, _ := args["--statusbar"]; val != nil {
-		rateStr, _ := args["--rate"].(string)
+	if val, _ := args["--export"]; val != nil {
+		conf.ExportPort = val.(string)
+	}
+	if val, _ := args["--rate"]; val != nil {
+		rateStr, _ := val.(string)
 		rate, err := strconv.ParseFloat(rateStr, 64)
 		if err != nil {
 			return fmt.Errorf("invalid rate parameter")
@ -330,7 +336,7 @@ func makeConfig() gotop.Config {
 		HelpVisible:          false,
 		UpdateInterval:       time.Second,
 		AverageLoad:          false,
-		PercpuLoad:           false,
+		PercpuLoad:           true,
 		TempScale:            w.Celsius,
 		Statusbar:            false,
 		NetInterface:         w.NET_INTERFACE_ALL,
@ -395,6 +401,12 @@ func main() {
 		ui.Render(bar)
 	}

+	if conf.ExportPort != "" {
+		go func() {
+			http.Handle("/metrics", promhttp.Handler())
+			http.ListenAndServe(conf.ExportPort, nil)
+		}()
+	}
 	eventLoop(conf, grid)
 }

--- a/config.go
+++ b/config.go
@ -38,6 +38,7 @@ type Config struct {
 	NetInterface   string
 	Layout         string
 	MaxLogSize     int64
+	ExportPort     string
 }

 func Parse(in io.Reader, conf *Config) error {
@ -117,6 +118,8 @@ func Parse(in io.Reader, conf *Config) error {
 				return err
 			}
 			conf.MaxLogSize = int64(iv)
+		case "export":
+			conf.ExportPort = kv[1]
 		}
 	}

--- a/layout/layout.go
+++ b/layout/layout.go
@ -108,8 +108,12 @@ func processRow(c gotop.Config, numRows int, rowDefs [][]widgetRule) (ui.GridIte
 	return ui.NewRow(1.0/float64(numRows), uiColumns...), rowDefs
 }

+type Metric interface {
+	EnableMetric()
+}
+
 func makeWidget(c gotop.Config, widRule widgetRule) interface{} {
-	var w interface{}
+	var w Metric
 	switch widRule.Widget {
 	case "cpu":
 		cpu := widgets.NewCpuWidget(c.UpdateInterval, c.GraphHorizontalScale, c.AverageLoad, c.PercpuLoad)
@ -130,7 +134,8 @@ func makeWidget(c gotop.Config, widRule widgetRule) interface{} {
 		}
 		w = cpu
 	case "disk":
-		w = widgets.NewDiskWidget()
+		dw := widgets.NewDiskWidget()
+		w = dw
 	case "mem":
 		m := widgets.NewMemWidget(c.UpdateInterval, c.GraphHorizontalScale)
 		m.LineColors["Main"] = ui.Color(c.Colorscheme.MainMem)
@ -174,6 +179,9 @@ func makeWidget(c gotop.Config, widRule widgetRule) interface{} {
 		log.Printf("Invalid widget name %s.  Must be one of %v", widRule.Widget, widgetNames)
 		return ui.NewBlock()
 	}
+	if c.ExportPort != "" {
+		w.EnableMetric()
+	}
 	return w
 }

--- a/widgets/battery.go
+++ b/widgets/battery.go
@ -8,6 +8,7 @@ import (
 	"time"

 	"github.com/distatus/battery"
+	"github.com/prometheus/client_golang/prometheus"

 	ui "github.com/xxxserxxx/gotop/termui"
 )
@ -15,6 +16,7 @@ import (
 type BatteryWidget struct {
 	*ui.LineGraph
 	updateInterval time.Duration
+	metric         []prometheus.Gauge
 }

 func NewBatteryWidget(horizontalScale int) *BatteryWidget {
@ -41,6 +43,25 @@ func NewBatteryWidget(horizontalScale int) *BatteryWidget {
 	return self
 }

+func (b *BatteryWidget) EnableMetric() {
+	bats, err := battery.GetAll()
+	if err != nil {
+		log.Printf("error setting up metrics: %v", err)
+		return
+	}
+	b.metric = make([]prometheus.Gauge, len(bats))
+	for i, bat := range bats {
+		gauge := prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: "gotop",
+			Subsystem: "battery",
+			Name:      fmt.Sprintf("%d", i),
+		})
+		gauge.Set(bat.Current / bat.Full)
+		b.metric[i] = gauge
+		prometheus.MustRegister(gauge)
+	}
+}
+
 func makeId(i int) string {
 	return "Batt" + strconv.Itoa(i)
 }
@ -57,8 +78,12 @@ func (self *BatteryWidget) update() {
 	}
 	for i, battery := range batteries {
 		id := makeId(i)
-		percentFull := math.Abs(battery.Current/battery.Full) * 100.0
+		perc := battery.Current / battery.Full
+		percentFull := math.Abs(perc) * 100.0
 		self.Data[id] = append(self.Data[id], percentFull)
 		self.Labels[id] = fmt.Sprintf("%3.0f%% %.0f/%.0f", percentFull, math.Abs(battery.Current), math.Abs(battery.Full))
+		if self.metric != nil {
+			self.metric[i].Set(perc)
+		}
 	}
 }
--- a/widgets/cpu.go
+++ b/widgets/cpu.go
@ -1,11 +1,13 @@
 package widgets

 import (
+	"context"
 	"fmt"
 	"log"
 	"sync"
 	"time"

+	"github.com/prometheus/client_golang/prometheus"
 	psCpu "github.com/shirou/gopsutil/cpu"

 	ui "github.com/xxxserxxx/gotop/termui"
@ -19,6 +21,7 @@ type CpuWidget struct {
 	updateInterval  time.Duration
 	formatString    string
 	updateLock      sync.Mutex
+	metric          []prometheus.Gauge
 }

 func NewCpuWidget(updateInterval time.Duration, horizontalScale int, showAverageLoad bool, showPerCpuLoad bool) *CpuWidget {
@ -71,6 +74,35 @@ func NewCpuWidget(updateInterval time.Duration, horizontalScale int, showAverage
 	return self
 }

+func (self *CpuWidget) EnableMetric() {
+	if self.ShowAverageLoad {
+		self.metric = make([]prometheus.Gauge, 1)
+		self.metric[0] = prometheus.NewGauge(prometheus.GaugeOpts{
+			Subsystem: "cpu",
+			Name:      "avg",
+		})
+	} else {
+		ctx, ccl := context.WithTimeout(context.Background(), time.Second*5)
+		defer ccl()
+		percents, err := psCpu.PercentWithContext(ctx, self.updateInterval, true)
+		if err != nil {
+			log.Printf("error setting up metrics: %v", err)
+			return
+		}
+		self.metric = make([]prometheus.Gauge, self.CpuCount)
+		for i, perc := range percents {
+			gauge := prometheus.NewGauge(prometheus.GaugeOpts{
+				Namespace: "gotop",
+				Subsystem: "cpu",
+				Name:      fmt.Sprintf("%d", i),
+			})
+			gauge.Set(perc)
+			prometheus.MustRegister(gauge)
+			self.metric[i] = gauge
+		}
+	}
+}
+
 func (b *CpuWidget) Scale(i int) {
 	b.LineGraph.HorizontalScale = i
 }
@ -88,6 +120,9 @@ func (self *CpuWidget) update() {
 				defer self.updateLock.Unlock()
 				self.Data["AVRG"] = append(self.Data["AVRG"], percent[0])
 				self.Labels["AVRG"] = fmt.Sprintf("%3.0f%%", percent[0])
+				if self.metric != nil {
+					self.metric[0].Set(percent[0])
+				}
 			}
 		}()
 	}
@ -109,6 +144,13 @@ func (self *CpuWidget) update() {
 						key := fmt.Sprintf(self.formatString, i)
 						self.Data[key] = append(self.Data[key], percent)
 						self.Labels[key] = fmt.Sprintf("%3.0f%%", percent)
+						if self.metric != nil {
+							if self.metric[i] == nil {
+								log.Printf("ERROR: not enough metrics %d", i)
+							} else {
+								self.metric[i].Set(percent)
+							}
+						}
 					}
 				}
 			}
--- a/widgets/disk.go
+++ b/widgets/disk.go
@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"

+	"github.com/prometheus/client_golang/prometheus"
 	psDisk "github.com/shirou/gopsutil/disk"

 	ui "github.com/xxxserxxx/gotop/termui"
@ -28,6 +29,7 @@ type DiskWidget struct {
 	*ui.Table
 	updateInterval time.Duration
 	Partitions     map[string]*Partition
+	metric         map[string]prometheus.Gauge
 }

 func NewDiskWidget() *DiskWidget {
@ -60,6 +62,21 @@ func NewDiskWidget() *DiskWidget {
 	return self
 }

+func (self *DiskWidget) EnableMetric() {
+	self.metric = make(map[string]prometheus.Gauge)
+	for key, part := range self.Partitions {
+		gauge := prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: "gotop",
+			Subsystem: "disk",
+			Name:      strings.ReplaceAll(key, "/", ":"),
+			//Name:      strings.Replace(strings.Replace(part.Device, "/dev/", "", -1), "mapper/", "", -1),
+		})
+		gauge.Set(float64(part.UsedPercent) / 100.0)
+		prometheus.MustRegister(gauge)
+		self.metric[key] = gauge
+	}
+}
+
 func (self *DiskWidget) update() {
 	partitions, err := psDisk.Partitions(false)
 	if err != nil {
@ -158,5 +175,12 @@ func (self *DiskWidget) update() {
 		self.Rows[i][3] = partition.Free
 		self.Rows[i][4] = partition.BytesReadRecently
 		self.Rows[i][5] = partition.BytesWrittenRecently
+		if self.metric != nil {
+			if self.metric[key] == nil {
+				log.Printf("ERROR: missing metric %s", key)
+			} else {
+				self.metric[key].Set(float64(partition.UsedPercent) / 100.0)
+			}
+		}
 	}
 }
--- a/widgets/mem.go
+++ b/widgets/mem.go
@ -5,6 +5,7 @@ import (
 	"log"
 	"time"

+	"github.com/prometheus/client_golang/prometheus"
 	psMem "github.com/shirou/gopsutil/mem"

 	ui "github.com/xxxserxxx/gotop/termui"
@ -14,6 +15,8 @@ import (
 type MemWidget struct {
 	*ui.LineGraph
 	updateInterval time.Duration
+	mainMetric     prometheus.Gauge
+	swapMetric     prometheus.Gauge
 }

 type MemoryInfo struct {
@ -45,6 +48,9 @@ func (self *MemWidget) updateMainMemory() {
 			Used:        mainMemory.Used,
 			UsedPercent: mainMemory.UsedPercent,
 		})
+		if self.mainMetric != nil {
+			self.mainMetric.Set(mainMemory.UsedPercent)
+		}
 	}
 }

@ -73,6 +79,30 @@ func NewMemWidget(updateInterval time.Duration, horizontalScale int) *MemWidget
 	return self
 }

+func (b *MemWidget) EnableMetric() {
+	b.mainMetric = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "gotop",
+		Subsystem: "memory",
+		Name:      "main",
+	})
+	mainMemory, err := psMem.VirtualMemory()
+	if err == nil {
+		b.mainMetric.Set(mainMemory.UsedPercent)
+	}
+	prometheus.MustRegister(b.mainMetric)
+
+	b.swapMetric = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "gotop",
+		Subsystem: "memory",
+		Name:      "swap",
+	})
+	swapMemory, err := psMem.SwapMemory()
+	if err == nil {
+		b.swapMetric.Set(swapMemory.UsedPercent)
+	}
+	prometheus.MustRegister(b.swapMetric)
+}
+
 func (b *MemWidget) Scale(i int) {
 	b.LineGraph.HorizontalScale = i
 }
--- a/widgets/mem_other.go
+++ b/widgets/mem_other.go
@ -18,5 +18,8 @@ func (self *MemWidget) updateSwapMemory() {
 			Used:        swapMemory.Used,
 			UsedPercent: swapMemory.UsedPercent,
 		})
+		if self.swapMetric != nil {
+			self.swapMetric.Set(swapMemory.UsedPercent)
+		}
 	}
 }
--- a/widgets/net.go
+++ b/widgets/net.go
@ -6,6 +6,7 @@ import (
 	"strings"
 	"time"

+	"github.com/prometheus/client_golang/prometheus"
 	psNet "github.com/shirou/gopsutil/net"

 	ui "github.com/xxxserxxx/gotop/termui"
@ -25,6 +26,8 @@ type NetWidget struct {
 	totalBytesRecv uint64
 	totalBytesSent uint64
 	NetInterface   []string
+	sentMetric     prometheus.Counter
+	recvMetric     prometheus.Counter
 }

 func NewNetWidget(netInterface string) *NetWidget {
@ -58,6 +61,22 @@ func NewNetWidget(netInterface string) *NetWidget {
 	return self
 }

+func (b *NetWidget) EnableMetric() {
+	b.recvMetric = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "gotop",
+		Subsystem: "net",
+		Name:      "recv",
+	})
+	prometheus.MustRegister(b.recvMetric)
+
+	b.sentMetric = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "gotop",
+		Subsystem: "net",
+		Name:      "sent",
+	})
+	prometheus.MustRegister(b.sentMetric)
+}
+
 func (self *NetWidget) update() {
 	interfaces, err := psNet.IOCounters(true)
 	if err != nil {
@ -114,6 +133,8 @@ func (self *NetWidget) update() {

 		self.Lines[0].Data = append(self.Lines[0].Data, int(recentBytesRecv))
 		self.Lines[1].Data = append(self.Lines[1].Data, int(recentBytesSent))
+		self.sentMetric.Add(float64(recentBytesSent))
+		self.recvMetric.Add(float64(recentBytesRecv))
 	}

 	// used in later calls to update
--- a/widgets/proc.go
+++ b/widgets/proc.go
@ -100,6 +100,10 @@ func NewProcWidget() *ProcWidget {
 	return self
 }

+func (p *ProcWidget) EnableMetric() {
+	// There's (currently) no metric for this
+}
+
 func (self *ProcWidget) SetEditingFilter(editing bool) {
 	self.entry.SetEditing(editing)
 }
--- a/widgets/temp.go
+++ b/widgets/temp.go
@ -7,6 +7,7 @@ import (
 	"time"

 	ui "github.com/gizak/termui/v3"
+	"github.com/prometheus/client_golang/prometheus"

 	"github.com/xxxserxxx/gotop/utils"
 )
@ -27,6 +28,7 @@ type TempWidget struct {
 	TempLowColor   ui.Color
 	TempHighColor  ui.Color
 	TempScale      TempScale
+	tempsMetric    map[string]prometheus.Gauge
 }

 func NewTempWidget(tempScale TempScale) *TempWidget {
@ -56,6 +58,20 @@ func NewTempWidget(tempScale TempScale) *TempWidget {
 	return self
 }

+func (self *TempWidget) EnableMetric() {
+	self.tempsMetric = make(map[string]prometheus.Gauge)
+	for k, v := range self.Data {
+		gauge := prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: "gotop",
+			Subsystem: "temp",
+			Name:      k,
+		})
+		gauge.Set(float64(v))
+		prometheus.MustRegister(gauge)
+		self.tempsMetric[k] = gauge
+	}
+}
+
 // Custom Draw method instead of inheriting from a generic Widget.
 func (self *TempWidget) Draw(buf *ui.Buffer) {
 	self.Block.Draw(buf)
@ -98,5 +114,6 @@ func (self *TempWidget) Draw(buf *ui.Buffer) {
 				image.Pt(self.Inner.Max.X-4, self.Inner.Min.Y+y),
 			)
 		}
+		self.tempsMetric[key].Set(float64(self.Data[key]))
 	}
 }