diff --git a/CHANGELOG.md b/CHANGELOG.md index 32bd5e4..1bb2384 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 > - **Fixed**: for any bug fixes. > - **Security**: in case of vulnerabilities. -## [3.3.0] - +## [3.x.x] - + +- Adds metrics. If run with the `--export :2112` flag (`:2112` is a port), + metrics are exposed as Prometheus metrics on that port and can be HTTP + GET-ted. + + +## [3.3.0] - 2020-02-17 - Added: Logs are now rotated. Settings are currently hard-coded at 4 files of 5MB each, so logs shouldn't take up more than 20MB. I'm going to see how many diff --git a/README.md b/README.md index a3ff80c..af66750 100644 --- a/README.md +++ b/README.md @@ -97,9 +97,9 @@ and these are separated by spaces. 4. Legal widget names are: cpu, disk, mem, temp, batt, net, procs 5. Widget names are not case sensitive 4. The simplest row is a single widget, by name, e.g. - ``` - cpu - ``` + ``` + cpu + ``` 5. **Weights** 1. Widgets with no weights have a weight of 1. 2. If multiple widgets are put on a row with no weights, they will all have @@ -117,19 +117,19 @@ and these are separated by spaces. second will be 5/7 ~= 67% wide (or, memory will be twice as wide as disk). 9. If prefixed by a number and colon, the widget will span that number of rows downward. E.g. - ``` - mem 2:cpu - net - ``` + ``` + mem 2:cpu + net + ``` Here, memory and network will be in the same row as CPU, one over the other, and each half as high as CPU; it'll look like this: - ``` - +------+------+ - | Mem | | - +------+ CPU | - | Net | | - +------+------+ - ``` + ``` + +------+------+ + | Mem | | + +------+ CPU | + | Net | | + +------+------+ + ``` 10. Negative, 0, or non-integer weights will be recorded as "1". Same for row spans. 11. Unrecognized widget names will cause the application to abort. @@ -146,23 +146,71 @@ and these are separated by spaces. Yes, you're clever enough to break the layout algorithm, but if you try to build massive edifices, you're in for disappointment. +### Metrics + +gotop can export widget data as Prometheus metrics. This allows users to take +snapshots of the current state of a machine running gotop, or to query gotop +remotely. + +All metrics are in the `gotop` namespace, and are tagged with +`goto__`. Metrics are only exported for widgets +that are enabled, and are updated with the same frequency as the configured +update interval. Most widgets are exported as Prometheus gauges. + +Metrics are disabled by default, and must be enabled with the `--export` flag. +The flag takes an interface port in the idiomatic Go format of +`:`; a common pattern is `-x :2112`. There is **no security** +on this feature; I recommend that you run this bound to a localhost interface, +e.g. `127.0.0.1:7653`, and if you want to access this remotely, run it behind +a proxy that provides SSL and authentication such as +[Caddy](https://caddyserver.com). + +Once enabled, any widgets that are enabled will appear in the HTTP payload of +a call to `http://:/metrics`. For example, + +``` +➜ ~ curl -s http://localhost:2112/metrics | egrep -e '^gotop' +gotop_battery_0 0.6387792286668692 +gotop_cpu_0 12.871287128721228 +gotop_cpu_1 11.000000000001364 +gotop_disk_:dev:nvme0n1p1 0.63 +gotop_memory_main 49.932259713701434 +gotop_memory_swap 0 +gotop_net_recv 129461 +gotop_net_sent 218525 +gotop_temp_coretemp_core0 37 +gotop_temp_coretemp_core1 37 +``` + +Disk metrics are reformatted to replace `/` with `:` which makes them legal +Prometheus names: + +``` +➜ ~ curl -s http://localhost:2112/metrics | egrep -e '^gotop_disk' | tr ':' '/' +gotop_disk_/dev/nvme0n1p1 0.63 +``` + +This feature satisfies a ticket request to provide a "snapshot" for comparison +with a known state, but it is also foundational for a future feature where +widgets can be configured with virtual devices fed by data from remote gotop +instances. The objective for that feature is to allow monitoring of multiple +remote VMs without having to have a wall of gotops running on a large monitor. + ### CLI Options `-c`, `--color=NAME` Set a colorscheme. -`-m`, `--minimal` Only show CPU, Mem and Process widgets. (DEPRECATED for `-l minimal`) +`-m`, `--minimal` Only show CPU, Mem and Process widgets. (DEPRECATED for `-l minimal`) `-r`, `--rate=RATE` Number of times per second to update CPU and Mem widgets [default: 1]. `-V`, `--version` Print version and exit. `-p`, `--percpu` Show each CPU in the CPU widget. `-a`, `--averagecpu` Show average CPU in the CPU widget. `-f`, `--fahrenheit` Show temperatures in fahrenheit. `-s`, `--statusbar` Show a statusbar with the time. -`-b`, `--battery` Show battery level widget (`minimal` turns off). [preview](./assets/screenshots/battery.png) (DEPRECATED for `-l battery`) -`-i`, `--interface=NAME` Select network interface [default: all]. -`-l`, `--layout=NAME` Choose a layout. gotop searches for a file by NAME in \$XDG_CONFIG_HOME/gotop, then relative to the current path. "-" reads a layout from stdin, allowing for simple, one-off layouts such as `echo net | gotop -l -` +`-b`, `--battery` Show battery level widget (`minimal` turns off). [preview](./assets/screenshots/battery.png) (DEPRECATED for `-l battery`) +`-i`, `--interface=NAME` Select network interface. Several interfaces can be defined using comma separated values. Interfaces can also be ignored by prefixing the interface with `!` [default: all]. +`-l`, `--layout=NAME` Choose a layout. gotop searches for a file by NAME in \$XDG_CONFIG_HOME/gotop, then relative to the current path. "-" reads a layout from stdin, allowing for simple, one-off layouts such as `echo net | gotop -l -` +`-x`, `--export=PORT` Enable metrics for export on the specified port. This feature is disabled by default. -Several interfaces can be defined using comma separated values. - -Interfaces can also be ignored using `!` ## Built With @@ -172,3 +220,4 @@ Interfaces can also be ignored using `!` - [shirou/gopsutil](https://github.com/shirou/gopsutil) - [goreleaser/nfpm](https://github.com/goreleaser/nfpm) - [distatus/battery](https://github.com/distatus/battery) +- [prometheus/client_golang](https://github.com/prometheus/client_golang) diff --git a/cmd/gotop/main.go b/cmd/gotop/main.go index 205a457..2462433 100644 --- a/cmd/gotop/main.go +++ b/cmd/gotop/main.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "log" + "net/http" "os" "os/signal" "path/filepath" @@ -14,6 +15,7 @@ import ( docopt "github.com/docopt/docopt.go" ui "github.com/gizak/termui/v3" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/xxxserxxx/gotop" "github.com/xxxserxxx/gotop/colorschemes" @@ -59,6 +61,7 @@ Options: -B, --bandwidth=bits Specify the number of bits per seconds. -l, --layout=NAME Name of layout spec file for the UI. Looks first in $XDG_CONFIG_HOME/gotop, then as a path. Use "-" to pipe. -i, --interface=NAME Select network interface [default: all]. + -x, --export=PORT Enable metrics for export on the specified port. Several interfaces can be defined using comma separated values. @@ -110,8 +113,11 @@ Colorschemes: if args["--minimal"].(bool) { conf.Layout = "minimal" } - if val, _ := args["--statusbar"]; val != nil { - rateStr, _ := args["--rate"].(string) + if val, _ := args["--export"]; val != nil { + conf.ExportPort = val.(string) + } + if val, _ := args["--rate"]; val != nil { + rateStr, _ := val.(string) rate, err := strconv.ParseFloat(rateStr, 64) if err != nil { return fmt.Errorf("invalid rate parameter") @@ -330,7 +336,7 @@ func makeConfig() gotop.Config { HelpVisible: false, UpdateInterval: time.Second, AverageLoad: false, - PercpuLoad: false, + PercpuLoad: true, TempScale: w.Celsius, Statusbar: false, NetInterface: w.NET_INTERFACE_ALL, @@ -395,6 +401,12 @@ func main() { ui.Render(bar) } + if conf.ExportPort != "" { + go func() { + http.Handle("/metrics", promhttp.Handler()) + http.ListenAndServe(conf.ExportPort, nil) + }() + } eventLoop(conf, grid) } diff --git a/config.go b/config.go index 0cd9e58..946ebdb 100644 --- a/config.go +++ b/config.go @@ -38,6 +38,7 @@ type Config struct { NetInterface string Layout string MaxLogSize int64 + ExportPort string } func Parse(in io.Reader, conf *Config) error { @@ -117,6 +118,8 @@ func Parse(in io.Reader, conf *Config) error { return err } conf.MaxLogSize = int64(iv) + case "export": + conf.ExportPort = kv[1] } } diff --git a/layout/layout.go b/layout/layout.go index c78bb67..f72cc0b 100644 --- a/layout/layout.go +++ b/layout/layout.go @@ -108,8 +108,12 @@ func processRow(c gotop.Config, numRows int, rowDefs [][]widgetRule) (ui.GridIte return ui.NewRow(1.0/float64(numRows), uiColumns...), rowDefs } +type Metric interface { + EnableMetric() +} + func makeWidget(c gotop.Config, widRule widgetRule) interface{} { - var w interface{} + var w Metric switch widRule.Widget { case "cpu": cpu := widgets.NewCpuWidget(c.UpdateInterval, c.GraphHorizontalScale, c.AverageLoad, c.PercpuLoad) @@ -130,7 +134,8 @@ func makeWidget(c gotop.Config, widRule widgetRule) interface{} { } w = cpu case "disk": - w = widgets.NewDiskWidget() + dw := widgets.NewDiskWidget() + w = dw case "mem": m := widgets.NewMemWidget(c.UpdateInterval, c.GraphHorizontalScale) m.LineColors["Main"] = ui.Color(c.Colorscheme.MainMem) @@ -174,6 +179,9 @@ func makeWidget(c gotop.Config, widRule widgetRule) interface{} { log.Printf("Invalid widget name %s. Must be one of %v", widRule.Widget, widgetNames) return ui.NewBlock() } + if c.ExportPort != "" { + w.EnableMetric() + } return w } diff --git a/widgets/battery.go b/widgets/battery.go index 89620bb..fb28971 100644 --- a/widgets/battery.go +++ b/widgets/battery.go @@ -8,6 +8,7 @@ import ( "time" "github.com/distatus/battery" + "github.com/prometheus/client_golang/prometheus" ui "github.com/xxxserxxx/gotop/termui" ) @@ -15,6 +16,7 @@ import ( type BatteryWidget struct { *ui.LineGraph updateInterval time.Duration + metric []prometheus.Gauge } func NewBatteryWidget(horizontalScale int) *BatteryWidget { @@ -41,6 +43,25 @@ func NewBatteryWidget(horizontalScale int) *BatteryWidget { return self } +func (b *BatteryWidget) EnableMetric() { + bats, err := battery.GetAll() + if err != nil { + log.Printf("error setting up metrics: %v", err) + return + } + b.metric = make([]prometheus.Gauge, len(bats)) + for i, bat := range bats { + gauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "gotop", + Subsystem: "battery", + Name: fmt.Sprintf("%d", i), + }) + gauge.Set(bat.Current / bat.Full) + b.metric[i] = gauge + prometheus.MustRegister(gauge) + } +} + func makeId(i int) string { return "Batt" + strconv.Itoa(i) } @@ -57,8 +78,12 @@ func (self *BatteryWidget) update() { } for i, battery := range batteries { id := makeId(i) - percentFull := math.Abs(battery.Current/battery.Full) * 100.0 + perc := battery.Current / battery.Full + percentFull := math.Abs(perc) * 100.0 self.Data[id] = append(self.Data[id], percentFull) self.Labels[id] = fmt.Sprintf("%3.0f%% %.0f/%.0f", percentFull, math.Abs(battery.Current), math.Abs(battery.Full)) + if self.metric != nil { + self.metric[i].Set(perc) + } } } diff --git a/widgets/cpu.go b/widgets/cpu.go index 8e8819c..37c4220 100644 --- a/widgets/cpu.go +++ b/widgets/cpu.go @@ -1,11 +1,13 @@ package widgets import ( + "context" "fmt" "log" "sync" "time" + "github.com/prometheus/client_golang/prometheus" psCpu "github.com/shirou/gopsutil/cpu" ui "github.com/xxxserxxx/gotop/termui" @@ -19,6 +21,7 @@ type CpuWidget struct { updateInterval time.Duration formatString string updateLock sync.Mutex + metric []prometheus.Gauge } func NewCpuWidget(updateInterval time.Duration, horizontalScale int, showAverageLoad bool, showPerCpuLoad bool) *CpuWidget { @@ -71,6 +74,35 @@ func NewCpuWidget(updateInterval time.Duration, horizontalScale int, showAverage return self } +func (self *CpuWidget) EnableMetric() { + if self.ShowAverageLoad { + self.metric = make([]prometheus.Gauge, 1) + self.metric[0] = prometheus.NewGauge(prometheus.GaugeOpts{ + Subsystem: "cpu", + Name: "avg", + }) + } else { + ctx, ccl := context.WithTimeout(context.Background(), time.Second*5) + defer ccl() + percents, err := psCpu.PercentWithContext(ctx, self.updateInterval, true) + if err != nil { + log.Printf("error setting up metrics: %v", err) + return + } + self.metric = make([]prometheus.Gauge, self.CpuCount) + for i, perc := range percents { + gauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "gotop", + Subsystem: "cpu", + Name: fmt.Sprintf("%d", i), + }) + gauge.Set(perc) + prometheus.MustRegister(gauge) + self.metric[i] = gauge + } + } +} + func (b *CpuWidget) Scale(i int) { b.LineGraph.HorizontalScale = i } @@ -88,6 +120,9 @@ func (self *CpuWidget) update() { defer self.updateLock.Unlock() self.Data["AVRG"] = append(self.Data["AVRG"], percent[0]) self.Labels["AVRG"] = fmt.Sprintf("%3.0f%%", percent[0]) + if self.metric != nil { + self.metric[0].Set(percent[0]) + } } }() } @@ -109,6 +144,13 @@ func (self *CpuWidget) update() { key := fmt.Sprintf(self.formatString, i) self.Data[key] = append(self.Data[key], percent) self.Labels[key] = fmt.Sprintf("%3.0f%%", percent) + if self.metric != nil { + if self.metric[i] == nil { + log.Printf("ERROR: not enough metrics %d", i) + } else { + self.metric[i].Set(percent) + } + } } } } diff --git a/widgets/disk.go b/widgets/disk.go index d20b078..d1da7a6 100644 --- a/widgets/disk.go +++ b/widgets/disk.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/prometheus/client_golang/prometheus" psDisk "github.com/shirou/gopsutil/disk" ui "github.com/xxxserxxx/gotop/termui" @@ -28,6 +29,7 @@ type DiskWidget struct { *ui.Table updateInterval time.Duration Partitions map[string]*Partition + metric map[string]prometheus.Gauge } func NewDiskWidget() *DiskWidget { @@ -60,6 +62,21 @@ func NewDiskWidget() *DiskWidget { return self } +func (self *DiskWidget) EnableMetric() { + self.metric = make(map[string]prometheus.Gauge) + for key, part := range self.Partitions { + gauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "gotop", + Subsystem: "disk", + Name: strings.ReplaceAll(key, "/", ":"), + //Name: strings.Replace(strings.Replace(part.Device, "/dev/", "", -1), "mapper/", "", -1), + }) + gauge.Set(float64(part.UsedPercent) / 100.0) + prometheus.MustRegister(gauge) + self.metric[key] = gauge + } +} + func (self *DiskWidget) update() { partitions, err := psDisk.Partitions(false) if err != nil { @@ -158,5 +175,12 @@ func (self *DiskWidget) update() { self.Rows[i][3] = partition.Free self.Rows[i][4] = partition.BytesReadRecently self.Rows[i][5] = partition.BytesWrittenRecently + if self.metric != nil { + if self.metric[key] == nil { + log.Printf("ERROR: missing metric %s", key) + } else { + self.metric[key].Set(float64(partition.UsedPercent) / 100.0) + } + } } } diff --git a/widgets/mem.go b/widgets/mem.go index d5bd67d..1c85f3b 100644 --- a/widgets/mem.go +++ b/widgets/mem.go @@ -5,6 +5,7 @@ import ( "log" "time" + "github.com/prometheus/client_golang/prometheus" psMem "github.com/shirou/gopsutil/mem" ui "github.com/xxxserxxx/gotop/termui" @@ -14,6 +15,8 @@ import ( type MemWidget struct { *ui.LineGraph updateInterval time.Duration + mainMetric prometheus.Gauge + swapMetric prometheus.Gauge } type MemoryInfo struct { @@ -45,6 +48,9 @@ func (self *MemWidget) updateMainMemory() { Used: mainMemory.Used, UsedPercent: mainMemory.UsedPercent, }) + if self.mainMetric != nil { + self.mainMetric.Set(mainMemory.UsedPercent) + } } } @@ -73,6 +79,30 @@ func NewMemWidget(updateInterval time.Duration, horizontalScale int) *MemWidget return self } +func (b *MemWidget) EnableMetric() { + b.mainMetric = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "gotop", + Subsystem: "memory", + Name: "main", + }) + mainMemory, err := psMem.VirtualMemory() + if err == nil { + b.mainMetric.Set(mainMemory.UsedPercent) + } + prometheus.MustRegister(b.mainMetric) + + b.swapMetric = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "gotop", + Subsystem: "memory", + Name: "swap", + }) + swapMemory, err := psMem.SwapMemory() + if err == nil { + b.swapMetric.Set(swapMemory.UsedPercent) + } + prometheus.MustRegister(b.swapMetric) +} + func (b *MemWidget) Scale(i int) { b.LineGraph.HorizontalScale = i } diff --git a/widgets/mem_other.go b/widgets/mem_other.go index 6b78acb..c42ef5a 100644 --- a/widgets/mem_other.go +++ b/widgets/mem_other.go @@ -18,5 +18,8 @@ func (self *MemWidget) updateSwapMemory() { Used: swapMemory.Used, UsedPercent: swapMemory.UsedPercent, }) + if self.swapMetric != nil { + self.swapMetric.Set(swapMemory.UsedPercent) + } } } diff --git a/widgets/net.go b/widgets/net.go index c4ac8e8..ae14e8a 100644 --- a/widgets/net.go +++ b/widgets/net.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/prometheus/client_golang/prometheus" psNet "github.com/shirou/gopsutil/net" ui "github.com/xxxserxxx/gotop/termui" @@ -25,6 +26,8 @@ type NetWidget struct { totalBytesRecv uint64 totalBytesSent uint64 NetInterface []string + sentMetric prometheus.Counter + recvMetric prometheus.Counter } func NewNetWidget(netInterface string) *NetWidget { @@ -58,6 +61,22 @@ func NewNetWidget(netInterface string) *NetWidget { return self } +func (b *NetWidget) EnableMetric() { + b.recvMetric = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "gotop", + Subsystem: "net", + Name: "recv", + }) + prometheus.MustRegister(b.recvMetric) + + b.sentMetric = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "gotop", + Subsystem: "net", + Name: "sent", + }) + prometheus.MustRegister(b.sentMetric) +} + func (self *NetWidget) update() { interfaces, err := psNet.IOCounters(true) if err != nil { @@ -114,6 +133,8 @@ func (self *NetWidget) update() { self.Lines[0].Data = append(self.Lines[0].Data, int(recentBytesRecv)) self.Lines[1].Data = append(self.Lines[1].Data, int(recentBytesSent)) + self.sentMetric.Add(float64(recentBytesSent)) + self.recvMetric.Add(float64(recentBytesRecv)) } // used in later calls to update diff --git a/widgets/proc.go b/widgets/proc.go index 9fed067..e4a8ace 100644 --- a/widgets/proc.go +++ b/widgets/proc.go @@ -100,6 +100,10 @@ func NewProcWidget() *ProcWidget { return self } +func (p *ProcWidget) EnableMetric() { + // There's (currently) no metric for this +} + func (self *ProcWidget) SetEditingFilter(editing bool) { self.entry.SetEditing(editing) } diff --git a/widgets/temp.go b/widgets/temp.go index 86e6e76..080531a 100644 --- a/widgets/temp.go +++ b/widgets/temp.go @@ -7,6 +7,7 @@ import ( "time" ui "github.com/gizak/termui/v3" + "github.com/prometheus/client_golang/prometheus" "github.com/xxxserxxx/gotop/utils" ) @@ -27,6 +28,7 @@ type TempWidget struct { TempLowColor ui.Color TempHighColor ui.Color TempScale TempScale + tempsMetric map[string]prometheus.Gauge } func NewTempWidget(tempScale TempScale) *TempWidget { @@ -56,6 +58,20 @@ func NewTempWidget(tempScale TempScale) *TempWidget { return self } +func (self *TempWidget) EnableMetric() { + self.tempsMetric = make(map[string]prometheus.Gauge) + for k, v := range self.Data { + gauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "gotop", + Subsystem: "temp", + Name: k, + }) + gauge.Set(float64(v)) + prometheus.MustRegister(gauge) + self.tempsMetric[k] = gauge + } +} + // Custom Draw method instead of inheriting from a generic Widget. func (self *TempWidget) Draw(buf *ui.Buffer) { self.Block.Draw(buf) @@ -98,5 +114,6 @@ func (self *TempWidget) Draw(buf *ui.Buffer) { image.Pt(self.Inner.Max.X-4, self.Inner.Min.Y+y), ) } + self.tempsMetric[key].Set(float64(self.Data[key])) } }