Change pprof labels to be prometheus compatible (#32865)

Enables scrapping pprof endpoint for continuous profiling

Closes: https://github.com/go-gitea/gitea/issues/32854
This commit is contained in:
TheFox0x7 2024-12-18 04:08:04 +01:00 committed by GitHub
parent 195fccd617
commit b945742293
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 16 additions and 10 deletions

View File

@ -136,7 +136,7 @@ func (g *Manager) doShutdown() {
} }
g.lock.Lock() g.lock.Lock()
g.shutdownCtxCancel() g.shutdownCtxCancel()
atShutdownCtx := pprof.WithLabels(g.hammerCtx, pprof.Labels("graceful-lifecycle", "post-shutdown")) atShutdownCtx := pprof.WithLabels(g.hammerCtx, pprof.Labels(LifecyclePProfLabel, "post-shutdown"))
pprof.SetGoroutineLabels(atShutdownCtx) pprof.SetGoroutineLabels(atShutdownCtx)
for _, fn := range g.toRunAtShutdown { for _, fn := range g.toRunAtShutdown {
go fn() go fn()
@ -167,7 +167,7 @@ func (g *Manager) doHammerTime(d time.Duration) {
default: default:
log.Warn("Setting Hammer condition") log.Warn("Setting Hammer condition")
g.hammerCtxCancel() g.hammerCtxCancel()
atHammerCtx := pprof.WithLabels(g.terminateCtx, pprof.Labels("graceful-lifecycle", "post-hammer")) atHammerCtx := pprof.WithLabels(g.terminateCtx, pprof.Labels(LifecyclePProfLabel, "post-hammer"))
pprof.SetGoroutineLabels(atHammerCtx) pprof.SetGoroutineLabels(atHammerCtx)
} }
g.lock.Unlock() g.lock.Unlock()
@ -183,7 +183,7 @@ func (g *Manager) doTerminate() {
default: default:
log.Warn("Terminating") log.Warn("Terminating")
g.terminateCtxCancel() g.terminateCtxCancel()
atTerminateCtx := pprof.WithLabels(g.managerCtx, pprof.Labels("graceful-lifecycle", "post-terminate")) atTerminateCtx := pprof.WithLabels(g.managerCtx, pprof.Labels(LifecyclePProfLabel, "post-terminate"))
pprof.SetGoroutineLabels(atTerminateCtx) pprof.SetGoroutineLabels(atTerminateCtx)
for _, fn := range g.toRunAtTerminate { for _, fn := range g.toRunAtTerminate {

View File

@ -22,6 +22,12 @@ const (
watchdogMsg systemdNotifyMsg = "WATCHDOG=1" watchdogMsg systemdNotifyMsg = "WATCHDOG=1"
) )
// LifecyclePProfLabel is a label marking manager lifecycle phase
// Making it compliant with prometheus key regex https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
// would enable someone interested to be able to to continuously gather profiles into pyroscope.
// Other labels for pprof (in "modules/process" package) should also follow this rule.
const LifecyclePProfLabel = "graceful_lifecycle"
func statusMsg(msg string) systemdNotifyMsg { func statusMsg(msg string) systemdNotifyMsg {
return systemdNotifyMsg("STATUS=" + msg) return systemdNotifyMsg("STATUS=" + msg)
} }
@ -65,10 +71,10 @@ func (g *Manager) prepare(ctx context.Context) {
g.hammerCtx, g.hammerCtxCancel = context.WithCancel(ctx) g.hammerCtx, g.hammerCtxCancel = context.WithCancel(ctx)
g.managerCtx, g.managerCtxCancel = context.WithCancel(ctx) g.managerCtx, g.managerCtxCancel = context.WithCancel(ctx)
g.terminateCtx = pprof.WithLabels(g.terminateCtx, pprof.Labels("graceful-lifecycle", "with-terminate")) g.terminateCtx = pprof.WithLabels(g.terminateCtx, pprof.Labels(LifecyclePProfLabel, "with-terminate"))
g.shutdownCtx = pprof.WithLabels(g.shutdownCtx, pprof.Labels("graceful-lifecycle", "with-shutdown")) g.shutdownCtx = pprof.WithLabels(g.shutdownCtx, pprof.Labels(LifecyclePProfLabel, "with-shutdown"))
g.hammerCtx = pprof.WithLabels(g.hammerCtx, pprof.Labels("graceful-lifecycle", "with-hammer")) g.hammerCtx = pprof.WithLabels(g.hammerCtx, pprof.Labels(LifecyclePProfLabel, "with-hammer"))
g.managerCtx = pprof.WithLabels(g.managerCtx, pprof.Labels("graceful-lifecycle", "with-manager")) g.managerCtx = pprof.WithLabels(g.managerCtx, pprof.Labels(LifecyclePProfLabel, "with-manager"))
if !g.setStateTransition(stateInit, stateRunning) { if !g.setStateTransition(stateInit, stateRunning) {
panic("invalid graceful manager state: transition from init to running failed") panic("invalid graceful manager state: transition from init to running failed")

View File

@ -32,7 +32,7 @@ func (c *Context) Value(key any) any {
} }
// ProcessContextKey is the key under which process contexts are stored // ProcessContextKey is the key under which process contexts are stored
var ProcessContextKey any = "process-context" var ProcessContextKey any = "process_context"
// GetContext will return a process context if one exists // GetContext will return a process context if one exists
func GetContext(ctx context.Context) *Context { func GetContext(ctx context.Context) *Context {

View File

@ -26,7 +26,7 @@ var (
) )
// DescriptionPProfLabel is a label set on goroutines that have a process attached // DescriptionPProfLabel is a label set on goroutines that have a process attached
const DescriptionPProfLabel = "process-description" const DescriptionPProfLabel = "process_description"
// PIDPProfLabel is a label set on goroutines that have a process attached // PIDPProfLabel is a label set on goroutines that have a process attached
const PIDPProfLabel = "pid" const PIDPProfLabel = "pid"
@ -35,7 +35,7 @@ const PIDPProfLabel = "pid"
const PPIDPProfLabel = "ppid" const PPIDPProfLabel = "ppid"
// ProcessTypePProfLabel is a label set on goroutines that have a process attached // ProcessTypePProfLabel is a label set on goroutines that have a process attached
const ProcessTypePProfLabel = "process-type" const ProcessTypePProfLabel = "process_type"
// IDType is a pid type // IDType is a pid type
type IDType string type IDType string