diff --git a/cmd/cmd.go b/cmd/cmd.go index 2aca5341d..24e8e5160 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -421,11 +421,18 @@ func initConfig() { } // Start the remote control server if configured - _, err = rcserver.Start(context.Background(), &rc.Opt) + _, err = rcserver.Start(ctx, &rc.Opt) if err != nil { log.Fatalf("Failed to start remote control: %v", err) } + // Start the metrics server if configured + _, err = rcserver.MetricsStart(ctx, &rc.Opt) + if err != nil { + log.Fatalf("Failed to start metrics server: %v", err) + + } + // Setup CPU profiling if desired if *cpuProfile != "" { fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile) diff --git a/docs/content/docs.md b/docs/content/docs.md index f8f5153d9..7f6e208e5 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -2836,6 +2836,17 @@ Rclone prefixes all log messages with their level in capitals, e.g. INFO which makes it easy to grep the log file for different kinds of information. +Metrics +------- + +Rclone can publish metrics in the OpenMetrics/Prometheus format. + +To enable the metrics endpoint, use the `--metrics-addr` flag. Metrics can also be published on the `--rc-addr` port if the `--rc` flag and `--rc-enable-metrics` flags are supplied or if using rclone rcd `--rc-enable-metrics` + +Rclone provides extensive configuration options for the metrics HTTP endpoint. These settings are grouped under the Metrics section and have a prefix `--metrics-*`. + +When metrics are enabled with `--rc-enable-metrics`, they will be published on the same port as the rc API. In this case, the `--metrics-*` flags will be ignored, and the HTTP endpoint configuration will be managed by the `--rc-*` parameters. + Exit Code --------- diff --git a/docs/content/rc.md b/docs/content/rc.md index 32e93867b..80c02a591 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -100,6 +100,7 @@ Default Off. ### --rc-enable-metrics Enable OpenMetrics/Prometheus compatible endpoint at `/metrics`. +If more control over the metrics is desired (for example running it on a different port or with different auth) then endpoint can be enabled with the `--metrics-*` flags instead. Default Off. diff --git a/fs/config/flags/flags.go b/fs/config/flags/flags.go index db76f5d71..6389026c8 100644 --- a/fs/config/flags/flags.go +++ b/fs/config/flags/flags.go @@ -124,6 +124,7 @@ func init() { All.NewGroup("Logging", "Flags for logging and statistics") All.NewGroup("Metadata", "Flags to control metadata") All.NewGroup("RC", "Flags to control the Remote Control API") + All.NewGroup("Metrics", "Flags to control the Metrics HTTP endpoint.") } // installFlag constructs a name from the flag passed in and diff --git a/fs/rc/rc.go b/fs/rc/rc.go index 55287ed6b..9c083f3b7 100644 --- a/fs/rc/rc.go +++ b/fs/rc/rc.go @@ -71,8 +71,8 @@ var OptionsInfo = fs.Options{{ }, { Name: "rc_enable_metrics", Default: false, - Help: "Enable prometheus metrics on /metrics", - Groups: "RC", + Help: "Enable the Prometheus metrics path at the remote control server", + Groups: "RC,Metrics", }, { Name: "rc_job_expire_duration", Default: 60 * time.Second, @@ -83,10 +83,18 @@ var OptionsInfo = fs.Options{{ Default: 10 * time.Second, Help: "Interval to check for expired async jobs", Groups: "RC", +}, { + Name: "metrics_addr", + Default: []string{""}, + Help: "IPaddress:Port or :Port to bind metrics server to", + Groups: "Metrics", }}. AddPrefix(libhttp.ConfigInfo, "rc", "RC"). AddPrefix(libhttp.AuthConfigInfo, "rc", "RC"). AddPrefix(libhttp.TemplateConfigInfo, "rc", "RC"). + AddPrefix(libhttp.ConfigInfo, "metrics", "Metrics"). + AddPrefix(libhttp.AuthConfigInfo, "metrics", "Metrics"). + AddPrefix(libhttp.TemplateConfigInfo, "metrics", "Metrics"). SetDefault("rc_addr", []string{"localhost:5572"}) func init() { @@ -109,6 +117,9 @@ type Options struct { WebGUINoOpenBrowser bool `config:"rc_web_gui_no_open_browser"` // set to disable auto opening browser WebGUIFetchURL string `config:"rc_web_fetch_url"` // set the default url for fetching webgui EnableMetrics bool `config:"rc_enable_metrics"` // set to disable prometheus metrics on /metrics + MetricsHTTP libhttp.Config `config:"metrics"` + MetricsAuth libhttp.AuthConfig `config:"metrics"` + MetricsTemplate libhttp.TemplateConfig `config:"metrics"` JobExpireDuration time.Duration `config:"rc_job_expire_duration"` JobExpireInterval time.Duration `config:"rc_job_expire_interval"` } diff --git a/fs/rc/rcserver/metrics.go b/fs/rc/rcserver/metrics.go new file mode 100644 index 000000000..502566f47 --- /dev/null +++ b/fs/rc/rcserver/metrics.go @@ -0,0 +1,97 @@ +// Package rcserver implements the HTTP endpoint to serve the remote control +package rcserver + +import ( + "context" + "fmt" + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/fs/rc" + "github.com/rclone/rclone/fs/rc/jobs" + libhttp "github.com/rclone/rclone/lib/http" +) + +const path = "/metrics" + +var promHandlerFunc http.HandlerFunc + +func init() { + rcloneCollector := accounting.NewRcloneCollector(context.Background()) + prometheus.MustRegister(rcloneCollector) + + m := fshttp.NewMetrics("rclone") + for _, c := range m.Collectors() { + prometheus.MustRegister(c) + } + fshttp.DefaultMetrics = m + + promHandlerFunc = promhttp.Handler().ServeHTTP +} + +// MetricsStart the remote control server if configured +// +// If the server wasn't configured the *Server returned may be nil +func MetricsStart(ctx context.Context, opt *rc.Options) (*MetricsServer, error) { + jobs.SetOpt(opt) // set the defaults for jobs + if opt.MetricsHTTP.ListenAddr[0] != "" { + // Serve on the DefaultServeMux so can have global registrations appear + s, err := newMetricsServer(ctx, opt) + if err != nil { + return nil, err + } + return s, s.Serve() + } + return nil, nil +} + +// MetricsServer contains everything to run the rc server +type MetricsServer struct { + ctx context.Context // for global config + server *libhttp.Server + promHandlerFunc http.Handler + opt *rc.Options +} + +func newMetricsServer(ctx context.Context, opt *rc.Options) (*MetricsServer, error) { + s := &MetricsServer{ + ctx: ctx, + opt: opt, + promHandlerFunc: promHandlerFunc, + } + + var err error + s.server, err = libhttp.NewServer(ctx, + libhttp.WithConfig(opt.MetricsHTTP), + libhttp.WithAuth(opt.MetricsAuth), + libhttp.WithTemplate(opt.MetricsTemplate), + ) + if err != nil { + return nil, fmt.Errorf("failed to init server: %w", err) + } + + router := s.server.Router() + router.Get(path, promHandlerFunc) + return s, nil +} + +// Serve runs the http server in the background. +// +// Use s.Close() and s.Wait() to shutdown server +func (s *MetricsServer) Serve() error { + s.server.Serve() + return nil +} + +// Wait blocks while the server is serving requests +func (s *MetricsServer) Wait() { + s.server.Wait() +} + +// Shutdown gracefully shuts down the server +func (s *MetricsServer) Shutdown() error { + return s.server.Shutdown() +} diff --git a/fs/rc/rcserver/metrics_test.go b/fs/rc/rcserver/metrics_test.go new file mode 100644 index 000000000..6141cc2a7 --- /dev/null +++ b/fs/rc/rcserver/metrics_test.go @@ -0,0 +1,88 @@ +package rcserver + +import ( + "context" + "fmt" + "net/http" + "regexp" + "testing" + + _ "github.com/rclone/rclone/backend/local" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/config/configfile" + "github.com/rclone/rclone/fs/rc" + "github.com/stretchr/testify/require" +) + +// Run a suite of tests +func testMetricsServer(t *testing.T, tests []testRun, opt *rc.Options) { + t.Helper() + ctx := context.Background() + configfile.Install() + rcServer, err := newMetricsServer(ctx, opt) + require.NoError(t, err) + testURL := rcServer.server.URLs()[0] + mux := rcServer.server.Router() + emulateCalls(t, tests, mux, testURL) +} + +// return an enabled rc +func newMetricsTestOpt() rc.Options { + opt := rc.Opt + opt.MetricsHTTP.ListenAddr = []string{testBindAddress} + return opt +} + +func TestMetrics(t *testing.T) { + stats := accounting.GlobalStats() + tests := makeMetricsTestCases(stats) + opt := newMetricsTestOpt() + testMetricsServer(t, tests, &opt) + + // Test changing a couple options + stats.Bytes(500) + for i := 0; i < 30; i++ { + require.NoError(t, stats.DeleteFile(context.Background(), 0)) + } + stats.Errors(2) + stats.Bytes(324) + + tests = makeMetricsTestCases(stats) + testMetricsServer(t, tests, &opt) +} + +func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) { + tests = []testRun{{ + Name: "Bytes Transferred Metric", + URL: "metrics", + Method: "GET", + Status: http.StatusOK, + Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())), + }, { + Name: "Checked Files Metric", + URL: "metrics", + Method: "GET", + Status: http.StatusOK, + Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())), + }, { + Name: "Errors Metric", + URL: "metrics", + Method: "GET", + Status: http.StatusOK, + Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())), + }, { + Name: "Deleted Files Metric", + URL: "metrics", + Method: "GET", + Status: http.StatusOK, + Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())), + }, { + Name: "Files Transferred Metric", + URL: "metrics", + Method: "GET", + Status: http.StatusOK, + Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())), + }, + } + return +} diff --git a/fs/rc/rcserver/rcserver.go b/fs/rc/rcserver/rcserver.go index 88f82a484..6c91931c7 100644 --- a/fs/rc/rcserver/rcserver.go +++ b/fs/rc/rcserver/rcserver.go @@ -18,13 +18,9 @@ import ( "time" "github.com/go-chi/chi/v5/middleware" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/config" - "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/fs/rc" "github.com/rclone/rclone/fs/rc/jobs" @@ -35,21 +31,6 @@ import ( "github.com/skratchdot/open-golang/open" ) -var promHandler http.Handler - -func init() { - rcloneCollector := accounting.NewRcloneCollector(context.Background()) - prometheus.MustRegister(rcloneCollector) - - m := fshttp.NewMetrics("rclone") - for _, c := range m.Collectors() { - prometheus.MustRegister(c) - } - fshttp.DefaultMetrics = m - - promHandler = promhttp.Handler() -} - // Start the remote control server if configured // // If the server wasn't configured the *Server returned may be nil @@ -376,7 +357,7 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string) s.serveRemote(w, r, fsMatchResult[2], fsMatchResult[1]) return case path == "metrics" && s.opt.EnableMetrics: - promHandler.ServeHTTP(w, r) + promHandlerFunc(w, r) return case path == "*" && s.opt.Serve: // Serve /* as the remote listing diff --git a/fs/rc/rcserver/rcserver_test.go b/fs/rc/rcserver/rcserver_test.go index 7b55f2e03..29738b614 100644 --- a/fs/rc/rcserver/rcserver_test.go +++ b/fs/rc/rcserver/rcserver_test.go @@ -15,9 +15,10 @@ import ( "testing" "time" + "github.com/go-chi/chi/v5" + _ "github.com/rclone/rclone/backend/local" "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/config/configfile" "github.com/rclone/rclone/fs/rc" "github.com/stretchr/testify/assert" @@ -115,6 +116,10 @@ func testServer(t *testing.T, tests []testRun, opt *rc.Options) { require.NoError(t, err) testURL := rcServer.server.URLs()[0] mux := rcServer.server.Router() + emulateCalls(t, tests, mux, testURL) +} + +func emulateCalls(t *testing.T, tests []testRun, mux chi.Router, testURL string) { for _, test := range tests { t.Run(test.Name, func(t *testing.T) { t.Helper() @@ -568,61 +573,6 @@ Unknown command testServer(t, tests, &opt) } -func TestMetrics(t *testing.T) { - stats := accounting.GlobalStats() - tests := makeMetricsTestCases(stats) - opt := newTestOpt() - opt.EnableMetrics = true - testServer(t, tests, &opt) - - // Test changing a couple options - stats.Bytes(500) - for i := 0; i < 30; i++ { - require.NoError(t, stats.DeleteFile(context.Background(), 0)) - } - stats.Errors(2) - stats.Bytes(324) - - tests = makeMetricsTestCases(stats) - testServer(t, tests, &opt) -} - -func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) { - tests = []testRun{{ - Name: "Bytes Transferred Metric", - URL: "/metrics", - Method: "GET", - Status: http.StatusOK, - Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())), - }, { - Name: "Checked Files Metric", - URL: "/metrics", - Method: "GET", - Status: http.StatusOK, - Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())), - }, { - Name: "Errors Metric", - URL: "/metrics", - Method: "GET", - Status: http.StatusOK, - Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())), - }, { - Name: "Deleted Files Metric", - URL: "/metrics", - Method: "GET", - Status: http.StatusOK, - Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())), - }, { - Name: "Files Transferred Metric", - URL: "/metrics", - Method: "GET", - Status: http.StatusOK, - Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())), - }, - } - return -} - var matchRemoteDirListing = regexp.MustCompile(`Directory listing of /`) func TestServingRoot(t *testing.T) {