You have reached the runsc metrics server page!
") - fmt.Fprintf(w, `To see actual metric data, head over to /metrics.
`) - fmt.Fprintf(w, "") - return httpOK -} - -// Metrics generated by the metrics server itself. -var ( - SandboxPresenceMetric = prometheus.Metric{ - Name: "sandbox_presence", - Type: prometheus.TypeGauge, - Help: "Boolean metric set to 1 for each known sandbox.", - } - SandboxRunningMetric = prometheus.Metric{ - Name: "sandbox_running", - Type: prometheus.TypeGauge, - Help: "Boolean metric set to 1 for each running sandbox.", - } - SandboxMetadataMetric = prometheus.Metric{ - Name: "sandbox_metadata", - Type: prometheus.TypeGauge, - Help: "Key-value pairs about per-sandbox metadata.", - } - SandboxCapabilitiesMetric = prometheus.Metric{ - Name: "sandbox_capabilities", - Type: prometheus.TypeGauge, - Help: "Linux capabilities added within containers of the sandbox.", - } - SandboxCapabilitiesMetricLabel = "capability" - SpecMetadataMetric = prometheus.Metric{ - Name: "spec_metadata", - Type: prometheus.TypeGauge, - Help: "Key-value pairs about OCI spec metadata.", - } - SandboxCreationMetric = prometheus.Metric{ - Name: "sandbox_creation_time_seconds", - Type: prometheus.TypeGauge, - Help: "When the sandbox was created, as a unix timestamp in seconds.", - } - NumRunningSandboxesMetric = prometheus.Metric{ - Name: "num_sandboxes_running", - Type: prometheus.TypeGauge, - Help: "Number of sandboxes running at present.", - } - NumCannotExportSandboxesMetric = prometheus.Metric{ - Name: "num_sandboxes_broken_metrics", - Type: prometheus.TypeGauge, - Help: "Number of sandboxes from which we cannot export metrics.", - } - NumTotalSandboxesMetric = prometheus.Metric{ - Name: "num_sandboxes_total", - Type: prometheus.TypeCounter, - Help: "Counter of sandboxes that have ever been started.", - } -) - -// Metrics is a list of metrics that the metric server generates. -var Metrics = []prometheus.Metric{ - SandboxPresenceMetric, - SandboxRunningMetric, - SandboxMetadataMetric, - SandboxCapabilitiesMetric, - SpecMetadataMetric, - SandboxCreationMetric, - NumRunningSandboxesMetric, - NumCannotExportSandboxesMetric, - NumTotalSandboxesMetric, - prometheus.ProcessStartTimeSeconds, -} - // serveMetrics serves metrics requests. func (m *metricServer) serveMetrics(w http.ResponseWriter, req *http.Request) httpResult { ctx, ctxCancel := context.WithTimeout(req.Context(), metricsExportTimeout) @@ -905,139 +747,6 @@ func (m *metricServer) servePID(w http.ResponseWriter, req *http.Request) httpRe return httpOK } -// profileCPU returns a CPU profile over HTTP. -func (m *metricServer) profileCPU(w http.ResponseWriter, req *http.Request) httpResult { - // Time to finish up profiling and flush out the results to the client. - const finishProfilingBuffer = 250 * time.Millisecond - - m.mu.Lock() - if m.shuttingDown { - m.mu.Unlock() - return httpResult{http.StatusServiceUnavailable, errors.New("server is shutting down already")} - } - m.mu.Unlock() - w.WriteHeader(http.StatusOK) - if err := pprof.StartCPUProfile(w); err != nil { - // We cannot return this as an error, because we've already sent the HTTP 200 OK status. - log.Warningf("Failed to start recording CPU profile: %v", err) - return httpOK - } - deadline := time.Now().Add(httpTimeout - finishProfilingBuffer) - if seconds, err := strconv.Atoi(req.URL.Query().Get("seconds")); err == nil && time.Duration(seconds)*time.Second < httpTimeout { - deadline = time.Now().Add(time.Duration(seconds) * time.Second) - } else if ctxDeadline, hasDeadline := req.Context().Deadline(); hasDeadline { - deadline = ctxDeadline.Add(-finishProfilingBuffer) - } - log.Infof("Profiling CPU until %v...", deadline) - var wasInterrupted bool - select { - case <-time.After(time.Until(deadline)): - wasInterrupted = false - case <-req.Context().Done(): - wasInterrupted = true - } - pprof.StopCPUProfile() - if wasInterrupted { - log.Warningf("Profiling CPU interrupted.") - } else { - log.Infof("Profiling CPU done.") - } - return httpOK -} - -// profileHeap returns a heap profile over HTTP. -func (m *metricServer) profileHeap(w http.ResponseWriter, req *http.Request) httpResult { - m.mu.Lock() - if m.shuttingDown { - m.mu.Unlock() - return httpResult{http.StatusServiceUnavailable, errors.New("server is shutting down already")} - } - m.mu.Unlock() - w.WriteHeader(http.StatusOK) - runtime.GC() // Run GC just before looking at the heap to get a clean view. - if err := pprof.Lookup("heap").WriteTo(w, 0); err != nil { - // We cannot return this as an error, because we've already sent the HTTP 200 OK status. - log.Warningf("Failed to record heap profile: %v", err) - } - return httpOK -} - -// shutdownLocked shuts down the server. It assumes mu is held. -func (m *metricServer) shutdownLocked(ctx context.Context) { - log.Infof("Server shutting down.") - m.shuttingDown = true - if m.udsPath != "" { - if err := os.Remove(m.udsPath); err != nil { - log.Warningf("Cannot remove UDS at %s: %v", m.udsPath, err) - } else { - m.udsPath = "" - } - } - if m.pidFile != "" { - if err := os.Remove(m.pidFile); err != nil { - log.Warningf("Cannot remove PID file at %s: %v", m.pidFile, err) - } - } - m.srv.Shutdown(ctx) -} - -// logRequest wraps an HTTP handler and adds logging to it. -func logRequest(f func(w http.ResponseWriter, req *http.Request) httpResult) func(w http.ResponseWriter, req *http.Request) { - return func(w http.ResponseWriter, req *http.Request) { - log.Infof("Request: %s %s", req.Method, req.URL.Path) - defer func() { - if r := recover(); r != nil { - log.Warningf("Request: %s %s: Panic:\n%v", req.Method, req.URL.Path, r) - } - }() - result := f(w, req) - if result.err != nil { - http.Error(w, result.err.Error(), result.code) - log.Warningf("Request: %s %s: Failed with HTTP code %d: %v", req.Method, req.URL.Path, result.code, result.err) - } - // Run GC after every request to keep memory usage as predictable and as flat as possible. - runtime.GC() - } -} - -// verify is one iteration of verifyLoop. -// It runs in a loop in the background which checks all sandboxes for liveness, tries to load -// their metadata if that hasn't been loaded yet, and tries to pick up new sandboxes that -// failed to register for whatever reason. -func (m *metricServer) verify(ctx context.Context) { - _, err := container.ListSandboxes(m.rootDir) - m.mu.Lock() - defer m.mu.Unlock() - if err != nil { - if !m.allowUnknownRoot { - log.Warningf("Cannot list sandboxes in root directory %s, it has likely gone away: %v. Server shutting down.", m.rootDir, err) - m.shutdownLocked(ctx) - } - return - } - m.refreshSandboxesLocked() -} - -// verifyLoop runs in the background and periodically calls verify. -func (m *metricServer) verifyLoop(ctx context.Context) { - ticker := time.NewTicker(verifyLoopInterval) - defer ticker.Stop() - for ctx.Err() == nil { - select { - case <-ctx.Done(): - return - case <-m.shutdownCh: - log.Infof("Received interrupt signal, shutting down server.") - m.mu.Lock() - m.shutdownLocked(ctx) - m.mu.Unlock() - return - case <-ticker.C: - m.verify(ctx) - } - } -} - // Server is the set of options to run a metric server. // Initialize this struct and then call Run on it to run the metric server. type Server struct { @@ -1081,7 +790,7 @@ func (s *Server) Run(ctx context.Context) error { } if _, err := container.ListSandboxes(conf.RootDir); err != nil { if !m.allowUnknownRoot { - return fmt.Errorf("invalid root directory %q: tried to list sandboxes within it and got: %v", conf.RootDir, err) + return fmt.Errorf("invalid root directory %q: tried to list sandboxes within it and got: %w", conf.RootDir, err) } log.Warningf("Invalid root directory %q: tried to list sandboxes within it and got: %v. Continuing anyway, as the server is configured to tolerate this.", conf.RootDir, err) } @@ -1089,7 +798,7 @@ func (s *Server) Run(ctx context.Context) error { // permission errors. Double-check by actually listing the directory. if _, err := ioutil.ReadDir(conf.RootDir); err != nil { if !m.allowUnknownRoot { - return fmt.Errorf("invalid root directory %q: tried to list all entries within it and got: %v", conf.RootDir, err) + return fmt.Errorf("invalid root directory %q: tried to list all entries within it and got: %w", conf.RootDir, err) } log.Warningf("Invalid root directory %q: tried to list all entries within it and got: %v. Continuing anyway, as the server is configured to tolerate this.", conf.RootDir, err) } @@ -1112,11 +821,11 @@ func (s *Server) Run(ctx context.Context) error { if strings.HasPrefix(conf.MetricServer, fmt.Sprintf("%c", os.PathSeparator)) { beforeBindSt, beforeBindErr := os.Stat(conf.MetricServer) if listener, listenErr = (&net.ListenConfig{}).Listen(ctx, "unix", conf.MetricServer); listenErr != nil { - return fmt.Errorf("cannot listen on unix domain socket %q: %v", conf.MetricServer, listenErr) + return fmt.Errorf("cannot listen on unix domain socket %q: %w", conf.MetricServer, listenErr) } afterBindSt, afterBindErr := os.Stat(conf.MetricServer) if afterBindErr != nil { - return fmt.Errorf("cannot stat our own unix domain socket %q: %v", conf.MetricServer, afterBindErr) + return fmt.Errorf("cannot stat our own unix domain socket %q: %w", conf.MetricServer, afterBindErr) } ownUDS := true if beforeBindErr == nil && beforeBindSt.Mode() == afterBindSt.Mode() { @@ -1144,7 +853,7 @@ func (s *Server) Run(ctx context.Context) error { log.Warningf("Binding on all interfaces. This will allow anyone to list all containers on your machine!") } if listener, listenErr = (&net.ListenConfig{}).Listen(ctx, "tcp", conf.MetricServer); listenErr != nil { - return fmt.Errorf("cannot listen on TCP address %q: %v", conf.MetricServer, listenErr) + return fmt.Errorf("cannot listen on TCP address %q: %w", conf.MetricServer, listenErr) } } @@ -1164,10 +873,12 @@ func (s *Server) Run(ctx context.Context) error { m.srv.Handler = mux m.srv.ReadTimeout = httpTimeout m.srv.WriteTimeout = httpTimeout - go m.verifyLoop(ctx) + if err := m.startVerifyLoop(ctx); err != nil { + return fmt.Errorf("cannot start background loop: %w", err) + } if m.pidFile != "" { if err := ioutil.WriteFile(m.pidFile, []byte(fmt.Sprintf("%d", m.pid)), 0644); err != nil { - return fmt.Errorf("cannot write PID to file %q: %v", m.pidFile, err) + return fmt.Errorf("cannot write PID to file %q: %w", m.pidFile, err) } defer os.Remove(m.pidFile) log.Infof("Wrote PID %d to file %v.", m.pid, m.pidFile) @@ -1193,7 +904,7 @@ func (s *Server) Run(ctx context.Context) error { if serveErr == http.ErrServerClosed { return nil } - return fmt.Errorf("cannot serve on address %s: %v", conf.MetricServer, serveErr) + return fmt.Errorf("cannot serve on address %s: %w", conf.MetricServer, serveErr) } // Per documentation, http.Server.Serve can never return a nil error, so this is not a success. return fmt.Errorf("HTTP server Serve() did not return expected error") diff --git a/runsc/metricserver/metricserver_http.go b/runsc/metricserver/metricserver_http.go new file mode 100644 index 000000000..2e861221d --- /dev/null +++ b/runsc/metricserver/metricserver_http.go @@ -0,0 +1,79 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metricserver + +import ( + "errors" + "fmt" + "net/http" + "runtime" + "strings" + "time" + + "gvisor.dev/gvisor/pkg/log" +) + +// httpTimeout is the timeout used for all connect/read/write operations of the HTTP server. +const httpTimeout = 1 * time.Minute + +// httpResult is returned by HTTP handlers. +type httpResult struct { + code int + err error +} + +// httpOK is the "everything went fine" HTTP result. +var httpOK = httpResult{code: http.StatusOK} + +// serveIndex serves the index page. +func (m *metricServer) serveIndex(w http.ResponseWriter, req *http.Request) httpResult { + if req.URL.Path != "/" { + if strings.HasPrefix(req.URL.Path, "/metrics?") { + // Prometheus's scrape_config.metrics_path takes in a query path and automatically encodes + // all special characters in it to %-form, including the "?" character. + // This can prevent use of query parameters, and we end up here instead. + // To address this, rewrite the URL to undo this transformation. + // This means requesting "/metrics%3Ffoo=bar" is rewritten to "/metrics?foo=bar". + req.URL.RawQuery = strings.TrimPrefix(req.URL.Path, "/metrics?") + req.URL.Path = "/metrics" + return m.serveMetrics(w, req) + } + return httpResult{http.StatusNotFound, errors.New("path not found")} + } + fmt.Fprintf(w, "You have reached the runsc metrics server page!
") + fmt.Fprintf(w, `To see actual metric data, head over to /metrics.
`) + fmt.Fprintf(w, "") + return httpOK +} + +// logRequest wraps an HTTP handler and adds logging to it. +func logRequest(f func(w http.ResponseWriter, req *http.Request) httpResult) func(w http.ResponseWriter, req *http.Request) { + return func(w http.ResponseWriter, req *http.Request) { + log.Infof("Request: %s %s", req.Method, req.URL.Path) + defer func() { + if r := recover(); r != nil { + log.Warningf("Request: %s %s: Panic:\n%v", req.Method, req.URL.Path, r) + } + }() + result := f(w, req) + if result.err != nil { + http.Error(w, result.err.Error(), result.code) + log.Warningf("Request: %s %s: Failed with HTTP code %d: %v", req.Method, req.URL.Path, result.code, result.err) + } + // Run GC after every request to keep memory usage as predictable and as flat as possible. + runtime.GC() + } +} diff --git a/runsc/metricserver/metricserver_lifecycle.go b/runsc/metricserver/metricserver_lifecycle.go new file mode 100644 index 000000000..76771e36b --- /dev/null +++ b/runsc/metricserver/metricserver_lifecycle.go @@ -0,0 +1,104 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.1 +// +build go1.1 + +package metricserver + +import ( + "context" + "os" + "time" + + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/runsc/container" +) + +// verifyLoopInterval is the interval at which we check whether there are any sandboxes we need +// to serve metrics for. +const verifyLoopInterval = 20 * time.Second + +// sandboxData contains additional per-sandbox data. +type sandboxData struct{} + +// load loads additional per-sandbox data. +func (s *sandboxData) load(*servedSandbox) error { + return nil +} + +// serverData contains additional server-wide data. +type serverData struct{} + +// verify is one iteration of verifyLoop. +// It runs in a loop in the background which checks all sandboxes for liveness, tries to load +// their metadata if that hasn't been loaded yet, and tries to pick up new sandboxes that +// failed to register for whatever reason. +func (m *metricServer) verify(ctx context.Context) { + _, err := container.ListSandboxes(m.rootDir) + m.mu.Lock() + defer m.mu.Unlock() + if err != nil { + if !m.allowUnknownRoot { + log.Warningf("Cannot list sandboxes in root directory %s, it has likely gone away: %v. Server shutting down.", m.rootDir, err) + m.shutdownLocked(ctx) + } + return + } + m.refreshSandboxesLocked() +} + +// startVerifyLoop runs in the background and periodically calls verify. +func (m *metricServer) startVerifyLoop(ctx context.Context) error { + go func() { + ticker := time.NewTicker(verifyLoopInterval) + defer ticker.Stop() + for ctx.Err() == nil { + select { + case <-ctx.Done(): + return + case <-m.shutdownCh: + log.Infof("Received interrupt signal, shutting down server.") + m.mu.Lock() + m.shutdownLocked(ctx) + m.mu.Unlock() + return + case <-ticker.C: + m.verify(ctx) + } + } + }() + return nil +} + +// shutdownLocked shuts down the server. It assumes mu is held. +func (m *metricServer) shutdownLocked(ctx context.Context) { + log.Infof("Server shutting down.") + m.shuttingDown = true + if m.udsPath != "" { + if err := os.Remove(m.udsPath); err != nil { + log.Warningf("Cannot remove UDS at %s: %v", m.udsPath, err) + } else { + m.udsPath = "" + } + } + if m.pidFile != "" { + if err := os.Remove(m.pidFile); err != nil { + log.Warningf("Cannot remove PID file at %s: %v", m.pidFile, err) + } else { + m.pidFile = "" + } + } + m.srv.Shutdown(ctx) +} diff --git a/runsc/metricserver/metricserver_metrics.go b/runsc/metricserver/metricserver_metrics.go new file mode 100644 index 000000000..fac3dc600 --- /dev/null +++ b/runsc/metricserver/metricserver_metrics.go @@ -0,0 +1,149 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metricserver + +import ( + "crypto/sha256" + "encoding/binary" + "io" + "strconv" + + "gvisor.dev/gvisor/pkg/prometheus" + "gvisor.dev/gvisor/runsc/container" +) + +// Metrics generated by the metrics server itself. +var ( + SandboxPresenceMetric = prometheus.Metric{ + Name: "sandbox_presence", + Type: prometheus.TypeGauge, + Help: "Boolean metric set to 1 for each known sandbox.", + } + SandboxRunningMetric = prometheus.Metric{ + Name: "sandbox_running", + Type: prometheus.TypeGauge, + Help: "Boolean metric set to 1 for each running sandbox.", + } + SandboxMetadataMetric = prometheus.Metric{ + Name: "sandbox_metadata", + Type: prometheus.TypeGauge, + Help: "Key-value pairs about per-sandbox metadata.", + } + SandboxCapabilitiesMetric = prometheus.Metric{ + Name: "sandbox_capabilities", + Type: prometheus.TypeGauge, + Help: "Linux capabilities added within containers of the sandbox.", + } + SandboxCapabilitiesMetricLabel = "capability" + SpecMetadataMetric = prometheus.Metric{ + Name: "spec_metadata", + Type: prometheus.TypeGauge, + Help: "Key-value pairs about OCI spec metadata.", + } + SandboxCreationMetric = prometheus.Metric{ + Name: "sandbox_creation_time_seconds", + Type: prometheus.TypeGauge, + Help: "When the sandbox was created, as a unix timestamp in seconds.", + } + NumRunningSandboxesMetric = prometheus.Metric{ + Name: "num_sandboxes_running", + Type: prometheus.TypeGauge, + Help: "Number of sandboxes running at present.", + } + NumCannotExportSandboxesMetric = prometheus.Metric{ + Name: "num_sandboxes_broken_metrics", + Type: prometheus.TypeGauge, + Help: "Number of sandboxes from which we cannot export metrics.", + } + NumTotalSandboxesMetric = prometheus.Metric{ + Name: "num_sandboxes_total", + Type: prometheus.TypeCounter, + Help: "Counter of sandboxes that have ever been started.", + } +) + +// Metrics is a list of metrics that the metric server generates. +var Metrics = []*prometheus.Metric{ + &SandboxPresenceMetric, + &SandboxRunningMetric, + &SandboxMetadataMetric, + &SandboxCapabilitiesMetric, + &SpecMetadataMetric, + &SandboxCreationMetric, + &NumRunningSandboxesMetric, + &NumCannotExportSandboxesMetric, + &NumTotalSandboxesMetric, + &prometheus.ProcessStartTimeSeconds, +} + +// SandboxPrometheusLabels returns a set of Prometheus labels that identifies the sandbox running +// the given root container. +func SandboxPrometheusLabels(rootContainer *container.Container) (map[string]string, error) { + s := rootContainer.Sandbox + labels := make(map[string]string, 4) + labels[prometheus.SandboxIDLabel] = s.ID + + // Compute iteration ID label in a stable manner. + // This uses sha256(ID + ":" + creation time). + h := sha256.New() + if _, err := io.WriteString(h, s.ID); err != nil { + return nil, err + } + if _, err := io.WriteString(h, ":"); err != nil { + return nil, err + } + if _, err := io.WriteString(h, rootContainer.CreatedAt.UTC().String()); err != nil { + return nil, err + } + labels[prometheus.IterationIDLabel] = strconv.FormatUint(binary.BigEndian.Uint64(h.Sum(nil)[:8]), 36) + + if s.PodName != "" { + labels[prometheus.PodNameLabel] = s.PodName + } + if s.Namespace != "" { + labels[prometheus.NamespaceLabel] = s.Namespace + } + return labels, nil +} + +// ComputeSpecMetadata returns the labels for the `spec_metadata` metric. +// It merges data from the Specs of multiple containers running within the +// same sandbox. +// This function must support being called with `allContainers` being nil. +// It must return the same set of label keys regardless of how many containers +// are in `allContainers`. +func ComputeSpecMetadata(allContainers []*container.Container) map[string]string { + const ( + unknownOCIVersion = "UNKNOWN" + inconsistentOCIVersion = "INCONSISTENT" + ) + + hasUID0Container := false + ociVersion := unknownOCIVersion + for _, cont := range allContainers { + if cont.RunsAsUID0() { + hasUID0Container = true + } + if ociVersion == unknownOCIVersion { + ociVersion = cont.Spec.Version + } else if ociVersion != cont.Spec.Version { + ociVersion = inconsistentOCIVersion + } + } + return map[string]string{ + "hasuid0": strconv.FormatBool(hasUID0Container), + "ociversion": ociVersion, + } +} diff --git a/runsc/metricserver/metricserver_profile.go b/runsc/metricserver/metricserver_profile.go new file mode 100644 index 000000000..ebf78819e --- /dev/null +++ b/runsc/metricserver/metricserver_profile.go @@ -0,0 +1,83 @@ +// Copyright 2023 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metricserver + +import ( + "errors" + "net/http" + "runtime" + "runtime/pprof" + "strconv" + "time" + + "gvisor.dev/gvisor/pkg/log" +) + +// profileCPU returns a CPU profile over HTTP. +func (m *metricServer) profileCPU(w http.ResponseWriter, req *http.Request) httpResult { + // Time to finish up profiling and flush out the results to the client. + const finishProfilingBuffer = 250 * time.Millisecond + + m.mu.Lock() + if m.shuttingDown { + m.mu.Unlock() + return httpResult{http.StatusServiceUnavailable, errors.New("server is shutting down already")} + } + m.mu.Unlock() + w.WriteHeader(http.StatusOK) + if err := pprof.StartCPUProfile(w); err != nil { + // We cannot return this as an error, because we've already sent the HTTP 200 OK status. + log.Warningf("Failed to start recording CPU profile: %v", err) + return httpOK + } + deadline := time.Now().Add(httpTimeout - finishProfilingBuffer) + if seconds, err := strconv.Atoi(req.URL.Query().Get("seconds")); err == nil && time.Duration(seconds)*time.Second < httpTimeout { + deadline = time.Now().Add(time.Duration(seconds) * time.Second) + } else if ctxDeadline, hasDeadline := req.Context().Deadline(); hasDeadline { + deadline = ctxDeadline.Add(-finishProfilingBuffer) + } + log.Infof("Profiling CPU until %v...", deadline) + var wasInterrupted bool + select { + case <-time.After(time.Until(deadline)): + wasInterrupted = false + case <-req.Context().Done(): + wasInterrupted = true + } + pprof.StopCPUProfile() + if wasInterrupted { + log.Warningf("Profiling CPU interrupted.") + } else { + log.Infof("Profiling CPU done.") + } + return httpOK +} + +// profileHeap returns a heap profile over HTTP. +func (m *metricServer) profileHeap(w http.ResponseWriter, req *http.Request) httpResult { + m.mu.Lock() + if m.shuttingDown { + m.mu.Unlock() + return httpResult{http.StatusServiceUnavailable, errors.New("server is shutting down already")} + } + m.mu.Unlock() + w.WriteHeader(http.StatusOK) + runtime.GC() // Run GC just before looking at the heap to get a clean view. + if err := pprof.Lookup("heap").WriteTo(w, 0); err != nil { + // We cannot return this as an error, because we've already sent the HTTP 200 OK status. + log.Warningf("Failed to record heap profile: %v", err) + } + return httpOK +}