Add Goroutine stack inspector to admin/monitor (#19207)
Continues on from #19202. Following the addition of pprof labels we can now more easily understand the relationship between a goroutine and the requests that spawn them. This PR takes advantage of the labels and adds a few others, then provides a mechanism for the monitoring page to query the pprof goroutine profile. The binary profile that results from this profile is immediately piped in to the google library for parsing this and then stack traces are formed for the goroutines. If the goroutine is within a context or has been created from a goroutine within a process context it will acquire the process description labels for that process. The goroutines are mapped with there associate pids and any that do not have an associated pid are placed in a group at the bottom as unbound. In this way we should be able to more easily examine goroutines that have been stuck. A manager command `gitea manager processes` is also provided that can export the processes (with or without stacktraces) to the command line. Signed-off-by: Andrew Thornton <art27@cantab.net>tokarchuk/v1.17
zeripath
3 years ago
committed by
GitHub
parent
9c349a4277
commit
c88547ce71
@ -0,0 +1,382 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"fmt" |
||||
"net/http" |
||||
"os" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/private" |
||||
"github.com/urfave/cli" |
||||
) |
||||
|
||||
var ( |
||||
defaultLoggingFlags = []cli.Flag{ |
||||
cli.StringFlag{ |
||||
Name: "group, g", |
||||
Usage: "Group to add logger to - will default to \"default\"", |
||||
}, cli.StringFlag{ |
||||
Name: "name, n", |
||||
Usage: "Name of the new logger - will default to mode", |
||||
}, cli.StringFlag{ |
||||
Name: "level, l", |
||||
Usage: "Logging level for the new logger", |
||||
}, cli.StringFlag{ |
||||
Name: "stacktrace-level, L", |
||||
Usage: "Stacktrace logging level", |
||||
}, cli.StringFlag{ |
||||
Name: "flags, F", |
||||
Usage: "Flags for the logger", |
||||
}, cli.StringFlag{ |
||||
Name: "expression, e", |
||||
Usage: "Matching expression for the logger", |
||||
}, cli.StringFlag{ |
||||
Name: "prefix, p", |
||||
Usage: "Prefix for the logger", |
||||
}, cli.BoolFlag{ |
||||
Name: "color", |
||||
Usage: "Use color in the logs", |
||||
}, cli.BoolFlag{ |
||||
Name: "debug", |
||||
}, |
||||
} |
||||
|
||||
subcmdLogging = cli.Command{ |
||||
Name: "logging", |
||||
Usage: "Adjust logging commands", |
||||
Subcommands: []cli.Command{ |
||||
{ |
||||
Name: "pause", |
||||
Usage: "Pause logging (Gitea will buffer logs up to a certain point and will drop them after that point)", |
||||
Flags: []cli.Flag{ |
||||
cli.BoolFlag{ |
||||
Name: "debug", |
||||
}, |
||||
}, |
||||
Action: runPauseLogging, |
||||
}, { |
||||
Name: "resume", |
||||
Usage: "Resume logging", |
||||
Flags: []cli.Flag{ |
||||
cli.BoolFlag{ |
||||
Name: "debug", |
||||
}, |
||||
}, |
||||
Action: runResumeLogging, |
||||
}, { |
||||
Name: "release-and-reopen", |
||||
Usage: "Cause Gitea to release and re-open files used for logging", |
||||
Flags: []cli.Flag{ |
||||
cli.BoolFlag{ |
||||
Name: "debug", |
||||
}, |
||||
}, |
||||
Action: runReleaseReopenLogging, |
||||
}, { |
||||
Name: "remove", |
||||
Usage: "Remove a logger", |
||||
ArgsUsage: "[name] Name of logger to remove", |
||||
Flags: []cli.Flag{ |
||||
cli.BoolFlag{ |
||||
Name: "debug", |
||||
}, cli.StringFlag{ |
||||
Name: "group, g", |
||||
Usage: "Group to add logger to - will default to \"default\"", |
||||
}, |
||||
}, |
||||
Action: runRemoveLogger, |
||||
}, { |
||||
Name: "add", |
||||
Usage: "Add a logger", |
||||
Subcommands: []cli.Command{ |
||||
{ |
||||
Name: "console", |
||||
Usage: "Add a console logger", |
||||
Flags: append(defaultLoggingFlags, |
||||
cli.BoolFlag{ |
||||
Name: "stderr", |
||||
Usage: "Output console logs to stderr - only relevant for console", |
||||
}), |
||||
Action: runAddConsoleLogger, |
||||
}, { |
||||
Name: "file", |
||||
Usage: "Add a file logger", |
||||
Flags: append(defaultLoggingFlags, []cli.Flag{ |
||||
cli.StringFlag{ |
||||
Name: "filename, f", |
||||
Usage: "Filename for the logger - this must be set.", |
||||
}, cli.BoolTFlag{ |
||||
Name: "rotate, r", |
||||
Usage: "Rotate logs", |
||||
}, cli.Int64Flag{ |
||||
Name: "max-size, s", |
||||
Usage: "Maximum size in bytes before rotation", |
||||
}, cli.BoolTFlag{ |
||||
Name: "daily, d", |
||||
Usage: "Rotate logs daily", |
||||
}, cli.IntFlag{ |
||||
Name: "max-days, D", |
||||
Usage: "Maximum number of daily logs to keep", |
||||
}, cli.BoolTFlag{ |
||||
Name: "compress, z", |
||||
Usage: "Compress rotated logs", |
||||
}, cli.IntFlag{ |
||||
Name: "compression-level, Z", |
||||
Usage: "Compression level to use", |
||||
}, |
||||
}...), |
||||
Action: runAddFileLogger, |
||||
}, { |
||||
Name: "conn", |
||||
Usage: "Add a net conn logger", |
||||
Flags: append(defaultLoggingFlags, []cli.Flag{ |
||||
cli.BoolFlag{ |
||||
Name: "reconnect-on-message, R", |
||||
Usage: "Reconnect to host for every message", |
||||
}, cli.BoolFlag{ |
||||
Name: "reconnect, r", |
||||
Usage: "Reconnect to host when connection is dropped", |
||||
}, cli.StringFlag{ |
||||
Name: "protocol, P", |
||||
Usage: "Set protocol to use: tcp, unix, or udp (defaults to tcp)", |
||||
}, cli.StringFlag{ |
||||
Name: "address, a", |
||||
Usage: "Host address and port to connect to (defaults to :7020)", |
||||
}, |
||||
}...), |
||||
Action: runAddConnLogger, |
||||
}, { |
||||
Name: "smtp", |
||||
Usage: "Add an SMTP logger", |
||||
Flags: append(defaultLoggingFlags, []cli.Flag{ |
||||
cli.StringFlag{ |
||||
Name: "username, u", |
||||
Usage: "Mail server username", |
||||
}, cli.StringFlag{ |
||||
Name: "password, P", |
||||
Usage: "Mail server password", |
||||
}, cli.StringFlag{ |
||||
Name: "host, H", |
||||
Usage: "Mail server host (defaults to: 127.0.0.1:25)", |
||||
}, cli.StringSliceFlag{ |
||||
Name: "send-to, s", |
||||
Usage: "Email address(es) to send to", |
||||
}, cli.StringFlag{ |
||||
Name: "subject, S", |
||||
Usage: "Subject header of sent emails", |
||||
}, |
||||
}...), |
||||
Action: runAddSMTPLogger, |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
} |
||||
) |
||||
|
||||
func runRemoveLogger(c *cli.Context) error { |
||||
setup("manager", c.Bool("debug")) |
||||
group := c.String("group") |
||||
if len(group) == 0 { |
||||
group = log.DEFAULT |
||||
} |
||||
name := c.Args().First() |
||||
ctx, cancel := installSignals() |
||||
defer cancel() |
||||
|
||||
statusCode, msg := private.RemoveLogger(ctx, group, name) |
||||
switch statusCode { |
||||
case http.StatusInternalServerError: |
||||
return fail("InternalServerError", msg) |
||||
} |
||||
|
||||
fmt.Fprintln(os.Stdout, msg) |
||||
return nil |
||||
} |
||||
|
||||
func runAddSMTPLogger(c *cli.Context) error { |
||||
setup("manager", c.Bool("debug")) |
||||
vals := map[string]interface{}{} |
||||
mode := "smtp" |
||||
if c.IsSet("host") { |
||||
vals["host"] = c.String("host") |
||||
} else { |
||||
vals["host"] = "127.0.0.1:25" |
||||
} |
||||
|
||||
if c.IsSet("username") { |
||||
vals["username"] = c.String("username") |
||||
} |
||||
if c.IsSet("password") { |
||||
vals["password"] = c.String("password") |
||||
} |
||||
|
||||
if !c.IsSet("send-to") { |
||||
return fmt.Errorf("Some recipients must be provided") |
||||
} |
||||
vals["sendTos"] = c.StringSlice("send-to") |
||||
|
||||
if c.IsSet("subject") { |
||||
vals["subject"] = c.String("subject") |
||||
} else { |
||||
vals["subject"] = "Diagnostic message from Gitea" |
||||
} |
||||
|
||||
return commonAddLogger(c, mode, vals) |
||||
} |
||||
|
||||
func runAddConnLogger(c *cli.Context) error { |
||||
setup("manager", c.Bool("debug")) |
||||
vals := map[string]interface{}{} |
||||
mode := "conn" |
||||
vals["net"] = "tcp" |
||||
if c.IsSet("protocol") { |
||||
switch c.String("protocol") { |
||||
case "udp": |
||||
vals["net"] = "udp" |
||||
case "unix": |
||||
vals["net"] = "unix" |
||||
} |
||||
} |
||||
if c.IsSet("address") { |
||||
vals["address"] = c.String("address") |
||||
} else { |
||||
vals["address"] = ":7020" |
||||
} |
||||
if c.IsSet("reconnect") { |
||||
vals["reconnect"] = c.Bool("reconnect") |
||||
} |
||||
if c.IsSet("reconnect-on-message") { |
||||
vals["reconnectOnMsg"] = c.Bool("reconnect-on-message") |
||||
} |
||||
return commonAddLogger(c, mode, vals) |
||||
} |
||||
|
||||
func runAddFileLogger(c *cli.Context) error { |
||||
setup("manager", c.Bool("debug")) |
||||
vals := map[string]interface{}{} |
||||
mode := "file" |
||||
if c.IsSet("filename") { |
||||
vals["filename"] = c.String("filename") |
||||
} else { |
||||
return fmt.Errorf("filename must be set when creating a file logger") |
||||
} |
||||
if c.IsSet("rotate") { |
||||
vals["rotate"] = c.Bool("rotate") |
||||
} |
||||
if c.IsSet("max-size") { |
||||
vals["maxsize"] = c.Int64("max-size") |
||||
} |
||||
if c.IsSet("daily") { |
||||
vals["daily"] = c.Bool("daily") |
||||
} |
||||
if c.IsSet("max-days") { |
||||
vals["maxdays"] = c.Int("max-days") |
||||
} |
||||
if c.IsSet("compress") { |
||||
vals["compress"] = c.Bool("compress") |
||||
} |
||||
if c.IsSet("compression-level") { |
||||
vals["compressionLevel"] = c.Int("compression-level") |
||||
} |
||||
return commonAddLogger(c, mode, vals) |
||||
} |
||||
|
||||
func runAddConsoleLogger(c *cli.Context) error { |
||||
setup("manager", c.Bool("debug")) |
||||
vals := map[string]interface{}{} |
||||
mode := "console" |
||||
if c.IsSet("stderr") && c.Bool("stderr") { |
||||
vals["stderr"] = c.Bool("stderr") |
||||
} |
||||
return commonAddLogger(c, mode, vals) |
||||
} |
||||
|
||||
func commonAddLogger(c *cli.Context, mode string, vals map[string]interface{}) error { |
||||
if len(c.String("level")) > 0 { |
||||
vals["level"] = log.FromString(c.String("level")).String() |
||||
} |
||||
if len(c.String("stacktrace-level")) > 0 { |
||||
vals["stacktraceLevel"] = log.FromString(c.String("stacktrace-level")).String() |
||||
} |
||||
if len(c.String("expression")) > 0 { |
||||
vals["expression"] = c.String("expression") |
||||
} |
||||
if len(c.String("prefix")) > 0 { |
||||
vals["prefix"] = c.String("prefix") |
||||
} |
||||
if len(c.String("flags")) > 0 { |
||||
vals["flags"] = log.FlagsFromString(c.String("flags")) |
||||
} |
||||
if c.IsSet("color") { |
||||
vals["colorize"] = c.Bool("color") |
||||
} |
||||
group := "default" |
||||
if c.IsSet("group") { |
||||
group = c.String("group") |
||||
} |
||||
name := mode |
||||
if c.IsSet("name") { |
||||
name = c.String("name") |
||||
} |
||||
ctx, cancel := installSignals() |
||||
defer cancel() |
||||
|
||||
statusCode, msg := private.AddLogger(ctx, group, name, mode, vals) |
||||
switch statusCode { |
||||
case http.StatusInternalServerError: |
||||
return fail("InternalServerError", msg) |
||||
} |
||||
|
||||
fmt.Fprintln(os.Stdout, msg) |
||||
return nil |
||||
} |
||||
|
||||
func runPauseLogging(c *cli.Context) error { |
||||
ctx, cancel := installSignals() |
||||
defer cancel() |
||||
|
||||
setup("manager", c.Bool("debug")) |
||||
statusCode, msg := private.PauseLogging(ctx) |
||||
switch statusCode { |
||||
case http.StatusInternalServerError: |
||||
return fail("InternalServerError", msg) |
||||
} |
||||
|
||||
fmt.Fprintln(os.Stdout, msg) |
||||
return nil |
||||
} |
||||
|
||||
func runResumeLogging(c *cli.Context) error { |
||||
ctx, cancel := installSignals() |
||||
defer cancel() |
||||
|
||||
setup("manager", c.Bool("debug")) |
||||
statusCode, msg := private.ResumeLogging(ctx) |
||||
switch statusCode { |
||||
case http.StatusInternalServerError: |
||||
return fail("InternalServerError", msg) |
||||
} |
||||
|
||||
fmt.Fprintln(os.Stdout, msg) |
||||
return nil |
||||
} |
||||
|
||||
func runReleaseReopenLogging(c *cli.Context) error { |
||||
ctx, cancel := installSignals() |
||||
defer cancel() |
||||
|
||||
setup("manager", c.Bool("debug")) |
||||
statusCode, msg := private.ReleaseReopenLogging(ctx) |
||||
switch statusCode { |
||||
case http.StatusInternalServerError: |
||||
return fail("InternalServerError", msg) |
||||
} |
||||
|
||||
fmt.Fprintln(os.Stdout, msg) |
||||
return nil |
||||
} |
||||
@ -0,0 +1,26 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package process |
||||
|
||||
import "fmt" |
||||
|
||||
// Error is a wrapped error describing the error results of Process Execution
|
||||
type Error struct { |
||||
PID IDType |
||||
Description string |
||||
Err error |
||||
CtxErr error |
||||
Stdout string |
||||
Stderr string |
||||
} |
||||
|
||||
func (err *Error) Error() string { |
||||
return fmt.Sprintf("exec(%s:%s) failed: %v(%v) stdout: %s stderr: %s", err.PID, err.Description, err.Err, err.CtxErr, err.Stdout, err.Stderr) |
||||
} |
||||
|
||||
// Unwrap implements the unwrappable implicit interface for go1.13 Unwrap()
|
||||
func (err *Error) Unwrap() error { |
||||
return err.Err |
||||
} |
||||
@ -0,0 +1,79 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package process |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"io" |
||||
"os/exec" |
||||
"time" |
||||
) |
||||
|
||||
// Exec a command and use the default timeout.
|
||||
func (pm *Manager) Exec(desc, cmdName string, args ...string) (string, string, error) { |
||||
return pm.ExecDir(DefaultContext, -1, "", desc, cmdName, args...) |
||||
} |
||||
|
||||
// ExecTimeout a command and use a specific timeout duration.
|
||||
func (pm *Manager) ExecTimeout(timeout time.Duration, desc, cmdName string, args ...string) (string, string, error) { |
||||
return pm.ExecDir(DefaultContext, timeout, "", desc, cmdName, args...) |
||||
} |
||||
|
||||
// ExecDir a command and use the default timeout.
|
||||
func (pm *Manager) ExecDir(ctx context.Context, timeout time.Duration, dir, desc, cmdName string, args ...string) (string, string, error) { |
||||
return pm.ExecDirEnv(ctx, timeout, dir, desc, nil, cmdName, args...) |
||||
} |
||||
|
||||
// ExecDirEnv runs a command in given path and environment variables, and waits for its completion
|
||||
// up to the given timeout (or DefaultTimeout if -1 is given).
|
||||
// Returns its complete stdout and stderr
|
||||
// outputs and an error, if any (including timeout)
|
||||
func (pm *Manager) ExecDirEnv(ctx context.Context, timeout time.Duration, dir, desc string, env []string, cmdName string, args ...string) (string, string, error) { |
||||
return pm.ExecDirEnvStdIn(ctx, timeout, dir, desc, env, nil, cmdName, args...) |
||||
} |
||||
|
||||
// ExecDirEnvStdIn runs a command in given path and environment variables with provided stdIN, and waits for its completion
|
||||
// up to the given timeout (or DefaultTimeout if timeout <= 0 is given).
|
||||
// Returns its complete stdout and stderr
|
||||
// outputs and an error, if any (including timeout)
|
||||
func (pm *Manager) ExecDirEnvStdIn(ctx context.Context, timeout time.Duration, dir, desc string, env []string, stdIn io.Reader, cmdName string, args ...string) (string, string, error) { |
||||
if timeout <= 0 { |
||||
timeout = 60 * time.Second |
||||
} |
||||
|
||||
stdOut := new(bytes.Buffer) |
||||
stdErr := new(bytes.Buffer) |
||||
|
||||
ctx, _, finished := pm.AddContextTimeout(ctx, timeout, desc) |
||||
defer finished() |
||||
|
||||
cmd := exec.CommandContext(ctx, cmdName, args...) |
||||
cmd.Dir = dir |
||||
cmd.Env = env |
||||
cmd.Stdout = stdOut |
||||
cmd.Stderr = stdErr |
||||
if stdIn != nil { |
||||
cmd.Stdin = stdIn |
||||
} |
||||
|
||||
if err := cmd.Start(); err != nil { |
||||
return "", "", err |
||||
} |
||||
|
||||
err := cmd.Wait() |
||||
if err != nil { |
||||
err = &Error{ |
||||
PID: GetPID(ctx), |
||||
Description: desc, |
||||
Err: err, |
||||
CtxErr: ctx.Err(), |
||||
Stdout: stdOut.String(), |
||||
Stderr: stdErr.String(), |
||||
} |
||||
} |
||||
|
||||
return stdOut.String(), stdErr.String(), err |
||||
} |
||||
@ -0,0 +1,355 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package process |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io" |
||||
"runtime/pprof" |
||||
"sort" |
||||
"time" |
||||
|
||||
"github.com/google/pprof/profile" |
||||
) |
||||
|
||||
// StackEntry is an entry on a stacktrace
|
||||
type StackEntry struct { |
||||
Function string |
||||
File string |
||||
Line int |
||||
} |
||||
|
||||
// Label represents a pprof label assigned to goroutine stack
|
||||
type Label struct { |
||||
Name string |
||||
Value string |
||||
} |
||||
|
||||
// Stack is a stacktrace relating to a goroutine. (Multiple goroutines may have the same stacktrace)
|
||||
type Stack struct { |
||||
Count int64 // Number of goroutines with this stack trace
|
||||
Description string |
||||
Labels []*Label `json:",omitempty"` |
||||
Entry []*StackEntry `json:",omitempty"` |
||||
} |
||||
|
||||
// A Process is a combined representation of a Process and a Stacktrace for the goroutines associated with it
|
||||
type Process struct { |
||||
PID IDType |
||||
ParentPID IDType |
||||
Description string |
||||
Start time.Time |
||||
Type string |
||||
|
||||
Children []*Process `json:",omitempty"` |
||||
Stacks []*Stack `json:",omitempty"` |
||||
} |
||||
|
||||
// Processes gets the processes in a thread safe manner
|
||||
func (pm *Manager) Processes(flat, noSystem bool) ([]*Process, int) { |
||||
pm.mutex.Lock() |
||||
processCount := len(pm.processMap) |
||||
processes := make([]*Process, 0, len(pm.processMap)) |
||||
if flat { |
||||
for _, process := range pm.processMap { |
||||
if noSystem && process.Type == SystemProcessType { |
||||
continue |
||||
} |
||||
processes = append(processes, process.toProcess()) |
||||
} |
||||
} else { |
||||
// We need our own processMap
|
||||
processMap := map[IDType]*Process{} |
||||
for _, internalProcess := range pm.processMap { |
||||
process, ok := processMap[internalProcess.PID] |
||||
if !ok { |
||||
process = internalProcess.toProcess() |
||||
processMap[process.PID] = process |
||||
} |
||||
|
||||
// Check its parent
|
||||
if process.ParentPID == "" { |
||||
processes = append(processes, process) |
||||
continue |
||||
} |
||||
|
||||
internalParentProcess, ok := pm.processMap[internalProcess.ParentPID] |
||||
if ok { |
||||
parentProcess, ok := processMap[process.ParentPID] |
||||
if !ok { |
||||
parentProcess = internalParentProcess.toProcess() |
||||
processMap[parentProcess.PID] = parentProcess |
||||
} |
||||
parentProcess.Children = append(parentProcess.Children, process) |
||||
continue |
||||
} |
||||
|
||||
processes = append(processes, process) |
||||
} |
||||
} |
||||
pm.mutex.Unlock() |
||||
|
||||
if !flat && noSystem { |
||||
for i := 0; i < len(processes); i++ { |
||||
process := processes[i] |
||||
if process.Type != SystemProcessType { |
||||
continue |
||||
} |
||||
processes[len(processes)-1], processes[i] = processes[i], processes[len(processes)-1] |
||||
processes = append(processes[:len(processes)-1], process.Children...) |
||||
i-- |
||||
} |
||||
} |
||||
|
||||
// Sort by process' start time. Oldest process appears first.
|
||||
sort.Slice(processes, func(i, j int) bool { |
||||
left, right := processes[i], processes[j] |
||||
|
||||
return left.Start.Before(right.Start) |
||||
}) |
||||
|
||||
return processes, processCount |
||||
} |
||||
|
||||
// ProcessStacktraces gets the processes and stacktraces in a thread safe manner
|
||||
func (pm *Manager) ProcessStacktraces(flat, noSystem bool) ([]*Process, int, int64, error) { |
||||
var stacks *profile.Profile |
||||
var err error |
||||
|
||||
// We cannot use the pm.ProcessMap here because we will release the mutex ...
|
||||
processMap := map[IDType]*Process{} |
||||
processCount := 0 |
||||
|
||||
// Lock the manager
|
||||
pm.mutex.Lock() |
||||
processCount = len(pm.processMap) |
||||
|
||||
// Add a defer to unlock in case there is a panic
|
||||
unlocked := false |
||||
defer func() { |
||||
if !unlocked { |
||||
pm.mutex.Unlock() |
||||
} |
||||
}() |
||||
|
||||
processes := make([]*Process, 0, len(pm.processMap)) |
||||
if flat { |
||||
for _, internalProcess := range pm.processMap { |
||||
process := internalProcess.toProcess() |
||||
processMap[process.PID] = process |
||||
if noSystem && internalProcess.Type == SystemProcessType { |
||||
continue |
||||
} |
||||
processes = append(processes, process) |
||||
} |
||||
} else { |
||||
for _, internalProcess := range pm.processMap { |
||||
process, ok := processMap[internalProcess.PID] |
||||
if !ok { |
||||
process = internalProcess.toProcess() |
||||
processMap[process.PID] = process |
||||
} |
||||
|
||||
// Check its parent
|
||||
if process.ParentPID == "" { |
||||
processes = append(processes, process) |
||||
continue |
||||
} |
||||
|
||||
internalParentProcess, ok := pm.processMap[internalProcess.ParentPID] |
||||
if ok { |
||||
parentProcess, ok := processMap[process.ParentPID] |
||||
if !ok { |
||||
parentProcess = internalParentProcess.toProcess() |
||||
processMap[parentProcess.PID] = parentProcess |
||||
} |
||||
parentProcess.Children = append(parentProcess.Children, process) |
||||
continue |
||||
} |
||||
|
||||
processes = append(processes, process) |
||||
} |
||||
} |
||||
|
||||
// Now from within the lock we need to get the goroutines.
|
||||
// Why? If we release the lock then between between filling the above map and getting
|
||||
// the stacktraces another process could be created which would then look like a dead process below
|
||||
reader, writer := io.Pipe() |
||||
defer reader.Close() |
||||
go func() { |
||||
err := pprof.Lookup("goroutine").WriteTo(writer, 0) |
||||
_ = writer.CloseWithError(err) |
||||
}() |
||||
stacks, err = profile.Parse(reader) |
||||
if err != nil { |
||||
return nil, 0, 0, err |
||||
} |
||||
|
||||
// Unlock the mutex
|
||||
pm.mutex.Unlock() |
||||
unlocked = true |
||||
|
||||
goroutineCount := int64(0) |
||||
|
||||
// Now walk through the "Sample" slice in the goroutines stack
|
||||
for _, sample := range stacks.Sample { |
||||
// In the "goroutine" pprof profile each sample represents one or more goroutines
|
||||
// with the same labels and stacktraces.
|
||||
|
||||
// We will represent each goroutine by a `Stack`
|
||||
stack := &Stack{} |
||||
|
||||
// Add the non-process associated labels from the goroutine sample to the Stack
|
||||
for name, value := range sample.Label { |
||||
if name == DescriptionPProfLabel || name == PIDPProfLabel || (!flat && name == PPIDPProfLabel) || name == ProcessTypePProfLabel { |
||||
continue |
||||
} |
||||
|
||||
// Labels from the "goroutine" pprof profile only have one value.
|
||||
// This is because the underlying representation is a map[string]string
|
||||
if len(value) != 1 { |
||||
// Unexpected...
|
||||
return nil, 0, 0, fmt.Errorf("label: %s in goroutine stack with unexpected number of values: %v", name, value) |
||||
} |
||||
|
||||
stack.Labels = append(stack.Labels, &Label{Name: name, Value: value[0]}) |
||||
} |
||||
|
||||
// The number of goroutines that this sample represents is the `stack.Value[0]`
|
||||
stack.Count = sample.Value[0] |
||||
goroutineCount += stack.Count |
||||
|
||||
// Now we want to associate this Stack with a Process.
|
||||
var process *Process |
||||
|
||||
// Try to get the PID from the goroutine labels
|
||||
if pidvalue, ok := sample.Label[PIDPProfLabel]; ok && len(pidvalue) == 1 { |
||||
pid := IDType(pidvalue[0]) |
||||
|
||||
// Now try to get the process from our map
|
||||
process, ok = processMap[pid] |
||||
if !ok && pid != "" { |
||||
// This means that no process has been found in the process map - but there was a process PID
|
||||
// Therefore this goroutine belongs to a dead process and it has escaped control of the process as it
|
||||
// should have died with the process context cancellation.
|
||||
|
||||
// We need to create a dead process holder for this process and label it appropriately
|
||||
|
||||
// get the parent PID
|
||||
ppid := IDType("") |
||||
if value, ok := sample.Label[PPIDPProfLabel]; ok && len(value) == 1 { |
||||
ppid = IDType(value[0]) |
||||
} |
||||
|
||||
// format the description
|
||||
description := "(dead process)" |
||||
if value, ok := sample.Label[DescriptionPProfLabel]; ok && len(value) == 1 { |
||||
description = value[0] + " " + description |
||||
} |
||||
|
||||
// override the type of the process to "code" but add the old type as a label on the first stack
|
||||
ptype := NoneProcessType |
||||
if value, ok := sample.Label[ProcessTypePProfLabel]; ok && len(value) == 1 { |
||||
stack.Labels = append(stack.Labels, &Label{Name: ProcessTypePProfLabel, Value: value[0]}) |
||||
} |
||||
process = &Process{ |
||||
PID: pid, |
||||
ParentPID: ppid, |
||||
Description: description, |
||||
Type: ptype, |
||||
} |
||||
|
||||
// Now add the dead process back to the map and tree so we don't go back through this again.
|
||||
processMap[process.PID] = process |
||||
added := false |
||||
if process.ParentPID != "" && !flat { |
||||
if parent, ok := processMap[process.ParentPID]; ok { |
||||
parent.Children = append(parent.Children, process) |
||||
added = true |
||||
} |
||||
} |
||||
if !added { |
||||
processes = append(processes, process) |
||||
} |
||||
} |
||||
} |
||||
|
||||
if process == nil { |
||||
// This means that the sample we're looking has no PID label
|
||||
var ok bool |
||||
process, ok = processMap[""] |
||||
if !ok { |
||||
// this is the first time we've come acrross an unassociated goroutine so create a "process" to hold them
|
||||
process = &Process{ |
||||
Description: "(unassociated)", |
||||
Type: NoneProcessType, |
||||
} |
||||
processMap[process.PID] = process |
||||
processes = append(processes, process) |
||||
} |
||||
} |
||||
|
||||
// The sample.Location represents a stack trace for this goroutine,
|
||||
// however each Location can represent multiple lines (mostly due to inlining)
|
||||
// so we need to walk the lines too
|
||||
for _, location := range sample.Location { |
||||
for _, line := range location.Line { |
||||
entry := &StackEntry{ |
||||
Function: line.Function.Name, |
||||
File: line.Function.Filename, |
||||
Line: int(line.Line), |
||||
} |
||||
stack.Entry = append(stack.Entry, entry) |
||||
} |
||||
} |
||||
|
||||
// Now we need a short-descriptive name to call the stack trace if when it is folded and
|
||||
// assuming the stack trace has some lines we'll choose the bottom of the stack (i.e. the
|
||||
// initial function that started the stack trace.) The top of the stack is unlikely to
|
||||
// be very helpful as a lot of the time it will be runtime.select or some other call into
|
||||
// a std library.
|
||||
stack.Description = "(unknown)" |
||||
if len(stack.Entry) > 0 { |
||||
stack.Description = stack.Entry[len(stack.Entry)-1].Function |
||||
} |
||||
|
||||
process.Stacks = append(process.Stacks, stack) |
||||
} |
||||
|
||||
// restrict to not show system processes
|
||||
if noSystem { |
||||
for i := 0; i < len(processes); i++ { |
||||
process := processes[i] |
||||
if process.Type != SystemProcessType && process.Type != NoneProcessType { |
||||
continue |
||||
} |
||||
processes[len(processes)-1], processes[i] = processes[i], processes[len(processes)-1] |
||||
processes = append(processes[:len(processes)-1], process.Children...) |
||||
i-- |
||||
} |
||||
} |
||||
|
||||
// Now finally re-sort the processes. Newest process appears first
|
||||
after := func(processes []*Process) func(i, j int) bool { |
||||
return func(i, j int) bool { |
||||
left, right := processes[i], processes[j] |
||||
return left.Start.After(right.Start) |
||||
} |
||||
} |
||||
sort.Slice(processes, after(processes)) |
||||
if !flat { |
||||
|
||||
var sortChildren func(process *Process) |
||||
|
||||
sortChildren = func(process *Process) { |
||||
sort.Slice(process.Children, after(process.Children)) |
||||
for _, child := range process.Children { |
||||
sortChildren(child) |
||||
} |
||||
} |
||||
} |
||||
|
||||
return processes, processCount, goroutineCount, err |
||||
} |
||||
@ -0,0 +1,161 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package private |
||||
|
||||
import ( |
||||
"bytes" |
||||
"fmt" |
||||
"io" |
||||
"net/http" |
||||
"runtime" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/modules/context" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/private" |
||||
process_module "code.gitea.io/gitea/modules/process" |
||||
) |
||||
|
||||
// Processes prints out the processes
|
||||
func Processes(ctx *context.PrivateContext) { |
||||
pid := ctx.FormString("cancel-pid") |
||||
if pid != "" { |
||||
process_module.GetManager().Cancel(process_module.IDType(pid)) |
||||
runtime.Gosched() |
||||
time.Sleep(100 * time.Millisecond) |
||||
} |
||||
|
||||
flat := ctx.FormBool("flat") |
||||
noSystem := ctx.FormBool("no-system") |
||||
stacktraces := ctx.FormBool("stacktraces") |
||||
json := ctx.FormBool("json") |
||||
|
||||
var processes []*process_module.Process |
||||
goroutineCount := int64(0) |
||||
processCount := 0 |
||||
var err error |
||||
if stacktraces { |
||||
processes, processCount, goroutineCount, err = process_module.GetManager().ProcessStacktraces(flat, noSystem) |
||||
if err != nil { |
||||
log.Error("Unable to get stacktrace: %v", err) |
||||
ctx.JSON(http.StatusInternalServerError, private.Response{ |
||||
Err: fmt.Sprintf("Failed to get stacktraces: %v", err), |
||||
}) |
||||
return |
||||
} |
||||
} else { |
||||
processes, processCount = process_module.GetManager().Processes(flat, noSystem) |
||||
} |
||||
|
||||
if json { |
||||
ctx.JSON(http.StatusOK, map[string]interface{}{ |
||||
"TotalNumberOfGoroutines": goroutineCount, |
||||
"TotalNumberOfProcesses": processCount, |
||||
"Processes": processes, |
||||
}) |
||||
return |
||||
} |
||||
|
||||
ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8") |
||||
ctx.Resp.WriteHeader(http.StatusOK) |
||||
|
||||
if err := writeProcesses(ctx.Resp, processes, processCount, goroutineCount, "", flat); err != nil { |
||||
log.Error("Unable to write out process stacktrace: %v", err) |
||||
if !ctx.Written() { |
||||
ctx.JSON(http.StatusInternalServerError, private.Response{ |
||||
Err: fmt.Sprintf("Failed to get stacktraces: %v", err), |
||||
}) |
||||
} |
||||
return |
||||
} |
||||
} |
||||
|
||||
func writeProcesses(out io.Writer, processes []*process_module.Process, processCount int, goroutineCount int64, indent string, flat bool) error { |
||||
if goroutineCount > 0 { |
||||
if _, err := fmt.Fprintf(out, "%sTotal Number of Goroutines: %d\n", indent, goroutineCount); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
if _, err := fmt.Fprintf(out, "%sTotal Number of Processes: %d\n", indent, processCount); err != nil { |
||||
return err |
||||
} |
||||
if len(processes) > 0 { |
||||
if err := writeProcess(out, processes[0], " ", flat); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
if len(processes) > 1 { |
||||
for _, process := range processes[1:] { |
||||
if _, err := fmt.Fprintf(out, "%s | \n", indent); err != nil { |
||||
return err |
||||
} |
||||
if err := writeProcess(out, process, " ", flat); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func writeProcess(out io.Writer, process *process_module.Process, indent string, flat bool) error { |
||||
sb := &bytes.Buffer{} |
||||
if flat { |
||||
if process.ParentPID != "" { |
||||
_, _ = fmt.Fprintf(sb, "%s+ PID: %s\t\tType: %s\n", indent, process.PID, process.Type) |
||||
} else { |
||||
_, _ = fmt.Fprintf(sb, "%s+ PID: %s:%s\tType: %s\n", indent, process.ParentPID, process.PID, process.Type) |
||||
} |
||||
} else { |
||||
_, _ = fmt.Fprintf(sb, "%s+ PID: %s\tType: %s\n", indent, process.PID, process.Type) |
||||
} |
||||
indent += "| " |
||||
|
||||
_, _ = fmt.Fprintf(sb, "%sDescription: %s\n", indent, process.Description) |
||||
_, _ = fmt.Fprintf(sb, "%sStart: %s\n", indent, process.Start) |
||||
|
||||
if len(process.Stacks) > 0 { |
||||
_, _ = fmt.Fprintf(sb, "%sGoroutines:\n", indent) |
||||
for _, stack := range process.Stacks { |
||||
indent := indent + " " |
||||
_, _ = fmt.Fprintf(sb, "%s+ Description: %s", indent, stack.Description) |
||||
if stack.Count > 1 { |
||||
_, _ = fmt.Fprintf(sb, "* %d", stack.Count) |
||||
} |
||||
_, _ = fmt.Fprintf(sb, "\n") |
||||
indent += "| " |
||||
if len(stack.Labels) > 0 { |
||||
_, _ = fmt.Fprintf(sb, "%sLabels: %q:%q", indent, stack.Labels[0].Name, stack.Labels[0].Value) |
||||
|
||||
if len(stack.Labels) > 1 { |
||||
for _, label := range stack.Labels[1:] { |
||||
_, _ = fmt.Fprintf(sb, ", %q:%q", label.Name, label.Value) |
||||
} |
||||
} |
||||
_, _ = fmt.Fprintf(sb, "\n") |
||||
} |
||||
_, _ = fmt.Fprintf(sb, "%sStack:\n", indent) |
||||
indent += " " |
||||
for _, entry := range stack.Entry { |
||||
_, _ = fmt.Fprintf(sb, "%s+ %s\n", indent, entry.Function) |
||||
_, _ = fmt.Fprintf(sb, "%s| %s:%d\n", indent, entry.File, entry.Line) |
||||
} |
||||
} |
||||
} |
||||
if _, err := out.Write(sb.Bytes()); err != nil { |
||||
return err |
||||
} |
||||
sb.Reset() |
||||
if len(process.Children) > 0 { |
||||
if _, err := fmt.Fprintf(out, "%sChildren:\n", indent); err != nil { |
||||
return err |
||||
} |
||||
for _, child := range process.Children { |
||||
if err := writeProcess(out, child, indent+" ", flat); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,66 @@ |
||||
<div class="item"> |
||||
<div class="df ac"> |
||||
<div class="icon ml-3 mr-3"> |
||||
{{if eq .Process.Type "request"}} |
||||
{{svg "octicon-globe" 16 }} |
||||
{{else if eq .Process.Type "system"}} |
||||
{{svg "octicon-cpu" 16 }} |
||||
{{else if eq .Process.Type "normal"}} |
||||
{{svg "octicon-terminal" 16 }} |
||||
{{else}} |
||||
{{svg "octicon-code" 16 }} |
||||
{{end}} |
||||
</div> |
||||
<div class="content f1"> |
||||
<div class="header">{{.Process.Description}}</div> |
||||
<div class="description">{{if ne .Process.Type "none"}}<span title="{{DateFmtLong .Process.Start}}">{{TimeSince .Process.Start .root.i18n.Lang}}</span>{{end}}</div> |
||||
</div> |
||||
<div> |
||||
{{if or (eq .Process.Type "request") (eq .Process.Type "normal") }} |
||||
<a class="delete-button icon" href="" data-url="{{.root.Link}}/cancel/{{.Process.PID}}" data-id="{{.Process.PID}}" data-name="{{.Process.Description}}">{{svg "octicon-trash" 16 "text-red"}}</a> |
||||
{{end}} |
||||
</div> |
||||
</div> |
||||
{{if .Process.Stacks}} |
||||
<div class="divided list ml-3"> |
||||
{{range .Process.Stacks}} |
||||
<div class="item"> |
||||
<details> |
||||
<summary> |
||||
<div class="dif content"> |
||||
<div class="header ml-3"> |
||||
<span class="icon mr-3">{{svg "octicon-code" 16 }}</span>{{.Description}}{{if gt .Count 1}} * {{.Count}}{{end}} |
||||
</div> |
||||
<div class="description"> |
||||
{{range .Labels}} |
||||
<div class="ui label">{{.Name}}<div class="detail">{{.Value}}</div></div> |
||||
{{end}} |
||||
</div> |
||||
</div> |
||||
</summary> |
||||
<div class="list"> |
||||
{{range .Entry}} |
||||
<div class="item df ac"> |
||||
<span class="icon mr-4">{{svg "octicon-dot-fill" 16 }}</span> |
||||
<div class="content f1"> |
||||
<div class="header"><code>{{.Function}}</code></div> |
||||
<div class="description"><code>{{.File}}:{{.Line}}</code></div> |
||||
</div> |
||||
</div> |
||||
{{end}} |
||||
</div> |
||||
</details> |
||||
</div> |
||||
{{end}} |
||||
</div> |
||||
{{end}} |
||||
|
||||
{{if .Process.Children}} |
||||
<div class="divided list"> |
||||
{{range .Process.Children}} |
||||
{{template "admin/stacktrace-row" dict "Process" . "root" $.root}} |
||||
{{end}} |
||||
</div> |
||||
{{end}} |
||||
|
||||
</div> |
||||
@ -0,0 +1,33 @@ |
||||
{{template "base/head" .}} |
||||
<div class="page-content admin monitor"> |
||||
{{template "admin/navbar" .}} |
||||
<div class="ui container"> |
||||
{{template "base/alert" .}} |
||||
<h4 class="ui top attached header"> |
||||
{{.i18n.Tr "admin.monitor.stacktrace"}}: {{.i18n.Tr "admin.monitor.goroutines" .GoroutineCount}} |
||||
<div class="ui right"> |
||||
<a class="ui blue tiny button" href="{{AppSubUrl}}/admin/monitor">{{.i18n.Tr "admin.monitor"}}</a> |
||||
</div> |
||||
</h4> |
||||
<div class="ui attached segment"> |
||||
<div class="ui relaxed divided list"> |
||||
{{range .ProcessStacks}} |
||||
{{template "admin/stacktrace-row" dict "Process" . "root" $}} |
||||
{{end}} |
||||
</div> |
||||
</div> |
||||
</div> |
||||
</div> |
||||
<div class="ui small basic delete modal"> |
||||
<div class="ui icon header"> |
||||
{{svg "octicon-x" 16 "close inside"}} |
||||
{{.i18n.Tr "admin.monitor.process.cancel"}} |
||||
</div> |
||||
<div class="content"> |
||||
<p>{{$.i18n.Tr "admin.monitor.process.cancel_notices" `<span class="name"></span>` | Safe}}</p> |
||||
<p>{{$.i18n.Tr "admin.monitor.process.cancel_desc"}}</p> |
||||
</div> |
||||
{{template "base/delete_modal_actions" .}} |
||||
</div> |
||||
|
||||
{{template "base/footer" .}} |
||||
Loading…
Reference in new issue