diff --git a/client/client.go b/client/client.go index 59698be6..fface241 100644 --- a/client/client.go +++ b/client/client.go @@ -21,6 +21,7 @@ import ( "github.com/jpillora/chisel/share/ccrypto" "github.com/jpillora/chisel/share/cio" "github.com/jpillora/chisel/share/cnet" + "github.com/jpillora/chisel/share/metrics" "github.com/jpillora/chisel/share/settings" "github.com/jpillora/chisel/share/tunnel" @@ -43,6 +44,8 @@ type Config struct { TLS TLSConfig DialContext func(ctx context.Context, network, addr string) (net.Conn, error) Verbose bool + MetricsAddr string + MetricsNamespace string } // TLSConfig for a Client @@ -67,6 +70,7 @@ type Client struct { stop func() eg *errgroup.Group tunnel *tunnel.Tunnel + metrics *metrics.Metrics } // NewClient creates a new client instance @@ -179,6 +183,18 @@ func NewClient(c *Config) (*Client, error) { HostKeyCallback: client.verifyServer, Timeout: settings.EnvDuration("SSH_TIMEOUT", 30*time.Second), } + //initialize metrics if enabled + if c.MetricsAddr != "" { + m, err := metrics.New(c.MetricsNamespace) + if err != nil { + return nil, err + } + client.metrics = m + if err := client.metrics.Start(c.MetricsAddr); err != nil { + return nil, err + } + client.Infof("Metrics server started on %s", c.MetricsAddr) + } //prepare client tunnel client.tunnel = tunnel.New(tunnel.Config{ Logger: client.Logger, @@ -186,6 +202,7 @@ func NewClient(c *Config) (*Client, error) { Outbound: hasReverse, Socks: hasReverse && hasSocks, KeepAlive: client.config.KeepAlive, + Metrics: client.metrics, }) return client, nil } diff --git a/client/client_connect.go b/client/client_connect.go index 884c7647..b3685a52 100644 --- a/client/client_connect.go +++ b/client/client_connect.go @@ -66,6 +66,13 @@ func (c *Client) connectionLoop(ctx context.Context) error { // connectionOnce connects to the chisel server and blocks func (c *Client) connectionOnce(ctx context.Context) (connected bool, err error) { + // Record connection attempt + if c.metrics != nil { + (*c.metrics.ClientConnectionAttempts).Inc() + if c.Debug { + c.Debugf("[metrics] client_connection_attempts_total incremented") + } + } //already closed? select { case <-ctx.Done(): @@ -92,6 +99,12 @@ func (c *Client) connectionOnce(ctx context.Context) (connected bool, err error) } wsConn, _, err := d.DialContext(ctx, c.server, c.config.Headers) if err != nil { + if c.metrics != nil { + c.metrics.ClientConnectionErrors.WithLabelValues("handshake_error").Inc() + if c.Debug { + c.Debugf("[metrics] client_connection_errors_total{cause=\"handshake_error\"} incremented") + } + } return false, err } conn := cnet.NewWebSocketConn(wsConn) @@ -103,8 +116,20 @@ func (c *Client) connectionOnce(ctx context.Context) (connected bool, err error) if strings.Contains(e, "unable to authenticate") { c.Infof("Authentication failed") c.Debugf(e) + if c.metrics != nil { + c.metrics.ClientConnectionErrors.WithLabelValues("auth_failure").Inc() + if c.Debug { + c.Debugf("[metrics] client_connection_errors_total{cause=\"auth_failure\"} incremented") + } + } } else { c.Infof(e) + if c.metrics != nil { + c.metrics.ClientConnectionErrors.WithLabelValues("handshake_error").Inc() + if c.Debug { + c.Debugf("[metrics] client_connection_errors_total{cause=\"handshake_error\"} incremented") + } + } } return false, err } @@ -123,12 +148,42 @@ func (c *Client) connectionOnce(ctx context.Context) (connected bool, err error) return false, err } if len(configerr) > 0 { + if c.metrics != nil { + c.metrics.ClientConnectionErrors.WithLabelValues("handshake_error").Inc() + if c.Debug { + c.Debugf("[metrics] client_connection_errors_total{cause=\"handshake_error\"} incremented") + } + } return false, errors.New(string(configerr)) } + // Record handshake duration and set connected status + if c.metrics != nil { + duration := time.Since(t0).Seconds() + c.metrics.ClientHandshakeDuration.Observe(duration) + c.metrics.ClientConnected.Set(1) + if c.Debug { + c.Debugf("[metrics] client_handshake_duration_seconds observed: %.6fs", duration) + c.Debugf("[metrics] client_connected set to 1") + } + } c.Infof("Connected (Latency %s)", time.Since(t0)) //connected, handover ssh connection for tunnel to use, and block err = c.tunnel.BindSSH(ctx, sshConn, reqs, chans) c.Infof("Disconnected") + // Set disconnected status + if c.metrics != nil { + c.metrics.ClientConnected.Set(0) + if c.Debug { + c.Debugf("[metrics] client_connected set to 0") + } + // Record transport error if it's not EOF + if err != nil && err != io.EOF && !strings.HasSuffix(err.Error(), "EOF") { + c.metrics.ClientConnectionErrors.WithLabelValues("transport_error").Inc() + if c.Debug { + c.Debugf("[metrics] client_connection_errors_total{cause=\"transport_error\"} incremented") + } + } + } connected = time.Since(t0) > 5*time.Second return connected, err } diff --git a/go.mod b/go.mod index 73cf3c04..5c7e3972 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/jpillora/backoff v1.0.0 github.com/jpillora/requestlog v1.0.0 github.com/jpillora/sizestr v1.0.0 + github.com/prometheus/client_golang v1.20.5 golang.org/x/crypto v0.53.0 golang.org/x/net v0.56.0 golang.org/x/sync v0.21.0 @@ -16,10 +17,18 @@ require ( require ( github.com/andrew-d/go-termutil v0.0.0-20150726205930-009166a695a2 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/jpillora/ansi v1.0.3 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.38.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) replace github.com/jpillora/chisel => ../chisel diff --git a/go.sum b/go.sum index a2e8edb8..c3b5dff3 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,14 @@ github.com/andrew-d/go-termutil v0.0.0-20150726205930-009166a695a2 h1:axBiC50cNZ github.com/andrew-d/go-termutil v0.0.0-20150726205930-009166a695a2/go.mod h1:jnzFpU88PccN/tPPhCpnNU8mZphvKxYM9lLNkd8e+os= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/jpillora/ansi v1.0.3 h1:nn4Jzti0EmRfDxm7JtEs5LzCbNwd5sv+0aE+LdS9/ZQ= @@ -14,6 +20,20 @@ github.com/jpillora/requestlog v1.0.0 h1:bg++eJ74T7DYL3DlIpiwknrtfdUA9oP/M4fL+Pp github.com/jpillora/requestlog v1.0.0/go.mod h1:HTWQb7QfDc2jtHnWe2XEIEeJB7gJPnVdpNn52HXPvy8= github.com/jpillora/sizestr v1.0.0 h1:4tr0FLxs1Mtq3TnsLDV+GYUWG7Q26a6s+tV5Zfw2ygw= github.com/jpillora/sizestr v1.0.0/go.mod h1:bUhLv4ctkknatr6gR42qPxirmd5+ds1u7mzD+MZ33f0= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= @@ -28,3 +48,5 @@ golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/main.go b/main.go index 7af1f45e..8a2e745b 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,15 @@ var help = ` server - runs chisel in server mode client - runs chisel in client mode + Global Options (must precede the command): + + --metrics, An optional "host:port" to serve Prometheus metrics on + at /metrics. When unset, no metrics are collected or served. + + --metrics-namespace, An optional prefix applied to every metric name + (defaults to "chisel"). Must match [a-zA-Z_][a-zA-Z0-9_]*. Only used + when --metrics is set. + Read more: https://github.com/jpillora/chisel @@ -37,6 +46,8 @@ func main() { version := flag.Bool("version", false, "") v := flag.Bool("v", false, "") + metricsAddr := flag.String("metrics", "", "") + metricsNamespace := flag.String("metrics-namespace", "", "") flag.Bool("help", false, "") flag.Bool("h", false, "") flag.Usage = func() {} @@ -57,9 +68,9 @@ func main() { switch subcmd { case "server": - server(args) + server(args, *metricsAddr, *metricsNamespace) case "client": - client(args) + client(args, *metricsAddr, *metricsNamespace) default: fmt.Print(help) os.Exit(0) @@ -176,11 +187,13 @@ var serverHelp = ` instead of the system roots. This is commonly used to implement mutual-TLS. ` + commonHelp -func server(args []string) { +func server(args []string, metricsAddr, metricsNamespace string) { flags := flag.NewFlagSet("server", flag.ContinueOnError) config := &chserver.Config{} + config.MetricsAddr = metricsAddr + config.MetricsNamespace = metricsNamespace flags.StringVar(&config.KeySeed, "key", "", "") flags.StringVar(&config.KeyFile, "keyfile", "", "") flags.StringVar(&config.AuthFile, "authfile", "", "") @@ -421,9 +434,11 @@ var clientHelp = ` enabled (mutual-TLS). ` + commonHelp -func client(args []string) { +func client(args []string, metricsAddr, metricsNamespace string) { flags := flag.NewFlagSet("client", flag.ContinueOnError) config := chclient.Config{Headers: http.Header{}} + config.MetricsAddr = metricsAddr + config.MetricsNamespace = metricsNamespace flags.StringVar(&config.Fingerprint, "fingerprint", "", "") flags.StringVar(&config.Auth, "auth", "", "") flags.DurationVar(&config.KeepAlive, "keepalive", 25*time.Second, "") diff --git a/server/server.go b/server/server.go index 8a702fce..0c0a4304 100644 --- a/server/server.go +++ b/server/server.go @@ -16,6 +16,7 @@ import ( "github.com/jpillora/chisel/share/ccrypto" "github.com/jpillora/chisel/share/cio" "github.com/jpillora/chisel/share/cnet" + "github.com/jpillora/chisel/share/metrics" "github.com/jpillora/chisel/share/settings" "github.com/jpillora/requestlog" "golang.org/x/crypto/ssh" @@ -23,15 +24,17 @@ import ( // Config is the configuration for the chisel service type Config struct { - KeySeed string - KeyFile string - AuthFile string - Auth string - Proxy string - Socks5 bool - Reverse bool - KeepAlive time.Duration - TLS TLSConfig + KeySeed string + KeyFile string + AuthFile string + Auth string + Proxy string + Socks5 bool + Reverse bool + KeepAlive time.Duration + TLS TLSConfig + MetricsAddr string + MetricsNamespace string } // Server respresent a chisel service @@ -45,6 +48,7 @@ type Server struct { sessions *settings.Users sshConfig *ssh.ServerConfig users *settings.UserIndex + metrics *metrics.Metrics } var upgrader = websocket.Upgrader{ @@ -140,6 +144,18 @@ func NewServer(c *Config) (*Server, error) { if c.Reverse { server.Infof("Reverse tunnelling enabled") } + //initialize metrics if enabled + if c.MetricsAddr != "" { + m, err := metrics.New(c.MetricsNamespace) + if err != nil { + return nil, err + } + server.metrics = m + if err := server.metrics.Start(c.MetricsAddr); err != nil { + return nil, err + } + server.Infof("Metrics server started on %s", c.MetricsAddr) + } return server, nil } @@ -199,6 +215,12 @@ func (s *Server) GetFingerprint() string { func (s *Server) authUser(c ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) { // check if user authentication is enabled and if not, allow all if s.users.Len() == 0 { + if s.metrics != nil { + s.metrics.ServerAuthAttempts.WithLabelValues("success").Inc() + if s.Debug { + s.Debugf("[metrics] server_auth_attempts_total{outcome=\"success\"} incremented") + } + } return nil, nil } // check the user exists and has matching password @@ -206,11 +228,23 @@ func (s *Server) authUser(c ssh.ConnMetadata, password []byte) (*ssh.Permissions user, found := s.users.Get(n) if !found || user.Pass != string(password) { s.Debugf("Login failed for user: %s", n) + if s.metrics != nil { + s.metrics.ServerAuthAttempts.WithLabelValues("failure").Inc() + if s.Debug { + s.Debugf("[metrics] server_auth_attempts_total{outcome=\"failure\"} incremented") + } + } return nil, errors.New("Invalid authentication for username: %s") } // insert the user session map // TODO this should probably have a lock on it given the map isn't thread-safe s.sessions.Set(string(c.SessionID()), user) + if s.metrics != nil { + s.metrics.ServerAuthAttempts.WithLabelValues("success").Inc() + if s.Debug { + s.Debugf("[metrics] server_auth_attempts_total{outcome=\"success\"} incremented") + } + } return nil, nil } diff --git a/server/server_handler.go b/server/server_handler.go index 8b5a68fd..755cecab 100644 --- a/server/server_handler.go +++ b/server/server_handler.go @@ -7,6 +7,7 @@ import ( "time" chshare "github.com/jpillora/chisel/share" + "github.com/jpillora/chisel/share/cio" "github.com/jpillora/chisel/share/cnet" "github.com/jpillora/chisel/share/settings" "github.com/jpillora/chisel/share/tunnel" @@ -50,18 +51,33 @@ func (s *Server) handleClientHandler(w http.ResponseWriter, r *http.Request) { // handleWebsocket is responsible for handling the websocket connection func (s *Server) handleWebsocket(w http.ResponseWriter, req *http.Request) { id := atomic.AddInt32(&s.sessCount, 1) - l := s.Fork("session#%d", id) + clientAddr := req.RemoteAddr + var l *cio.Logger + if s.Debug { + l = s.Fork("session#%d@%s", id, clientAddr) + } else { + l = s.Fork("session#%d", id) + } wsConn, err := upgrader.Upgrade(w, req, nil) if err != nil { l.Debugf("Failed to upgrade (%s)", err) return } + // Start timer for session setup duration + sessionStart := time.Now() + conn := cnet.NewWebSocketConn(wsConn) // perform SSH handshake on net.Conn l.Debugf("Handshaking with %s...", req.RemoteAddr) sshConn, chans, reqs, err := ssh.NewServerConn(conn, s.sshConfig) if err != nil { s.Debugf("Failed to handshake (%s)", err) + if s.metrics != nil { + s.metrics.ServerSessions.WithLabelValues("auth_failure").Inc() + if s.Debug { + l.Debugf("[metrics] server_sessions_total{outcome=\"auth_failure\"} incremented (client: %s)", clientAddr) + } + } return } // pull the users from the session map @@ -93,11 +109,23 @@ func (s *Server) handleWebsocket(w http.ResponseWriter, req *http.Request) { } if r.Type != "config" { failed(s.Errorf("expecting config request")) + if s.metrics != nil { + s.metrics.ServerSessions.WithLabelValues("config_error").Inc() + if s.Debug { + l.Debugf("[metrics] server_sessions_total{outcome=\"config_error\"} incremented (client: %s)", clientAddr) + } + } return } c, err := settings.DecodeConfig(r.Payload) if err != nil { failed(s.Errorf("invalid config")) + if s.metrics != nil { + s.metrics.ServerSessions.WithLabelValues("config_error").Inc() + if s.Debug { + l.Debugf("[metrics] server_sessions_total{outcome=\"config_error\"} incremented (client: %s)", clientAddr) + } + } return } //print if client and server versions dont match @@ -117,6 +145,12 @@ func (s *Server) handleWebsocket(w http.ResponseWriter, req *http.Request) { addr := r.UserAddr() if !user.HasAccess(addr) { failed(s.Errorf("access to '%s' denied", addr)) + if s.metrics != nil { + s.metrics.ServerSessions.WithLabelValues("acl_denied").Inc() + if s.Debug { + l.Debugf("[metrics] server_sessions_total{outcome=\"acl_denied\"} incremented (client: %s)", clientAddr) + } + } return } } @@ -134,6 +168,16 @@ func (s *Server) handleWebsocket(w http.ResponseWriter, req *http.Request) { } //successfuly validated config! r.Reply(true, nil) + // Record session setup duration and established session + if s.metrics != nil { + duration := time.Since(sessionStart).Seconds() + s.metrics.ServerSessionSetupDuration.Observe(duration) + s.metrics.ServerSessions.WithLabelValues("established").Inc() + if s.Debug { + l.Debugf("[metrics] server_session_setup_duration_seconds observed: %.6fs (client: %s)", duration, clientAddr) + l.Debugf("[metrics] server_sessions_total{outcome=\"established\"} incremented (client: %s)", clientAddr) + } + } //tunnel per ssh connection tunnelConfig := tunnel.Config{ Logger: l, @@ -141,6 +185,7 @@ func (s *Server) handleWebsocket(w http.ResponseWriter, req *http.Request) { Outbound: true, //server always accepts outbound Socks: s.config.Socks5, KeepAlive: s.config.KeepAlive, + Metrics: s.metrics, } //enforce ACL on every channel, not just the initial config if user != nil { @@ -165,6 +210,12 @@ func (s *Server) handleWebsocket(w http.ResponseWriter, req *http.Request) { err = eg.Wait() if err != nil && !strings.HasSuffix(err.Error(), "EOF") { l.Debugf("Closed connection (%s)", err) + if s.metrics != nil { + s.metrics.ServerSessions.WithLabelValues("transport_error").Inc() + if s.Debug { + l.Debugf("[metrics] server_sessions_total{outcome=\"transport_error\"} incremented (client: %s)", clientAddr) + } + } } else { l.Debugf("Closed connection") } diff --git a/share/metrics/metrics.go b/share/metrics/metrics.go new file mode 100644 index 00000000..337a2958 --- /dev/null +++ b/share/metrics/metrics.go @@ -0,0 +1,250 @@ +package metrics + +import ( + "fmt" + "net/http" + "regexp" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var namespaceRe = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +// Metrics holds all Prometheus metrics for chisel server and client +type Metrics struct { + namespace string + registry *prometheus.Registry + verbose bool + + // Server metrics + ServerAuthAttempts *prometheus.CounterVec + ServerSessionSetupDuration prometheus.Histogram + ServerSessions *prometheus.CounterVec + + // Client metrics + ClientConnectionAttempts *prometheus.Counter + ClientConnectionErrors *prometheus.CounterVec + ClientHandshakeDuration prometheus.Histogram + ClientConnected prometheus.Gauge + + // Tunnel metrics (shared by server and client) + TunnelConnections *prometheus.Counter + TunnelConnectionErrors *prometheus.Counter + TunnelActiveConnections prometheus.Gauge + TunnelBytes *prometheus.CounterVec + TunnelKeepalivePings *prometheus.CounterVec +} + +// New creates a new Metrics instance with the given namespace +func New(namespace string) (*Metrics, error) { + if namespace == "" { + namespace = "chisel" + } + if !namespaceRe.MatchString(namespace) { + return nil, fmt.Errorf("invalid metrics namespace %q: must match %s", namespace, namespaceRe.String()) + } + + registry := prometheus.NewRegistry() + m := &Metrics{ + namespace: namespace, + registry: registry, + } + + // Server metrics + m.ServerAuthAttempts = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "server_auth_attempts_total", + Help: "Total SSH authentication attempts, labeled by outcome", + }, + []string{"outcome"}, + ) + + m.ServerSessionSetupDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: namespace, + Name: "server_session_setup_duration_seconds", + Help: "Seconds from WebSocket upgrade to successful config handshake", + Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5}, + }, + ) + + m.ServerSessions = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "server_sessions_total", + Help: "Total session attempts by terminal outcome", + }, + []string{"outcome"}, + ) + + // Client metrics + clientConnectionAttempts := prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "client_connection_attempts_total", + Help: "Total connection attempts including retries after disconnect", + }, + ) + m.ClientConnectionAttempts = &clientConnectionAttempts + + m.ClientConnectionErrors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "client_connection_errors_total", + Help: "Total connection failures by category", + }, + []string{"cause"}, + ) + + m.ClientHandshakeDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: namespace, + Name: "client_handshake_duration_seconds", + Help: "Seconds for config round-trip: send SSH config request to receive server reply", + Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0}, + }, + ) + + m.ClientConnected = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "client_connected", + Help: "1 when tunnel is active (after config verified), 0 when disconnected", + }, + ) + + // Tunnel metrics + tunnelConnections := prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "tunnel_connections_total", + Help: "Total tunnel connections opened", + }, + ) + m.TunnelConnections = &tunnelConnections + + tunnelConnectionErrors := prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "tunnel_connection_errors_total", + Help: "Total tunnel connection failures (SSH OpenChannel + remote dial errors)", + }, + ) + m.TunnelConnectionErrors = &tunnelConnectionErrors + + m.TunnelActiveConnections = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "tunnel_active_connections", + Help: "Number of connections currently piping data", + }, + ) + + m.TunnelBytes = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "tunnel_bytes_total", + Help: "Total bytes transferred through the tunnel", + }, + []string{"direction"}, + ) + + m.TunnelKeepalivePings = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "tunnel_keepalive_pings_total", + Help: "Total SSH keepalive pings sent, labeled by outcome", + }, + []string{"outcome"}, + ) + + // Register all metrics + registry.MustRegister( + m.ServerAuthAttempts, + m.ServerSessionSetupDuration, + m.ServerSessions, + *m.ClientConnectionAttempts, + m.ClientConnectionErrors, + m.ClientHandshakeDuration, + m.ClientConnected, + *m.TunnelConnections, + *m.TunnelConnectionErrors, + m.TunnelActiveConnections, + m.TunnelBytes, + m.TunnelKeepalivePings, + ) + + return m, nil +} + +// SetVerbose enables verbose logging of metric changes +func (m *Metrics) SetVerbose(verbose bool) { + m.verbose = verbose +} + +// Start starts the metrics HTTP server on the given address +func (m *Metrics) Start(addr string) error { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{})) + + go func() { + if err := http.ListenAndServe(addr, mux); err != nil && err != http.ErrServerClosed { + // Log error but don't crash the main application + println("metrics server error:", err.Error()) + } + }() + + return nil +} + +// RecordTunnelConnection increments the tunnel connections counter +func (m *Metrics) RecordTunnelConnection() { + (*m.TunnelConnections).Inc() + if m.verbose { + println("[metrics] tunnel_connections_total incremented") + } +} + +// RecordTunnelConnectionError increments the tunnel connection errors counter +func (m *Metrics) RecordTunnelConnectionError() { + (*m.TunnelConnectionErrors).Inc() + if m.verbose { + println("[metrics] tunnel_connection_errors_total incremented") + } +} + +// RecordTunnelActiveConnectionsInc increments the active connections gauge +func (m *Metrics) RecordTunnelActiveConnectionsInc() { + m.TunnelActiveConnections.Inc() + if m.verbose { + println("[metrics] tunnel_active_connections incremented") + } +} + +// RecordTunnelActiveConnectionsDec decrements the active connections gauge +func (m *Metrics) RecordTunnelActiveConnectionsDec() { + m.TunnelActiveConnections.Dec() + if m.verbose { + println("[metrics] tunnel_active_connections decremented") + } +} + +// RecordTunnelBytes records bytes sent and received +func (m *Metrics) RecordTunnelBytes(sent, received int64) { + m.TunnelBytes.WithLabelValues("sent").Add(float64(sent)) + m.TunnelBytes.WithLabelValues("received").Add(float64(received)) + if m.verbose { + println("[metrics] tunnel_bytes_total{direction=\"sent\"} +=", sent) + println("[metrics] tunnel_bytes_total{direction=\"received\"} +=", received) + } +} + +// RecordTunnelKeepalivePing records a keepalive ping outcome +func (m *Metrics) RecordTunnelKeepalivePing(outcome string) { + m.TunnelKeepalivePings.WithLabelValues(outcome).Inc() + if m.verbose { + println("[metrics] tunnel_keepalive_pings_total{outcome=\"" + outcome + "\"} incremented") + } +} diff --git a/share/metrics/metrics_test.go b/share/metrics/metrics_test.go new file mode 100644 index 00000000..bfc6dd57 --- /dev/null +++ b/share/metrics/metrics_test.go @@ -0,0 +1,173 @@ +package metrics + +import ( + "io" + "net/http" + "strings" + "testing" + "time" +) + +func TestNewDefaultsNamespace(t *testing.T) { + m, err := New("") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if m.namespace != "chisel" { + t.Fatalf("expected default namespace 'chisel', got %q", m.namespace) + } +} + +func TestNewCustomNamespace(t *testing.T) { + m, err := New("outsystemscc") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if m.namespace != "outsystemscc" { + t.Fatalf("expected namespace 'outsystemscc', got %q", m.namespace) + } +} + +func TestNewInvalidNamespace(t *testing.T) { + for _, ns := range []string{"1bad", "bad-name", "bad.name", "bad name"} { + if _, err := New(ns); err == nil { + t.Fatalf("expected error for invalid namespace %q, got nil", ns) + } + } +} + +func TestRecordTunnelConnection(t *testing.T) { + m, err := New("chisel") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + m.RecordTunnelConnection() + m.RecordTunnelConnection() + + mfs, gatherErr := m.registry.Gather() + if gatherErr != nil { + t.Fatalf("gather failed: %s", gatherErr) + } + var found bool + for _, mf := range mfs { + if mf.GetName() == "chisel_tunnel_connections_total" { + found = true + if got := mf.GetMetric()[0].GetCounter().GetValue(); got != 2 { + t.Fatalf("expected counter value 2, got %v", got) + } + } + } + if !found { + t.Fatal("chisel_tunnel_connections_total not found in registry") + } +} + +func TestRecordTunnelBytes(t *testing.T) { + m, err := New("chisel") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + m.RecordTunnelBytes(100, 50) + + if got := testutilGetCounterValue(t, m, "chisel_tunnel_bytes_total", "direction", "sent"); got != 100 { + t.Fatalf("expected sent=100, got %v", got) + } + if got := testutilGetCounterValue(t, m, "chisel_tunnel_bytes_total", "direction", "received"); got != 50 { + t.Fatalf("expected received=50, got %v", got) + } +} + +func TestRecordTunnelActiveConnections(t *testing.T) { + m, err := New("chisel") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + m.RecordTunnelActiveConnectionsInc() + m.RecordTunnelActiveConnectionsInc() + m.RecordTunnelActiveConnectionsDec() + + mfs, _ := m.registry.Gather() + for _, mf := range mfs { + if mf.GetName() == "chisel_tunnel_active_connections" { + if got := mf.GetMetric()[0].GetGauge().GetValue(); got != 1 { + t.Fatalf("expected gauge value 1, got %v", got) + } + return + } + } + t.Fatal("chisel_tunnel_active_connections not found") +} + +func TestRecordTunnelKeepalivePing(t *testing.T) { + m, err := New("chisel") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + m.RecordTunnelKeepalivePing("success") + m.RecordTunnelKeepalivePing("timeout") + m.RecordTunnelKeepalivePing("success") + + if got := testutilGetCounterValue(t, m, "chisel_tunnel_keepalive_pings_total", "outcome", "success"); got != 2 { + t.Fatalf("expected success=2, got %v", got) + } + if got := testutilGetCounterValue(t, m, "chisel_tunnel_keepalive_pings_total", "outcome", "timeout"); got != 1 { + t.Fatalf("expected timeout=1, got %v", got) + } +} + +func TestStartServesMetricsEndpoint(t *testing.T) { + m, err := New("chisel") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + m.RecordTunnelConnection() + + addr := "127.0.0.1:19191" + if err := m.Start(addr); err != nil { + t.Fatalf("Start failed: %s", err) + } + // give the goroutine a moment to bind the listener + var resp *http.Response + for i := 0; i < 20; i++ { + resp, err = http.Get("http://" + addr + "/metrics") + if err == nil { + break + } + time.Sleep(25 * time.Millisecond) + } + if err != nil { + t.Fatalf("GET /metrics failed: %s", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + if !strings.Contains(string(body), "chisel_tunnel_connections_total 1") { + t.Fatalf("expected metric in body, got: %s", body) + } +} + +// testutilGetCounterValue extracts a CounterVec's value for a single label pair. +func testutilGetCounterValue(t *testing.T, m *Metrics, name, labelName, labelValue string) float64 { + t.Helper() + mfs, err := m.registry.Gather() + if err != nil { + t.Fatalf("gather failed: %s", err) + } + for _, mf := range mfs { + if mf.GetName() != name { + continue + } + for _, metric := range mf.GetMetric() { + for _, l := range metric.GetLabel() { + if l.GetName() == labelName && l.GetValue() == labelValue { + return metric.GetCounter().GetValue() + } + } + } + } + t.Fatalf("metric %s{%s=%q} not found", name, labelName, labelValue) + return 0 +} diff --git a/share/tunnel/metrics_helper.go b/share/tunnel/metrics_helper.go new file mode 100644 index 00000000..b73ab13f --- /dev/null +++ b/share/tunnel/metrics_helper.go @@ -0,0 +1,43 @@ +package tunnel + +func (t *Tunnel) recordTunnelConnection() { + if t.metrics == nil { + return + } + t.metrics.RecordTunnelConnection() +} + +func (t *Tunnel) recordTunnelConnectionError() { + if t.metrics == nil { + return + } + t.metrics.RecordTunnelConnectionError() +} + +func (t *Tunnel) recordTunnelActiveConnectionsInc() { + if t.metrics == nil { + return + } + t.metrics.RecordTunnelActiveConnectionsInc() +} + +func (t *Tunnel) recordTunnelActiveConnectionsDec() { + if t.metrics == nil { + return + } + t.metrics.RecordTunnelActiveConnectionsDec() +} + +func (t *Tunnel) recordTunnelBytes(sent, received int64) { + if t.metrics == nil { + return + } + t.metrics.RecordTunnelBytes(sent, received) +} + +func (t *Tunnel) recordTunnelKeepalivePing(outcome string) { + if t.metrics == nil { + return + } + t.metrics.RecordTunnelKeepalivePing(outcome) +} diff --git a/share/tunnel/metrics_helper_test.go b/share/tunnel/metrics_helper_test.go new file mode 100644 index 00000000..fc832335 --- /dev/null +++ b/share/tunnel/metrics_helper_test.go @@ -0,0 +1,18 @@ +package tunnel + +import "testing" + +// TestRecordMetricsNilSafe guards against a regression where a *Tunnel +// constructed without metrics (the common case, --metrics unset) would +// panic on the first connection because a nil metrics pointer, once +// wrapped, was mistaken for a non-nil value. +func TestRecordMetricsNilSafe(t *testing.T) { + tun := &Tunnel{} + + tun.recordTunnelConnection() + tun.recordTunnelConnectionError() + tun.recordTunnelActiveConnectionsInc() + tun.recordTunnelActiveConnectionsDec() + tun.recordTunnelBytes(10, 20) + tun.recordTunnelKeepalivePing("success") +} diff --git a/share/tunnel/tunnel.go b/share/tunnel/tunnel.go index a42f56d1..db3f3830 100644 --- a/share/tunnel/tunnel.go +++ b/share/tunnel/tunnel.go @@ -13,6 +13,7 @@ import ( "github.com/armon/go-socks5" "github.com/jpillora/chisel/share/cio" "github.com/jpillora/chisel/share/cnet" + "github.com/jpillora/chisel/share/metrics" "github.com/jpillora/chisel/share/settings" "golang.org/x/crypto/ssh" "golang.org/x/sync/errgroup" @@ -27,7 +28,8 @@ type Config struct { KeepAlive time.Duration //ACL optionally checks if a given address (host:port) is allowed. //When set, outbound connections are denied if this returns false. - ACL func(addr string) bool + ACL func(addr string) bool + Metrics *metrics.Metrics // nil = disabled } // Tunnel represents an SSH tunnel with proxy capabilities. @@ -48,13 +50,15 @@ type Tunnel struct { //internals connStats cnet.ConnCount socksServer *socks5.Server + metrics *metrics.Metrics } // New Tunnel from the given Config func New(c Config) *Tunnel { c.Logger = c.Logger.Fork("tun") t := &Tunnel{ - Config: c, + Config: c, + metrics: c.Metrics, } t.activatingConn.Add(1) //setup socks server (not listening on any port!) @@ -213,6 +217,15 @@ func (t *Tunnel) keepAliveLoop(sshConn ssh.Conn) { // as SendRequest will be unblocked on connection closure, the goroutine will send // an error message and finish, therefore releasing the channel. + // Record keepalive ping outcome + if err == nil { + t.recordTunnelKeepalivePing("success") + } else if err.Error() == "KEEPALIVE REPLY TIMEOUT ERROR" { + t.recordTunnelKeepalivePing("timeout") + } else { + t.recordTunnelKeepalivePing("error") + } + if err != nil { break } diff --git a/share/tunnel/tunnel_out_ssh.go b/share/tunnel/tunnel_out_ssh.go index 3b380fa8..da4ab220 100644 --- a/share/tunnel/tunnel_out_ssh.go +++ b/share/tunnel/tunnel_out_ssh.go @@ -57,6 +57,7 @@ func (t *Tunnel) handleSSHChannel(ch ssh.NewChannel) { sshChan, reqs, err := ch.Accept() if err != nil { t.Debugf("Failed to accept stream: %s", err) + t.recordTunnelConnectionError() return } stream := io.ReadWriteCloser(sshChan) @@ -66,6 +67,8 @@ func (t *Tunnel) handleSSHChannel(ch ssh.NewChannel) { l := t.Logger.Fork("conn#%d", t.connStats.New()) //ready to handle t.connStats.Open() + t.recordTunnelConnection() + t.recordTunnelActiveConnectionsInc() l.Debugf("Open %s", t.connStats.String()) if socks { err = t.handleSocks(stream) @@ -75,6 +78,7 @@ func (t *Tunnel) handleSSHChannel(ch ssh.NewChannel) { err = t.handleTCP(l, stream, hostPort) } t.connStats.Close() + t.recordTunnelActiveConnectionsDec() errmsg := "" if err != nil && !strings.HasSuffix(err.Error(), "EOF") { errmsg = fmt.Sprintf(" (error %s)", err) @@ -89,9 +93,11 @@ func (t *Tunnel) handleSocks(src io.ReadWriteCloser) error { func (t *Tunnel) handleTCP(l *cio.Logger, src io.ReadWriteCloser, hostPort string) error { dst, err := net.Dial("tcp", hostPort) if err != nil { + t.recordTunnelConnectionError() return err } s, r := cio.Pipe(src, dst) + t.recordTunnelBytes(s, r) l.Debugf("sent %s received %s", sizestr.ToString(s), sizestr.ToString(r)) return nil }