From 1336b83142ca27a13181abba684a1fb2fa6e9ad2 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 16:10:26 +0800 Subject: [PATCH 1/7] api prommetrics --- go.mod | 6 +- go.sum | 4 +- internal/api/init.go | 7 +- internal/api/router.go | 20 +- pkg/common/ginprometheus/ginprometheus.go | 855 +++++++++++----------- pkg/common/prommetrics/api.go | 106 +++ pkg/common/prommetrics/api_test.go | 56 ++ pkg/common/prommetrics/gin_api.go | 18 +- pkg/common/prommetrics/prommetrics.go | 17 +- 9 files changed, 636 insertions(+), 453 deletions(-) create mode 100644 pkg/common/prommetrics/api.go create mode 100644 pkg/common/prommetrics/api_test.go diff --git a/go.mod b/go.mod index 285f0d965..58a9478ac 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.24 - github.com/openimsdk/tools v0.0.49-alpha.39 + github.com/openimsdk/tools v0.0.49-alpha.41 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -176,3 +176,7 @@ require ( golang.org/x/crypto v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +//replace ( +// github.com/openimsdk/tools => /Users/chao/Desktop/withchao/tools +//) diff --git a/go.sum b/go.sum index 8d2eb40bb..d741a538a 100644 --- a/go.sum +++ b/go.sum @@ -272,8 +272,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.39 h1:bl5+q7xHsc/j1NnkN8/gYmn23RsNNbRizDY58d2EY1w= -github.com/openimsdk/tools v0.0.49-alpha.39/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= +github.com/openimsdk/tools v0.0.49-alpha.41 h1:rLnwW/yYqtuonDB61U8KB/HXj0BMP4chzF0GLgJL+Uo= +github.com/openimsdk/tools v0.0.49-alpha.41/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/init.go b/internal/api/init.go index 23866c4a0..e6fe46d22 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -29,7 +29,6 @@ import ( "time" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" - ginprom "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" @@ -72,9 +71,9 @@ func Start(ctx context.Context, index int, config *Config) error { netDone <- struct{}{} return } - p := ginprom.NewPrometheus("app", prommetrics.GetGinCusMetrics("Api")) - p.SetListenAddress(fmt.Sprintf(":%d", prometheusPort)) - if err = p.Use(router); err != nil && err != http.ErrServerClosed { + srv := http.NewServeMux() + srv.Handle(prommetrics.ApiPath, prommetrics.ApiHandler()) + if err := http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv); err != nil && err != http.ErrServerClosed { netErr = errs.WrapMsg(err, fmt.Sprintf("prometheus start err: %d", prometheusPort)) netDone <- struct{}{} } diff --git a/internal/api/router.go b/internal/api/router.go index 0f46f26ba..b519c30ff 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,8 +2,10 @@ package api import ( "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "net/http" "strings" + "time" "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" @@ -19,6 +21,22 @@ import ( "google.golang.org/grpc/credentials/insecure" ) +func prommetricsGin() gin.HandlerFunc { + return func(c *gin.Context) { + start := time.Now() + c.Next() + path := c.FullPath() + prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status(), time.Since(start)) + if c.Request.Method == http.MethodPost { + if resp := apiresp.GetGinApiResponse(c); resp == nil { + prommetrics.APICall(path, -1, "NO_GIN_RESPONSE_FOUND") + } else { + prommetrics.APICall(path, resp.ErrCode, resp.ErrMsg) + } + } + } +} + func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { disCov.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) @@ -37,7 +55,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En authRpc := rpcclient.NewAuth(disCov, config.Share.RpcRegisterName.Auth) thirdRpc := rpcclient.NewThird(disCov, config.Share.RpcRegisterName.Third, config.API.Prometheus.GrafanaURL) - r.Use(gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) + r.Use(prommetricsGin(), gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) userRouterGroup := r.Group("/user") diff --git a/pkg/common/ginprometheus/ginprometheus.go b/pkg/common/ginprometheus/ginprometheus.go index c2e6bdcca..64f8a0d8a 100644 --- a/pkg/common/ginprometheus/ginprometheus.go +++ b/pkg/common/ginprometheus/ginprometheus.go @@ -14,430 +14,431 @@ package ginprometheus -import ( - "bytes" - "fmt" - "io" - "net/http" - "os" - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -var defaultMetricPath = "/metrics" - -// counter, counter_vec, gauge, gauge_vec, -// histogram, histogram_vec, summary, summary_vec. -var ( - reqCounter = &Metric{ - ID: "reqCnt", - Name: "requests_total", - Description: "How many HTTP requests processed, partitioned by status code and HTTP method.", - Type: "counter_vec", - Args: []string{"code", "method", "handler", "host", "url"}} - - reqDuration = &Metric{ - ID: "reqDur", - Name: "request_duration_seconds", - Description: "The HTTP request latencies in seconds.", - Type: "histogram_vec", - Args: []string{"code", "method", "url"}, - } - - resSize = &Metric{ - ID: "resSz", - Name: "response_size_bytes", - Description: "The HTTP response sizes in bytes.", - Type: "summary"} - - reqSize = &Metric{ - ID: "reqSz", - Name: "request_size_bytes", - Description: "The HTTP request sizes in bytes.", - Type: "summary"} - - standardMetrics = []*Metric{ - reqCounter, - reqDuration, - resSize, - reqSize, - } -) - -/* -RequestCounterURLLabelMappingFn is a function which can be supplied to the middleware to control -the cardinality of the request counter's "url" label, which might be required in some contexts. -For instance, if for a "/customer/:name" route you don't want to generate a time series for every -possible customer name, you could use this function: - - func(c *gin.Context) string { - url := c.Request.URL.Path - for _, p := range c.Params { - if p.Key == "name" { - url = strings.Replace(url, p.Value, ":name", 1) - break - } - } - return url - } - -which would map "/customer/alice" and "/customer/bob" to their template "/customer/:name". -*/ -type RequestCounterURLLabelMappingFn func(c *gin.Context) string - -// Metric is a definition for the name, description, type, ID, and -// prometheus.Collector type (i.e. CounterVec, Summary, etc) of each metric. -type Metric struct { - MetricCollector prometheus.Collector - ID string - Name string - Description string - Type string - Args []string -} - -// Prometheus contains the metrics gathered by the instance and its path. -type Prometheus struct { - reqCnt *prometheus.CounterVec - reqDur *prometheus.HistogramVec - reqSz, resSz prometheus.Summary - router *gin.Engine - listenAddress string - Ppg PrometheusPushGateway - - MetricsList []*Metric - MetricsPath string - - ReqCntURLLabelMappingFn RequestCounterURLLabelMappingFn - - // gin.Context string to use as a prometheus URL label - URLLabelFromContext string -} - -// PrometheusPushGateway contains the configuration for pushing to a Prometheus pushgateway (optional). -type PrometheusPushGateway struct { - - // Push interval in seconds - PushIntervalSeconds time.Duration - - // Push Gateway URL in format http://domain:port - // where JOBNAME can be any string of your choice - PushGatewayURL string - - // Local metrics URL where metrics are fetched from, this could be omitted in the future - // if implemented using prometheus common/expfmt instead - MetricsURL string - - // pushgateway job name, defaults to "gin" - Job string -} - -// NewPrometheus generates a new set of metrics with a certain subsystem name. -func NewPrometheus(subsystem string, customMetricsList ...[]*Metric) *Prometheus { - if subsystem == "" { - subsystem = "app" - } - - var metricsList []*Metric - - if len(customMetricsList) > 1 { - panic("Too many args. NewPrometheus( string, ).") - } else if len(customMetricsList) == 1 { - metricsList = customMetricsList[0] - } - metricsList = append(metricsList, standardMetrics...) - - p := &Prometheus{ - MetricsList: metricsList, - MetricsPath: defaultMetricPath, - ReqCntURLLabelMappingFn: func(c *gin.Context) string { - return c.FullPath() // e.g. /user/:id , /user/:id/info - }, - } - - p.registerMetrics(subsystem) - - return p -} - -// SetPushGateway sends metrics to a remote pushgateway exposed on pushGatewayURL -// every pushIntervalSeconds. Metrics are fetched from metricsURL. -func (p *Prometheus) SetPushGateway(pushGatewayURL, metricsURL string, pushIntervalSeconds time.Duration) { - p.Ppg.PushGatewayURL = pushGatewayURL - p.Ppg.MetricsURL = metricsURL - p.Ppg.PushIntervalSeconds = pushIntervalSeconds - p.startPushTicker() -} - -// SetPushGatewayJob job name, defaults to "gin". -func (p *Prometheus) SetPushGatewayJob(j string) { - p.Ppg.Job = j -} - -// SetListenAddress for exposing metrics on address. If not set, it will be exposed at the -// same address of the gin engine that is being used. -func (p *Prometheus) SetListenAddress(address string) { - p.listenAddress = address - if p.listenAddress != "" { - p.router = gin.Default() - } -} - -// SetListenAddressWithRouter for using a separate router to expose metrics. (this keeps things like GET /metrics out of -// your content's access log). -func (p *Prometheus) SetListenAddressWithRouter(listenAddress string, r *gin.Engine) { - p.listenAddress = listenAddress - if len(p.listenAddress) > 0 { - p.router = r - } -} - -// SetMetricsPath set metrics paths. -func (p *Prometheus) SetMetricsPath(e *gin.Engine) error { - - if p.listenAddress != "" { - p.router.GET(p.MetricsPath, prometheusHandler()) - return p.runServer() - } else { - e.GET(p.MetricsPath, prometheusHandler()) - return nil - } -} - -// SetMetricsPathWithAuth set metrics paths with authentication. -func (p *Prometheus) SetMetricsPathWithAuth(e *gin.Engine, accounts gin.Accounts) error { - - if p.listenAddress != "" { - p.router.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) - return p.runServer() - } else { - e.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) - return nil - } - -} - -func (p *Prometheus) runServer() error { - return p.router.Run(p.listenAddress) -} - -func (p *Prometheus) getMetrics() []byte { - response, err := http.Get(p.Ppg.MetricsURL) - if err != nil { - return nil - } - - defer response.Body.Close() - - body, _ := io.ReadAll(response.Body) - return body -} - -var hostname, _ = os.Hostname() - -func (p *Prometheus) getPushGatewayURL() string { - if p.Ppg.Job == "" { - p.Ppg.Job = "gin" - } - return p.Ppg.PushGatewayURL + "/metrics/job/" + p.Ppg.Job + "/instance/" + hostname -} - -func (p *Prometheus) sendMetricsToPushGateway(metrics []byte) { - req, err := http.NewRequest("POST", p.getPushGatewayURL(), bytes.NewBuffer(metrics)) - if err != nil { - return - } - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - fmt.Println("Error sending to push gateway error:", err.Error()) - } - - resp.Body.Close() -} - -func (p *Prometheus) startPushTicker() { - ticker := time.NewTicker(time.Second * p.Ppg.PushIntervalSeconds) - go func() { - for range ticker.C { - p.sendMetricsToPushGateway(p.getMetrics()) - } - }() -} - -// NewMetric associates prometheus.Collector based on Metric.Type. -func NewMetric(m *Metric, subsystem string) prometheus.Collector { - var metric prometheus.Collector - switch m.Type { - case "counter_vec": - metric = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "counter": - metric = prometheus.NewCounter( - prometheus.CounterOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - case "gauge_vec": - metric = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "gauge": - metric = prometheus.NewGauge( - prometheus.GaugeOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - case "histogram_vec": - metric = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "histogram": - metric = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - case "summary_vec": - metric = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - m.Args, - ) - case "summary": - metric = prometheus.NewSummary( - prometheus.SummaryOpts{ - Subsystem: subsystem, - Name: m.Name, - Help: m.Description, - }, - ) - } - return metric -} - -func (p *Prometheus) registerMetrics(subsystem string) { - for _, metricDef := range p.MetricsList { - metric := NewMetric(metricDef, subsystem) - if err := prometheus.Register(metric); err != nil { - fmt.Println("could not be registered in Prometheus,metricDef.Name:", metricDef.Name, " error:", err.Error()) - } - - switch metricDef { - case reqCounter: - p.reqCnt = metric.(*prometheus.CounterVec) - case reqDuration: - p.reqDur = metric.(*prometheus.HistogramVec) - case resSize: - p.resSz = metric.(prometheus.Summary) - case reqSize: - p.reqSz = metric.(prometheus.Summary) - } - metricDef.MetricCollector = metric - } -} - -// Use adds the middleware to a gin engine. -func (p *Prometheus) Use(e *gin.Engine) error { - e.Use(p.HandlerFunc()) - return p.SetMetricsPath(e) -} - -// UseWithAuth adds the middleware to a gin engine with BasicAuth. -func (p *Prometheus) UseWithAuth(e *gin.Engine, accounts gin.Accounts) error { - e.Use(p.HandlerFunc()) - return p.SetMetricsPathWithAuth(e, accounts) -} - -// HandlerFunc defines handler function for middleware. -func (p *Prometheus) HandlerFunc() gin.HandlerFunc { - return func(c *gin.Context) { - if c.Request.URL.Path == p.MetricsPath { - c.Next() - return - } - - start := time.Now() - reqSz := computeApproximateRequestSize(c.Request) - - c.Next() - - status := strconv.Itoa(c.Writer.Status()) - elapsed := float64(time.Since(start)) / float64(time.Second) - resSz := float64(c.Writer.Size()) - - url := p.ReqCntURLLabelMappingFn(c) - if len(p.URLLabelFromContext) > 0 { - u, found := c.Get(p.URLLabelFromContext) - if !found { - u = "unknown" - } - url = u.(string) - } - p.reqDur.WithLabelValues(status, c.Request.Method, url).Observe(elapsed) - p.reqCnt.WithLabelValues(status, c.Request.Method, c.HandlerName(), c.Request.Host, url).Inc() - p.reqSz.Observe(float64(reqSz)) - p.resSz.Observe(resSz) - } -} - -func prometheusHandler() gin.HandlerFunc { - h := promhttp.Handler() - return func(c *gin.Context) { - h.ServeHTTP(c.Writer, c.Request) - } -} - -func computeApproximateRequestSize(r *http.Request) int { - var s int - if r.URL != nil { - s = len(r.URL.Path) - } - - s += len(r.Method) - s += len(r.Proto) - for name, values := range r.Header { - s += len(name) - for _, value := range values { - s += len(value) - } - } - s += len(r.Host) - - // r.FormData and r.MultipartForm are assumed to be included in r.URL. - - if r.ContentLength != -1 { - s += int(r.ContentLength) - } - return s -} +// +//import ( +// "bytes" +// "fmt" +// "io" +// "net/http" +// "os" +// "strconv" +// "time" +// +// "github.com/gin-gonic/gin" +// "github.com/prometheus/client_golang/prometheus" +// "github.com/prometheus/client_golang/prometheus/promhttp" +//) +// +//var defaultMetricPath = "/metrics" +// +//// counter, counter_vec, gauge, gauge_vec, +//// histogram, histogram_vec, summary, summary_vec. +//var ( +// reqCounter = &Metric{ +// ID: "reqCnt", +// Name: "requests_total", +// Description: "How many HTTP requests processed, partitioned by status code and HTTP method.", +// Type: "counter_vec", +// Args: []string{"code", "method", "handler", "host", "url"}} +// +// reqDuration = &Metric{ +// ID: "reqDur", +// Name: "request_duration_seconds", +// Description: "The HTTP request latencies in seconds.", +// Type: "histogram_vec", +// Args: []string{"code", "method", "url"}, +// } +// +// resSize = &Metric{ +// ID: "resSz", +// Name: "response_size_bytes", +// Description: "The HTTP response sizes in bytes.", +// Type: "summary"} +// +// reqSize = &Metric{ +// ID: "reqSz", +// Name: "request_size_bytes", +// Description: "The HTTP request sizes in bytes.", +// Type: "summary"} +// +// standardMetrics = []*Metric{ +// reqCounter, +// reqDuration, +// resSize, +// reqSize, +// } +//) +// +///* +//RequestCounterURLLabelMappingFn is a function which can be supplied to the middleware to control +//the cardinality of the request counter's "url" label, which might be required in some contexts. +//For instance, if for a "/customer/:name" route you don't want to generate a time series for every +//possible customer name, you could use this function: +// +// func(c *gin.Context) string { +// url := c.Request.URL.Path +// for _, p := range c.Params { +// if p.Key == "name" { +// url = strings.Replace(url, p.Value, ":name", 1) +// break +// } +// } +// return url +// } +// +//which would map "/customer/alice" and "/customer/bob" to their template "/customer/:name". +//*/ +//type RequestCounterURLLabelMappingFn func(c *gin.Context) string +// +//// Metric is a definition for the name, description, type, ID, and +//// prometheus.Collector type (i.e. CounterVec, Summary, etc) of each metric. +//type Metric struct { +// MetricCollector prometheus.Collector +// ID string +// Name string +// Description string +// Type string +// Args []string +//} +// +//// Prometheus contains the metrics gathered by the instance and its path. +//type Prometheus struct { +// reqCnt *prometheus.CounterVec +// reqDur *prometheus.HistogramVec +// reqSz, resSz prometheus.Summary +// router *gin.Engine +// listenAddress string +// Ppg PrometheusPushGateway +// +// MetricsList []*Metric +// MetricsPath string +// +// ReqCntURLLabelMappingFn RequestCounterURLLabelMappingFn +// +// // gin.Context string to use as a prometheus URL label +// URLLabelFromContext string +//} +// +//// PrometheusPushGateway contains the configuration for pushing to a Prometheus pushgateway (optional). +//type PrometheusPushGateway struct { +// +// // Push interval in seconds +// PushIntervalSeconds time.Duration +// +// // Push Gateway URL in format http://domain:port +// // where JOBNAME can be any string of your choice +// PushGatewayURL string +// +// // Local metrics URL where metrics are fetched from, this could be omitted in the future +// // if implemented using prometheus common/expfmt instead +// MetricsURL string +// +// // pushgateway job name, defaults to "gin" +// Job string +//} +// +//// NewPrometheus generates a new set of metrics with a certain subsystem name. +//func NewPrometheus(subsystem string, customMetricsList ...[]*Metric) *Prometheus { +// if subsystem == "" { +// subsystem = "app" +// } +// +// var metricsList []*Metric +// +// if len(customMetricsList) > 1 { +// panic("Too many args. NewPrometheus( string, ).") +// } else if len(customMetricsList) == 1 { +// metricsList = customMetricsList[0] +// } +// metricsList = append(metricsList, standardMetrics...) +// +// p := &Prometheus{ +// MetricsList: metricsList, +// MetricsPath: defaultMetricPath, +// ReqCntURLLabelMappingFn: func(c *gin.Context) string { +// return c.FullPath() // e.g. /user/:id , /user/:id/info +// }, +// } +// +// p.registerMetrics(subsystem) +// +// return p +//} +// +//// SetPushGateway sends metrics to a remote pushgateway exposed on pushGatewayURL +//// every pushIntervalSeconds. Metrics are fetched from metricsURL. +//func (p *Prometheus) SetPushGateway(pushGatewayURL, metricsURL string, pushIntervalSeconds time.Duration) { +// p.Ppg.PushGatewayURL = pushGatewayURL +// p.Ppg.MetricsURL = metricsURL +// p.Ppg.PushIntervalSeconds = pushIntervalSeconds +// p.startPushTicker() +//} +// +//// SetPushGatewayJob job name, defaults to "gin". +//func (p *Prometheus) SetPushGatewayJob(j string) { +// p.Ppg.Job = j +//} +// +//// SetListenAddress for exposing metrics on address. If not set, it will be exposed at the +//// same address of the gin engine that is being used. +//func (p *Prometheus) SetListenAddress(address string) { +// p.listenAddress = address +// if p.listenAddress != "" { +// p.router = gin.Default() +// } +//} +// +//// SetListenAddressWithRouter for using a separate router to expose metrics. (this keeps things like GET /metrics out of +//// your content's access log). +//func (p *Prometheus) SetListenAddressWithRouter(listenAddress string, r *gin.Engine) { +// p.listenAddress = listenAddress +// if len(p.listenAddress) > 0 { +// p.router = r +// } +//} +// +//// SetMetricsPath set metrics paths. +//func (p *Prometheus) SetMetricsPath(e *gin.Engine) error { +// +// if p.listenAddress != "" { +// p.router.GET(p.MetricsPath, prometheusHandler()) +// return p.runServer() +// } else { +// e.GET(p.MetricsPath, prometheusHandler()) +// return nil +// } +//} +// +//// SetMetricsPathWithAuth set metrics paths with authentication. +//func (p *Prometheus) SetMetricsPathWithAuth(e *gin.Engine, accounts gin.Accounts) error { +// +// if p.listenAddress != "" { +// p.router.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) +// return p.runServer() +// } else { +// e.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) +// return nil +// } +// +//} +// +//func (p *Prometheus) runServer() error { +// return p.router.Run(p.listenAddress) +//} +// +//func (p *Prometheus) getMetrics() []byte { +// response, err := http.Get(p.Ppg.MetricsURL) +// if err != nil { +// return nil +// } +// +// defer response.Body.Close() +// +// body, _ := io.ReadAll(response.Body) +// return body +//} +// +//var hostname, _ = os.Hostname() +// +//func (p *Prometheus) getPushGatewayURL() string { +// if p.Ppg.Job == "" { +// p.Ppg.Job = "gin" +// } +// return p.Ppg.PushGatewayURL + "/metrics/job/" + p.Ppg.Job + "/instance/" + hostname +//} +// +//func (p *Prometheus) sendMetricsToPushGateway(metrics []byte) { +// req, err := http.NewRequest("POST", p.getPushGatewayURL(), bytes.NewBuffer(metrics)) +// if err != nil { +// return +// } +// +// client := &http.Client{} +// resp, err := client.Do(req) +// if err != nil { +// fmt.Println("Error sending to push gateway error:", err.Error()) +// } +// +// resp.Body.Close() +//} +// +//func (p *Prometheus) startPushTicker() { +// ticker := time.NewTicker(time.Second * p.Ppg.PushIntervalSeconds) +// go func() { +// for range ticker.C { +// p.sendMetricsToPushGateway(p.getMetrics()) +// } +// }() +//} +// +//// NewMetric associates prometheus.Collector based on Metric.Type. +//func NewMetric(m *Metric, subsystem string) prometheus.Collector { +// var metric prometheus.Collector +// switch m.Type { +// case "counter_vec": +// metric = prometheus.NewCounterVec( +// prometheus.CounterOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "counter": +// metric = prometheus.NewCounter( +// prometheus.CounterOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// case "gauge_vec": +// metric = prometheus.NewGaugeVec( +// prometheus.GaugeOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "gauge": +// metric = prometheus.NewGauge( +// prometheus.GaugeOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// case "histogram_vec": +// metric = prometheus.NewHistogramVec( +// prometheus.HistogramOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "histogram": +// metric = prometheus.NewHistogram( +// prometheus.HistogramOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// case "summary_vec": +// metric = prometheus.NewSummaryVec( +// prometheus.SummaryOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// m.Args, +// ) +// case "summary": +// metric = prometheus.NewSummary( +// prometheus.SummaryOpts{ +// Subsystem: subsystem, +// Name: m.Name, +// Help: m.Description, +// }, +// ) +// } +// return metric +//} +// +//func (p *Prometheus) registerMetrics(subsystem string) { +// for _, metricDef := range p.MetricsList { +// metric := NewMetric(metricDef, subsystem) +// if err := prometheus.Register(metric); err != nil { +// fmt.Println("could not be registered in Prometheus,metricDef.Name:", metricDef.Name, " error:", err.Error()) +// } +// +// switch metricDef { +// case reqCounter: +// p.reqCnt = metric.(*prometheus.CounterVec) +// case reqDuration: +// p.reqDur = metric.(*prometheus.HistogramVec) +// case resSize: +// p.resSz = metric.(prometheus.Summary) +// case reqSize: +// p.reqSz = metric.(prometheus.Summary) +// } +// metricDef.MetricCollector = metric +// } +//} +// +//// Use adds the middleware to a gin engine. +//func (p *Prometheus) Use(e *gin.Engine) error { +// e.Use(p.HandlerFunc()) +// return p.SetMetricsPath(e) +//} +// +//// UseWithAuth adds the middleware to a gin engine with BasicAuth. +//func (p *Prometheus) UseWithAuth(e *gin.Engine, accounts gin.Accounts) error { +// e.Use(p.HandlerFunc()) +// return p.SetMetricsPathWithAuth(e, accounts) +//} +// +//// HandlerFunc defines handler function for middleware. +//func (p *Prometheus) HandlerFunc() gin.HandlerFunc { +// return func(c *gin.Context) { +// if c.Request.URL.Path == p.MetricsPath { +// c.Next() +// return +// } +// +// start := time.Now() +// reqSz := computeApproximateRequestSize(c.Request) +// +// c.Next() +// +// status := strconv.Itoa(c.Writer.Status()) +// elapsed := float64(time.Since(start)) / float64(time.Second) +// resSz := float64(c.Writer.Size()) +// +// url := p.ReqCntURLLabelMappingFn(c) +// if len(p.URLLabelFromContext) > 0 { +// u, found := c.Get(p.URLLabelFromContext) +// if !found { +// u = "unknown" +// } +// url = u.(string) +// } +// p.reqDur.WithLabelValues(status, c.Request.Method, url).Observe(elapsed) +// p.reqCnt.WithLabelValues(status, c.Request.Method, c.HandlerName(), c.Request.Host, url).Inc() +// p.reqSz.Observe(float64(reqSz)) +// p.resSz.Observe(resSz) +// } +//} +// +//func prometheusHandler() gin.HandlerFunc { +// h := promhttp.Handler() +// return func(c *gin.Context) { +// h.ServeHTTP(c.Writer, c.Request) +// } +//} +// +//func computeApproximateRequestSize(r *http.Request) int { +// var s int +// if r.URL != nil { +// s = len(r.URL.Path) +// } +// +// s += len(r.Method) +// s += len(r.Proto) +// for name, values := range r.Header { +// s += len(name) +// for _, value := range values { +// s += len(value) +// } +// } +// s += len(r.Host) +// +// // r.FormData and r.MultipartForm are assumed to be included in r.URL. +// +// if r.ContentLength != -1 { +// s += int(r.ContentLength) +// } +// return s +//} diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go new file mode 100644 index 000000000..ce45290ad --- /dev/null +++ b/pkg/common/prommetrics/api.go @@ -0,0 +1,106 @@ +package prommetrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" + "strconv" + "time" +) + +const ApiPath = "/metrics" + +var ( + apiRegistry = prometheus.NewRegistry() + apiCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "api_count", + Help: "Total number of API calls", + }, + []string{"path", "code", "type"}, + ) + httpCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "http_count", + Help: "Total number of HTTP calls", + }, + []string{"path", "method", "status", "duration"}, + ) +) + +func init() { + apiRegistry.MustRegister(apiCounter, httpCounter) +} + +func APICall(path string, apiCode int, apiType string) { + apiCounter.With(prometheus.Labels{"path": path, "code": strconv.Itoa(apiCode), "type": apiType}).Inc() +} + +func HttpCall(path string, method string, status int, duration time.Duration) { + httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status), "duration": durationRange(duration)}).Inc() +} + +var ( + durations = [...]time.Duration{ + time.Millisecond * 1, + time.Millisecond * 2, + time.Millisecond * 3, + time.Millisecond * 4, + time.Millisecond * 5, + time.Millisecond * 6, + time.Millisecond * 7, + time.Millisecond * 8, + time.Millisecond * 9, + time.Millisecond * 10, + time.Millisecond * 20, + time.Millisecond * 30, + time.Millisecond * 40, + time.Millisecond * 50, + time.Millisecond * 60, + time.Millisecond * 70, + time.Millisecond * 80, + time.Millisecond * 90, + time.Millisecond * 100, + time.Millisecond * 200, + time.Millisecond * 300, + time.Millisecond * 400, + time.Millisecond * 500, + time.Millisecond * 600, + time.Millisecond * 700, + time.Millisecond * 800, + time.Millisecond * 900, + time.Second * 1, + time.Second * 2, + time.Second * 3, + time.Second * 4, + time.Second * 5, + time.Second * 6, + time.Second * 7, + time.Second * 8, + time.Second * 9, + time.Second * 10, + time.Second * 20, + time.Second * 30, + time.Second * 40, + time.Second * 50, + time.Second * 60, + time.Second * 70, + time.Second * 80, + time.Second * 90, + time.Second * 100, + } + maxDuration = durations[len(durations)-1] +) + +func durationRange(duration time.Duration) string { + for _, d := range durations { + if duration <= d { + return d.String() + } + } + return ">" + maxDuration.String() +} + +func ApiHandler() http.Handler { + return promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}) +} diff --git a/pkg/common/prommetrics/api_test.go b/pkg/common/prommetrics/api_test.go new file mode 100644 index 000000000..3a691c794 --- /dev/null +++ b/pkg/common/prommetrics/api_test.go @@ -0,0 +1,56 @@ +package prommetrics + +import ( + "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" + "testing" + "time" +) + +var ( + apiCallCounter1 = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "api_calls_total", + Help: "Total number of API calls", + }, + []string{"endpoint", "status", "code", "error"}, + ) + registerer *prometheus.Registry +) + +func init() { + registerer = prometheus.NewRegistry() + registerer.MustRegister(apiCallCounter1) +} + +func recordAPICall(endpoint string, status string) { + apiCallCounter1.With(prometheus.Labels{"endpoint": endpoint, "status": status, "code": "200", "error": "ArgsError"}).Inc() +} + +func TestName(t *testing.T) { + go func() { + for i := 0; ; i++ { + recordAPICall("/api/test", "success") + time.Sleep(time.Second) + } + }() + + go func() { + for i := 0; ; i++ { + recordAPICall("/api/test", "failed") + time.Sleep(time.Second * 3) + } + }() + http.Handle("/metrics", promhttp.HandlerFor(registerer, promhttp.HandlerOpts{})) + if err := http.ListenAndServe(":2112", nil); err != nil { + panic(err) + } +} + +func TestName2(t *testing.T) { + var d time.Duration + d = time.Second / 900 + fmt.Println(durationRange(d)) +} diff --git a/pkg/common/prommetrics/gin_api.go b/pkg/common/prommetrics/gin_api.go index 9f2e4c99d..47b043f64 100644 --- a/pkg/common/prommetrics/gin_api.go +++ b/pkg/common/prommetrics/gin_api.go @@ -14,17 +14,17 @@ package prommetrics -import ginprom "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" +//import ginprom "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" /* labels := prometheus.Labels{"label_one": "any", "label_two": "value"} ApiCustomCnt.MetricCollector.(*prometheus.CounterVec).With(labels).Inc(). */ -var ( - ApiCustomCnt = &ginprom.Metric{ - Name: "custom_total", - Description: "Custom counter events.", - Type: "counter_vec", - Args: []string{"label_one", "label_two"}, - } -) +//var ( +// ApiCustomCnt = &ginprom.Metric{ +// Name: "custom_total", +// Description: "Custom counter events.", +// Type: "counter_vec", +// Args: []string{"label_one", "label_two"}, +// } +//) diff --git a/pkg/common/prommetrics/prommetrics.go b/pkg/common/prommetrics/prommetrics.go index 47e5d02b8..3955d8ea5 100644 --- a/pkg/common/prommetrics/prommetrics.go +++ b/pkg/common/prommetrics/prommetrics.go @@ -17,7 +17,6 @@ package prommetrics import ( gp "github.com/grpc-ecosystem/go-grpc-prometheus" config2 "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" ) @@ -48,11 +47,11 @@ func GetGrpcCusMetrics(registerName string, share *config2.Share) []prometheus.C } } -func GetGinCusMetrics(name string) []*ginprometheus.Metric { - switch name { - case "Api": - return []*ginprometheus.Metric{ApiCustomCnt} - default: - return []*ginprometheus.Metric{ApiCustomCnt} - } -} +//func GetGinCusMetrics(name string) []*ginprometheus.Metric { +// switch name { +// case "Api": +// return []*ginprometheus.Metric{ApiCustomCnt} +// default: +// return []*ginprometheus.Metric{ApiCustomCnt} +// } +//} From 671398688714dc754839c723066f0c6c2aad7a9d Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 16:15:27 +0800 Subject: [PATCH 2/7] api prommetrics --- internal/api/router.go | 13 ++++++------- pkg/common/prommetrics/api.go | 11 ++++++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index b519c30ff..936d6ab0b 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,14 +2,10 @@ package api import ( "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" - "net/http" - "strings" - "time" - "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" @@ -19,14 +15,17 @@ import ( "github.com/openimsdk/tools/mw" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "net/http" + "strings" ) func prommetricsGin() gin.HandlerFunc { return func(c *gin.Context) { - start := time.Now() + //start := time.Now() c.Next() path := c.FullPath() - prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status(), time.Since(start)) + prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status()) + //prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status(), time.Since(start)) if c.Request.Method == http.MethodPost { if resp := apiresp.GetGinApiResponse(c); resp == nil { prommetrics.APICall(path, -1, "NO_GIN_RESPONSE_FOUND") diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go index ce45290ad..aa5e0a60e 100644 --- a/pkg/common/prommetrics/api.go +++ b/pkg/common/prommetrics/api.go @@ -24,7 +24,8 @@ var ( Name: "http_count", Help: "Total number of HTTP calls", }, - []string{"path", "method", "status", "duration"}, + //[]string{"path", "method", "status", "duration"}, + []string{"path", "method", "status"}, ) ) @@ -36,8 +37,12 @@ func APICall(path string, apiCode int, apiType string) { apiCounter.With(prometheus.Labels{"path": path, "code": strconv.Itoa(apiCode), "type": apiType}).Inc() } -func HttpCall(path string, method string, status int, duration time.Duration) { - httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status), "duration": durationRange(duration)}).Inc() +//func HttpCall(path string, method string, status int, duration time.Duration) { +// httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status), "duration": durationRange(duration)}).Inc() +//} + +func HttpCall(path string, method string, status int) { + httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status)}).Inc() } var ( From 9151b56fe9b1eb2ea52b92801ac183f2f5c30cd6 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 16:21:24 +0800 Subject: [PATCH 3/7] api prommetrics --- internal/api/router.go | 4 ++-- pkg/common/prommetrics/api.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index 936d6ab0b..1c70c6bc5 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -28,9 +28,9 @@ func prommetricsGin() gin.HandlerFunc { //prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status(), time.Since(start)) if c.Request.Method == http.MethodPost { if resp := apiresp.GetGinApiResponse(c); resp == nil { - prommetrics.APICall(path, -1, "NO_GIN_RESPONSE_FOUND") + prommetrics.APICall(path, -1) } else { - prommetrics.APICall(path, resp.ErrCode, resp.ErrMsg) + prommetrics.APICall(path, resp.ErrCode) } } } diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go index aa5e0a60e..15f8a3e31 100644 --- a/pkg/common/prommetrics/api.go +++ b/pkg/common/prommetrics/api.go @@ -17,7 +17,7 @@ var ( Name: "api_count", Help: "Total number of API calls", }, - []string{"path", "code", "type"}, + []string{"path", "code"}, ) httpCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -33,8 +33,8 @@ func init() { apiRegistry.MustRegister(apiCounter, httpCounter) } -func APICall(path string, apiCode int, apiType string) { - apiCounter.With(prometheus.Labels{"path": path, "code": strconv.Itoa(apiCode), "type": apiType}).Inc() +func APICall(path string, apiCode int) { + apiCounter.With(prometheus.Labels{"path": path, "code": strconv.Itoa(apiCode)}).Inc() } //func HttpCall(path string, method string, status int, duration time.Duration) { From c1d66faa109199cb7695ddc0cb37f2d8668e213a Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 16:36:20 +0800 Subject: [PATCH 4/7] api prommetrics --- internal/api/router.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index 1c70c6bc5..0a42d82bd 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -21,17 +21,15 @@ import ( func prommetricsGin() gin.HandlerFunc { return func(c *gin.Context) { - //start := time.Now() c.Next() path := c.FullPath() - prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status()) - //prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status(), time.Since(start)) - if c.Request.Method == http.MethodPost { - if resp := apiresp.GetGinApiResponse(c); resp == nil { - prommetrics.APICall(path, -1) - } else { - prommetrics.APICall(path, resp.ErrCode) - } + if c.Writer.Status() == http.StatusNotFound { + prommetrics.HttpCall("<404>", c.Request.Method, c.Writer.Status()) + } else { + prommetrics.HttpCall(path, c.Request.Method, c.Writer.Status()) + } + if resp := apiresp.GetGinApiResponse(c); resp != nil { + prommetrics.APICall(path, c.Request.Method, resp.ErrCode) } } } From e3ee24df3129313ec6d9420c975bb46b786b6cc9 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 16:40:09 +0800 Subject: [PATCH 5/7] api prommetrics --- pkg/common/prommetrics/api.go | 77 +++--------------------------- pkg/common/prommetrics/api_test.go | 1 + 2 files changed, 7 insertions(+), 71 deletions(-) diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go index 15f8a3e31..b80b8df85 100644 --- a/pkg/common/prommetrics/api.go +++ b/pkg/common/prommetrics/api.go @@ -5,7 +5,6 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "net/http" "strconv" - "time" ) const ApiPath = "/metrics" @@ -17,14 +16,13 @@ var ( Name: "api_count", Help: "Total number of API calls", }, - []string{"path", "code"}, + []string{"path", "method", "code"}, ) httpCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "http_count", Help: "Total number of HTTP calls", }, - //[]string{"path", "method", "status", "duration"}, []string{"path", "method", "status"}, ) ) @@ -33,79 +31,16 @@ func init() { apiRegistry.MustRegister(apiCounter, httpCounter) } -func APICall(path string, apiCode int) { - apiCounter.With(prometheus.Labels{"path": path, "code": strconv.Itoa(apiCode)}).Inc() +func APICall(path string, method string, apiCode int) { + apiCounter.With(prometheus.Labels{"path": path, "method": method, "code": strconv.Itoa(apiCode)}).Inc() } -//func HttpCall(path string, method string, status int, duration time.Duration) { -// httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status), "duration": durationRange(duration)}).Inc() -//} - func HttpCall(path string, method string, status int) { httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status)}).Inc() } -var ( - durations = [...]time.Duration{ - time.Millisecond * 1, - time.Millisecond * 2, - time.Millisecond * 3, - time.Millisecond * 4, - time.Millisecond * 5, - time.Millisecond * 6, - time.Millisecond * 7, - time.Millisecond * 8, - time.Millisecond * 9, - time.Millisecond * 10, - time.Millisecond * 20, - time.Millisecond * 30, - time.Millisecond * 40, - time.Millisecond * 50, - time.Millisecond * 60, - time.Millisecond * 70, - time.Millisecond * 80, - time.Millisecond * 90, - time.Millisecond * 100, - time.Millisecond * 200, - time.Millisecond * 300, - time.Millisecond * 400, - time.Millisecond * 500, - time.Millisecond * 600, - time.Millisecond * 700, - time.Millisecond * 800, - time.Millisecond * 900, - time.Second * 1, - time.Second * 2, - time.Second * 3, - time.Second * 4, - time.Second * 5, - time.Second * 6, - time.Second * 7, - time.Second * 8, - time.Second * 9, - time.Second * 10, - time.Second * 20, - time.Second * 30, - time.Second * 40, - time.Second * 50, - time.Second * 60, - time.Second * 70, - time.Second * 80, - time.Second * 90, - time.Second * 100, - } - maxDuration = durations[len(durations)-1] -) - -func durationRange(duration time.Duration) string { - for _, d := range durations { - if duration <= d { - return d.String() - } - } - return ">" + maxDuration.String() -} - func ApiHandler() http.Handler { - return promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}) + return promhttp.InstrumentMetricHandler( + apiRegistry, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), + ) } diff --git a/pkg/common/prommetrics/api_test.go b/pkg/common/prommetrics/api_test.go index 3a691c794..b0e0f7b20 100644 --- a/pkg/common/prommetrics/api_test.go +++ b/pkg/common/prommetrics/api_test.go @@ -43,6 +43,7 @@ func TestName(t *testing.T) { time.Sleep(time.Second * 3) } }() + promhttp.Handler() http.Handle("/metrics", promhttp.HandlerFor(registerer, promhttp.HandlerOpts{})) if err := http.ListenAndServe(":2112", nil); err != nil { panic(err) From 0a9b53bbe313140699808275895eead9bbae96dd Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 5 Jul 2024 14:19:34 +0800 Subject: [PATCH 6/7] rpc prommetrics --- internal/api/init.go | 2 +- pkg/common/prommetrics/api.go | 10 ++++-- pkg/common/prommetrics/api_test.go | 57 ------------------------------ pkg/common/prommetrics/gin_api.go | 30 ---------------- pkg/common/prommetrics/rpc.go | 33 +++++++++++++++++ pkg/common/startrpc/start.go | 51 ++++++++++++++++++-------- 6 files changed, 78 insertions(+), 105 deletions(-) delete mode 100644 pkg/common/prommetrics/api_test.go delete mode 100644 pkg/common/prommetrics/gin_api.go create mode 100644 pkg/common/prommetrics/rpc.go diff --git a/internal/api/init.go b/internal/api/init.go index e6fe46d22..679d901f4 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -74,7 +74,7 @@ func Start(ctx context.Context, index int, config *Config) error { srv := http.NewServeMux() srv.Handle(prommetrics.ApiPath, prommetrics.ApiHandler()) if err := http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv); err != nil && err != http.ErrServerClosed { - netErr = errs.WrapMsg(err, fmt.Sprintf("prometheus start err: %d", prometheusPort)) + netErr = errs.WrapMsg(err, fmt.Sprintf("api prometheus start err: %d", prometheusPort)) netDone <- struct{}{} } }() diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go index b80b8df85..806198977 100644 --- a/pkg/common/prommetrics/api.go +++ b/pkg/common/prommetrics/api.go @@ -39,8 +39,12 @@ func HttpCall(path string, method string, status int) { httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status)}).Inc() } +//func ApiHandler() http.Handler { +// return promhttp.InstrumentMetricHandler( +// apiRegistry, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), +// ) +//} + func ApiHandler() http.Handler { - return promhttp.InstrumentMetricHandler( - apiRegistry, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), - ) + return promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}) } diff --git a/pkg/common/prommetrics/api_test.go b/pkg/common/prommetrics/api_test.go deleted file mode 100644 index b0e0f7b20..000000000 --- a/pkg/common/prommetrics/api_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package prommetrics - -import ( - "fmt" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "net/http" - "testing" - "time" -) - -var ( - apiCallCounter1 = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "api_calls_total", - Help: "Total number of API calls", - }, - []string{"endpoint", "status", "code", "error"}, - ) - registerer *prometheus.Registry -) - -func init() { - registerer = prometheus.NewRegistry() - registerer.MustRegister(apiCallCounter1) -} - -func recordAPICall(endpoint string, status string) { - apiCallCounter1.With(prometheus.Labels{"endpoint": endpoint, "status": status, "code": "200", "error": "ArgsError"}).Inc() -} - -func TestName(t *testing.T) { - go func() { - for i := 0; ; i++ { - recordAPICall("/api/test", "success") - time.Sleep(time.Second) - } - }() - - go func() { - for i := 0; ; i++ { - recordAPICall("/api/test", "failed") - time.Sleep(time.Second * 3) - } - }() - promhttp.Handler() - http.Handle("/metrics", promhttp.HandlerFor(registerer, promhttp.HandlerOpts{})) - if err := http.ListenAndServe(":2112", nil); err != nil { - panic(err) - } -} - -func TestName2(t *testing.T) { - var d time.Duration - d = time.Second / 900 - fmt.Println(durationRange(d)) -} diff --git a/pkg/common/prommetrics/gin_api.go b/pkg/common/prommetrics/gin_api.go deleted file mode 100644 index 47b043f64..000000000 --- a/pkg/common/prommetrics/gin_api.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package prommetrics - -//import ginprom "github.com/openimsdk/open-im-server/v3/pkg/common/ginprometheus" - -/* -labels := prometheus.Labels{"label_one": "any", "label_two": "value"} -ApiCustomCnt.MetricCollector.(*prometheus.CounterVec).With(labels).Inc(). -*/ -//var ( -// ApiCustomCnt = &ginprom.Metric{ -// Name: "custom_total", -// Description: "Custom counter events.", -// Type: "counter_vec", -// Args: []string{"label_one", "label_two"}, -// } -//) diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go new file mode 100644 index 000000000..834f76a9f --- /dev/null +++ b/pkg/common/prommetrics/rpc.go @@ -0,0 +1,33 @@ +package prommetrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" + "strconv" +) + +const RpcPath = "/metrics" + +var ( + rpcRegistry = prometheus.NewRegistry() + rpcCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "rpc_count", + Help: "Total number of RPC calls", + }, + []string{"name", "path", "code"}, + ) +) + +func init() { + rpcRegistry.MustRegister(rpcCounter) +} + +func RPCCall(name string, path string, code int) { + rpcCounter.With(prometheus.Labels{"name": name, "path": path, "code": strconv.Itoa(code)}).Inc() +} + +func RPCHandler() http.Handler { + return promhttp.HandlerFor(rpcRegistry, promhttp.HandlerOpts{}) +} diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index b531daa47..a88797793 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -19,7 +19,7 @@ import ( "fmt" config2 "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/utils/datautil" - "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc/status" "net" "net/http" "os" @@ -29,7 +29,6 @@ import ( "syscall" "time" - grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -38,7 +37,6 @@ import ( "github.com/openimsdk/tools/mw" "github.com/openimsdk/tools/system/program" "github.com/openimsdk/tools/utils/network" - "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -77,13 +75,14 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC return err } - var reg *prometheus.Registry - var metric *grpcprometheus.ServerMetrics + //var reg *prometheus.Registry + //var metric *grpcprometheus.ServerMetrics if prometheusConfig.Enable { - cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) - reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) - options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), - grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) + //cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + //reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) + //options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), + // grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) + options = append(options, mw.GrpcServer(), prommetricsInterceptor(rpcRegisterName)) } else { options = append(options, mw.GrpcServer()) } @@ -122,13 +121,19 @@ func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusC netDone <- struct{}{} return } - metric.InitializeMetrics(srv) - // Create a HTTP server for prometheus. - httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} - if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) + srv := http.NewServeMux() + srv.Handle(prommetrics.RpcPath, prommetrics.RPCHandler()) + if err := http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv); err != nil && err != http.ErrServerClosed { + netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) netDone <- struct{}{} } + //metric.InitializeMetrics(srv) + // Create a HTTP server for prometheus. + //httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} + //if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + // netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) + // netDone <- struct{}{} + //} }() } @@ -175,3 +180,21 @@ func gracefulStopWithCtx(ctx context.Context, f func()) error { return nil } } + +func prommetricsInterceptor(rpcRegisterName string) grpc.ServerOption { + getCode := func(err error) int { + if err == nil { + return 0 + } + rpcErr, ok := err.(interface{ GRPCStatus() *status.Status }) + if !ok { + return -1 + } + return int(rpcErr.GRPCStatus().Code()) + } + return grpc.ChainUnaryInterceptor(func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + resp, err := handler(ctx, req) + prommetrics.RPCCall(rpcRegisterName, info.FullMethod, getCode(err)) + return resp, err + }) +} From 2052e1c7d569be86c3af6b67c4030f35e4d532de Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 5 Jul 2024 14:59:41 +0800 Subject: [PATCH 7/7] rpc prommetrics --- go.sum | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/go.sum b/go.sum index a5a620a14..577154a1b 100644 --- a/go.sum +++ b/go.sum @@ -200,8 +200,6 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= -github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kelindar/bitmap v1.5.2 h1:XwX7CTvJtetQZ64zrOkApoZZHBJRkjE23NfqUALA/HE= @@ -222,12 +220,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= -github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= -github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= -github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= -github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/likexian/gokit v0.25.13 h1:p2Uw3+6fGG53CwdU2Dz0T6bOycdb2+bAFAa3ymwWVkM= github.com/likexian/gokit v0.25.13/go.mod h1:qQhEWFBEfqLCO3/vOEo2EDKd+EycekVtUK4tex+l2H4= github.com/lithammer/shortuuid v3.0.0+incompatible h1:NcD0xWW/MZYXEHa6ITy6kaXN5nwm/V115vj2YXfhS0w= @@ -270,10 +262,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= -github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.28 h1:1CfdFxvKzyOIvgNMVMq4ZB2upAJ0evLbbigOhWQzhu8= -github.com/openimsdk/tools v0.0.49-alpha.28/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= +github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= +github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= +github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -527,6 +519,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=