Add metrics for internal API requests (#2310)

* Add response size and requests total to internal handler

* Move MustRegister calls to New* funcs

* Move MustRegister back to init

* Init at some place, minimize changes

clientapi/routing/register.go

@@ -64,11 +64,6 @@ const (
 	sessionIDLength   = 24
 )
 
-func init() {
-	// Register prometheus metrics. They must be registered to be exposed.
-	prometheus.MustRegister(amtRegUsers)
-}
-
 // sessionsDict keeps track of completed auth stages for each session.
 // It shouldn't be passed by value because it contains a mutex.
 type sessionsDict struct {

clientapi/routing/routing.go

@@ -37,6 +37,7 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/nats-io/nats.go"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
 
@@ -60,6 +61,8 @@ func Setup(
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 	mscCfg *config.MSCs, natsClient *nats.Conn,
 ) {
+	prometheus.MustRegister(amtRegUsers, sendEventDuration)
+
 	rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
 	userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg)
 

clientapi/routing/sendevent.go

@@ -46,10 +46,6 @@ var (
 	userRoomSendMutexes sync.Map // (roomID+userID) -> mutex. mutexes to ensure correct ordering of sendEvents
 )
 
-func init() {
-	prometheus.MustRegister(sendEventDuration)
-}
-
 var sendEventDuration = prometheus.NewHistogramVec(
 	prometheus.HistogramOpts{
 		Namespace: "dendrite",

federationapi/routing/routing.go

@@ -29,6 +29,7 @@ import (
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
 
@@ -53,6 +54,10 @@ func Setup(
 	servers federationAPI.ServersInRoomProvider,
 	producer *producers.SyncAPIProducer,
 ) {
+	prometheus.MustRegister(
+		pduCountTotal, eduCountTotal,
+	)
+
 	v2keysmux := keyMux.PathPrefix("/v2").Subrouter()
 	v1fedmux := fedMux.PathPrefix("/v1").Subrouter()
 	v2fedmux := fedMux.PathPrefix("/v2").Subrouter()

federationapi/routing/send.go

@@ -74,12 +74,6 @@ var (
 	)
 )
 
-func init() {
-	prometheus.MustRegister(
-		pduCountTotal, eduCountTotal,
-	)
-}
-
 var inFlightTxnsPerOrigin sync.Map // transaction ID -> chan util.JSONResponse
 
 // Send implements /_matrix/federation/v1/send/{txnID}

internal/httputil/httpapi.go

@@ -171,6 +171,7 @@ func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request)
 			prometheus.CounterOpts{
 				Name:      metricsName,
 				Help:      "Total number of http requests for HTML resources",
+				Namespace: "dendrite",
 			},
 			[]string{"code"},
 		),
@@ -201,7 +202,28 @@ func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse
 		h.ServeHTTP(w, req)
 	}
 
-	return http.HandlerFunc(withSpan)
+	return promhttp.InstrumentHandlerCounter(
+		promauto.NewCounterVec(
+			prometheus.CounterOpts{
+				Name:      metricsName + "_requests_total",
+				Help:      "Total number of internal API calls",
+				Namespace: "dendrite",
+			},
+			[]string{"code"},
+		),
+		promhttp.InstrumentHandlerResponseSize(
+			promauto.NewHistogramVec(
+				prometheus.HistogramOpts{
+					Namespace: "dendrite",
+					Name:      metricsName + "_response_size_bytes",
+					Help:      "A histogram of response sizes for requests.",
+					Buckets:   []float64{200, 500, 900, 1500, 5000, 15000, 50000, 100000},
+				},
+				[]string{},
+			),
+			http.HandlerFunc(withSpan),
+		),
+	)
 }
 
 // MakeFedAPI makes an http.Handler that checks matrix federation authentication.

roomserver/internal/input/input.go

@@ -167,6 +167,7 @@ func (r *Inputer) startWorkerForRoom(roomID string) {
 // will look to see if we have a worker for that room which has its
 // own consumer. If we don't, we'll start one.
 func (r *Inputer) Start() error {
+	prometheus.MustRegister(roomserverInputBackpressure, processRoomEventDuration)
 	_, err := r.JetStream.Subscribe(
 		"", // This is blank because we specified it in BindStream.
 		func(m *nats.Msg) {
@@ -421,10 +422,6 @@ func (r *Inputer) WriteOutputEvents(roomID string, updates []api.OutputEvent) er
 	return nil
 }
 
-func init() {
-	prometheus.MustRegister(roomserverInputBackpressure)
-}
-
 var roomserverInputBackpressure = prometheus.NewGaugeVec(
 	prometheus.GaugeOpts{
 		Namespace: "dendrite",

roomserver/internal/input/input_events.go

@@ -37,10 +37,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func init() {
-	prometheus.MustRegister(processRoomEventDuration)
-}
-
 // TODO: Does this value make sense?
 const MaximumMissingProcessingTime = time.Minute * 2
 

syncapi/sync/requestpool.go

@@ -67,6 +67,9 @@ func NewRequestPool(
 	streams *streams.Streams, notifier *notifier.Notifier,
 	producer PresencePublisher,
 ) *RequestPool {
+	prometheus.MustRegister(
+		activeSyncRequests, waitingSyncRequests,
+	)
 	rp := &RequestPool{
 		db:       db,
 		cfg:      cfg,
@@ -183,12 +186,6 @@ func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device)
 	rp.lastseen.Store(device.UserID+device.ID, time.Now())
 }
 
-func init() {
-	prometheus.MustRegister(
-		activeSyncRequests, waitingSyncRequests,
-	)
-}
-
 var activeSyncRequests = prometheus.NewGauge(
 	prometheus.GaugeOpts{
 		Namespace: "dendrite",