hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2025-08-01 13:52:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add message stats to reporting (#2748)

Browse source 
Since we're now listening on the `OutputRoomEvent` stream, we are able
to store messages stats.
This commit is contained in:
Till 2022-11-02 11:18:11 +01:00 • committed by GitHub
parent b367cfeddf
commit 86b25a6337
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 383 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

4
userapi/storage/interface.go
View file

@ -19,6 +19,8 @@ import (
"encoding/json"
"errors"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/internal/pushrules"
"github.com/matrix-org/dendrite/userapi/api"
@ -144,6 +146,8 @@ type Database interface {
type Statistics interface {
UserStatistics(ctx context.Context) (*types.UserStatistics, *types.DatabaseEngine, error)
DailyRoomsMessages(ctx context.Context, serverName gomatrixserverlib.ServerName) (stats types.MessageStats, activeRooms, activeE2EERooms int64, err error)
UpsertDailyRoomsMessages(ctx context.Context, serverName gomatrixserverlib.ServerName, stats types.MessageStats, activeRooms, activeE2EERooms int64) error
}
// Err3PIDInUse is the error returned when trying to save an association involving

73
userapi/storage/postgres/stats_table.go
View file

@ -20,13 +20,14 @@ import (
"time"
"github.com/lib/pq"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/tables"
"github.com/matrix-org/dendrite/userapi/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
)
const userDailyVisitsSchema = `
@ -43,6 +44,35 @@ CREATE INDEX IF NOT EXISTS userapi_daily_visits_timestamp_idx ON userapi_daily_v
CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON userapi_daily_visits(localpart, timestamp);
`
const messagesDailySchema = `
CREATE TABLE IF NOT EXISTS userapi_daily_stats (
timestamp BIGINT NOT NULL,
server_name TEXT NOT NULL,
messages BIGINT NOT NULL,
sent_messages BIGINT NOT NULL,
e2ee_messages BIGINT NOT NULL,
sent_e2ee_messages BIGINT NOT NULL,
active_rooms BIGINT NOT NULL,
active_e2ee_rooms BIGINT NOT NULL,
CONSTRAINT daily_stats_unique UNIQUE (timestamp, server_name)
);
`
const upsertDailyMessagesSQL = `
INSERT INTO userapi_daily_stats AS u (timestamp, server_name, messages, sent_messages, e2ee_messages, sent_e2ee_messages, active_rooms, active_e2ee_rooms)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT ON CONSTRAINT daily_stats_unique
DO UPDATE SET
messages=u.messages+excluded.messages, sent_messages=u.sent_messages+excluded.sent_messages,
e2ee_messages=u.e2ee_messages+excluded.e2ee_messages, sent_e2ee_messages=u.sent_e2ee_messages+excluded.sent_e2ee_messages,
active_rooms=GREATEST($7, u.active_rooms), active_e2ee_rooms=GREATEST($8, u.active_e2ee_rooms)
`
const selectDailyMessagesSQL = `
SELECT messages, sent_messages, e2ee_messages, sent_e2ee_messages, active_rooms, active_e2ee_rooms
FROM userapi_daily_stats
WHERE server_name = $1 AND timestamp = $2;
`
const countUsersLastSeenAfterSQL = "" +
"SELECT COUNT(*) FROM (" +
" SELECT localpart FROM userapi_devices WHERE last_seen_ts > $1 " +
@ -170,6 +200,8 @@ type statsStatements struct {
countUserByAccountTypeStmt *sql.Stmt
countRegisteredUserByTypeStmt *sql.Stmt
dbEngineVersionStmt *sql.Stmt
upsertMessagesStmt *sql.Stmt
selectDailyMessagesStmt *sql.Stmt
}
func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) {
@ -182,6 +214,10 @@ func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName)
if err != nil {
return nil, err
}
_, err = db.Exec(messagesDailySchema)
if err != nil {
return nil, err
}
go s.startTimers()
return s, sqlutil.StatementList{
{&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL},
@ -191,6 +227,8 @@ func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName)
{&s.countUserByAccountTypeStmt, countUserByAccountTypeSQL},
{&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeStmt},
{&s.dbEngineVersionStmt, queryDBEngineVersion},
{&s.upsertMessagesStmt, upsertDailyMessagesSQL},
{&s.selectDailyMessagesStmt, selectDailyMessagesSQL},
}.Prepare(db)
}
@ -435,3 +473,34 @@ func (s *statsStatements) UpdateUserDailyVisits(
}
return err
}
func (s *statsStatements) UpsertDailyStats(
ctx context.Context, txn *sql.Tx,
serverName gomatrixserverlib.ServerName, stats types.MessageStats,
activeRooms, activeE2EERooms int64,
) error {
stmt := sqlutil.TxStmt(txn, s.upsertMessagesStmt)
timestamp := time.Now().Truncate(time.Hour * 24)
_, err := stmt.ExecContext(ctx,
gomatrixserverlib.AsTimestamp(timestamp),
serverName,
stats.Messages, stats.SentMessages, stats.MessagesE2EE, stats.SentMessagesE2EE,
activeRooms, activeE2EERooms,
)
return err
}
func (s *statsStatements) DailyRoomsMessages(
ctx context.Context, txn *sql.Tx,
serverName gomatrixserverlib.ServerName,
) (msgStats types.MessageStats, activeRooms, activeE2EERooms int64, err error) {
stmt := sqlutil.TxStmt(txn, s.selectDailyMessagesStmt)
timestamp := time.Now().Truncate(time.Hour * 24)
err = stmt.QueryRowContext(ctx, serverName, gomatrixserverlib.AsTimestamp(timestamp)).
Scan(&msgStats.Messages, &msgStats.SentMessages, &msgStats.MessagesE2EE, &msgStats.SentMessagesE2EE, &activeRooms, &activeE2EERooms)
if err != nil && err != sql.ErrNoRows {
return msgStats, 0, 0, err
}
return msgStats, activeRooms, activeE2EERooms, nil
}

15
userapi/storage/shared/storage.go
View file

@ -29,13 +29,12 @@ import (
"github.com/matrix-org/gomatrixserverlib"
"golang.org/x/crypto/bcrypt"
"github.com/matrix-org/dendrite/userapi/types"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/internal/pushrules"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/tables"
"github.com/matrix-org/dendrite/userapi/types"
)
// Database represents an account database
@ -808,3 +807,15 @@ func (d *Database) RemovePushers(
func (d *Database) UserStatistics(ctx context.Context) (*types.UserStatistics, *types.DatabaseEngine, error) {
return d.Stats.UserStatistics(ctx, nil)
}
func (d *Database) UpsertDailyRoomsMessages(ctx context.Context, serverName gomatrixserverlib.ServerName, stats types.MessageStats, activeRooms, activeE2EERooms int64) error {
return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
return d.Stats.UpsertDailyStats(ctx, txn, serverName, stats, activeRooms, activeE2EERooms)
})
}
func (d *Database) DailyRoomsMessages(
ctx context.Context, serverName gomatrixserverlib.ServerName,
) (stats types.MessageStats, activeRooms, activeE2EERooms int64, err error) {
return d.Stats.DailyRoomsMessages(ctx, nil, serverName)
}

68
userapi/storage/sqlite3/stats_table.go
View file

@ -44,6 +44,35 @@ CREATE INDEX IF NOT EXISTS userapi_daily_visits_timestamp_idx ON userapi_daily_v
CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON userapi_daily_visits(localpart, timestamp);
`
const messagesDailySchema = `
CREATE TABLE IF NOT EXISTS userapi_daily_stats (
timestamp BIGINT NOT NULL,
server_name TEXT NOT NULL,
messages BIGINT NOT NULL,
sent_messages BIGINT NOT NULL,
e2ee_messages BIGINT NOT NULL,
sent_e2ee_messages BIGINT NOT NULL,
active_rooms BIGINT NOT NULL,
active_e2ee_rooms BIGINT NOT NULL,
CONSTRAINT daily_stats_unique UNIQUE (timestamp, server_name)
);
`
const upsertDailyMessagesSQL = `
INSERT INTO userapi_daily_stats (timestamp, server_name, messages, sent_messages, e2ee_messages, sent_e2ee_messages, active_rooms, active_e2ee_rooms)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (timestamp, server_name)
DO UPDATE SET
messages=messages+excluded.messages, sent_messages=sent_messages+excluded.sent_messages,
e2ee_messages=e2ee_messages+excluded.e2ee_messages, sent_e2ee_messages=sent_e2ee_messages+excluded.sent_e2ee_messages,
active_rooms=MAX($7, active_rooms), active_e2ee_rooms=MAX($8, active_e2ee_rooms)
`
const selectDailyMessagesSQL = `
SELECT messages, sent_messages, e2ee_messages, sent_e2ee_messages, active_rooms, active_e2ee_rooms
FROM userapi_daily_stats
WHERE server_name = $1 AND timestamp = $2;
`
const countUsersLastSeenAfterSQL = "" +
"SELECT COUNT(*) FROM (" +
" SELECT localpart FROM userapi_devices WHERE last_seen_ts > $1 " +
@ -176,6 +205,8 @@ type statsStatements struct {
countUserByAccountTypeStmt *sql.Stmt
countRegisteredUserByTypeStmt *sql.Stmt
dbEngineVersionStmt *sql.Stmt
upsertMessagesStmt *sql.Stmt
selectDailyMessagesStmt *sql.Stmt
}
func NewSQLiteStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) {
@ -189,6 +220,10 @@ func NewSQLiteStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (t
if err != nil {
return nil, err
}
_, err = db.Exec(messagesDailySchema)
if err != nil {
return nil, err
}
go s.startTimers()
return s, sqlutil.StatementList{
{&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL},
@ -198,6 +233,8 @@ func NewSQLiteStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (t
{&s.countUserByAccountTypeStmt, countUserByAccountTypeSQL},
{&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeSQL},
{&s.dbEngineVersionStmt, queryDBEngineVersion},
{&s.upsertMessagesStmt, upsertDailyMessagesSQL},
{&s.selectDailyMessagesStmt, selectDailyMessagesSQL},
}.Prepare(db)
}
@ -451,3 +488,34 @@ func (s *statsStatements) UpdateUserDailyVisits(
}
return err
}
func (s *statsStatements) UpsertDailyStats(
ctx context.Context, txn *sql.Tx,
serverName gomatrixserverlib.ServerName, stats types.MessageStats,
activeRooms, activeE2EERooms int64,
) error {
stmt := sqlutil.TxStmt(txn, s.upsertMessagesStmt)
timestamp := time.Now().Truncate(time.Hour * 24)
_, err := stmt.ExecContext(ctx,
gomatrixserverlib.AsTimestamp(timestamp),
serverName,
stats.Messages, stats.SentMessages, stats.MessagesE2EE, stats.SentMessagesE2EE,
activeRooms, activeE2EERooms,
)
return err
}
func (s *statsStatements) DailyRoomsMessages(
ctx context.Context, txn *sql.Tx,
serverName gomatrixserverlib.ServerName,
) (msgStats types.MessageStats, activeRooms, activeE2EERooms int64, err error) {
stmt := sqlutil.TxStmt(txn, s.selectDailyMessagesStmt)
timestamp := time.Now().Truncate(time.Hour * 24)
err = stmt.QueryRowContext(ctx, serverName, gomatrixserverlib.AsTimestamp(timestamp)).
Scan(&msgStats.Messages, &msgStats.SentMessages, &msgStats.MessagesE2EE, &msgStats.SentMessagesE2EE, &activeRooms, &activeE2EERooms)
if err != nil && err != sql.ErrNoRows {
return msgStats, 0, 0, err
}
return msgStats, activeRooms, activeE2EERooms, nil
}

4
userapi/storage/tables/interface.go
View file

@ -20,6 +20,8 @@ import (
"encoding/json"
"time"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/types"
@ -115,7 +117,9 @@ type NotificationTable interface {
type StatsTable interface {
UserStatistics(ctx context.Context, txn *sql.Tx) (*types.UserStatistics, *types.DatabaseEngine, error)
DailyRoomsMessages(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) (msgStats types.MessageStats, activeRooms, activeE2EERooms int64, err error)
UpdateUserDailyVisits(ctx context.Context, txn *sql.Tx, startTime, lastUpdate time.Time) error
UpsertDailyStats(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, stats types.MessageStats, activeRooms, activeE2EERooms int64) error
}
type NotificationFilter uint32