hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2025-07-31 13:22:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove BaseDendrite (#3023)

Browse source 
Removes `BaseDendrite` to, hopefully, make testing and composing of
components easier in the future.
This commit is contained in:
Till 2023-03-22 09:21:32 +01:00 • committed by GitHub
parent ec6879e5ae
commit 5e85a00cb3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
68 changed files with 1186 additions and 1002 deletions
Download patch file Download diff file Expand all files Collapse all files

2
userapi/consumers/roomserver_test.go
View file

@ -21,7 +21,7 @@ import (
func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, func()) {
t.Helper()
connStr, close := test.PrepareDBConnectionString(t, dbType)
cm := sqlutil.NewConnectionManager()
cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{})
db, err := storage.NewUserDatabase(context.Background(), cm, &config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, "", 4, 0, 0, "")

2
userapi/internal/device_list_update_test.go
View file

@ -364,7 +364,7 @@ func mustCreateKeyserverDB(t *testing.T, dbType test.DBType) (storage.KeyDatabas
t.Helper()
connStr, clearDB := test.PrepareDBConnectionString(t, dbType)
cm := sqlutil.NewConnectionManager()
cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{})
db, err := storage.NewKeyDatabase(cm, &config.DatabaseOptions{ConnectionString: config.DataSource(connStr)})
if err != nil {
t.Fatal(err)

2
userapi/internal/key_api_test.go
View file

@ -16,7 +16,7 @@ import (
func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) {
t.Helper()
connStr, close := test.PrepareDBConnectionString(t, dbType)
cm := sqlutil.NewConnectionManager()
cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{})
db, err := storage.NewKeyDatabase(cm, &config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
})

7
userapi/storage/storage_test.go
View file

@ -35,7 +35,7 @@ var (
func mustCreateUserDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, func()) {
connStr, close := test.PrepareDBConnectionString(t, dbType)
cm := sqlutil.NewConnectionManager()
cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{})
db, err := storage.NewUserDatabase(context.Background(), cm, &config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, "localhost", bcrypt.MinCost, openIDLifetimeMS, loginTokenLifetime, "_server")
@ -576,8 +576,9 @@ func Test_Notification(t *testing.T) {
}
func mustCreateKeyDatabase(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) {
base, close := testrig.CreateBaseDendrite(t, dbType)
db, err := storage.NewKeyDatabase(base.ConnectionManager, &base.Cfg.KeyServer.Database)
cfg, processCtx, close := testrig.CreateConfig(t, dbType)
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
db, err := storage.NewKeyDatabase(cm, &cfg.KeyServer.Database)
if err != nil {
t.Fatalf("failed to create new database: %v", err)
}

58
userapi/userapi.go
View file

@ -19,10 +19,12 @@ import (
fedsenderapi "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/internal/pushgateway"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/process"
"github.com/sirupsen/logrus"
rsapi "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/consumers"
@ -35,31 +37,33 @@ import (
// NewInternalAPI returns a concrete implementation of the internal API. Callers
// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
func NewInternalAPI(
base *base.BaseDendrite,
processContext *process.ProcessContext,
dendriteCfg *config.Dendrite,
cm sqlutil.Connections,
natsInstance *jetstream.NATSInstance,
rsAPI rsapi.UserRoomserverAPI,
fedClient fedsenderapi.KeyserverFederationAPI,
) *internal.UserInternalAPI {
cfg := &base.Cfg.UserAPI
js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
appServices := base.Cfg.Derived.ApplicationServices
js, _ := natsInstance.Prepare(processContext, &dendriteCfg.Global.JetStream)
appServices := dendriteCfg.Derived.ApplicationServices
pgClient := pushgateway.NewHTTPClient(cfg.PushGatewayDisableTLSValidation)
pgClient := pushgateway.NewHTTPClient(dendriteCfg.UserAPI.PushGatewayDisableTLSValidation)
db, err := storage.NewUserDatabase(
base.ProcessContext.Context(),
base.ConnectionManager,
&cfg.AccountDatabase,
cfg.Matrix.ServerName,
cfg.BCryptCost,
cfg.OpenIDTokenLifetimeMS,
processContext.Context(),
cm,
&dendriteCfg.UserAPI.AccountDatabase,
dendriteCfg.Global.ServerName,
dendriteCfg.UserAPI.BCryptCost,
dendriteCfg.UserAPI.OpenIDTokenLifetimeMS,
api.DefaultLoginTokenLifetime,
cfg.Matrix.ServerNotices.LocalPart,
dendriteCfg.UserAPI.Matrix.ServerNotices.LocalPart,
)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to accounts db")
}
keyDB, err := storage.NewKeyDatabase(base.ConnectionManager, &base.Cfg.KeyServer.Database)
keyDB, err := storage.NewKeyDatabase(cm, &dendriteCfg.KeyServer.Database)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to key db")
}
@ -70,11 +74,11 @@ func NewInternalAPI(
// it's handled by clientapi, and hence uses its topic. When user
// API handles it for all account data, we can remove it from
// here.
cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData),
cfg.Matrix.JetStream.Prefixed(jetstream.OutputNotificationData),
dendriteCfg.Global.JetStream.Prefixed(jetstream.OutputClientData),
dendriteCfg.Global.JetStream.Prefixed(jetstream.OutputNotificationData),
)
keyChangeProducer := &producers.KeyChange{
Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
Topic: dendriteCfg.Global.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
JetStream: js,
DB: keyDB,
}
@ -84,15 +88,15 @@ func NewInternalAPI(
KeyDatabase: keyDB,
SyncProducer: syncProducer,
KeyChangeProducer: keyChangeProducer,
Config: cfg,
Config: &dendriteCfg.UserAPI,
AppServices: appServices,
RSAPI: rsAPI,
DisableTLSValidation: cfg.PushGatewayDisableTLSValidation,
DisableTLSValidation: dendriteCfg.UserAPI.PushGatewayDisableTLSValidation,
PgClient: pgClient,
FedClient: fedClient,
}
updater := internal.NewDeviceListUpdater(base.ProcessContext, keyDB, userAPI, keyChangeProducer, fedClient, 8, rsAPI, cfg.Matrix.ServerName) // 8 workers TODO: configurable
updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, 8, rsAPI, dendriteCfg.Global.ServerName) // 8 workers TODO: configurable
userAPI.Updater = updater
// Remove users which we don't share a room with anymore
if err := updater.CleanUp(); err != nil {
@ -106,28 +110,28 @@ func NewInternalAPI(
}()
dlConsumer := consumers.NewDeviceListUpdateConsumer(
base.ProcessContext, cfg, js, updater,
processContext, &dendriteCfg.UserAPI, js, updater,
)
if err := dlConsumer.Start(); err != nil {
logrus.WithError(err).Panic("failed to start device list consumer")
}
sigConsumer := consumers.NewSigningKeyUpdateConsumer(
base.ProcessContext, cfg, js, userAPI,
processContext, &dendriteCfg.UserAPI, js, userAPI,
)
if err := sigConsumer.Start(); err != nil {
logrus.WithError(err).Panic("failed to start signing key consumer")
}
receiptConsumer := consumers.NewOutputReceiptEventConsumer(
base.ProcessContext, cfg, js, db, syncProducer, pgClient,
processContext, &dendriteCfg.UserAPI, js, db, syncProducer, pgClient,
)
if err := receiptConsumer.Start(); err != nil {
logrus.WithError(err).Panic("failed to start user API receipt consumer")
}
eventConsumer := consumers.NewOutputRoomEventConsumer(
base.ProcessContext, cfg, js, db, pgClient, rsAPI, syncProducer,
processContext, &dendriteCfg.UserAPI, js, db, pgClient, rsAPI, syncProducer,
)
if err := eventConsumer.Start(); err != nil {
logrus.WithError(err).Panic("failed to start user API streamed event consumer")
@ -136,15 +140,15 @@ func NewInternalAPI(
var cleanOldNotifs func()
cleanOldNotifs = func() {
logrus.Infof("Cleaning old notifications")
if err := db.DeleteOldNotifications(base.Context()); err != nil {
if err := db.DeleteOldNotifications(processContext.Context()); err != nil {
logrus.WithError(err).Error("Failed to clean old notifications")
}
time.AfterFunc(time.Hour, cleanOldNotifs)
}
time.AfterFunc(time.Minute, cleanOldNotifs)
if base.Cfg.Global.ReportStats.Enabled {
go util.StartPhoneHomeCollector(time.Now(), base.Cfg, db)
if dendriteCfg.Global.ReportStats.Enabled {
go util.StartPhoneHomeCollector(time.Now(), dendriteCfg, db)
}
return userAPI

26
userapi/userapi_test.go
View file

@ -68,34 +68,25 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType, pub
if opts.loginTokenLifetime == 0 {
opts.loginTokenLifetime = api.DefaultLoginTokenLifetime * time.Millisecond
}
base, baseclose := testrig.CreateBaseDendrite(t, dbType)
connStr, close := test.PrepareDBConnectionString(t, dbType)
cfg, ctx, close := testrig.CreateConfig(t, dbType)
sName := serverName
if opts.serverName != "" {
sName = gomatrixserverlib.ServerName(opts.serverName)
}
cm := sqlutil.NewConnectionManager()
ctx := context.Background()
accountDB, err := storage.NewUserDatabase(ctx, cm, &config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, sName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "")
cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions)
accountDB, err := storage.NewUserDatabase(ctx.Context(), cm, &cfg.UserAPI.AccountDatabase, sName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "")
if err != nil {
t.Fatalf("failed to create account DB: %s", err)
}
keyDB, err := storage.NewKeyDatabase(base.ConnectionManager, &config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
})
keyDB, err := storage.NewKeyDatabase(cm, &cfg.KeyServer.Database)
if err != nil {
t.Fatalf("failed to create key DB: %s", err)
}
cfg := &config.UserAPI{
Matrix: &config.Global{
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: sName,
},
},
cfg.Global.SigningIdentity = gomatrixserverlib.SigningIdentity{
ServerName: sName,
}
if publisher == nil {
@ -107,12 +98,11 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType, pub
return &internal.UserInternalAPI{
DB: accountDB,
KeyDatabase: keyDB,
Config: cfg,
Config: &cfg.UserAPI,
SyncProducer: syncProducer,
KeyChangeProducer: keyChangeProducer,
}, accountDB, func() {
close()
baseclose()
}
}

2
userapi/util/notify_test.go
View file

@ -77,7 +77,7 @@ func TestNotifyUserCountsAsync(t *testing.T) {
// Create DB and Dendrite base
connStr, close := test.PrepareDBConnectionString(t, dbType)
defer close()
cm := sqlutil.NewConnectionManager()
cm := sqlutil.NewConnectionManager(nil, config.DatabaseOptions{})
db, err := storage.NewUserDatabase(ctx, cm, &config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, "test", bcrypt.MinCost, 0, 0, "")

14
userapi/util/phonehomestats_test.go
View file

@ -7,10 +7,10 @@ import (
"testing"
"time"
"github.com/matrix-org/dendrite/internal/sqlutil"
"golang.org/x/crypto/bcrypt"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/dendrite/userapi/storage"
@ -18,12 +18,10 @@ import (
func TestCollect(t *testing.T) {
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
b, _, _ := testrig.Base(nil)
connStr, closeDB := test.PrepareDBConnectionString(t, dbType)
cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
defer closeDB()
db, err := storage.NewUserDatabase(b.Context(), b.ConnectionManager, &config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, "localhost", bcrypt.MinCost, 1000, 1000, "")
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
db, err := storage.NewUserDatabase(processCtx.Context(), cm, &cfg.UserAPI.AccountDatabase, "localhost", bcrypt.MinCost, 1000, 1000, "")
if err != nil {
t.Error(err)
}
@ -62,12 +60,12 @@ func TestCollect(t *testing.T) {
}))
defer srv.Close()
b.Cfg.Global.ReportStats.Endpoint = srv.URL
cfg.Global.ReportStats.Endpoint = srv.URL
stats := phoneHomeStats{
prevData: timestampToRUUsage{},
serverName: "localhost",
startTime: time.Now(),
cfg: b.Cfg,
cfg: cfg,
db: db,
isMonolith: false,
client: &http.Client{Timeout: time.Second},