hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2025-07-31 13:22:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Preparations for removing BaseDendrite (#3016)

Browse source 
Preparations to actually remove/replace `BaseDendrite`.
Quite a few changes:
- SyncAPI accepts an `fulltext.Indexer` interface (fulltext is removed
from `BaseDendrite`)
- Caches are removed from `BaseDendrite`
- Introduces a `Router` struct (likely to change)
  - also fixes #2903
- Introduces a `sqlutil.ConnectionManager`, which should remove
`base.DatabaseConnection` later on
- probably more
This commit is contained in:
Till 2023-03-17 12:09:45 +01:00 • committed by GitHub
parent d88f71ab71
commit 5579121c6f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
85 changed files with 722 additions and 470 deletions
Download patch file Download diff file Expand all files Collapse all files

6
roomserver/internal/api.go
View file

@ -60,7 +60,7 @@ type RoomserverInternalAPI struct {
func NewRoomserverAPI(
base *base.BaseDendrite, roomserverDB storage.Database,
js nats.JetStreamContext, nc *nats.Conn,
js nats.JetStreamContext, nc *nats.Conn, caches caching.RoomServerCaches,
) *RoomserverInternalAPI {
var perspectiveServerNames []gomatrixserverlib.ServerName
for _, kp := range base.Cfg.FederationAPI.KeyPerspectives {
@ -78,7 +78,7 @@ func NewRoomserverAPI(
DB: roomserverDB,
Base: base,
Cfg: &base.Cfg.RoomServer,
Cache: base.Caches,
Cache: caches,
ServerName: base.Cfg.Global.ServerName,
PerspectiveServerNames: perspectiveServerNames,
InputRoomEventTopic: base.Cfg.Global.JetStream.Prefixed(jetstream.InputRoomEvent),
@ -89,7 +89,7 @@ func NewRoomserverAPI(
ServerACLs: serverACLs,
Queryer: &query.Queryer{
DB: roomserverDB,
Cache: base.Caches,
Cache: caches,
IsLocalServerName: base.Cfg.Global.IsLocalServerName,
ServerACLs: serverACLs,
},

22
roomserver/internal/helpers/helpers_test.go
View file

@ -3,26 +3,28 @@ package helpers
import (
"context"
"testing"
"time"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/stretchr/testify/assert"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/dendrite/roomserver/storage"
"github.com/matrix-org/dendrite/test"
)
func mustCreateDatabase(t *testing.T, dbType test.DBType) (*base.BaseDendrite, storage.Database, func()) {
base, close := testrig.CreateBaseDendrite(t, dbType)
db, err := storage.Open(base, &base.Cfg.RoomServer.Database, base.Caches)
func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
conStr, close := test.PrepareDBConnectionString(t, dbType)
caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, caching.DisableMetrics)
cm := sqlutil.NewConnectionManager()
db, err := storage.Open(context.Background(), cm, &config.DatabaseOptions{ConnectionString: config.DataSource(conStr)}, caches)
if err != nil {
t.Fatalf("failed to create Database: %v", err)
}
return base, db, close
return db, close
}
func TestIsInvitePendingWithoutNID(t *testing.T) {
@ -32,7 +34,7 @@ func TestIsInvitePendingWithoutNID(t *testing.T) {
room := test.NewRoom(t, alice, test.RoomPreset(test.PresetPublicChat))
_ = bob
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
_, db, close := mustCreateDatabase(t, dbType)
db, close := mustCreateDatabase(t, dbType)
defer close()
// store all events

6
roomserver/internal/input/input_test.go
View file

@ -7,6 +7,7 @@ import (
"time"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/roomserver/internal/input"
"github.com/matrix-org/dendrite/roomserver/storage"
@ -48,14 +49,15 @@ func TestSingleTransactionOnInput(t *testing.T) {
Kind: api.KindOutlier, // don't panic if we generate an output event
Event: event.Headered(gomatrixserverlib.RoomVersionV6),
}
cm := sqlutil.NewConnectionManager()
db, err := storage.Open(
nil,
context.Background(), cm,
&config.DatabaseOptions{
ConnectionString: "",
MaxOpenConnections: 1,
MaxIdleConnections: 1,
},
caching.NewRistrettoCache(8*1024*1024, time.Hour, false),
caching.NewRistrettoCache(8*1024*1024, time.Hour, caching.DisableMetrics),
)
if err != nil {
t.Logf("PostgreSQL not available (%s), skipping", err)

6
roomserver/roomserver.go
View file

@ -15,6 +15,7 @@
package roomserver
import (
"github.com/matrix-org/dendrite/internal/caching"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/roomserver/api"
@ -26,10 +27,11 @@ import (
// NewInternalAPI returns a concrete implementation of the internal API.
func NewInternalAPI(
base *base.BaseDendrite,
caches caching.RoomServerCaches,
) api.RoomserverInternalAPI {
cfg := &base.Cfg.RoomServer
roomserverDB, err := storage.Open(base, &cfg.Database, base.Caches)
roomserverDB, err := storage.Open(base.ProcessContext.Context(), base.ConnectionManager, &cfg.Database, caches)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to room server db")
}
@ -37,6 +39,6 @@ func NewInternalAPI(
js, nc := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
return internal.NewRoomserverAPI(
base, roomserverDB, js, nc,
base, roomserverDB, js, nc, caches,
)
}

17
roomserver/roomserver_test.go
View file

@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/stretchr/testify/assert"
"github.com/matrix-org/dendrite/roomserver/state"
@ -32,7 +33,8 @@ import (
func mustCreateDatabase(t *testing.T, dbType test.DBType) (*base.BaseDendrite, storage.Database, func()) {
t.Helper()
base, close := testrig.CreateBaseDendrite(t, dbType)
db, err := storage.Open(base, &base.Cfg.RoomServer.Database, base.Caches)
caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics)
db, err := storage.Open(base.ProcessContext.Context(), base.ConnectionManager, &base.Cfg.RoomServer.Database, caches)
if err != nil {
t.Fatalf("failed to create Database: %v", err)
}
@ -43,7 +45,8 @@ func TestUsers(t *testing.T) {
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
base, close := testrig.CreateBaseDendrite(t, dbType)
defer close()
rsAPI := roomserver.NewInternalAPI(base)
caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics)
rsAPI := roomserver.NewInternalAPI(base, caches)
// SetFederationAPI starts the room event input consumer
rsAPI.SetFederationAPI(nil, nil)
@ -181,7 +184,8 @@ func Test_QueryLeftUsers(t *testing.T) {
base, _, close := mustCreateDatabase(t, dbType)
defer close()
rsAPI := roomserver.NewInternalAPI(base)
caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics)
rsAPI := roomserver.NewInternalAPI(base, caches)
// SetFederationAPI starts the room event input consumer
rsAPI.SetFederationAPI(nil, nil)
// Create the room
@ -232,12 +236,13 @@ func TestPurgeRoom(t *testing.T) {
defer jetstream.DeleteAllStreams(jsCtx, &base.Cfg.Global.JetStream)
fedClient := base.CreateFederationClient()
rsAPI := roomserver.NewInternalAPI(base)
caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics)
rsAPI := roomserver.NewInternalAPI(base, caches)
userAPI := userapi.NewInternalAPI(base, rsAPI, nil)
// this starts the JetStream consumers
syncapi.AddPublicRoutes(base, userAPI, rsAPI)
federationapi.NewInternalAPI(base, fedClient, rsAPI, base.Caches, nil, true)
syncapi.AddPublicRoutes(base, userAPI, rsAPI, caches)
federationapi.NewInternalAPI(base, fedClient, rsAPI, caches, nil, true)
rsAPI.SetFederationAPI(nil, nil)
// Create the room

7
roomserver/storage/postgres/storage.go
View file

@ -28,7 +28,6 @@ import (
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/storage/postgres/deltas"
"github.com/matrix-org/dendrite/roomserver/storage/shared"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
)
@ -38,10 +37,10 @@ type Database struct {
}
// Open a postgres database.
func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
func Open(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
var d Database
var err error
db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter())
db, writer, err := conMan.Connection(dbProperties)
if err != nil {
return nil, fmt.Errorf("sqlutil.Open: %w", err)
}
@ -53,7 +52,7 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c
// Special case, since this migration uses several tables, so it needs to
// be sure that all tables are created first.
if err = executeMigration(base.Context(), db); err != nil {
if err = executeMigration(ctx, db); err != nil {
return nil, err
}

7
roomserver/storage/sqlite3/storage.go
View file

@ -28,7 +28,6 @@ import (
"github.com/matrix-org/dendrite/roomserver/storage/shared"
"github.com/matrix-org/dendrite/roomserver/storage/sqlite3/deltas"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
)
@ -38,10 +37,10 @@ type Database struct {
}
// Open a sqlite database.
func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
func Open(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
var d Database
var err error
db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter())
db, writer, err := conMan.Connection(dbProperties)
if err != nil {
return nil, fmt.Errorf("sqlutil.Open: %w", err)
}
@ -62,7 +61,7 @@ func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache c
// Special case, since this migration uses several tables, so it needs to
// be sure that all tables are created first.
if err = executeMigration(base.Context(), db); err != nil {
if err = executeMigration(ctx, db); err != nil {
return nil, err
}

9
roomserver/storage/storage.go
View file

@ -18,22 +18,23 @@
package storage
import (
"context"
"fmt"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/storage/postgres"
"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
)
// Open opens a database connection.
func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) {
func Open(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) {
switch {
case dbProperties.ConnectionString.IsSQLite():
return sqlite3.Open(base, dbProperties, cache)
return sqlite3.Open(ctx, conMan, dbProperties, cache)
case dbProperties.ConnectionString.IsPostgres():
return postgres.Open(base, dbProperties, cache)
return postgres.Open(ctx, conMan, dbProperties, cache)
default:
return nil, fmt.Errorf("unexpected database type")
}

7
roomserver/storage/storage_wasm.go
View file

@ -15,19 +15,20 @@
package storage
import (
"context"
"fmt"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
)
// NewPublicRoomsServerDatabase opens a database connection.
func Open(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) {
func Open(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) {
switch {
case dbProperties.ConnectionString.IsSQLite():
return sqlite3.Open(base, dbProperties, cache)
return sqlite3.Open(ctx, conMan, dbProperties, cache)
case dbProperties.ConnectionString.IsPostgres():
return nil, fmt.Errorf("can't use Postgres implementation")
default: