hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2025-07-30 21:12:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Preparations for removing BaseDendrite (#3016)

Browse source 
Preparations to actually remove/replace `BaseDendrite`.
Quite a few changes:
- SyncAPI accepts an `fulltext.Indexer` interface (fulltext is removed
from `BaseDendrite`)
- Caches are removed from `BaseDendrite`
- Introduces a `Router` struct (likely to change)
  - also fixes #2903
- Introduces a `sqlutil.ConnectionManager`, which should remove
`base.DatabaseConnection` later on
- probably more
This commit is contained in:
Till 2023-03-17 12:09:45 +01:00 • committed by GitHub
parent d88f71ab71
commit 5579121c6f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
85 changed files with 722 additions and 470 deletions
Download patch file Download diff file Expand all files Collapse all files

2
federationapi/federationapi.go
View file

@ -94,7 +94,7 @@ func NewInternalAPI(
) api.FederationInternalAPI {
cfg := &base.Cfg.FederationAPI
federationDB, err := storage.NewDatabase(base, &cfg.Database, base.Caches, base.Cfg.Global.IsLocalServerName)
federationDB, err := storage.NewDatabase(base.ProcessContext.Context(), base.ConnectionManager, &cfg.Database, caches, base.Cfg.Global.IsLocalServerName)
if err != nil {
logrus.WithError(err).Panic("failed to connect to federation sender db")
}

6
federationapi/federationapi_test.go
View file

@ -10,6 +10,7 @@ import (
"testing"
"time"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
@ -163,6 +164,7 @@ func TestFederationAPIJoinThenKeyUpdate(t *testing.T) {
func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
base, close := testrig.CreateBaseDendrite(t, dbType)
caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics)
base.Cfg.FederationAPI.PreferDirectFetch = true
base.Cfg.FederationAPI.KeyPerspectives = nil
defer close()
@ -212,7 +214,7 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
},
},
}
fsapi := federationapi.NewInternalAPI(base, fc, rsapi, base.Caches, nil, false)
fsapi := federationapi.NewInternalAPI(base, fc, rsapi, caches, nil, false)
var resp api.PerformJoinResponse
fsapi.PerformJoin(context.Background(), &api.PerformJoinRequest{
@ -278,7 +280,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
federationapi.AddPublicRoutes(b, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil)
baseURL, cancel := test.ListenAndServe(t, b.PublicFederationAPIMux, true)
baseURL, cancel := test.ListenAndServe(t, b.Routers.Federation, true)
defer cancel()
serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))

13
federationapi/queue/queue_test.go
View file

@ -21,6 +21,8 @@ import (
"testing"
"time"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/test/testrig"
"go.uber.org/atomic"
"gotest.tools/v3/poll"
@ -34,17 +36,17 @@ import (
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/process"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
)
func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase bool) (storage.Database, *process.ProcessContext, func()) {
if realDatabase {
// Real Database/s
b, baseClose := testrig.CreateBaseDendrite(t, dbType)
caches := caching.NewRistrettoCache(b.Cfg.Global.Cache.EstimatedMaxSize, b.Cfg.Global.Cache.MaxAge, caching.DisableMetrics)
connStr, dbClose := test.PrepareDBConnectionString(t, dbType)
db, err := storage.NewDatabase(b, &config.DatabaseOptions{
db, err := storage.NewDatabase(b.ProcessContext.Context(), b.ConnectionManager, &config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, b.Caches, b.Cfg.Global.IsLocalServerName)
}, caches, b.Cfg.Global.IsLocalServerName)
if err != nil {
t.Fatalf("NewDatabase returned %s", err)
}
@ -55,10 +57,7 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase
} else {
// Fake Database
db := test.NewInMemoryFederationDatabase()
b := struct {
ProcessContext *process.ProcessContext
}{ProcessContext: process.NewProcessContext()}
return db, b.ProcessContext, func() {}
return db, process.NewProcessContext(), func() {}
}
}

2
federationapi/routing/profile_test.go
View file

@ -50,7 +50,7 @@ func TestHandleQueryProfile(t *testing.T) {
defer close()
fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
base.PublicFederationAPIMux = fedMux
base.Routers.Federation = fedMux
base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false
fedClient := fakeFedClient{}

2
federationapi/routing/query_test.go
View file

@ -50,7 +50,7 @@ func TestHandleQueryDirectory(t *testing.T) {
defer close()
fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
base.PublicFederationAPIMux = fedMux
base.Routers.Federation = fedMux
base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false
fedClient := fakeFedClient{}

6
federationapi/routing/routing.go
View file

@ -65,9 +65,9 @@ func Setup(
servers federationAPI.ServersInRoomProvider,
producer *producers.SyncAPIProducer,
) {
fedMux := base.PublicFederationAPIMux
keyMux := base.PublicKeyAPIMux
wkMux := base.PublicWellKnownAPIMux
fedMux := base.Routers.Federation
keyMux := base.Routers.Keys
wkMux := base.Routers.WellKnown
cfg := &base.Cfg.FederationAPI
if base.EnableMetrics {

2
federationapi/routing/send_test.go
View file

@ -48,7 +48,7 @@ func TestHandleSend(t *testing.T) {
defer close()
fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
base.PublicFederationAPIMux = fedMux
base.Routers.Federation = fedMux
base.Cfg.FederationAPI.Matrix.SigningIdentity.ServerName = testOrigin
base.Cfg.FederationAPI.Matrix.Metrics.Enabled = false
fedapi := fedAPI.NewInternalAPI(base, nil, nil, nil, nil, true)

8
federationapi/storage/postgres/storage.go
View file

@ -16,6 +16,7 @@
package postgres
import (
"context"
"database/sql"
"fmt"
@ -23,7 +24,6 @@ import (
"github.com/matrix-org/dendrite/federationapi/storage/shared"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)
@ -36,10 +36,10 @@ type Database struct {
}
// NewDatabase opens a new database
func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (*Database, error) {
func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (*Database, error) {
var d Database
var err error
if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil {
if d.db, d.writer, err = conMan.Connection(dbProperties); err != nil {
return nil, err
}
blacklist, err := NewPostgresBlacklistTable(d.db)
@ -95,7 +95,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
Version: "federationsender: drop federationsender_rooms",
Up: deltas.UpRemoveRoomsTable,
})
err = m.Up(base.Context())
err = m.Up(ctx)
if err != nil {
return nil, err
}

8
federationapi/storage/sqlite3/storage.go
View file

@ -15,13 +15,13 @@
package sqlite3
import (
"context"
"database/sql"
"github.com/matrix-org/dendrite/federationapi/storage/shared"
"github.com/matrix-org/dendrite/federationapi/storage/sqlite3/deltas"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)
@ -34,10 +34,10 @@ type Database struct {
}
// NewDatabase opens a new database
func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (*Database, error) {
func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (*Database, error) {
var d Database
var err error
if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil {
if d.db, d.writer, err = conMan.Connection(dbProperties); err != nil {
return nil, err
}
blacklist, err := NewSQLiteBlacklistTable(d.db)
@ -93,7 +93,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions,
Version: "federationsender: drop federationsender_rooms",
Up: deltas.UpRemoveRoomsTable,
})
err = m.Up(base.Context())
err = m.Up(ctx)
if err != nil {
return nil, err
}

9
federationapi/storage/storage.go
View file

@ -18,23 +18,24 @@
package storage
import (
"context"
"fmt"
"github.com/matrix-org/dendrite/federationapi/storage/postgres"
"github.com/matrix-org/dendrite/federationapi/storage/sqlite3"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)
// NewDatabase opens a new database
func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (Database, error) {
func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (Database, error) {
switch {
case dbProperties.ConnectionString.IsSQLite():
return sqlite3.NewDatabase(base, dbProperties, cache, isLocalServerName)
return sqlite3.NewDatabase(ctx, conMan, dbProperties, cache, isLocalServerName)
case dbProperties.ConnectionString.IsPostgres():
return postgres.NewDatabase(base, dbProperties, cache, isLocalServerName)
return postgres.NewDatabase(ctx, conMan, dbProperties, cache, isLocalServerName)
default:
return nil, fmt.Errorf("unexpected database type")
}

12
federationapi/storage/storage_test.go
View file

@ -7,26 +7,28 @@ import (
"time"
"github.com/matrix-org/dendrite/federationapi/storage"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/stretchr/testify/assert"
)
func mustCreateFederationDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
b, baseClose := testrig.CreateBaseDendrite(t, dbType)
caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, false)
connStr, dbClose := test.PrepareDBConnectionString(t, dbType)
db, err := storage.NewDatabase(b, &config.DatabaseOptions{
ctx := context.Background()
cm := sqlutil.NewConnectionManager()
db, err := storage.NewDatabase(ctx, cm, &config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
}, b.Caches, func(server gomatrixserverlib.ServerName) bool { return server == "localhost" })
}, caches, func(server gomatrixserverlib.ServerName) bool { return server == "localhost" })
if err != nil {
t.Fatalf("NewDatabase returned %s", err)
}
return db, func() {
dbClose()
baseClose()
}
}

7
federationapi/storage/storage_wasm.go
View file

@ -15,20 +15,21 @@
package storage
import (
"context"
"fmt"
"github.com/matrix-org/dendrite/federationapi/storage/sqlite3"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)
// NewDatabase opens a new database
func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (Database, error) {
func NewDatabase(ctx context.Context, conMan sqlutil.Connections, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (Database, error) {
switch {
case dbProperties.ConnectionString.IsSQLite():
return sqlite3.NewDatabase(base, dbProperties, cache, serverName)
return sqlite3.NewDatabase(ctx, conMan, dbProperties, cache, isLocalServerName)
case dbProperties.ConnectionString.IsPostgres():
return nil, fmt.Errorf("can't use Postgres implementation")
default: