hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2025-07-29 12:42:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Roomserver NID caches (#1335)

Browse source 
* Initial work on roomserver NID caches

* Give caches to roomserver storage

* Populate caches

* Fix bugs

* Fix WASM build

* Don't hit cache twice in RoomNIDExcludingStubs

* Store reverse room ID-room NID mapping, consult caches when assigning NIDs
This commit is contained in:
Neil Alexander 2020-08-25 12:32:29 +01:00 • committed by GitHub
parent 05242096a1
commit c8b873abc8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 239 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

2
roomserver/internal/api.go
View file

@ -16,7 +16,7 @@ type RoomserverInternalAPI struct {
DB storage.Database
Cfg *config.RoomServer
Producer sarama.SyncProducer
Cache caching.RoomVersionCache
Cache caching.RoomServerCaches
ServerName gomatrixserverlib.ServerName
KeyRing gomatrixserverlib.JSONVerifier
FedClient *gomatrixserverlib.FederationClient

2
roomserver/roomserver.go
View file

@ -42,7 +42,7 @@ func NewInternalAPI(
) api.RoomserverInternalAPI {
cfg := &base.Cfg.RoomServer
roomserverDB, err := storage.Open(&cfg.Database)
roomserverDB, err := storage.Open(&cfg.Database, base.Caches)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to room server db")
}

4
roomserver/storage/postgres/storage.go
View file

@ -18,6 +18,7 @@ package postgres
import (
"database/sql"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/sqlutil"
@ -33,7 +34,7 @@ type Database struct {
// Open a postgres database.
// nolint: gocyclo
func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
var d Database
var db *sql.DB
var err error
@ -98,6 +99,7 @@ func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
}
d.Database = shared.Database{
DB: db,
Cache: cache,
Writer: sqlutil.NewDummyWriter(),
EventTypesTable: eventTypes,
EventStateKeysTable: eventStateKeys,

80
roomserver/storage/shared/storage.go
View file

@ -6,6 +6,7 @@ import (
"encoding/json"
"fmt"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
@ -27,6 +28,7 @@ const redactionsArePermanent = false
type Database struct {
DB *sql.DB
Cache caching.RoomServerCaches
Writer sqlutil.Writer
EventsTable tables.Events
EventJSONTable tables.EventJSON
@ -51,7 +53,26 @@ func (d *Database) SupportsConcurrentRoomInputs() bool {
func (d *Database) EventTypeNIDs(
ctx context.Context, eventTypes []string,
) (map[string]types.EventTypeNID, error) {
return d.EventTypesTable.BulkSelectEventTypeNID(ctx, eventTypes)
result := make(map[string]types.EventTypeNID)
remaining := []string{}
for _, eventType := range eventTypes {
if nid, ok := d.Cache.GetRoomServerEventTypeNID(eventType); ok {
result[eventType] = nid
} else {
remaining = append(remaining, eventType)
}
}
if len(remaining) > 0 {
nids, err := d.EventTypesTable.BulkSelectEventTypeNID(ctx, remaining)
if err != nil {
return nil, err
}
for eventType, nid := range nids {
result[eventType] = nid
d.Cache.StoreRoomServerEventTypeNID(eventType, nid)
}
}
return result, nil
}
func (d *Database) EventStateKeys(
@ -63,7 +84,26 @@ func (d *Database) EventStateKeys(
func (d *Database) EventStateKeyNIDs(
ctx context.Context, eventStateKeys []string,
) (map[string]types.EventStateKeyNID, error) {
return d.EventStateKeysTable.BulkSelectEventStateKeyNID(ctx, eventStateKeys)
result := make(map[string]types.EventStateKeyNID)
remaining := []string{}
for _, eventStateKey := range eventStateKeys {
if nid, ok := d.Cache.GetRoomServerStateKeyNID(eventStateKey); ok {
result[eventStateKey] = nid
} else {
remaining = append(remaining, eventStateKey)
}
}
if len(remaining) > 0 {
nids, err := d.EventStateKeysTable.BulkSelectEventStateKeyNID(ctx, remaining)
if err != nil {
return nil, err
}
for eventStateKey, nid := range nids {
result[eventStateKey] = nid
d.Cache.StoreRoomServerStateKeyNID(eventStateKey, nid)
}
}
return result, nil
}
func (d *Database) StateEntriesForEventIDs(
@ -157,10 +197,14 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type
}
func (d *Database) RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) {
if nid, ok := d.Cache.GetRoomServerRoomNID(roomID); ok {
return nid, nil
}
roomNID, err := d.RoomsTable.SelectRoomNID(ctx, nil, roomID)
if err == sql.ErrNoRows {
return 0, nil
}
d.Cache.StoreRoomServerRoomNID(roomID, roomNID)
return roomNID, err
}
@ -214,6 +258,9 @@ func (d *Database) StateEntries(
func (d *Database) GetRoomVersionForRoom(
ctx context.Context, roomID string,
) (gomatrixserverlib.RoomVersion, error) {
if roomVersion, ok := d.Cache.GetRoomVersion(roomID); ok {
return roomVersion, nil
}
return d.RoomsTable.SelectRoomVersionForRoomID(
ctx, nil, roomID,
)
@ -222,6 +269,11 @@ func (d *Database) GetRoomVersionForRoom(
func (d *Database) GetRoomVersionForRoomNID(
ctx context.Context, roomNID types.RoomNID,
) (gomatrixserverlib.RoomVersion, error) {
if roomID, ok := d.Cache.GetRoomServerRoomID(roomNID); ok {
if roomVersion, ok := d.Cache.GetRoomVersion(roomID); ok {
return roomVersion, nil
}
}
return d.RoomsTable.SelectRoomVersionForRoomNID(
ctx, roomNID,
)
@ -488,6 +540,9 @@ func (d *Database) assignRoomNID(
ctx context.Context, txn *sql.Tx,
roomID string, roomVersion gomatrixserverlib.RoomVersion,
) (types.RoomNID, error) {
if roomNID, ok := d.Cache.GetRoomServerRoomNID(roomID); ok {
return roomNID, nil
}
// Check if we already have a numeric ID in the database.
roomNID, err := d.RoomsTable.SelectRoomNID(ctx, txn, roomID)
if err == sql.ErrNoRows {
@ -498,14 +553,20 @@ func (d *Database) assignRoomNID(
roomNID, err = d.RoomsTable.SelectRoomNID(ctx, txn, roomID)
}
}
if err == nil {
d.Cache.StoreRoomServerRoomNID(roomID, roomNID)
}
return roomNID, err
}
func (d *Database) assignEventTypeNID(
ctx context.Context, txn *sql.Tx, eventType string,
) (eventTypeNID types.EventTypeNID, err error) {
) (types.EventTypeNID, error) {
if eventTypeNID, ok := d.Cache.GetRoomServerEventTypeNID(eventType); ok {
return eventTypeNID, nil
}
// Check if we already have a numeric ID in the database.
eventTypeNID, err = d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType)
eventTypeNID, err := d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType)
if err == sql.ErrNoRows {
// We don't have a numeric ID so insert one into the database.
eventTypeNID, err = d.EventTypesTable.InsertEventTypeNID(ctx, txn, eventType)
@ -514,12 +575,18 @@ func (d *Database) assignEventTypeNID(
eventTypeNID, err = d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType)
}
}
return
if err == nil {
d.Cache.StoreRoomServerEventTypeNID(eventType, eventTypeNID)
}
return eventTypeNID, err
}
func (d *Database) assignStateKeyNID(
ctx context.Context, txn *sql.Tx, eventStateKey string,
) (types.EventStateKeyNID, error) {
if eventStateKeyNID, ok := d.Cache.GetRoomServerStateKeyNID(eventStateKey); ok {
return eventStateKeyNID, nil
}
// Check if we already have a numeric ID in the database.
eventStateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, txn, eventStateKey)
if err == sql.ErrNoRows {
@ -530,6 +597,9 @@ func (d *Database) assignStateKeyNID(
eventStateKeyNID, err = d.EventStateKeysTable.SelectEventStateKeyNID(ctx, txn, eventStateKey)
}
}
if err == nil {
d.Cache.StoreRoomServerStateKeyNID(eventStateKey, eventStateKeyNID)
}
return eventStateKeyNID, err
}

4
roomserver/storage/sqlite3/storage.go
View file

@ -19,6 +19,7 @@ import (
"context"
"database/sql"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/storage/shared"
@ -46,7 +47,7 @@ type Database struct {
// Open a sqlite database.
// nolint: gocyclo
func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
var d Database
var err error
if d.db, err = sqlutil.Open(dbProperties); err != nil {
@ -120,6 +121,7 @@ func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
}
d.Database = shared.Database{
DB: d.db,
Cache: cache,
Writer: sqlutil.NewExclusiveWriter(),
EventsTable: d.events,
EventTypesTable: d.eventTypes,

7
roomserver/storage/storage.go
View file

@ -19,18 +19,19 @@ package storage
import (
"fmt"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/roomserver/storage/postgres"
"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
)
// Open opens a database connection.
func Open(dbProperties *config.DatabaseOptions) (Database, error) {
func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) {
switch {
case dbProperties.ConnectionString.IsSQLite():
return sqlite3.Open(dbProperties)
return sqlite3.Open(dbProperties, cache)
case dbProperties.ConnectionString.IsPostgres():
return postgres.Open(dbProperties)
return postgres.Open(dbProperties, cache)
default:
return nil, fmt.Errorf("unexpected database type")
}

5
roomserver/storage/storage_wasm.go
View file

@ -17,15 +17,16 @@ package storage
import (
"fmt"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
)
// NewPublicRoomsServerDatabase opens a database connection.
func Open(dbProperties *config.DatabaseOptions) (Database, error) {
func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (Database, error) {
switch {
case dbProperties.ConnectionString.IsSQLite():
return sqlite3.Open(dbProperties)
return sqlite3.Open(dbProperties, cache)
case dbProperties.ConnectionString.IsPostgres():
return nil, fmt.Errorf("can't use Postgres implementation")
default: