hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2024-12-27 07:28:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Limit database connections (#980, #564) (#998)

Browse source 
* Limit database connections (#564)

- Add new options to the config file database:
      max_open_conns: 100
      max_idle_conns: 2
      conn_max_lifetime: -1
- Implement connection parameter setup on the *DB (database/sql) in internal/sqlutil/trace.go:Open()
- Propagate the values in the form of DbProperties interface via all the
  Open() and NewDatabase() functions

Signed-off-by: Tomas Jirka <tomas.jirka@email.cz>

* Fix wasm builds

* Remove file accidentally added from working tree

Co-authored-by: Tomas Jirka <tomas.jirka@email.cz>
This commit is contained in:
Neil Alexander 2020-05-01 13:34:53 +01:00 committed by GitHub
parent 908108c23e
commit f7cfa75886
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
51 changed files with 213 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
appservice/appservice.go
View file

@ -48,7 +48,7 @@ func SetupAppServiceAPIComponent(
transactionsCache *transactions.Cache,
) appserviceAPI.AppServiceQueryAPI {
// Create a connection to the appservice postgres DB
appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService))
appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService), base.Cfg.DbProperties())
if err != nil {
logrus.WithError(err).Panicf("failed to connect to appservice db")
}

5
appservice/storage/postgres/storage.go
View file

@ -21,6 +21,7 @@ import (
// Import postgres database driver
_ "github.com/lib/pq"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
)
@ -33,10 +34,10 @@ type Database struct {
}
// NewDatabase opens a new database
func NewDatabase(dataSourceName string) (*Database, error) {
func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) {
var result Database
var err error
if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
return nil, err
}
if err = result.prepare(); err != nil {

2
appservice/storage/sqlite3/storage.go
View file

@ -37,7 +37,7 @@ type Database struct {
func NewDatabase(dataSourceName string) (*Database, error) {
var result Database
var err error
if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
return nil, err
}
if err = result.prepare(); err != nil {

11
appservice/storage/storage.go
View file

@ -21,19 +21,22 @@ import (
"github.com/matrix-org/dendrite/appservice/storage/postgres"
"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
"github.com/matrix-org/dendrite/common"
)
func NewDatabase(dataSourceName string) (Database, error) {
// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
// and sets DB connection parameters
func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgres.NewDatabase(dataSourceName)
return postgres.NewDatabase(dataSourceName, dbProperties)
}
switch uri.Scheme {
case "postgres":
return postgres.NewDatabase(dataSourceName)
return postgres.NewDatabase(dataSourceName, dbProperties)
case "file":
return sqlite3.NewDatabase(dataSourceName)
default:
return postgres.NewDatabase(dataSourceName)
return postgres.NewDatabase(dataSourceName, dbProperties)
}
}

6
appservice/storage/storage_wasm.go
View file

@ -19,9 +19,13 @@ import (
"net/url"
"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
"github.com/matrix-org/dendrite/common"
)
func NewDatabase(dataSourceName string) (Database, error) {
func NewDatabase(
dataSourceName string,
dbProperties common.DbProperties, // nolint:unparam
) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return nil, fmt.Errorf("Cannot use postgres implementation")

4
clientapi/auth/storage/accounts/postgres/storage.go
View file

@ -44,10 +44,10 @@ type Database struct {
}
// NewDatabase creates a new accounts and profiles database
func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) {
var db *sql.DB
var err error
if db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
return nil, err
}
partitions := common.PartitionOffsetStatements{}

2
clientapi/auth/storage/accounts/sqlite3/storage.go
View file

@ -50,7 +50,7 @@ type Database struct {
func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
var db *sql.DB
var err error
if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
return nil, err
}
partitions := common.PartitionOffsetStatements{}

11
clientapi/auth/storage/accounts/storage.go
View file

@ -21,20 +21,23 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres"
"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/gomatrixserverlib"
)
func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) {
// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
// and sets postgres connection parameters
func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgres.NewDatabase(dataSourceName, serverName)
return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
}
switch uri.Scheme {
case "postgres":
return postgres.NewDatabase(dataSourceName, serverName)
return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
case "file":
return sqlite3.NewDatabase(dataSourceName, serverName)
default:
return postgres.NewDatabase(dataSourceName, serverName)
return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
}
}

7
clientapi/auth/storage/accounts/storage_wasm.go
View file

@ -19,10 +19,15 @@ import (
"net/url"
"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/gomatrixserverlib"
)
func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) {
func NewDatabase(
dataSourceName string,
dbProperties common.DbProperties, // nolint:unparam
serverName gomatrixserverlib.ServerName,
) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return nil, fmt.Errorf("Cannot use postgres implementation")

4
clientapi/auth/storage/devices/postgres/storage.go
View file

@ -36,10 +36,10 @@ type Database struct {
}
// NewDatabase creates a new device database
func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) {
var db *sql.DB
var err error
if db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
return nil, err
}
d := devicesStatements{}

2
clientapi/auth/storage/devices/sqlite3/storage.go
View file

@ -41,7 +41,7 @@ type Database struct {
func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
var db *sql.DB
var err error
if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
return nil, err
}
d := devicesStatements{}

11
clientapi/auth/storage/devices/storage.go
View file

@ -21,20 +21,23 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres"
"github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/gomatrixserverlib"
)
func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) {
// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
// and sets postgres connection parameters
func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgres.NewDatabase(dataSourceName, serverName)
return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
}
switch uri.Scheme {
case "postgres":
return postgres.NewDatabase(dataSourceName, serverName)
return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
case "file":
return sqlite3.NewDatabase(dataSourceName, serverName)
default:
return postgres.NewDatabase(dataSourceName, serverName)
return postgres.NewDatabase(dataSourceName, dbProperties, serverName)
}
}

7
clientapi/auth/storage/devices/storage_wasm.go
View file

@ -19,10 +19,15 @@ import (
"net/url"
"github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/gomatrixserverlib"
)
func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) {
func NewDatabase(
dataSourceName string,
dbProperties common.DbProperties, // nolint:unparam
serverName gomatrixserverlib.ServerName,
) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return nil, fmt.Errorf("Cannot use postgres implementation")

4
cmd/create-account/main.go
View file

@ -63,7 +63,7 @@ func main() {
serverName := gomatrixserverlib.ServerName(*serverNameStr)
accountDB, err := accounts.NewDatabase(*database, serverName)
accountDB, err := accounts.NewDatabase(*database, nil, serverName)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
@ -78,7 +78,7 @@ func main() {
os.Exit(1)
}
deviceDB, err := devices.NewDatabase(*database, serverName)
deviceDB, err := devices.NewDatabase(*database, nil, serverName)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)

1
cmd/dendrite-demo-libp2p/main.go
View file

@ -56,6 +56,7 @@ func createKeyDB(
) keydb.Database {
db, err := keydb.NewDatabase(
string(base.Base.Cfg.Database.ServerKey),
base.Base.Cfg.DbProperties(),
base.Base.Cfg.Matrix.ServerName,
base.Base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
base.Base.Cfg.Matrix.KeyID,

2
cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go
View file

@ -45,7 +45,7 @@ type PublicRoomsServerDatabase struct {
// NewPublicRoomsServerDatabase creates a new public rooms server database.
func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT) (*PublicRoomsServerDatabase, error) {
pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName)
pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil)
if err != nil {
return nil, err
}

2
cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go
View file

@ -48,7 +48,7 @@ type PublicRoomsServerDatabase struct {
// NewPublicRoomsServerDatabase creates a new public rooms server database.
func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub) (*PublicRoomsServerDatabase, error) {
pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName)
pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil)
if err != nil {
return nil, err
}

2
cmd/dendrite-monolith-server/main.go
View file

@ -79,7 +79,7 @@ func main() {
eduProducer := producers.NewEDUServerProducer(eduInputAPI)
federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
mediaapi.SetupMediaAPIComponent(base, deviceDB)
publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
if err != nil {
logrus.WithError(err).Panicf("failed to connect to public rooms db")
}

2
cmd/dendrite-public-rooms-api-server/main.go
View file

@ -32,7 +32,7 @@ func main() {
rsAPI := base.CreateHTTPRoomserverAPIs()
rsAPI.SetFederationSenderAPI(fsAPI)
publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
if err != nil {
logrus.WithError(err).Panicf("failed to connect to public rooms db")
}

9
common/basecomponent/base.go
View file

@ -150,7 +150,7 @@ func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.Fede
// CreateDeviceDB creates a new instance of the device database. Should only be
// called once per component.
func (b *BaseDendrite) CreateDeviceDB() devices.Database {
db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.Matrix.ServerName)
db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to devices db")
}
@ -161,7 +161,7 @@ func (b *BaseDendrite) CreateDeviceDB() devices.Database {
// CreateAccountsDB creates a new instance of the accounts database. Should only
// be called once per component.
func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.Matrix.ServerName)
db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to accounts db")
}
@ -174,6 +174,7 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
func (b *BaseDendrite) CreateKeyDB() keydb.Database {
db, err := keydb.NewDatabase(
string(b.Cfg.Database.ServerKey),
b.Cfg.DbProperties(),
b.Cfg.Matrix.ServerName,
b.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
b.Cfg.Matrix.KeyID,
@ -244,7 +245,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
uri, err := url.Parse(string(cfg.Database.Naffka))
if err != nil || uri.Scheme == "file" {
db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka))
db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka), nil)
if err != nil {
logrus.WithError(err).Panic("Failed to open naffka database")
}
@ -254,7 +255,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
logrus.WithError(err).Panic("Failed to setup naffka database")
}
} else {
db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka))
db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka), nil)
if err != nil {
logrus.WithError(err).Panic("Failed to open naffka database")
}

42
common/config/config.go
View file

@ -188,6 +188,12 @@ type Dendrite struct {
PublicRoomsAPI DataSource `yaml:"public_rooms_api"`
// The Naffka database is used internally by the naffka library, if used.
Naffka DataSource `yaml:"naffka,omitempty"`
// Maximum open connections to the DB (0 = use default, negative means unlimited)
MaxOpenConns int `yaml:"max_open_conns"`
// Maximum idle connections to the DB (0 = use default, negative means unlimited)
MaxIdleConns int `yaml:"max_idle_conns"`
// maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited)
ConnMaxLifetimeSec int `yaml:"conn_max_lifetime"`
} `yaml:"database"`
// TURN Server Config
@ -484,6 +490,15 @@ func (config *Dendrite) SetDefaults() {
defaultMaxFileSizeBytes := FileSizeBytes(10485760)
config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes
}
if config.Database.MaxIdleConns == 0 {
config.Database.MaxIdleConns = 2
}
if config.Database.MaxOpenConns == 0 {
config.Database.MaxOpenConns = 100
}
}
// Error returns a string detailing how many errors were contained within a
@ -746,6 +761,33 @@ func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err
)
}
// MaxIdleConns returns maximum idle connections to the DB
func (config Dendrite) MaxIdleConns() int {
return config.Database.MaxIdleConns
}
// MaxOpenConns returns maximum open connections to the DB
func (config Dendrite) MaxOpenConns() int {
return config.Database.MaxOpenConns
}
// ConnMaxLifetime returns maximum amount of time a connection may be reused
func (config Dendrite) ConnMaxLifetime() time.Duration {
return time.Duration(config.Database.ConnMaxLifetimeSec) * time.Second
}
// DbProperties functions return properties used by database/sql/DB
type DbProperties interface {
MaxIdleConns() int
MaxOpenConns() int
ConnMaxLifetime() time.Duration
}
// DbProperties returns cfg as a DbProperties interface
func (config Dendrite) DbProperties() DbProperties {
return config
}
// logrusLogger is a small wrapper that implements jaeger.Logger using logrus.
type logrusLogger struct {
l *logrus.Logger

8
common/keydb/keydb.go
View file

@ -21,6 +21,7 @@ import (
"golang.org/x/crypto/ed25519"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/keydb/postgres"
"github.com/matrix-org/dendrite/common/keydb/sqlite3"
"github.com/matrix-org/gomatrixserverlib"
@ -29,20 +30,21 @@ import (
// NewDatabase opens a database connection.
func NewDatabase(
dataSourceName string,
dbProperties common.DbProperties,
serverName gomatrixserverlib.ServerName,
serverKey ed25519.PublicKey,
serverKeyID gomatrixserverlib.KeyID,
) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
}
switch uri.Scheme {
case "postgres":
return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
case "file":
return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
default:
return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
}
}

2
common/keydb/keydb_wasm.go
View file

@ -20,6 +20,7 @@ import (
"golang.org/x/crypto/ed25519"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/keydb/sqlite3"
"github.com/matrix-org/gomatrixserverlib"
)
@ -27,6 +28,7 @@ import (
// NewDatabase opens a database connection.
func NewDatabase(
dataSourceName string,
dbProperties common.DbProperties, // nolint:unparam
serverName gomatrixserverlib.ServerName,
serverKey ed25519.PublicKey,
serverKeyID gomatrixserverlib.KeyID,

4
common/keydb/postgres/keydb.go
View file

@ -21,6 +21,7 @@ import (
"golang.org/x/crypto/ed25519"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
)
@ -37,11 +38,12 @@ type Database struct {
// Returns an error if there was a problem talking to the database.
func NewDatabase(
dataSourceName string,
dbProperties common.DbProperties,
serverName gomatrixserverlib.ServerName,
serverKey ed25519.PublicKey,
serverKeyID gomatrixserverlib.KeyID,
) (*Database, error) {
db, err := sqlutil.Open("postgres", dataSourceName)
db, err := sqlutil.Open("postgres", dataSourceName, dbProperties)
if err != nil {
return nil, err
}

2
common/keydb/sqlite3/keydb.go
View file

@ -44,7 +44,7 @@ func NewDatabase(
serverKey ed25519.PublicKey,
serverKeyID gomatrixserverlib.KeyID,
) (*Database, error) {
db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName)
db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil)
if err != nil {
return nil, err
}

8
common/sql.go
View file

@ -18,6 +18,7 @@ import (
"database/sql"
"fmt"
"runtime"
"time"
)
// A Transaction is something that can be committed or rolledback.
@ -99,3 +100,10 @@ func SQLiteDriverName() string {
}
return "sqlite3"
}
// DbProperties functions return properties used by database/sql/DB
type DbProperties interface {
MaxIdleConns() int
MaxOpenConns() int
ConnMaxLifetime() time.Duration
}

3
dendrite-config.yaml
View file

@ -116,6 +116,9 @@ database:
federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable"
appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable"
public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable"
max_open_conns: 100
max_idle_conns: 2
conn_max_lifetime: -1
# If using naffka you need to specify a naffka database
# naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable"

2
federationsender/federationsender.go
View file

@ -37,7 +37,7 @@ func SetupFederationSenderComponent(
rsAPI roomserverAPI.RoomserverInternalAPI,
keyRing *gomatrixserverlib.KeyRing,
) api.FederationSenderInternalAPI {
federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender))
federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender), base.Cfg.DbProperties())
if err != nil {
logrus.WithError(err).Panic("failed to connect to federation sender db")
}

4
federationsender/storage/postgres/storage.go
View file

@ -33,10 +33,10 @@ type Database struct {
}
// NewDatabase opens a new database
func NewDatabase(dataSourceName string) (*Database, error) {
func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) {
var result Database
var err error
if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
return nil, err
}
if err = result.prepare(); err != nil {

2
federationsender/storage/sqlite3/storage.go
View file

@ -38,7 +38,7 @@ type Database struct {
func NewDatabase(dataSourceName string) (*Database, error) {
var result Database
var err error
if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
return nil, err
}
if err = result.prepare(); err != nil {

9
federationsender/storage/storage.go
View file

@ -19,22 +19,23 @@ package storage
import (
"net/url"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/federationsender/storage/postgres"
"github.com/matrix-org/dendrite/federationsender/storage/sqlite3"
)
// NewDatabase opens a new database
func NewDatabase(dataSourceName string) (Database, error) {
func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgres.NewDatabase(dataSourceName)
return postgres.NewDatabase(dataSourceName, dbProperties)
}
switch uri.Scheme {
case "file":
return sqlite3.NewDatabase(dataSourceName)
case "postgres":
return postgres.NewDatabase(dataSourceName)
return postgres.NewDatabase(dataSourceName, dbProperties)
default:
return postgres.NewDatabase(dataSourceName)
return postgres.NewDatabase(dataSourceName, dbProperties)
}
}

6
federationsender/storage/storage_wasm.go
View file

@ -18,11 +18,15 @@ import (
"fmt"
"net/url"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/federationsender/storage/sqlite3"
)
// NewDatabase opens a new database
func NewDatabase(dataSourceName string) (Database, error) {
func NewDatabase(
dataSourceName string,
dbProperties common.DbProperties, // nolint:unparam
) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return nil, fmt.Errorf("Cannot use postgres implementation")

21
internal/sqlutil/trace.go
View file

@ -21,9 +21,11 @@ import (
"fmt"
"io"
"os"
"regexp"
"strings"
"time"
"github.com/matrix-org/dendrite/common"
"github.com/ngrok/sqlmw"
"github.com/sirupsen/logrus"
)
@ -76,12 +78,27 @@ func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest [
// Open opens a database specified by its database driver name and a driver-specific data source name,
// usually consisting of at least a database name and connection information. Includes tracing driver
// if DENDRITE_TRACE_SQL=1
func Open(driverName, dsn string) (*sql.DB, error) {
func Open(driverName, dsn string, dbProperties common.DbProperties) (*sql.DB, error) {
if tracingEnabled {
// install the wrapped driver
driverName += "-trace"
}
return sql.Open(driverName, dsn)
db, err := sql.Open(driverName, dsn)
if err != nil {
return nil, err
}
if driverName != common.SQLiteDriverName() && dbProperties != nil {
logrus.WithFields(logrus.Fields{
"MaxOpenConns": dbProperties.MaxOpenConns(),
"MaxIdleConns": dbProperties.MaxIdleConns(),
"ConnMaxLifetime": dbProperties.ConnMaxLifetime(),
"dataSourceName": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"),
}).Debug("Setting DB connection limits")
db.SetMaxOpenConns(dbProperties.MaxOpenConns())
db.SetMaxIdleConns(dbProperties.MaxIdleConns())
db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime())
}
return db, nil
}
func init() {

2
mediaapi/mediaapi.go
View file

@ -29,7 +29,7 @@ func SetupMediaAPIComponent(
base *basecomponent.BaseDendrite,
deviceDB devices.Database,
) {
mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI))
mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI), base.Cfg.DbProperties())
if err != nil {
logrus.WithError(err).Panicf("failed to connect to media db")
}

5
mediaapi/storage/postgres/storage.go
View file

@ -21,6 +21,7 @@ import (
// Import the postgres database driver.
_ "github.com/lib/pq"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/mediaapi/types"
"github.com/matrix-org/gomatrixserverlib"
@ -33,10 +34,10 @@ type Database struct {
}
// Open opens a postgres database.
func Open(dataSourceName string) (*Database, error) {
func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) {
var d Database
var err error
if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
return nil, err
}
if err = d.statements.prepare(d.db); err != nil {

2
mediaapi/storage/sqlite3/storage.go
View file

@ -37,7 +37,7 @@ type Database struct {
func Open(dataSourceName string) (*Database, error) {
var d Database
var err error
if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
return nil, err
}
if err = d.statements.prepare(d.db); err != nil {

9
mediaapi/storage/storage.go
View file

@ -19,22 +19,23 @@ package storage
import (
"net/url"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/mediaapi/storage/postgres"
"github.com/matrix-org/dendrite/mediaapi/storage/sqlite3"
)
// Open opens a postgres database.
func Open(dataSourceName string) (Database, error) {
func Open(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgres.Open(dataSourceName)
return postgres.Open(dataSourceName, dbProperties)
}
switch uri.Scheme {
case "postgres":
return postgres.Open(dataSourceName)
return postgres.Open(dataSourceName, dbProperties)
case "file":
return sqlite3.Open(dataSourceName)
default:
return postgres.Open(dataSourceName)
return postgres.Open(dataSourceName, dbProperties)
}
}

6
mediaapi/storage/storage_wasm.go
View file

@ -18,11 +18,15 @@ import (
"fmt"
"net/url"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/mediaapi/storage/sqlite3"
)
// Open opens a postgres database.
func Open(dataSourceName string) (Database, error) {
func Open(
dataSourceName string,
dbProperties common.DbProperties, // nolint:unparam
) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return nil, fmt.Errorf("Cannot use postgres implementation")

4
publicroomsapi/storage/postgres/storage.go
View file

@ -36,10 +36,10 @@ type PublicRoomsServerDatabase struct {
type attributeValue interface{}
// NewPublicRoomsServerDatabase creates a new public rooms server database.
func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) {
func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties common.DbProperties) (*PublicRoomsServerDatabase, error) {
var db *sql.DB
var err error
if db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
return nil, err
}
storage := PublicRoomsServerDatabase{

2
publicroomsapi/storage/sqlite3/storage.go
View file

@ -41,7 +41,7 @@ type attributeValue interface{}
func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) {
var db *sql.DB
var err error
if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil {
return nil, err
}
storage := PublicRoomsServerDatabase{

9
publicroomsapi/storage/storage.go
View file

@ -19,6 +19,7 @@ package storage
import (
"net/url"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/publicroomsapi/storage/postgres"
"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
)
@ -27,17 +28,17 @@ const schemePostgres = "postgres"
const schemeFile = "file"
// NewPublicRoomsServerDatabase opens a database connection.
func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) {
func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgres.NewPublicRoomsServerDatabase(dataSourceName)
return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties)
}
switch uri.Scheme {
case schemePostgres:
return postgres.NewPublicRoomsServerDatabase(dataSourceName)
return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties)
case schemeFile:
return sqlite3.NewPublicRoomsServerDatabase(dataSourceName)
default:
return postgres.NewPublicRoomsServerDatabase(dataSourceName)
return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties)
}
}

2
roomserver/roomserver.go
View file

@ -35,7 +35,7 @@ func SetupRoomServerComponent(
keyRing gomatrixserverlib.JSONVerifier,
fedClient *gomatrixserverlib.FederationClient,
) api.RoomserverInternalAPI {
roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer))
roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer), base.Cfg.DbProperties())
if err != nil {
logrus.WithError(err).Panicf("failed to connect to room server db")
}

5
roomserver/storage/postgres/storage.go
View file

@ -20,6 +20,7 @@ import (
"database/sql"
"encoding/json"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/internal/sqlutil"
// Import the postgres database driver.
@ -36,10 +37,10 @@ type Database struct {
}
// Open a postgres database.
func Open(dataSourceName string) (*Database, error) {
func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) {
var d Database
var err error
if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
return nil, err
}
if err = d.statements.prepare(d.db); err != nil {

4
roomserver/storage/sqlite3/storage.go
View file

@ -37,7 +37,7 @@ type Database struct {
db *sql.DB
}
// Open a postgres database.
// Open a sqlite database.
func Open(dataSourceName string) (*Database, error) {
var d Database
uri, err := url.Parse(dataSourceName)
@ -52,7 +52,7 @@ func Open(dataSourceName string) (*Database, error) {
} else {
return nil, errors.New("no filename or path in connect string")
}
if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil {
if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs, nil); err != nil {
return nil, err
}
//d.db.Exec("PRAGMA journal_mode=WAL;")

11
roomserver/storage/storage.go
View file

@ -19,22 +19,23 @@ package storage
import (
"net/url"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/roomserver/storage/postgres"
"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
)
// NewPublicRoomsServerDatabase opens a database connection.
func Open(dataSourceName string) (Database, error) {
// Open opens a database connection.
func Open(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgres.Open(dataSourceName)
return postgres.Open(dataSourceName, dbProperties)
}
switch uri.Scheme {
case "postgres":
return postgres.Open(dataSourceName)
return postgres.Open(dataSourceName, dbProperties)
case "file":
return sqlite3.Open(dataSourceName)
default:
return postgres.Open(dataSourceName)
return postgres.Open(dataSourceName, dbProperties)
}
}

6
roomserver/storage/storage_wasm.go
View file

@ -18,11 +18,15 @@ import (
"fmt"
"net/url"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
)
// NewPublicRoomsServerDatabase opens a database connection.
func Open(dataSourceName string) (Database, error) {
func Open(
dataSourceName string,
dbProperties common.DbProperties, // nolint:unparam
) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return nil, fmt.Errorf("Cannot use postgres implementation")

4
syncapi/storage/postgres/syncserver.go
View file

@ -61,10 +61,10 @@ type SyncServerDatasource struct {
}
// NewSyncServerDatasource creates a new sync server database
func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) {
func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProperties) (*SyncServerDatasource, error) {
var d SyncServerDatasource
var err error
if d.db, err = sqlutil.Open("postgres", dbDataSourceName); err != nil {
if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil {
return nil, err
}
if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil {

2
syncapi/storage/sqlite3/syncserver.go
View file

@ -80,7 +80,7 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro
} else {
return nil, errors.New("no filename or path in connect string")
}
if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil {
if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs, nil); err != nil {
return nil, err
}
if err = d.prepare(); err != nil {

11
syncapi/storage/storage.go
View file

@ -19,22 +19,23 @@ package storage
import (
"net/url"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/syncapi/storage/postgres"
"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
)
// NewPublicRoomsServerDatabase opens a database connection.
func NewSyncServerDatasource(dataSourceName string) (Database, error) {
// NewSyncServerDatasource opens a database connection.
func NewSyncServerDatasource(dataSourceName string, dbProperties common.DbProperties) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgres.NewSyncServerDatasource(dataSourceName)
return postgres.NewSyncServerDatasource(dataSourceName, dbProperties)
}
switch uri.Scheme {
case "postgres":
return postgres.NewSyncServerDatasource(dataSourceName)
return postgres.NewSyncServerDatasource(dataSourceName, dbProperties)
case "file":
return sqlite3.NewSyncServerDatasource(dataSourceName)
default:
return postgres.NewSyncServerDatasource(dataSourceName)
return postgres.NewSyncServerDatasource(dataSourceName, dbProperties)
}
}

6
syncapi/storage/storage_wasm.go
View file

@ -18,11 +18,15 @@ import (
"fmt"
"net/url"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
)
// NewPublicRoomsServerDatabase opens a database connection.
func NewSyncServerDatasource(dataSourceName string) (Database, error) {
func NewSyncServerDatasource(
dataSourceName string,
dbProperties common.DbProperties, // nolint:unparam
) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return nil, fmt.Errorf("Cannot use postgres implementation")

2
syncapi/syncapi.go
View file

@ -42,7 +42,7 @@ func SetupSyncAPIComponent(
federation *gomatrixserverlib.FederationClient,
cfg *config.Dendrite,
) {
syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI))
syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI), base.Cfg.DbProperties())
if err != nil {
logrus.WithError(err).Panicf("failed to connect to sync db")
}