hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2025-07-31 13:22:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Limit database connections (#980, #564) (#998)

Browse source 
* Limit database connections (#564)

- Add new options to the config file database:
      max_open_conns: 100
      max_idle_conns: 2
      conn_max_lifetime: -1
- Implement connection parameter setup on the *DB (database/sql) in internal/sqlutil/trace.go:Open()
- Propagate the values in the form of DbProperties interface via all the
  Open() and NewDatabase() functions

Signed-off-by: Tomas Jirka <tomas.jirka@email.cz>

* Fix wasm builds

* Remove file accidentally added from working tree

Co-authored-by: Tomas Jirka <tomas.jirka@email.cz>
This commit is contained in:
Neil Alexander 2020-05-01 13:34:53 +01:00 • committed by GitHub
parent 908108c23e
commit f7cfa75886
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
51 changed files with 213 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

9
common/basecomponent/base.go
View file

@ -150,7 +150,7 @@ func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.Fede
// CreateDeviceDB creates a new instance of the device database. Should only be
// called once per component.
func (b *BaseDendrite) CreateDeviceDB() devices.Database {
db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.Matrix.ServerName)
db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to devices db")
}
@ -161,7 +161,7 @@ func (b *BaseDendrite) CreateDeviceDB() devices.Database {
// CreateAccountsDB creates a new instance of the accounts database. Should only
// be called once per component.
func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.Matrix.ServerName)
db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to accounts db")
}
@ -174,6 +174,7 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
func (b *BaseDendrite) CreateKeyDB() keydb.Database {
db, err := keydb.NewDatabase(
string(b.Cfg.Database.ServerKey),
b.Cfg.DbProperties(),
b.Cfg.Matrix.ServerName,
b.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
b.Cfg.Matrix.KeyID,
@ -244,7 +245,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
uri, err := url.Parse(string(cfg.Database.Naffka))
if err != nil || uri.Scheme == "file" {
db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka))
db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka), nil)
if err != nil {
logrus.WithError(err).Panic("Failed to open naffka database")
}
@ -254,7 +255,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
logrus.WithError(err).Panic("Failed to setup naffka database")
}
} else {
db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka))
db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka), nil)
if err != nil {
logrus.WithError(err).Panic("Failed to open naffka database")
}

42
common/config/config.go
View file

@ -188,6 +188,12 @@ type Dendrite struct {
PublicRoomsAPI DataSource `yaml:"public_rooms_api"`
// The Naffka database is used internally by the naffka library, if used.
Naffka DataSource `yaml:"naffka,omitempty"`
// Maximum open connections to the DB (0 = use default, negative means unlimited)
MaxOpenConns int `yaml:"max_open_conns"`
// Maximum idle connections to the DB (0 = use default, negative means unlimited)
MaxIdleConns int `yaml:"max_idle_conns"`
// maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited)
ConnMaxLifetimeSec int `yaml:"conn_max_lifetime"`
} `yaml:"database"`
// TURN Server Config
@ -484,6 +490,15 @@ func (config *Dendrite) SetDefaults() {
defaultMaxFileSizeBytes := FileSizeBytes(10485760)
config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes
}
if config.Database.MaxIdleConns == 0 {
config.Database.MaxIdleConns = 2
}
if config.Database.MaxOpenConns == 0 {
config.Database.MaxOpenConns = 100
}
}
// Error returns a string detailing how many errors were contained within a
@ -746,6 +761,33 @@ func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err
)
}
// MaxIdleConns returns maximum idle connections to the DB
func (config Dendrite) MaxIdleConns() int {
return config.Database.MaxIdleConns
}
// MaxOpenConns returns maximum open connections to the DB
func (config Dendrite) MaxOpenConns() int {
return config.Database.MaxOpenConns
}
// ConnMaxLifetime returns maximum amount of time a connection may be reused
func (config Dendrite) ConnMaxLifetime() time.Duration {
return time.Duration(config.Database.ConnMaxLifetimeSec) * time.Second
}
// DbProperties functions return properties used by database/sql/DB
type DbProperties interface {
MaxIdleConns() int
MaxOpenConns() int
ConnMaxLifetime() time.Duration
}
// DbProperties returns cfg as a DbProperties interface
func (config Dendrite) DbProperties() DbProperties {
return config
}
// logrusLogger is a small wrapper that implements jaeger.Logger using logrus.
type logrusLogger struct {
l *logrus.Logger

8
common/keydb/keydb.go
View file

@ -21,6 +21,7 @@ import (
"golang.org/x/crypto/ed25519"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/keydb/postgres"
"github.com/matrix-org/dendrite/common/keydb/sqlite3"
"github.com/matrix-org/gomatrixserverlib"
@ -29,20 +30,21 @@ import (
// NewDatabase opens a database connection.
func NewDatabase(
dataSourceName string,
dbProperties common.DbProperties,
serverName gomatrixserverlib.ServerName,
serverKey ed25519.PublicKey,
serverKeyID gomatrixserverlib.KeyID,
) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
}
switch uri.Scheme {
case "postgres":
return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
case "file":
return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
default:
return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID)
}
}

2
common/keydb/keydb_wasm.go
View file

@ -20,6 +20,7 @@ import (
"golang.org/x/crypto/ed25519"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/keydb/sqlite3"
"github.com/matrix-org/gomatrixserverlib"
)
@ -27,6 +28,7 @@ import (
// NewDatabase opens a database connection.
func NewDatabase(
dataSourceName string,
dbProperties common.DbProperties, // nolint:unparam
serverName gomatrixserverlib.ServerName,
serverKey ed25519.PublicKey,
serverKeyID gomatrixserverlib.KeyID,

4
common/keydb/postgres/keydb.go
View file

@ -21,6 +21,7 @@ import (
"golang.org/x/crypto/ed25519"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
)
@ -37,11 +38,12 @@ type Database struct {
// Returns an error if there was a problem talking to the database.
func NewDatabase(
dataSourceName string,
dbProperties common.DbProperties,
serverName gomatrixserverlib.ServerName,
serverKey ed25519.PublicKey,
serverKeyID gomatrixserverlib.KeyID,
) (*Database, error) {
db, err := sqlutil.Open("postgres", dataSourceName)
db, err := sqlutil.Open("postgres", dataSourceName, dbProperties)
if err != nil {
return nil, err
}

2
common/keydb/sqlite3/keydb.go
View file

@ -44,7 +44,7 @@ func NewDatabase(
serverKey ed25519.PublicKey,
serverKeyID gomatrixserverlib.KeyID,
) (*Database, error) {
db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName)
db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil)
if err != nil {
return nil, err
}

8
common/sql.go
View file

@ -18,6 +18,7 @@ import (
"database/sql"
"fmt"
"runtime"
"time"
)
// A Transaction is something that can be committed or rolledback.
@ -99,3 +100,10 @@ func SQLiteDriverName() string {
}
return "sqlite3"
}
// DbProperties functions return properties used by database/sql/DB
type DbProperties interface {
MaxIdleConns() int
MaxOpenConns() int
ConnMaxLifetime() time.Duration
}