Configuration format v1 (#1230)

* Initial pass at refactoring config (not finished)

* Don't forget current state and EDU servers

* More shifting around

* Update server key API tests

* Fix roomserver test

* Fix more tests

* Further tweaks

* Fix current state server test (sort of)

* Maybe fix appservices

* Fix client API test

* Include database connection string in database options

* Fix sync API build

* Update config test

* Fix unit tests

* Fix federation sender build

* Fix gobind build

* Set Listen address for all services in HTTP monolith mode

* Validate config, reinstate appservice derived in directory, tweaks

* Tweak federation API test

* Set MaxOpenConnections/MaxIdleConnections to previous values

* Update generate-config

roomserver/internal/api.go

@@ -14,7 +14,7 @@ import (
 // RoomserverInternalAPI is an implementation of api.RoomserverInternalAPI
 type RoomserverInternalAPI struct {
 	DB                   storage.Database
-	Cfg                  *config.Dendrite
+	Cfg                  *config.RoomServer
 	Producer             sarama.SyncProducer
 	Cache                caching.RoomVersionCache
 	ServerName           gomatrixserverlib.ServerName

roomserver/internal/perform_join.go

@@ -189,12 +189,12 @@ func (r *RoomserverInternalAPI) performJoinRoomByID(
 	// but everyone has since left. I suspect it does the wrong thing.
 	buildRes := api.QueryLatestEventsAndStateResponse{}
 	event, err := eventutil.BuildEvent(
-		ctx,        // the request context
-		&eb,        // the template join event
-		r.Cfg,      // the server configuration
-		time.Now(), // the event timestamp to use
-		r,          // the roomserver API to use
-		&buildRes,  // the query response
+		ctx,          // the request context
+		&eb,          // the template join event
+		r.Cfg.Matrix, // the server configuration
+		time.Now(),   // the event timestamp to use
+		r,            // the roomserver API to use
+		&buildRes,    // the query response
 	)
 
 	switch err {

roomserver/internal/perform_leave.go

@@ -99,12 +99,12 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID(
 	// but everyone has since left. I suspect it does the wrong thing.
 	buildRes := api.QueryLatestEventsAndStateResponse{}
 	event, err := eventutil.BuildEvent(
-		ctx,        // the request context
-		&eb,        // the template leave event
-		r.Cfg,      // the server configuration
-		time.Now(), // the event timestamp to use
-		r,          // the roomserver API to use
-		&buildRes,  // the query response
+		ctx,          // the request context
+		&eb,          // the template leave event
+		r.Cfg.Matrix, // the server configuration
+		time.Now(),   // the event timestamp to use
+		r,            // the roomserver API to use
+		&buildRes,    // the query response
 	)
 	if err != nil {
 		return fmt.Errorf("eventutil.BuildEvent: %w", err)

roomserver/roomserver.go

@@ -39,18 +39,20 @@ func NewInternalAPI(
 	keyRing gomatrixserverlib.JSONVerifier,
 	fedClient *gomatrixserverlib.FederationClient,
 ) api.RoomserverInternalAPI {
-	roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer), base.Cfg.DbProperties())
+	cfg := &base.Cfg.RoomServer
+
+	roomserverDB, err := storage.Open(&cfg.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to room server db")
 	}
 
 	return &internal.RoomserverInternalAPI{
 		DB:                   roomserverDB,
-		Cfg:                  base.Cfg,
+		Cfg:                  cfg,
 		Producer:             base.KafkaProducer,
-		OutputRoomEventTopic: string(base.Cfg.Kafka.Topics.OutputRoomEvent),
+		OutputRoomEventTopic: string(cfg.Matrix.Kafka.Topics.OutputRoomEvent),
 		Cache:                base.Caches,
-		ServerName:           base.Cfg.Matrix.ServerName,
+		ServerName:           cfg.Matrix.ServerName,
 		FedClient:            fedClient,
 		KeyRing:              keyRing,
 	}

roomserver/roomserver_test.go

@@ -95,12 +95,12 @@ func mustLoadEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events []js
 
 func mustSendEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events []json.RawMessage) (api.RoomserverInternalAPI, *dummyProducer, []gomatrixserverlib.HeaderedEvent) {
 	cfg := &config.Dendrite{}
-	cfg.Database.RoomServer = roomserverDBFileURI
-	cfg.Kafka.Topics.OutputRoomEvent = "output_room_event"
-	cfg.Matrix.ServerName = testOrigin
-	cfg.Kafka.UseNaffka = true
+	cfg.Defaults()
+	cfg.Global.ServerName = testOrigin
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.RoomServer.Database.ConnectionString = config.DataSource(roomserverDBFileURI)
 	dp := &dummyProducer{
-		topic: string(cfg.Kafka.Topics.OutputRoomEvent),
+		topic: string(cfg.Global.Kafka.Topics.OutputRoomEvent),
 	}
 	cache, err := caching.NewInMemoryLRUCache(true)
 	if err != nil {

roomserver/storage/postgres/storage.go

@@ -18,6 +18,7 @@ package postgres
 import (
 	"database/sql"
 
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 
 	// Import the postgres database driver.
@@ -32,11 +33,11 @@ type Database struct {
 
 // Open a postgres database.
 // nolint: gocyclo
-func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) {
+func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
 	var d Database
 	var db *sql.DB
 	var err error
-	if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
+	if db, err = sqlutil.Open(dbProperties); err != nil {
 		return nil, err
 	}
 	eventStateKeys, err := NewPostgresEventStateKeysTable(db)

roomserver/storage/sqlite3/storage.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"database/sql"
 
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
@@ -44,13 +45,10 @@ type Database struct {
 
 // Open a sqlite database.
 // nolint: gocyclo
-func Open(dataSourceName string) (*Database, error) {
+func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
 	var d Database
-	cs, err := sqlutil.ParseFileURI(dataSourceName)
-	if err != nil {
-		return nil, err
-	}
-	if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
+	var err error
+	if d.db, err = sqlutil.Open(dbProperties); err != nil {
 		return nil, err
 	}
 	//d.db.Exec("PRAGMA journal_mode=WAL;")

roomserver/storage/storage.go

@@ -17,25 +17,21 @@
 package storage
 
 import (
-	"net/url"
+	"fmt"
 
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/storage/postgres"
 	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
 )
 
 // Open opens a database connection.
-func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return postgres.Open(dataSourceName, dbProperties)
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return postgres.Open(dataSourceName, dbProperties)
-	case "file":
-		return sqlite3.Open(dataSourceName)
+func Open(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.Open(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return postgres.Open(dbProperties)
 	default:
-		return postgres.Open(dataSourceName, dbProperties)
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }

roomserver/storage/storage_wasm.go

@@ -16,27 +16,19 @@ package storage
 
 import (
 	"fmt"
-	"net/url"
 
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3"
 )
 
 // NewPublicRoomsServerDatabase opens a database connection.
-func Open(
-	dataSourceName string,
-	dbProperties sqlutil.DbProperties, // nolint:unparam
-) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	case "file":
-		return sqlite3.Open(dataSourceName)
+func Open(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.Open(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
-		return nil, fmt.Errorf("Cannot use postgres implementation")
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }