Configuration format v1 (#1230)

* Initial pass at refactoring config (not finished)

* Don't forget current state and EDU servers

* More shifting around

* Update server key API tests

* Fix roomserver test

* Fix more tests

* Further tweaks

* Fix current state server test (sort of)

* Maybe fix appservices

* Fix client API test

* Include database connection string in database options

* Fix sync API build

* Update config test

* Fix unit tests

* Fix federation sender build

* Fix gobind build

* Set Listen address for all services in HTTP monolith mode

* Validate config, reinstate appservice derived in directory, tweaks

* Tweak federation API test

* Set MaxOpenConnections/MaxIdleConnections to previous values

* Update generate-config

keyserver/keyserver.go

@@ -36,17 +36,14 @@ func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) {
 // NewInternalAPI returns a concerete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
-	cfg *config.Dendrite, fedClient *gomatrixserverlib.FederationClient, producer sarama.SyncProducer,
+	cfg *config.KeyServer, fedClient *gomatrixserverlib.FederationClient, producer sarama.SyncProducer,
 ) api.KeyInternalAPI {
-	db, err := storage.NewDatabase(
-		string(cfg.Database.E2EKey),
-		cfg.DbProperties(),
-	)
+	db, err := storage.NewDatabase(&cfg.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to key server database")
 	}
 	keyChangeProducer := &producers.KeyChange{
-		Topic:    string(cfg.Kafka.Topics.OutputKeyChangeEvent),
+		Topic:    string(cfg.Matrix.Kafka.Topics.OutputKeyChangeEvent),
 		Producer: producer,
 		DB:       db,
 	}

keyserver/storage/postgres/storage.go

@@ -15,14 +15,15 @@
 package postgres
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/keyserver/storage/shared"
 )
 
 // NewDatabase creates a new sync server database
-func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*shared.Database, error) {
+func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) {
 	var err error
-	db, err := sqlutil.Open("postgres", dbDataSourceName, dbProperties)
+	db, err := sqlutil.Open(dbProperties)
 	if err != nil {
 		return nil, err
 	}

keyserver/storage/sqlite3/storage.go

@@ -15,17 +15,13 @@
 package sqlite3
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/keyserver/storage/shared"
 )
 
-func NewDatabase(dataSourceName string) (*shared.Database, error) {
-	var err error
-	cs, err := sqlutil.ParseFileURI(dataSourceName)
-	if err != nil {
-		return nil, err
-	}
-	db, err := sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil)
+func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) {
+	db, err := sqlutil.Open(dbProperties)
 	if err != nil {
 		return nil, err
 	}

keyserver/storage/storage.go

@@ -17,26 +17,22 @@
 package storage
 
 import (
-	"net/url"
+	"fmt"
 
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/keyserver/storage/postgres"
 	"github.com/matrix-org/dendrite/keyserver/storage/sqlite3"
 )
 
 // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
 // and sets postgres connection parameters
-func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return postgres.NewDatabase(dataSourceName, dbProperties)
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return postgres.NewDatabase(dataSourceName, dbProperties)
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName)
+func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return postgres.NewDatabase(dbProperties)
 	default:
-		return postgres.NewDatabase(dataSourceName, dbProperties)
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }

keyserver/storage/storage_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/Shopify/sarama"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/keyserver/api"
 )
 
@@ -21,7 +22,9 @@ func MustCreateDatabase(t *testing.T) (Database, func()) {
 		log.Fatal(err)
 	}
 	t.Logf("Database %s", tmpfile.Name())
-	db, err := NewDatabase(fmt.Sprintf("file://%s", tmpfile.Name()), nil)
+	db, err := NewDatabase(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(fmt.Sprintf("file://%s", tmpfile.Name())),
+	})
 	if err != nil {
 		t.Fatalf("Failed to NewDatabase: %s", err)
 	}

keyserver/storage/storage_wasm.go

@@ -16,26 +16,18 @@ package storage
 
 import (
 	"fmt"
-	"net/url"
 
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/keyserver/storage/sqlite3"
 )
 
-func NewDatabase(
-	dataSourceName string,
-	dbProperties sqlutil.DbProperties, // nolint:unparam
-) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName)
+func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
-		return nil, fmt.Errorf("Cannot use postgres implementation")
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }