hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2025-07-29 12:42:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Configuration format v1 (#1230)

Browse source 
* Initial pass at refactoring config (not finished)

* Don't forget current state and EDU servers

* More shifting around

* Update server key API tests

* Fix roomserver test

* Fix more tests

* Further tweaks

* Fix current state server test (sort of)

* Maybe fix appservices

* Fix client API test

* Include database connection string in database options

* Fix sync API build

* Update config test

* Fix unit tests

* Fix federation sender build

* Fix gobind build

* Set Listen address for all services in HTTP monolith mode

* Validate config, reinstate appservice derived in directory, tweaks

* Tweak federation API test

* Set MaxOpenConnections/MaxIdleConnections to previous values

* Update generate-config
This commit is contained in:
Neil Alexander 2020-08-10 14:18:04 +01:00 • committed by GitHub
parent fdabba1851
commit 4b09f445c9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
155 changed files with 1716 additions and 1503 deletions
Download patch file Download diff file Expand all files Collapse all files

4
mediaapi/mediaapi.go
View file

@ -26,11 +26,11 @@ import (
// AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component.
func AddPublicRoutes(
router *mux.Router, cfg *config.Dendrite,
router *mux.Router, cfg *config.MediaAPI,
userAPI userapi.UserInternalAPI,
client *gomatrixserverlib.Client,
) {
mediaDB, err := storage.Open(string(cfg.Database.MediaAPI), cfg.DbProperties())
mediaDB, err := storage.Open(&cfg.Database)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to media db")
}

18
mediaapi/routing/download.go
View file

@ -73,7 +73,7 @@ func Download(
req *http.Request,
origin gomatrixserverlib.ServerName,
mediaID types.MediaID,
cfg *config.Dendrite,
cfg *config.MediaAPI,
db storage.Database,
client *gomatrixserverlib.Client,
activeRemoteRequests *types.ActiveRemoteRequests,
@ -203,7 +203,7 @@ func (r *downloadRequest) Validate() *util.JSONResponse {
func (r *downloadRequest) doDownload(
ctx context.Context,
w http.ResponseWriter,
cfg *config.Dendrite,
cfg *config.MediaAPI,
db storage.Database,
client *gomatrixserverlib.Client,
activeRemoteRequests *types.ActiveRemoteRequests,
@ -233,9 +233,9 @@ func (r *downloadRequest) doDownload(
r.MediaMetadata = mediaMetadata
}
return r.respondFromLocalFile(
ctx, w, cfg.Media.AbsBasePath, activeThumbnailGeneration,
cfg.Media.MaxThumbnailGenerators, db,
cfg.Media.DynamicThumbnails, cfg.Media.ThumbnailSizes,
ctx, w, cfg.AbsBasePath, activeThumbnailGeneration,
cfg.MaxThumbnailGenerators, db,
cfg.DynamicThumbnails, cfg.ThumbnailSizes,
)
}
@ -514,7 +514,7 @@ func (r *downloadRequest) generateThumbnail(
func (r *downloadRequest) getRemoteFile(
ctx context.Context,
client *gomatrixserverlib.Client,
cfg *config.Dendrite,
cfg *config.MediaAPI,
db storage.Database,
activeRemoteRequests *types.ActiveRemoteRequests,
activeThumbnailGeneration *types.ActiveThumbnailGeneration,
@ -550,9 +550,9 @@ func (r *downloadRequest) getRemoteFile(
// If we do not have a record, we need to fetch the remote file first and then respond from the local file
err := r.fetchRemoteFileAndStoreMetadata(
ctx, client,
cfg.Media.AbsBasePath, *cfg.Media.MaxFileSizeBytes, db,
cfg.Media.ThumbnailSizes, activeThumbnailGeneration,
cfg.Media.MaxThumbnailGenerators,
cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db,
cfg.ThumbnailSizes, activeThumbnailGeneration,
cfg.MaxThumbnailGenerators,
)
if err != nil {
return errors.Wrap(err, "error querying the database.")

4
mediaapi/routing/routing.go
View file

@ -42,7 +42,7 @@ const pathPrefixV1 = "/media/v1" // TODO: remove when synapse is fixed
// nolint: gocyclo
func Setup(
publicAPIMux *mux.Router,
cfg *config.Dendrite,
cfg *config.MediaAPI,
db storage.Database,
userAPI userapi.UserInternalAPI,
client *gomatrixserverlib.Client,
@ -81,7 +81,7 @@ func Setup(
func makeDownloadAPI(
name string,
cfg *config.Dendrite,
cfg *config.MediaAPI,
db storage.Database,
client *gomatrixserverlib.Client,
activeRemoteRequests *types.ActiveRemoteRequests,

16
mediaapi/routing/upload.go
View file

@ -54,7 +54,7 @@ type uploadResponse struct {
// This implementation supports a configurable maximum file size limit in bytes. If a user tries to upload more than this, they will receive an error that their upload is too large.
// Uploaded files are processed piece-wise to avoid DoS attacks which would starve the server of memory.
// TODO: We should time out requests if they have not received any data within a configured timeout period.
func Upload(req *http.Request, cfg *config.Dendrite, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse {
func Upload(req *http.Request, cfg *config.MediaAPI, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse {
r, resErr := parseAndValidateRequest(req, cfg)
if resErr != nil {
return *resErr
@ -75,7 +75,7 @@ func Upload(req *http.Request, cfg *config.Dendrite, db storage.Database, active
// parseAndValidateRequest parses the incoming upload request to validate and extract
// all the metadata about the media being uploaded.
// Returns either an uploadRequest or an error formatted as a util.JSONResponse
func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRequest, *util.JSONResponse) {
func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRequest, *util.JSONResponse) {
r := &uploadRequest{
MediaMetadata: &types.MediaMetadata{
Origin: cfg.Matrix.ServerName,
@ -86,7 +86,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRe
Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.Matrix.ServerName),
}
if resErr := r.Validate(*cfg.Media.MaxFileSizeBytes); resErr != nil {
if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil {
return nil, resErr
}
@ -96,7 +96,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRe
func (r *uploadRequest) doUpload(
ctx context.Context,
reqReader io.Reader,
cfg *config.Dendrite,
cfg *config.MediaAPI,
db storage.Database,
activeThumbnailGeneration *types.ActiveThumbnailGeneration,
) *util.JSONResponse {
@ -110,10 +110,10 @@ func (r *uploadRequest) doUpload(
// method of deduplicating files to save storage, as well as a way to conduct
// integrity checks on the file data in the repository.
// Data is truncated to maxFileSizeBytes. Content-Length was reported as 0 < Content-Length <= maxFileSizeBytes so this is OK.
hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.Media.MaxFileSizeBytes, cfg.Media.AbsBasePath)
hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.MaxFileSizeBytes, cfg.AbsBasePath)
if err != nil {
r.Logger.WithError(err).WithFields(log.Fields{
"MaxFileSizeBytes": *cfg.Media.MaxFileSizeBytes,
"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
}).Warn("Error while transferring file")
fileutils.RemoveDir(tmpDir, r.Logger)
return &util.JSONResponse{
@ -159,8 +159,8 @@ func (r *uploadRequest) doUpload(
}
return r.storeFileAndMetadata(
ctx, tmpDir, cfg.Media.AbsBasePath, db, cfg.Media.ThumbnailSizes,
activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators,
ctx, tmpDir, cfg.AbsBasePath, db, cfg.ThumbnailSizes,
activeThumbnailGeneration, cfg.MaxThumbnailGenerators,
)
}

5
mediaapi/storage/postgres/storage.go
View file

@ -21,6 +21,7 @@ import (
// Import the postgres database driver.
_ "github.com/lib/pq"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/mediaapi/types"
"github.com/matrix-org/gomatrixserverlib"
@ -33,10 +34,10 @@ type Database struct {
}
// Open opens a postgres database.
func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) {
func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
var d Database
var err error
if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
if d.db, err = sqlutil.Open(dbProperties); err != nil {
return nil, err
}
if err = d.statements.prepare(d.db); err != nil {

9
mediaapi/storage/sqlite3/storage.go
View file

@ -20,6 +20,7 @@ import (
"database/sql"
// Import the postgres database driver.
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/mediaapi/types"
"github.com/matrix-org/gomatrixserverlib"
@ -33,14 +34,10 @@ type Database struct {
}
// Open opens a postgres database.
func Open(dataSourceName string) (*Database, error) {
func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
var d Database
var err error
cs, err := sqlutil.ParseFileURI(dataSourceName)
if err != nil {
return nil, err
}
if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
if d.db, err = sqlutil.Open(dbProperties); err != nil {
return nil, err
}
if err = d.statements.prepare(d.db); err != nil {

22
mediaapi/storage/storage.go
View file

@ -17,25 +17,21 @@
package storage
import (
"net/url"
"fmt"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/mediaapi/storage/postgres"
"github.com/matrix-org/dendrite/mediaapi/storage/sqlite3"
)
// Open opens a postgres database.
func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgres.Open(dataSourceName, dbProperties)
}
switch uri.Scheme {
case "postgres":
return postgres.Open(dataSourceName, dbProperties)
case "file":
return sqlite3.Open(dataSourceName)
func Open(dbProperties *config.DatabaseOptions) (Database, error) {
switch {
case dbProperties.ConnectionString.IsSQLite():
return sqlite3.Open(dbProperties)
case dbProperties.ConnectionString.IsPostgres():
return postgres.Open(dbProperties)
default:
return postgres.Open(dataSourceName, dbProperties)
return nil, fmt.Errorf("unexpected database type")
}
}

24
mediaapi/storage/storage_wasm.go
View file

@ -16,27 +16,19 @@ package storage
import (
"fmt"
"net/url"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/mediaapi/storage/sqlite3"
)
// Open opens a postgres database.
func Open(
dataSourceName string,
dbProperties sqlutil.DbProperties, // nolint:unparam
) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return nil, fmt.Errorf("Cannot use postgres implementation")
}
switch uri.Scheme {
case "postgres":
return nil, fmt.Errorf("Cannot use postgres implementation")
case "file":
return sqlite3.Open(dataSourceName)
func Open(dbProperties *config.DatabaseOptions) (Database, error) {
switch {
case dbProperties.ConnectionString.IsSQLite():
return sqlite3.Open(dbProperties)
case dbProperties.ConnectionString.IsPostgres():
return nil, fmt.Errorf("can't use Postgres implementation")
default:
return nil, fmt.Errorf("Cannot use postgres implementation")
return nil, fmt.Errorf("unexpected database type")
}
}