Implement storage interfaces (#841)

* Implement interfaces for federationsender storage

* Implement interfaces for mediaapi storage

* Implement interfaces for publicroomsapi storage

* Implement interfaces for roomserver storage

* Implement interfaces for syncapi storage

* Implement interfaces for keydb storage

* common.PartitionStorer in publicroomsapi interface

* Update copyright notices

mediaapi/routing/download.go

@@ -67,7 +67,7 @@ func Download(
 	origin gomatrixserverlib.ServerName,
 	mediaID types.MediaID,
 	cfg *config.Dendrite,
-	db *storage.Database,
+	db storage.Database,
 	client *gomatrixserverlib.Client,
 	activeRemoteRequests *types.ActiveRemoteRequests,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
@@ -192,7 +192,7 @@ func (r *downloadRequest) doDownload(
 	ctx context.Context,
 	w http.ResponseWriter,
 	cfg *config.Dendrite,
-	db *storage.Database,
+	db storage.Database,
 	client *gomatrixserverlib.Client,
 	activeRemoteRequests *types.ActiveRemoteRequests,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
@@ -235,7 +235,7 @@ func (r *downloadRequest) respondFromLocalFile(
 	absBasePath config.Path,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 	maxThumbnailGenerators int,
-	db *storage.Database,
+	db storage.Database,
 	dynamicThumbnails bool,
 	thumbnailSizes []config.ThumbnailSize,
 ) (*types.MediaMetadata, error) {
@@ -325,7 +325,7 @@ func (r *downloadRequest) getThumbnailFile(
 	filePath types.Path,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 	maxThumbnailGenerators int,
-	db *storage.Database,
+	db storage.Database,
 	dynamicThumbnails bool,
 	thumbnailSizes []config.ThumbnailSize,
 ) (*os.File, *types.ThumbnailMetadata, error) {
@@ -407,7 +407,7 @@ func (r *downloadRequest) generateThumbnail(
 	thumbnailSize types.ThumbnailSize,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 	maxThumbnailGenerators int,
-	db *storage.Database,
+	db storage.Database,
 ) (*types.ThumbnailMetadata, error) {
 	r.Logger.WithFields(log.Fields{
 		"Width":        thumbnailSize.Width,
@@ -443,7 +443,7 @@ func (r *downloadRequest) getRemoteFile(
 	ctx context.Context,
 	client *gomatrixserverlib.Client,
 	cfg *config.Dendrite,
-	db *storage.Database,
+	db storage.Database,
 	activeRemoteRequests *types.ActiveRemoteRequests,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 ) (errorResponse error) {
@@ -545,7 +545,7 @@ func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(
 	client *gomatrixserverlib.Client,
 	absBasePath config.Path,
 	maxFileSizeBytes config.FileSizeBytes,
-	db *storage.Database,
+	db storage.Database,
 	thumbnailSizes []config.ThumbnailSize,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 	maxThumbnailGenerators int,

mediaapi/routing/routing.go

@@ -43,7 +43,7 @@ const pathPrefixR0 = "/_matrix/media/r0"
 func Setup(
 	apiMux *mux.Router,
 	cfg *config.Dendrite,
-	db *storage.Database,
+	db storage.Database,
 	deviceDB *devices.Database,
 	client *gomatrixserverlib.Client,
 ) {
@@ -80,7 +80,7 @@ func Setup(
 func makeDownloadAPI(
 	name string,
 	cfg *config.Dendrite,
-	db *storage.Database,
+	db storage.Database,
 	client *gomatrixserverlib.Client,
 	activeRemoteRequests *types.ActiveRemoteRequests,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,

mediaapi/routing/upload.go

@@ -53,7 +53,7 @@ type uploadResponse struct {
 // This implementation supports a configurable maximum file size limit in bytes. If a user tries to upload more than this, they will receive an error that their upload is too large.
 // Uploaded files are processed piece-wise to avoid DoS attacks which would starve the server of memory.
 // TODO: We should time out requests if they have not received any data within a configured timeout period.
-func Upload(req *http.Request, cfg *config.Dendrite, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse {
+func Upload(req *http.Request, cfg *config.Dendrite, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse {
 	r, resErr := parseAndValidateRequest(req, cfg)
 	if resErr != nil {
 		return *resErr
@@ -96,7 +96,7 @@ func (r *uploadRequest) doUpload(
 	ctx context.Context,
 	reqReader io.Reader,
 	cfg *config.Dendrite,
-	db *storage.Database,
+	db storage.Database,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 ) *util.JSONResponse {
 	r.Logger.WithFields(log.Fields{
@@ -214,7 +214,7 @@ func (r *uploadRequest) storeFileAndMetadata(
 	ctx context.Context,
 	tmpDir types.Path,
 	absBasePath config.Path,
-	db *storage.Database,
+	db storage.Database,
 	thumbnailSizes []config.ThumbnailSize,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 	maxThumbnailGenerators int,

mediaapi/storage/postgres/media_repository_table.go

@@ -1,4 +1,5 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package storage
+package postgres
 
 import (
 	"context"

mediaapi/storage/postgres/prepare.go

@@ -1,4 +1,5 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,7 +15,7 @@
 
 // FIXME: This should be made common!
 
-package storage
+package postgres
 
 import (
 	"database/sql"

mediaapi/storage/postgres/sql.go

@@ -1,4 +1,5 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package storage
+package postgres
 
 import (
 	"database/sql"

mediaapi/storage/postgres/storage.go

@@ -0,0 +1,106 @@
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	// Import the postgres database driver.
+	_ "github.com/lib/pq"
+	"github.com/matrix-org/dendrite/mediaapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// Database is used to store metadata about a repository of media files.
+type Database struct {
+	statements statements
+	db         *sql.DB
+}
+
+// Open opens a postgres database.
+func Open(dataSourceName string) (*Database, error) {
+	var d Database
+	var err error
+	if d.db, err = sql.Open("postgres", dataSourceName); err != nil {
+		return nil, err
+	}
+	if err = d.statements.prepare(d.db); err != nil {
+		return nil, err
+	}
+	return &d, nil
+}
+
+// StoreMediaMetadata inserts the metadata about the uploaded media into the database.
+// Returns an error if the combination of MediaID and Origin are not unique in the table.
+func (d *Database) StoreMediaMetadata(
+	ctx context.Context, mediaMetadata *types.MediaMetadata,
+) error {
+	return d.statements.media.insertMedia(ctx, mediaMetadata)
+}
+
+// GetMediaMetadata returns metadata about media stored on this server.
+// The media could have been uploaded to this server or fetched from another server and cached here.
+// Returns nil metadata if there is no metadata associated with this media.
+func (d *Database) GetMediaMetadata(
+	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
+) (*types.MediaMetadata, error) {
+	mediaMetadata, err := d.statements.media.selectMedia(ctx, mediaID, mediaOrigin)
+	if err != nil && err == sql.ErrNoRows {
+		return nil, nil
+	}
+	return mediaMetadata, err
+}
+
+// StoreThumbnail inserts the metadata about the thumbnail into the database.
+// Returns an error if the combination of MediaID and Origin are not unique in the table.
+func (d *Database) StoreThumbnail(
+	ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata,
+) error {
+	return d.statements.thumbnail.insertThumbnail(ctx, thumbnailMetadata)
+}
+
+// GetThumbnail returns metadata about a specific thumbnail.
+// The media could have been uploaded to this server or fetched from another server and cached here.
+// Returns nil metadata if there is no metadata associated with this thumbnail.
+func (d *Database) GetThumbnail(
+	ctx context.Context,
+	mediaID types.MediaID,
+	mediaOrigin gomatrixserverlib.ServerName,
+	width, height int,
+	resizeMethod string,
+) (*types.ThumbnailMetadata, error) {
+	thumbnailMetadata, err := d.statements.thumbnail.selectThumbnail(
+		ctx, mediaID, mediaOrigin, width, height, resizeMethod,
+	)
+	if err != nil && err == sql.ErrNoRows {
+		return nil, nil
+	}
+	return thumbnailMetadata, err
+}
+
+// GetThumbnails returns metadata about all thumbnails for a specific media stored on this server.
+// The media could have been uploaded to this server or fetched from another server and cached here.
+// Returns nil metadata if there are no thumbnails associated with this media.
+func (d *Database) GetThumbnails(
+	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
+) ([]*types.ThumbnailMetadata, error) {
+	thumbnails, err := d.statements.thumbnail.selectThumbnails(ctx, mediaID, mediaOrigin)
+	if err != nil && err == sql.ErrNoRows {
+		return nil, nil
+	}
+	return thumbnails, err
+}

mediaapi/storage/postgres/thumbnail_table.go

@@ -1,4 +1,5 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package storage
+package postgres
 
 import (
 	"context"

mediaapi/storage/storage.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,90 +16,32 @@ package storage
 
 import (
 	"context"
-	"database/sql"
+	"errors"
+	"net/url"
 
-	// Import the postgres database driver.
-	_ "github.com/lib/pq"
+	"github.com/matrix-org/dendrite/mediaapi/storage/postgres"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-// Database is used to store metadata about a repository of media files.
-type Database struct {
-	statements statements
-	db         *sql.DB
+type Database interface {
+	StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error
+	GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error)
+	StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error
+	GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error)
+	GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error)
 }
 
 // Open opens a postgres database.
-func Open(dataSourceName string) (*Database, error) {
-	var d Database
-	var err error
-	if d.db, err = sql.Open("postgres", dataSourceName); err != nil {
+func Open(dataSourceName string) (Database, error) {
+	uri, err := url.Parse(dataSourceName)
+	if err != nil {
 		return nil, err
 	}
-	if err = d.statements.prepare(d.db); err != nil {
-		return nil, err
+	switch uri.Scheme {
+	case "postgres":
+		return postgres.Open(dataSourceName)
+	default:
+		return nil, errors.New("unknown schema")
 	}
-	return &d, nil
-}
-
-// StoreMediaMetadata inserts the metadata about the uploaded media into the database.
-// Returns an error if the combination of MediaID and Origin are not unique in the table.
-func (d *Database) StoreMediaMetadata(
-	ctx context.Context, mediaMetadata *types.MediaMetadata,
-) error {
-	return d.statements.media.insertMedia(ctx, mediaMetadata)
-}
-
-// GetMediaMetadata returns metadata about media stored on this server.
-// The media could have been uploaded to this server or fetched from another server and cached here.
-// Returns nil metadata if there is no metadata associated with this media.
-func (d *Database) GetMediaMetadata(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
-) (*types.MediaMetadata, error) {
-	mediaMetadata, err := d.statements.media.selectMedia(ctx, mediaID, mediaOrigin)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
-	}
-	return mediaMetadata, err
-}
-
-// StoreThumbnail inserts the metadata about the thumbnail into the database.
-// Returns an error if the combination of MediaID and Origin are not unique in the table.
-func (d *Database) StoreThumbnail(
-	ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata,
-) error {
-	return d.statements.thumbnail.insertThumbnail(ctx, thumbnailMetadata)
-}
-
-// GetThumbnail returns metadata about a specific thumbnail.
-// The media could have been uploaded to this server or fetched from another server and cached here.
-// Returns nil metadata if there is no metadata associated with this thumbnail.
-func (d *Database) GetThumbnail(
-	ctx context.Context,
-	mediaID types.MediaID,
-	mediaOrigin gomatrixserverlib.ServerName,
-	width, height int,
-	resizeMethod string,
-) (*types.ThumbnailMetadata, error) {
-	thumbnailMetadata, err := d.statements.thumbnail.selectThumbnail(
-		ctx, mediaID, mediaOrigin, width, height, resizeMethod,
-	)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
-	}
-	return thumbnailMetadata, err
-}
-
-// GetThumbnails returns metadata about all thumbnails for a specific media stored on this server.
-// The media could have been uploaded to this server or fetched from another server and cached here.
-// Returns nil metadata if there are no thumbnails associated with this media.
-func (d *Database) GetThumbnails(
-	ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName,
-) ([]*types.ThumbnailMetadata, error) {
-	thumbnails, err := d.statements.thumbnail.selectThumbnails(ctx, mediaID, mediaOrigin)
-	if err != nil && err == sql.ErrNoRows {
-		return nil, nil
-	}
-	return thumbnails, err
 }

mediaapi/thumbnailer/thumbnailer.go

@@ -136,7 +136,7 @@ func isThumbnailExists(
 	dst types.Path,
 	config types.ThumbnailSize,
 	mediaMetadata *types.MediaMetadata,
-	db *storage.Database,
+	db storage.Database,
 	logger *log.Entry,
 ) (bool, error) {
 	thumbnailMetadata, err := db.GetThumbnail(

mediaapi/thumbnailer/thumbnailer_nfnt.go

@@ -45,7 +45,7 @@ func GenerateThumbnails(
 	mediaMetadata *types.MediaMetadata,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 	maxThumbnailGenerators int,
-	db *storage.Database,
+	db storage.Database,
 	logger *log.Entry,
 ) (busy bool, errorReturn error) {
 	img, err := readFile(string(src))
@@ -78,7 +78,7 @@ func GenerateThumbnail(
 	mediaMetadata *types.MediaMetadata,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 	maxThumbnailGenerators int,
-	db *storage.Database,
+	db storage.Database,
 	logger *log.Entry,
 ) (busy bool, errorReturn error) {
 	img, err := readFile(string(src))
@@ -142,7 +142,7 @@ func createThumbnail(
 	mediaMetadata *types.MediaMetadata,
 	activeThumbnailGeneration *types.ActiveThumbnailGeneration,
 	maxThumbnailGenerators int,
-	db *storage.Database,
+	db storage.Database,
 	logger *log.Entry,
 ) (busy bool, errorReturn error) {
 	logger = logger.WithFields(log.Fields{