mediaapi: Add thumbnail support (#132)

* vendor: Add bimg image processing library

bimg is MIT licensed. It depends on the C library libvips which is LGPL
v2.1+ licensed. libvips must be installed separately.

* mediaapi: Add YAML config file support

* mediaapi: Add thumbnail support

* mediaapi: Add missing thumbnail files

* travis: Add ppa and install libvips-dev

* travis: Another ppa and install libvips-dev attempt

* travis: Add sudo: required for sudo apt* usage

* mediaapi/thumbnailer: Make comparison code more readable

* mediaapi: Simplify logging of thumbnail properties

* mediaapi/thumbnailer: Rename metrics to fitness

Metrics is used in the context of monitoring with Prometheus so renaming
to avoid confusion.

* mediaapi/thumbnailer: Use math.Inf() for max aspect and size

* mediaapi/thumbnailer: Limit number of parallel generators

Fall back to selecting from already-/pre-generated thumbnails or serving
the original.

* mediaapi/thumbnailer: Split bimg code into separate file

* vendor: Add github.com/nfnt/resize pure go image scaler

* mediaapi/thumbnailer: Add nfnt/resize thumbnailer

* travis: Don't install libvips-dev via ppa

* mediaapi: Add notes to README about resizers

* mediaapi: Elaborate on scaling libs in README

src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go

@@ -15,10 +15,14 @@
 package main
 
 import (
+	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os"
+	"os/user"
 	"path/filepath"
 	"strconv"
+	"strings"
 
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/mediaapi/config"
@@ -28,6 +32,7 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	log "github.com/Sirupsen/logrus"
+	yaml "gopkg.in/yaml.v2"
 )
 
 var (
@@ -38,36 +43,25 @@ var (
 	basePath   = os.Getenv("BASE_PATH")
 	// Note: if the MAX_FILE_SIZE_BYTES is set to 0, it will be unlimited
 	maxFileSizeBytesString = os.Getenv("MAX_FILE_SIZE_BYTES")
+	configPath             = os.Getenv("CONFIG_PATH")
 )
 
 func main() {
 	common.SetupLogging(logDir)
 
-	if bindAddr == "" {
-		log.Panic("No BIND_ADDRESS environment variable found.")
-	}
-	if basePath == "" {
-		log.Panic("No BASE_PATH environment variable found.")
-	}
-	absBasePath, err := filepath.Abs(basePath)
-	if err != nil {
-		log.WithError(err).WithField("BASE_PATH", basePath).Panic("BASE_PATH is invalid (must be able to make absolute)")
-	}
+	log.WithFields(log.Fields{
+		"BIND_ADDRESS":        bindAddr,
+		"DATABASE":            dataSource,
+		"LOG_DIR":             logDir,
+		"SERVER_NAME":         serverName,
+		"BASE_PATH":           basePath,
+		"MAX_FILE_SIZE_BYTES": maxFileSizeBytesString,
+		"CONFIG_PATH":         configPath,
+	}).Info("Loading configuration based on config file and environment variables")
 
-	if serverName == "" {
-		serverName = "localhost"
-	}
-	maxFileSizeBytes, err := strconv.ParseInt(maxFileSizeBytesString, 10, 64)
+	cfg, err := configureServer()
 	if err != nil {
-		maxFileSizeBytes = 10 * 1024 * 1024
-		log.WithError(err).WithField("MAX_FILE_SIZE_BYTES", maxFileSizeBytesString).Warnf("Failed to parse MAX_FILE_SIZE_BYTES. Defaulting to %v bytes.", maxFileSizeBytes)
-	}
-
-	cfg := &config.MediaAPI{
-		ServerName:       gomatrixserverlib.ServerName(serverName),
-		AbsBasePath:      types.Path(absBasePath),
-		MaxFileSizeBytes: types.FileSizeBytes(maxFileSizeBytes),
-		DataSource:       dataSource,
+		log.WithError(err).Fatal("Invalid configuration")
 	}
 
 	db, err := storage.Open(cfg.DataSource)
@@ -76,14 +70,182 @@ func main() {
 	}
 
 	log.WithFields(log.Fields{
-		"BASE_PATH":           absBasePath,
-		"BIND_ADDRESS":        bindAddr,
-		"DATABASE":            dataSource,
-		"LOG_DIR":             logDir,
-		"MAX_FILE_SIZE_BYTES": maxFileSizeBytes,
-		"SERVER_NAME":         serverName,
-	}).Info("Starting mediaapi")
+		"BIND_ADDRESS":           bindAddr,
+		"LOG_DIR":                logDir,
+		"CONFIG_PATH":            configPath,
+		"ServerName":             cfg.ServerName,
+		"AbsBasePath":            cfg.AbsBasePath,
+		"MaxFileSizeBytes":       *cfg.MaxFileSizeBytes,
+		"DataSource":             cfg.DataSource,
+		"DynamicThumbnails":      cfg.DynamicThumbnails,
+		"MaxThumbnailGenerators": cfg.MaxThumbnailGenerators,
+		"ThumbnailSizes":         cfg.ThumbnailSizes,
+	}).Info("Starting mediaapi server with configuration")
 
 	routing.Setup(http.DefaultServeMux, http.DefaultClient, cfg, db)
 	log.Fatal(http.ListenAndServe(bindAddr, nil))
 }
+
+// configureServer loads configuration from a yaml file and overrides with environment variables
+func configureServer() (*config.MediaAPI, error) {
+	cfg, err := loadConfig(configPath)
+	if err != nil {
+		log.WithError(err).Fatal("Invalid config file")
+	}
+
+	// override values from environment variables
+	applyOverrides(cfg)
+
+	if err := validateConfig(cfg); err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
+}
+
+// FIXME: make common somehow? copied from sync api
+func loadConfig(configPath string) (*config.MediaAPI, error) {
+	contents, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		return nil, err
+	}
+	var cfg config.MediaAPI
+	if err = yaml.Unmarshal(contents, &cfg); err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}
+
+func applyOverrides(cfg *config.MediaAPI) {
+	if serverName != "" {
+		if cfg.ServerName != "" {
+			log.WithFields(log.Fields{
+				"server_name": cfg.ServerName,
+				"SERVER_NAME": serverName,
+			}).Info("Overriding server_name from config file with environment variable")
+		}
+		cfg.ServerName = gomatrixserverlib.ServerName(serverName)
+	}
+	if cfg.ServerName == "" {
+		log.Info("ServerName not set. Defaulting to 'localhost'.")
+		cfg.ServerName = "localhost"
+	}
+
+	if basePath != "" {
+		if cfg.BasePath != "" {
+			log.WithFields(log.Fields{
+				"base_path": cfg.BasePath,
+				"BASE_PATH": basePath,
+			}).Info("Overriding base_path from config file with environment variable")
+		}
+		cfg.BasePath = types.Path(basePath)
+	}
+
+	if maxFileSizeBytesString != "" {
+		if cfg.MaxFileSizeBytes != nil {
+			log.WithFields(log.Fields{
+				"max_file_size_bytes": *cfg.MaxFileSizeBytes,
+				"MAX_FILE_SIZE_BYTES": maxFileSizeBytesString,
+			}).Info("Overriding max_file_size_bytes from config file with environment variable")
+		}
+		maxFileSizeBytesInt, err := strconv.ParseInt(maxFileSizeBytesString, 10, 64)
+		if err != nil {
+			maxFileSizeBytesInt = 10 * 1024 * 1024
+			log.WithError(err).WithField(
+				"MAX_FILE_SIZE_BYTES", maxFileSizeBytesString,
+			).Infof("MAX_FILE_SIZE_BYTES not set? Defaulting to %v bytes.", maxFileSizeBytesInt)
+		}
+		maxFileSizeBytes := types.FileSizeBytes(maxFileSizeBytesInt)
+		cfg.MaxFileSizeBytes = &maxFileSizeBytes
+	}
+
+	if dataSource != "" {
+		if cfg.DataSource != "" {
+			log.WithFields(log.Fields{
+				"database": cfg.DataSource,
+				"DATABASE": dataSource,
+			}).Info("Overriding database from config file with environment variable")
+		}
+		cfg.DataSource = dataSource
+	}
+
+	if cfg.MaxThumbnailGenerators == 0 {
+		log.WithField(
+			"max_thumbnail_generators", cfg.MaxThumbnailGenerators,
+		).Info("Using default max_thumbnail_generators")
+		cfg.MaxThumbnailGenerators = 10
+	}
+}
+
+func validateConfig(cfg *config.MediaAPI) error {
+	if bindAddr == "" {
+		return fmt.Errorf("no BIND_ADDRESS environment variable found")
+	}
+
+	absBasePath, err := getAbsolutePath(cfg.BasePath)
+	if err != nil {
+		return fmt.Errorf("invalid base path (%v): %q", cfg.BasePath, err)
+	}
+	cfg.AbsBasePath = types.Path(absBasePath)
+
+	if *cfg.MaxFileSizeBytes < 0 {
+		return fmt.Errorf("invalid max file size bytes (%v)", *cfg.MaxFileSizeBytes)
+	}
+
+	if cfg.DataSource == "" {
+		return fmt.Errorf("invalid database (%v)", cfg.DataSource)
+	}
+
+	for _, config := range cfg.ThumbnailSizes {
+		if config.Width <= 0 || config.Height <= 0 {
+			return fmt.Errorf("invalid thumbnail size %vx%v", config.Width, config.Height)
+		}
+	}
+
+	return nil
+}
+
+func getAbsolutePath(basePath types.Path) (types.Path, error) {
+	var err error
+	if basePath == "" {
+		var wd string
+		wd, err = os.Getwd()
+		return types.Path(wd), err
+	}
+	// Note: If we got here len(basePath) >= 1
+	if basePath[0] == '~' {
+		basePath, err = expandHomeDir(basePath)
+		if err != nil {
+			return "", err
+		}
+	}
+	absBasePath, err := filepath.Abs(string(basePath))
+	return types.Path(absBasePath), err
+}
+
+// expandHomeDir parses paths beginning with ~/path or ~user/path and replaces the home directory part
+func expandHomeDir(basePath types.Path) (types.Path, error) {
+	slash := strings.Index(string(basePath), "/")
+	if slash == -1 {
+		// pretend the slash is after the path as none was found within the string
+		// simplifies code using slash below
+		slash = len(basePath)
+	}
+	var usr *user.User
+	var err error
+	if slash == 1 {
+		// basePath is ~ or ~/path
+		usr, err = user.Current()
+		if err != nil {
+			return "", fmt.Errorf("failed to get user's home directory: %q", err)
+		}
+	} else {
+		// slash > 1
+		// basePath is ~user or ~user/path
+		usr, err = user.Lookup(string(basePath[1:slash]))
+		if err != nil {
+			return "", fmt.Errorf("failed to get user's home directory: %q", err)
+		}
+	}
+	return types.Path(filepath.Join(usr.HomeDir, string(basePath[slash:]))), nil
+}

src/github.com/matrix-org/dendrite/mediaapi/README.md

@@ -0,0 +1,27 @@
+# Media API
+
+This server is responsible for serving `/media` requests as per:
+
+http://matrix.org/docs/spec/client_server/r0.2.0.html#id43
+
+## Scaling libraries
+
+### nfnt/resize (default)
+
+Thumbnailing uses https://github.com/nfnt/resize by default which is a pure golang image scaling library relying on image codecs from the standard library. It is ISC-licensed.
+
+It is multi-threaded and uses Lanczos3 so produces sharp images. Using Lanczos3 all the way makes it slower than some other approaches like bimg. (~845ms in total for pre-generating 32x32-crop, 96x96-crop, 320x240-scale, 640x480-scale and 800x600-scale from a given JPEG image on a given machine.)
+
+See the sample below for image quality with nfnt/resize:
+
+![](nfnt-96x96-crop.jpg)
+
+### bimg (uses libvips C library)
+
+Alternatively one can use `gb build -tags bimg` to use bimg from https://github.com/h2non/bimg (MIT-licensed) which uses libvips from https://github.com/jcupitt/libvips (LGPL v2.1+ -licensed). libvips is a C library and must be installed/built separately. See the github page for details. Also note that libvips in turn has dependencies with a selection of FOSS licenses.
+
+bimg and libvips have significantly better performance than nfnt/resize but produce slightly less-sharp images. bimg uses a box filter for downscaling to within about 200% of the target scale and then uses Lanczos3 for the last bit. This is a much faster approach but comes at the expense of sharpness. (~295ms in total for pre-generating 32x32-crop, 96x96-crop, 320x240-scale, 640x480-scale and 800x600-scale from a given JPEG image on a given machine.)
+
+See the sample below for image quality with bimg:
+
+![](bimg-96x96-crop.jpg)

src/github.com/matrix-org/dendrite/mediaapi/config/config.go

@@ -23,12 +23,20 @@ import (
 type MediaAPI struct {
 	// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
 	ServerName gomatrixserverlib.ServerName `yaml:"server_name"`
+	// The base path to where the media files will be stored. May be relative or absolute.
+	BasePath types.Path `yaml:"base_path"`
 	// The absolute base path to where media files will be stored.
-	AbsBasePath types.Path `yaml:"abs_base_path"`
+	AbsBasePath types.Path `yaml:"-"`
 	// The maximum file size in bytes that is allowed to be stored on this server.
 	// Note: if MaxFileSizeBytes is set to 0, the size is unlimited.
 	// Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
-	MaxFileSizeBytes types.FileSizeBytes `yaml:"max_file_size_bytes"`
+	MaxFileSizeBytes *types.FileSizeBytes `yaml:"max_file_size_bytes,omitempty"`
 	// The postgres connection config for connecting to the database e.g a postgres:// URI
 	DataSource string `yaml:"database"`
+	// Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
+	DynamicThumbnails bool `yaml:"dynamic_thumbnails"`
+	// The maximum number of simultaneous thumbnail generators. default: 10
+	MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"`
+	// A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
+	ThumbnailSizes []types.ThumbnailSize `yaml:"thumbnail_sizes"`
 }

src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go

@@ -35,28 +35,40 @@ const pathPrefixR0 = "/_matrix/media/v1"
 func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.MediaAPI, db *storage.Database) {
 	apiMux := mux.NewRouter()
 	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
+
+	activeThumbnailGeneration := &types.ActiveThumbnailGeneration{
+		PathToResult: map[string]*types.ThumbnailGenerationResult{},
+	}
+
 	// FIXME: /upload should use common.MakeAuthAPI()
 	r0mux.Handle("/upload", common.MakeAPI("upload", func(req *http.Request) util.JSONResponse {
-		return writers.Upload(req, cfg, db)
+		return writers.Upload(req, cfg, db, activeThumbnailGeneration)
 	}))
 
 	activeRemoteRequests := &types.ActiveRemoteRequests{
 		MXCToResult: map[string]*types.RemoteRequestResult{},
 	}
 	r0mux.Handle("/download/{serverName}/{mediaId}",
-		prometheus.InstrumentHandler("download", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-			req = util.RequestWithLogging(req)
-
-			// Set common headers returned regardless of the outcome of the request
-			util.SetCORSHeaders(w)
-			// Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors
-			w.Header().Set("Content-Type", "application/json")
-
-			vars := mux.Vars(req)
-			writers.Download(w, req, gomatrixserverlib.ServerName(vars["serverName"]), types.MediaID(vars["mediaId"]), cfg, db, activeRemoteRequests)
-		})),
+		makeDownloadAPI("download", cfg, db, activeRemoteRequests, activeThumbnailGeneration),
+	)
+	r0mux.Handle("/thumbnail/{serverName}/{mediaId}",
+		makeDownloadAPI("thumbnail", cfg, db, activeRemoteRequests, activeThumbnailGeneration),
 	)
 
 	servMux.Handle("/metrics", prometheus.Handler())
 	servMux.Handle("/api/", http.StripPrefix("/api", apiMux))
 }
+
+func makeDownloadAPI(name string, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) http.HandlerFunc {
+	return prometheus.InstrumentHandler(name, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		req = util.RequestWithLogging(req)
+
+		// Set common headers returned regardless of the outcome of the request
+		util.SetCORSHeaders(w)
+		// Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors
+		w.Header().Set("Content-Type", "application/json")
+
+		vars := mux.Vars(req)
+		writers.Download(w, req, gomatrixserverlib.ServerName(vars["serverName"]), types.MediaID(vars["mediaId"]), cfg, db, activeRemoteRequests, activeThumbnailGeneration, name == "thumbnail")
+	}))
+}

src/github.com/matrix-org/dendrite/mediaapi/storage/sql.go

@@ -19,13 +19,17 @@ import (
 )
 
 type statements struct {
-	mediaStatements
+	media     mediaStatements
+	thumbnail thumbnailStatements
 }
 
 func (s *statements) prepare(db *sql.DB) error {
 	var err error
 
-	if err = s.mediaStatements.prepare(db); err != nil {
+	if err = s.media.prepare(db); err != nil {
+		return err
+	}
+	if err = s.thumbnail.prepare(db); err != nil {
 		return err
 	}
 

src/github.com/matrix-org/dendrite/mediaapi/storage/storage.go

@@ -45,16 +45,44 @@ func Open(dataSourceName string) (*Database, error) {
 // StoreMediaMetadata inserts the metadata about the uploaded media into the database.
 // Returns an error if the combination of MediaID and Origin are not unique in the table.
 func (d *Database) StoreMediaMetadata(mediaMetadata *types.MediaMetadata) error {
-	return d.statements.insertMedia(mediaMetadata)
+	return d.statements.media.insertMedia(mediaMetadata)
 }
 
 // GetMediaMetadata returns metadata about media stored on this server.
 // The media could have been uploaded to this server or fetched from another server and cached here.
 // Returns nil metadata if there is no metadata associated with this media.
 func (d *Database) GetMediaMetadata(mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) {
-	mediaMetadata, err := d.statements.selectMedia(mediaID, mediaOrigin)
+	mediaMetadata, err := d.statements.media.selectMedia(mediaID, mediaOrigin)
 	if err != nil && err == sql.ErrNoRows {
 		return nil, nil
 	}
 	return mediaMetadata, err
 }
+
+// StoreThumbnail inserts the metadata about the thumbnail into the database.
+// Returns an error if the combination of MediaID and Origin are not unique in the table.
+func (d *Database) StoreThumbnail(thumbnailMetadata *types.ThumbnailMetadata) error {
+	return d.statements.thumbnail.insertThumbnail(thumbnailMetadata)
+}
+
+// GetThumbnail returns metadata about a specific thumbnail.
+// The media could have been uploaded to this server or fetched from another server and cached here.
+// Returns nil metadata if there is no metadata associated with this thumbnail.
+func (d *Database) GetThumbnail(mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) {
+	thumbnailMetadata, err := d.statements.thumbnail.selectThumbnail(mediaID, mediaOrigin, width, height, resizeMethod)
+	if err != nil && err == sql.ErrNoRows {
+		return nil, nil
+	}
+	return thumbnailMetadata, err
+}
+
+// GetThumbnails returns metadata about all thumbnails for a specific media stored on this server.
+// The media could have been uploaded to this server or fetched from another server and cached here.
+// Returns nil metadata if there are no thumbnails associated with this media.
+func (d *Database) GetThumbnails(mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error) {
+	thumbnails, err := d.statements.thumbnail.selectThumbnails(mediaID, mediaOrigin)
+	if err != nil && err == sql.ErrNoRows {
+		return nil, nil
+	}
+	return thumbnails, err
+}

src/github.com/matrix-org/dendrite/mediaapi/storage/thumbnail_table.go

@@ -0,0 +1,157 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"database/sql"
+	"time"
+
+	"github.com/matrix-org/dendrite/mediaapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const thumbnailSchema = `
+-- The thumbnail table holds metadata for each thumbnail file stored and accessible to the local server,
+-- the actual file is stored separately.
+CREATE TABLE IF NOT EXISTS thumbnail (
+    -- The id used to refer to the media.
+    -- For uploads to this server this is a base64-encoded sha256 hash of the file data
+    -- For media from remote servers, this can be any unique identifier string
+    media_id TEXT NOT NULL,
+    -- The origin of the media as requested by the client. Should be a homeserver domain.
+    media_origin TEXT NOT NULL,
+    -- The MIME-type of the thumbnail file.
+    content_type TEXT NOT NULL,
+    -- Size of the thumbnail file in bytes.
+    file_size_bytes BIGINT NOT NULL,
+    -- When the thumbnail was generated in UNIX epoch ms.
+    creation_ts BIGINT NOT NULL,
+    -- The width of the thumbnail
+    width INTEGER NOT NULL,
+    -- The height of the thumbnail
+    height INTEGER NOT NULL,
+    -- The resize method used to generate the thumbnail. Can be crop or scale.
+    resize_method TEXT NOT NULL
+);
+CREATE UNIQUE INDEX IF NOT EXISTS thumbnail_index ON thumbnail (media_id, media_origin, width, height, resize_method);
+`
+
+const insertThumbnailSQL = `
+INSERT INTO thumbnail (media_id, media_origin, content_type, file_size_bytes, creation_ts, width, height, resize_method)
+    VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+`
+
+// Note: this selects one specific thumbnail
+const selectThumbnailSQL = `
+SELECT content_type, file_size_bytes, creation_ts FROM thumbnail WHERE media_id = $1 AND media_origin = $2 AND width = $3 AND height = $4 AND resize_method = $5
+`
+
+// Note: this selects all thumbnails for a media_origin and media_id
+const selectThumbnailsSQL = `
+SELECT content_type, file_size_bytes, creation_ts, width, height, resize_method FROM thumbnail WHERE media_id = $1 AND media_origin = $2
+`
+
+type thumbnailStatements struct {
+	insertThumbnailStmt  *sql.Stmt
+	selectThumbnailStmt  *sql.Stmt
+	selectThumbnailsStmt *sql.Stmt
+}
+
+func (s *thumbnailStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(thumbnailSchema)
+	if err != nil {
+		return
+	}
+
+	return statementList{
+		{&s.insertThumbnailStmt, insertThumbnailSQL},
+		{&s.selectThumbnailStmt, selectThumbnailSQL},
+		{&s.selectThumbnailsStmt, selectThumbnailsSQL},
+	}.prepare(db)
+}
+
+func (s *thumbnailStatements) insertThumbnail(thumbnailMetadata *types.ThumbnailMetadata) error {
+	thumbnailMetadata.MediaMetadata.CreationTimestamp = types.UnixMs(time.Now().UnixNano() / 1000000)
+	_, err := s.insertThumbnailStmt.Exec(
+		thumbnailMetadata.MediaMetadata.MediaID,
+		thumbnailMetadata.MediaMetadata.Origin,
+		thumbnailMetadata.MediaMetadata.ContentType,
+		thumbnailMetadata.MediaMetadata.FileSizeBytes,
+		thumbnailMetadata.MediaMetadata.CreationTimestamp,
+		thumbnailMetadata.ThumbnailSize.Width,
+		thumbnailMetadata.ThumbnailSize.Height,
+		thumbnailMetadata.ThumbnailSize.ResizeMethod,
+	)
+	return err
+}
+
+func (s *thumbnailStatements) selectThumbnail(mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) {
+	thumbnailMetadata := types.ThumbnailMetadata{
+		MediaMetadata: &types.MediaMetadata{
+			MediaID: mediaID,
+			Origin:  mediaOrigin,
+		},
+		ThumbnailSize: types.ThumbnailSize{
+			Width:        width,
+			Height:       height,
+			ResizeMethod: resizeMethod,
+		},
+	}
+	err := s.selectThumbnailStmt.QueryRow(
+		thumbnailMetadata.MediaMetadata.MediaID,
+		thumbnailMetadata.MediaMetadata.Origin,
+		thumbnailMetadata.ThumbnailSize.Width,
+		thumbnailMetadata.ThumbnailSize.Height,
+		thumbnailMetadata.ThumbnailSize.ResizeMethod,
+	).Scan(
+		&thumbnailMetadata.MediaMetadata.ContentType,
+		&thumbnailMetadata.MediaMetadata.FileSizeBytes,
+		&thumbnailMetadata.MediaMetadata.CreationTimestamp,
+	)
+	return &thumbnailMetadata, err
+}
+
+func (s *thumbnailStatements) selectThumbnails(mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error) {
+	rows, err := s.selectThumbnailsStmt.Query(
+		mediaID, mediaOrigin,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	var thumbnails []*types.ThumbnailMetadata
+	for rows.Next() {
+		thumbnailMetadata := types.ThumbnailMetadata{
+			MediaMetadata: &types.MediaMetadata{
+				MediaID: mediaID,
+				Origin:  mediaOrigin,
+			},
+		}
+		err = rows.Scan(
+			&thumbnailMetadata.MediaMetadata.ContentType,
+			&thumbnailMetadata.MediaMetadata.FileSizeBytes,
+			&thumbnailMetadata.MediaMetadata.CreationTimestamp,
+			&thumbnailMetadata.ThumbnailSize.Width,
+			&thumbnailMetadata.ThumbnailSize.Height,
+			&thumbnailMetadata.ThumbnailSize.ResizeMethod,
+		)
+		if err != nil {
+			return nil, err
+		}
+		thumbnails = append(thumbnails, &thumbnailMetadata)
+	}
+
+	return thumbnails, err
+}

src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go

@@ -0,0 +1,221 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package thumbnailer
+
+import (
+	"fmt"
+	"math"
+	"path/filepath"
+	"sync"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/matrix-org/dendrite/mediaapi/types"
+)
+
+type thumbnailFitness struct {
+	isSmaller      int
+	aspect         float64
+	size           float64
+	methodMismatch int
+	fileSize       types.FileSizeBytes
+}
+
+// thumbnailTemplate is the filename template for thumbnails
+const thumbnailTemplate = "thumbnail-%vx%v-%v"
+
+// GetThumbnailPath returns the path to a thumbnail given the absolute src path and thumbnail size configuration
+func GetThumbnailPath(src types.Path, config types.ThumbnailSize) types.Path {
+	srcDir := filepath.Dir(string(src))
+	return types.Path(filepath.Join(
+		srcDir,
+		fmt.Sprintf(thumbnailTemplate, config.Width, config.Height, config.ResizeMethod),
+	))
+}
+
+// SelectThumbnail compares the (potentially) available thumbnails with the desired thumbnail and returns the best match
+// The algorithm is very similar to what was implemented in Synapse
+// In order of priority unless absolute, the following metrics are compared; the image is:
+// * the same size or larger than requested
+// * if a cropped image is desired, has an aspect ratio close to requested
+// * has a size close to requested
+// * if a cropped image is desired, prefer the same method, if scaled is desired, absolutely require scaled
+// * has a small file size
+// If a pre-generated thumbnail size is the best match, but it has not been generated yet, the caller can use the returned size to generate it.
+// Returns nil if no thumbnail matches the criteria
+func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailMetadata, thumbnailSizes []types.ThumbnailSize) (*types.ThumbnailMetadata, *types.ThumbnailSize) {
+	var chosenThumbnail *types.ThumbnailMetadata
+	var chosenThumbnailSize *types.ThumbnailSize
+	bestFit := newThumbnailFitness()
+
+	for _, thumbnail := range thumbnails {
+		if desired.ResizeMethod == "scale" && thumbnail.ThumbnailSize.ResizeMethod != "scale" {
+			continue
+		}
+		fitness := calcThumbnailFitness(thumbnail.ThumbnailSize, thumbnail.MediaMetadata, desired)
+		if isBetter := fitness.betterThan(bestFit, desired.ResizeMethod == "crop"); isBetter {
+			bestFit = fitness
+			chosenThumbnail = thumbnail
+		}
+	}
+
+	for _, thumbnailSize := range thumbnailSizes {
+		if desired.ResizeMethod == "scale" && thumbnailSize.ResizeMethod != "scale" {
+			continue
+		}
+		fitness := calcThumbnailFitness(thumbnailSize, nil, desired)
+		if isBetter := fitness.betterThan(bestFit, desired.ResizeMethod == "crop"); isBetter {
+			bestFit = fitness
+			chosenThumbnailSize = &types.ThumbnailSize{
+				Width:        thumbnailSize.Width,
+				Height:       thumbnailSize.Height,
+				ResizeMethod: thumbnailSize.ResizeMethod,
+			}
+		}
+	}
+
+	return chosenThumbnail, chosenThumbnailSize
+}
+
+// getActiveThumbnailGeneration checks for active thumbnail generation
+func getActiveThumbnailGeneration(dst types.Path, config types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, logger *log.Entry) (isActive bool, busy bool, errorReturn error) {
+	// Check if there is active thumbnail generation.
+	activeThumbnailGeneration.Lock()
+	defer activeThumbnailGeneration.Unlock()
+	if activeThumbnailGenerationResult, ok := activeThumbnailGeneration.PathToResult[string(dst)]; ok {
+		logger.Info("Waiting for another goroutine to generate the thumbnail.")
+
+		// NOTE: Wait unlocks and locks again internally. There is still a deferred Unlock() that will unlock this.
+		activeThumbnailGenerationResult.Cond.Wait()
+		// Note: either there is an error or it is nil, either way returning it is correct
+		return false, false, activeThumbnailGenerationResult.Err
+	}
+
+	// Only allow thumbnail generation up to a maximum configured number. Above this we fall back to serving the
+	// original. Or in the case of pre-generation, they maybe get generated on the first request for a thumbnail if
+	// load has subsided.
+	if len(activeThumbnailGeneration.PathToResult) >= maxThumbnailGenerators {
+		return false, true, nil
+	}
+
+	// No active thumbnail generation so create one
+	activeThumbnailGeneration.PathToResult[string(dst)] = &types.ThumbnailGenerationResult{
+		Cond: &sync.Cond{L: activeThumbnailGeneration},
+	}
+
+	return true, false, nil
+}
+
+// broadcastGeneration broadcasts that thumbnail generation completed and the error to all waiting goroutines
+// Note: This should only be called by the owner of the activeThumbnailGenerationResult
+func broadcastGeneration(dst types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, config types.ThumbnailSize, errorReturn error, logger *log.Entry) {
+	activeThumbnailGeneration.Lock()
+	defer activeThumbnailGeneration.Unlock()
+	if activeThumbnailGenerationResult, ok := activeThumbnailGeneration.PathToResult[string(dst)]; ok {
+		logger.Info("Signalling other goroutines waiting for this goroutine to generate the thumbnail.")
+		// Note: errorReturn is a named return value error that is signalled from here to waiting goroutines
+		activeThumbnailGenerationResult.Err = errorReturn
+		activeThumbnailGenerationResult.Cond.Broadcast()
+	}
+	delete(activeThumbnailGeneration.PathToResult, string(dst))
+}
+
+// init with worst values
+func newThumbnailFitness() thumbnailFitness {
+	return thumbnailFitness{
+		isSmaller:      1,
+		aspect:         math.Inf(1),
+		size:           math.Inf(1),
+		methodMismatch: 0,
+		fileSize:       types.FileSizeBytes(math.MaxInt64),
+	}
+}
+
+func calcThumbnailFitness(size types.ThumbnailSize, metadata *types.MediaMetadata, desired types.ThumbnailSize) thumbnailFitness {
+	dW := desired.Width
+	dH := desired.Height
+	tW := size.Width
+	tH := size.Height
+
+	fitness := newThumbnailFitness()
+	// In all cases, a larger metric value is a worse fit.
+	// compare size: thumbnail smaller is true and gives 1, larger is false and gives 0
+	fitness.isSmaller = boolToInt(tW < dW || tH < dH)
+	// comparison of aspect ratios only makes sense for a request for desired cropped
+	fitness.aspect = math.Abs(float64(dW*tH - dH*tW))
+	// compare sizes
+	fitness.size = math.Abs(float64((dW - tW) * (dH - tH)))
+	// compare resize method
+	fitness.methodMismatch = boolToInt(size.ResizeMethod != desired.ResizeMethod)
+	if metadata != nil {
+		// file size
+		fitness.fileSize = metadata.FileSizeBytes
+	}
+
+	return fitness
+}
+
+func boolToInt(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+func (a thumbnailFitness) betterThan(b thumbnailFitness, desiredCrop bool) bool {
+	// preference means returning -1
+
+	// prefer images that are not smaller
+	// e.g. isSmallerDiff > 0 means b is smaller than desired and a is not smaller
+	if a.isSmaller > b.isSmaller {
+		return false
+	} else if a.isSmaller < b.isSmaller {
+		return true
+	}
+
+	// prefer aspect ratios closer to desired only if desired cropped
+	// only cropped images have differing aspect ratios
+	// desired scaled only accepts scaled images
+	if desiredCrop {
+		if a.aspect > b.aspect {
+			return false
+		} else if a.aspect < b.aspect {
+			return true
+		}
+	}
+
+	// prefer closer in size
+	if a.size > b.size {
+		return false
+	} else if a.size < b.size {
+		return true
+	}
+
+	// prefer images using the same method
+	// e.g. methodMismatchDiff > 0 means b's method is different from desired and a's matches the desired method
+	if a.methodMismatch > b.methodMismatch {
+		return false
+	} else if a.methodMismatch < b.methodMismatch {
+		return true
+	}
+
+	// prefer smaller files
+	if a.fileSize > b.fileSize {
+		return false
+	} else if a.fileSize < b.fileSize {
+		return true
+	}
+
+	return false
+}

src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go

@@ -0,0 +1,217 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build bimg
+
+package thumbnailer
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/matrix-org/dendrite/mediaapi/storage"
+	"github.com/matrix-org/dendrite/mediaapi/types"
+	"gopkg.in/h2non/bimg.v1"
+)
+
+// GenerateThumbnails generates the configured thumbnail sizes for the source file
+func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
+	buffer, err := bimg.Read(string(src))
+	if err != nil {
+		logger.WithError(err).WithField("src", src).Error("Failed to read src file")
+		return false, err
+	}
+	for _, config := range configs {
+		// Note: createThumbnail does locking based on activeThumbnailGeneration
+		busy, err = createThumbnail(src, buffer, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
+		if err != nil {
+			logger.WithError(err).WithField("src", src).Error("Failed to generate thumbnails")
+			return false, err
+		}
+		if busy {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// GenerateThumbnail generates the configured thumbnail size for the source file
+func GenerateThumbnail(src types.Path, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
+	buffer, err := bimg.Read(string(src))
+	if err != nil {
+		logger.WithError(err).WithFields(log.Fields{
+			"src": src,
+		}).Error("Failed to read src file")
+		return false, err
+	}
+	// Note: createThumbnail does locking based on activeThumbnailGeneration
+	busy, err = createThumbnail(src, buffer, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
+	if err != nil {
+		logger.WithError(err).WithFields(log.Fields{
+			"src": src,
+		}).Error("Failed to generate thumbnails")
+		return false, err
+	}
+	if busy {
+		return true, nil
+	}
+	return false, nil
+}
+
+// createThumbnail checks if the thumbnail exists, and if not, generates it
+// Thumbnail generation is only done once for each non-existing thumbnail.
+func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
+	logger = logger.WithFields(log.Fields{
+		"Width":        config.Width,
+		"Height":       config.Height,
+		"ResizeMethod": config.ResizeMethod,
+	})
+
+	dst := GetThumbnailPath(src, config)
+
+	// Note: getActiveThumbnailGeneration uses mutexes and conditions from activeThumbnailGeneration
+	isActive, busy, err := getActiveThumbnailGeneration(dst, config, activeThumbnailGeneration, maxThumbnailGenerators, logger)
+	if err != nil {
+		return false, err
+	}
+	if busy {
+		return true, nil
+	}
+
+	if isActive {
+		// Note: This is an active request that MUST broadcastGeneration to wake up waiting goroutines!
+		// Note: broadcastGeneration uses mutexes and conditions from activeThumbnailGeneration
+		defer func() {
+			// Note: errorReturn is the named return variable so we wrap this in a closure to re-evaluate the arguments at defer-time
+			if err := recover(); err != nil {
+				broadcastGeneration(dst, activeThumbnailGeneration, config, err.(error), logger)
+				panic(err)
+			}
+			broadcastGeneration(dst, activeThumbnailGeneration, config, errorReturn, logger)
+		}()
+	}
+
+	// Check if the thumbnail exists.
+	thumbnailMetadata, err := db.GetThumbnail(mediaMetadata.MediaID, mediaMetadata.Origin, config.Width, config.Height, config.ResizeMethod)
+	if err != nil {
+		logger.Error("Failed to query database for thumbnail.")
+		return false, err
+	}
+	if thumbnailMetadata != nil {
+		return false, nil
+	}
+	// Note: The double-negative is intentional as os.IsExist(err) != !os.IsNotExist(err).
+	// The functions are error checkers to be used in different cases.
+	if _, err = os.Stat(string(dst)); !os.IsNotExist(err) {
+		// Thumbnail exists
+		return false, nil
+	}
+
+	if isActive == false {
+		// Note: This should not happen, but we check just in case.
+		logger.Error("Failed to stat file but this is not the active thumbnail generator. This should not happen.")
+		return false, fmt.Errorf("Not active thumbnail generator. Stat error: %q", err)
+	}
+
+	start := time.Now()
+	width, height, err := resize(dst, buffer, config.Width, config.Height, config.ResizeMethod == "crop", logger)
+	if err != nil {
+		return false, err
+	}
+	logger.WithFields(log.Fields{
+		"ActualWidth":  width,
+		"ActualHeight": height,
+		"processTime":  time.Now().Sub(start),
+	}).Info("Generated thumbnail")
+
+	stat, err := os.Stat(string(dst))
+	if err != nil {
+		return false, err
+	}
+
+	thumbnailMetadata = &types.ThumbnailMetadata{
+		MediaMetadata: &types.MediaMetadata{
+			MediaID: mediaMetadata.MediaID,
+			Origin:  mediaMetadata.Origin,
+			// Note: the code currently always creates a JPEG thumbnail
+			ContentType:   types.ContentType("image/jpeg"),
+			FileSizeBytes: types.FileSizeBytes(stat.Size()),
+		},
+		ThumbnailSize: types.ThumbnailSize{
+			Width:        width,
+			Height:       height,
+			ResizeMethod: config.ResizeMethod,
+		},
+	}
+
+	err = db.StoreThumbnail(thumbnailMetadata)
+	if err != nil {
+		logger.WithError(err).WithFields(log.Fields{
+			"ActualWidth":  width,
+			"ActualHeight": height,
+		}).Error("Failed to store thumbnail metadata in database.")
+		return false, err
+	}
+
+	return false, nil
+}
+
+// resize scales an image to fit within the provided width and height
+// If the source aspect ratio is different to the target dimensions, one edge will be smaller than requested
+// If crop is set to true, the image will be scaled to fill the width and height with any excess being cropped off
+func resize(dst types.Path, buffer []byte, w, h int, crop bool, logger *log.Entry) (int, int, error) {
+	inImage := bimg.NewImage(buffer)
+
+	inSize, err := inImage.Size()
+	if err != nil {
+		return -1, -1, err
+	}
+
+	options := bimg.Options{
+		Type:    bimg.JPEG,
+		Quality: 85,
+	}
+	if crop {
+		options.Width = w
+		options.Height = h
+		options.Crop = true
+	} else {
+		inAR := float64(inSize.Width) / float64(inSize.Height)
+		outAR := float64(w) / float64(h)
+
+		if inAR > outAR {
+			// input has wider AR than requested output so use requested width and calculate height to match input AR
+			options.Width = w
+			options.Height = int(float64(w) / inAR)
+		} else {
+			// input has narrower AR than requested output so use requested height and calculate width to match input AR
+			options.Width = int(float64(h) * inAR)
+			options.Height = h
+		}
+	}
+
+	newImage, err := inImage.Process(options)
+	if err != nil {
+		return -1, -1, err
+	}
+
+	if err = bimg.Write(string(dst), newImage); err != nil {
+		logger.WithError(err).Error("Failed to resize image")
+		return -1, -1, err
+	}
+
+	return options.Width, options.Height, nil
+}

src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go

@@ -0,0 +1,249 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !bimg
+
+package thumbnailer
+
+import (
+	"fmt"
+	"image"
+	"image/draw"
+	// Imported for gif codec
+	_ "image/gif"
+	"image/jpeg"
+	// Imported for png codec
+	_ "image/png"
+	"os"
+	"time"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/matrix-org/dendrite/mediaapi/storage"
+	"github.com/matrix-org/dendrite/mediaapi/types"
+	"github.com/nfnt/resize"
+)
+
+// GenerateThumbnails generates the configured thumbnail sizes for the source file
+func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
+	img, err := readFile(string(src))
+	if err != nil {
+		logger.WithError(err).WithField("src", src).Error("Failed to read src file")
+		return false, err
+	}
+	for _, config := range configs {
+		// Note: createThumbnail does locking based on activeThumbnailGeneration
+		busy, err = createThumbnail(src, img, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
+		if err != nil {
+			logger.WithError(err).WithField("src", src).Error("Failed to generate thumbnails")
+			return false, err
+		}
+		if busy {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// GenerateThumbnail generates the configured thumbnail size for the source file
+func GenerateThumbnail(src types.Path, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
+	img, err := readFile(string(src))
+	if err != nil {
+		logger.WithError(err).WithFields(log.Fields{
+			"src": src,
+		}).Error("Failed to read src file")
+		return false, err
+	}
+	// Note: createThumbnail does locking based on activeThumbnailGeneration
+	busy, err = createThumbnail(src, img, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
+	if err != nil {
+		logger.WithError(err).WithFields(log.Fields{
+			"src": src,
+		}).Error("Failed to generate thumbnails")
+		return false, err
+	}
+	if busy {
+		return true, nil
+	}
+	return false, nil
+}
+
+func readFile(src string) (image.Image, error) {
+	file, err := os.Open(src)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	img, _, err := image.Decode(file)
+	if err != nil {
+		return nil, err
+	}
+
+	return img, nil
+}
+
+func writeFile(img image.Image, dst string) error {
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	return jpeg.Encode(out, img, &jpeg.Options{
+		Quality: 85,
+	})
+}
+
+// createThumbnail checks if the thumbnail exists, and if not, generates it
+// Thumbnail generation is only done once for each non-existing thumbnail.
+func createThumbnail(src types.Path, img image.Image, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
+	logger = logger.WithFields(log.Fields{
+		"Width":        config.Width,
+		"Height":       config.Height,
+		"ResizeMethod": config.ResizeMethod,
+	})
+
+	dst := GetThumbnailPath(src, config)
+
+	// Note: getActiveThumbnailGeneration uses mutexes and conditions from activeThumbnailGeneration
+	isActive, busy, err := getActiveThumbnailGeneration(dst, config, activeThumbnailGeneration, maxThumbnailGenerators, logger)
+	if err != nil {
+		return false, err
+	}
+	if busy {
+		return true, nil
+	}
+
+	if isActive {
+		// Note: This is an active request that MUST broadcastGeneration to wake up waiting goroutines!
+		// Note: broadcastGeneration uses mutexes and conditions from activeThumbnailGeneration
+		defer func() {
+			// Note: errorReturn is the named return variable so we wrap this in a closure to re-evaluate the arguments at defer-time
+			// if err := recover(); err != nil {
+			// 	broadcastGeneration(dst, activeThumbnailGeneration, config, err.(error), logger)
+			// 	panic(err)
+			// }
+			broadcastGeneration(dst, activeThumbnailGeneration, config, errorReturn, logger)
+		}()
+	}
+
+	// Check if the thumbnail exists.
+	thumbnailMetadata, err := db.GetThumbnail(mediaMetadata.MediaID, mediaMetadata.Origin, config.Width, config.Height, config.ResizeMethod)
+	if err != nil {
+		logger.Error("Failed to query database for thumbnail.")
+		return false, err
+	}
+	if thumbnailMetadata != nil {
+		return false, nil
+	}
+	// Note: The double-negative is intentional as os.IsExist(err) != !os.IsNotExist(err).
+	// The functions are error checkers to be used in different cases.
+	if _, err = os.Stat(string(dst)); !os.IsNotExist(err) {
+		// Thumbnail exists
+		return false, nil
+	}
+
+	if isActive == false {
+		// Note: This should not happen, but we check just in case.
+		logger.Error("Failed to stat file but this is not the active thumbnail generator. This should not happen.")
+		return false, fmt.Errorf("Not active thumbnail generator. Stat error: %q", err)
+	}
+
+	start := time.Now()
+	width, height, err := adjustSize(dst, img, config.Width, config.Height, config.ResizeMethod == "crop", logger)
+	if err != nil {
+		return false, err
+	}
+	logger.WithFields(log.Fields{
+		"ActualWidth":  width,
+		"ActualHeight": height,
+		"processTime":  time.Now().Sub(start),
+	}).Info("Generated thumbnail")
+
+	stat, err := os.Stat(string(dst))
+	if err != nil {
+		return false, err
+	}
+
+	thumbnailMetadata = &types.ThumbnailMetadata{
+		MediaMetadata: &types.MediaMetadata{
+			MediaID: mediaMetadata.MediaID,
+			Origin:  mediaMetadata.Origin,
+			// Note: the code currently always creates a JPEG thumbnail
+			ContentType:   types.ContentType("image/jpeg"),
+			FileSizeBytes: types.FileSizeBytes(stat.Size()),
+		},
+		ThumbnailSize: types.ThumbnailSize{
+			Width:        width,
+			Height:       height,
+			ResizeMethod: config.ResizeMethod,
+		},
+	}
+
+	err = db.StoreThumbnail(thumbnailMetadata)
+	if err != nil {
+		logger.WithError(err).WithFields(log.Fields{
+			"ActualWidth":  width,
+			"ActualHeight": height,
+		}).Error("Failed to store thumbnail metadata in database.")
+		return false, err
+	}
+
+	return false, nil
+}
+
+// adjustSize scales an image to fit within the provided width and height
+// If the source aspect ratio is different to the target dimensions, one edge will be smaller than requested
+// If crop is set to true, the image will be scaled to fill the width and height with any excess being cropped off
+func adjustSize(dst types.Path, img image.Image, w, h int, crop bool, logger *log.Entry) (int, int, error) {
+	var out image.Image
+	var err error
+	if crop {
+		inAR := float64(img.Bounds().Dx()) / float64(img.Bounds().Dy())
+		outAR := float64(w) / float64(h)
+
+		var scaleW, scaleH uint
+		if inAR > outAR {
+			// input has shorter AR than requested output so use requested height and calculate width to match input AR
+			scaleW = uint(float64(h) * inAR)
+			scaleH = uint(h)
+		} else {
+			// input has taller AR than requested output so use requested width and calculate height to match input AR
+			scaleW = uint(w)
+			scaleH = uint(float64(w) / inAR)
+		}
+
+		scaled := resize.Resize(scaleW, scaleH, img, resize.Lanczos3)
+
+		xoff := (scaled.Bounds().Dx() - w) / 2
+		yoff := (scaled.Bounds().Dy() - h) / 2
+
+		tr := image.Rect(0, 0, w, h)
+		target := image.NewRGBA(tr)
+		draw.Draw(target, tr, scaled, image.Pt(xoff, yoff), draw.Src)
+		out = target
+	} else {
+		out = resize.Thumbnail(uint(w), uint(h), img, resize.Lanczos3)
+		if err != nil {
+			return -1, -1, err
+		}
+	}
+
+	if err = writeFile(out, string(dst)); err != nil {
+		logger.WithError(err).Error("Failed to encode and write image")
+		return -1, -1, err
+	}
+
+	return out.Bounds().Max.X, out.Bounds().Max.Y, nil
+}

src/github.com/matrix-org/dendrite/mediaapi/types/types.go

@@ -77,3 +77,37 @@ type ActiveRemoteRequests struct {
 	// The string key is an mxc:// URL
 	MXCToResult map[string]*RemoteRequestResult
 }
+
+// ThumbnailSize contains a single thumbnail size configuration
+type ThumbnailSize struct {
+	// Maximum width of the thumbnail image
+	Width int `yaml:"width"`
+	// Maximum height of the thumbnail image
+	Height int `yaml:"height"`
+	// ResizeMethod is one of crop or scale.
+	// crop scales to fill the requested dimensions and crops the excess.
+	// scale scales to fit the requested dimensions and one dimension may be smaller than requested.
+	ResizeMethod string `yaml:"method,omitempty"`
+}
+
+// ThumbnailMetadata contains the metadata about an individual thumbnail
+type ThumbnailMetadata struct {
+	MediaMetadata *MediaMetadata
+	ThumbnailSize ThumbnailSize
+}
+
+// ThumbnailGenerationResult is used for broadcasting the result of thumbnail generation to routines waiting on the condition
+type ThumbnailGenerationResult struct {
+	// Condition used for the generator to signal the result to all other routines waiting on this condition
+	Cond *sync.Cond
+	// Resulting error from the generation attempt
+	Err error
+}
+
+// ActiveThumbnailGeneration is a lockable map of file paths being thumbnailed
+// It is used to ensure thumbnails are only generated once.
+type ActiveThumbnailGeneration struct {
+	sync.Mutex
+	// The string key is a thumbnail file path
+	PathToResult map[string]*ThumbnailGenerationResult
+}

src/github.com/matrix-org/dendrite/mediaapi/writers/download.go

@@ -24,6 +24,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strconv"
+	"strings"
 	"sync"
 
 	log "github.com/Sirupsen/logrus"
@@ -31,6 +32,7 @@ import (
 	"github.com/matrix-org/dendrite/mediaapi/config"
 	"github.com/matrix-org/dendrite/mediaapi/fileutils"
 	"github.com/matrix-org/dendrite/mediaapi/storage"
+	"github.com/matrix-org/dendrite/mediaapi/thumbnailer"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -41,31 +43,56 @@ const mediaIDCharacters = "A-Za-z0-9_=-"
 // Note: unfortunately regex.MustCompile() cannot be assigned to a const
 var mediaIDRegex = regexp.MustCompile("[" + mediaIDCharacters + "]+")
 
-// downloadRequest metadata included in or derivable from an download request
+// downloadRequest metadata included in or derivable from a download or thumbnail request
 // https://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-media-r0-download-servername-mediaid
+// http://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-media-r0-thumbnail-servername-mediaid
 type downloadRequest struct {
-	MediaMetadata *types.MediaMetadata
-	Logger        *log.Entry
+	MediaMetadata      *types.MediaMetadata
+	IsThumbnailRequest bool
+	ThumbnailSize      types.ThumbnailSize
+	Logger             *log.Entry
 }
 
-// Download implements /download
+// Download implements /download amd /thumbnail
 // Files from this server (i.e. origin == cfg.ServerName) are served directly
 // Files from remote servers (i.e. origin != cfg.ServerName) are cached locally.
 // If they are present in the cache, they are served directly.
 // If they are not present in the cache, they are obtained from the remote server and
 // simultaneously served back to the client and written into the cache.
-func Download(w http.ResponseWriter, req *http.Request, origin gomatrixserverlib.ServerName, mediaID types.MediaID, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests) {
+func Download(w http.ResponseWriter, req *http.Request, origin gomatrixserverlib.ServerName, mediaID types.MediaID, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, isThumbnailRequest bool) {
 	r := &downloadRequest{
 		MediaMetadata: &types.MediaMetadata{
 			MediaID: mediaID,
 			Origin:  origin,
 		},
+		IsThumbnailRequest: isThumbnailRequest,
 		Logger: util.GetLogger(req.Context()).WithFields(log.Fields{
 			"Origin":  origin,
 			"MediaID": mediaID,
 		}),
 	}
 
+	if r.IsThumbnailRequest {
+		width, err := strconv.Atoi(req.FormValue("width"))
+		if err != nil {
+			width = -1
+		}
+		height, err := strconv.Atoi(req.FormValue("height"))
+		if err != nil {
+			height = -1
+		}
+		r.ThumbnailSize = types.ThumbnailSize{
+			Width:        width,
+			Height:       height,
+			ResizeMethod: strings.ToLower(req.FormValue("method")),
+		}
+		r.Logger.WithFields(log.Fields{
+			"RequestedWidth":        r.ThumbnailSize.Width,
+			"RequestedHeight":       r.ThumbnailSize.Height,
+			"RequestedResizeMethod": r.ThumbnailSize.ResizeMethod,
+		})
+	}
+
 	// request validation
 	if req.Method != "GET" {
 		r.jsonErrorResponse(w, util.JSONResponse{
@@ -80,7 +107,7 @@ func Download(w http.ResponseWriter, req *http.Request, origin gomatrixserverlib
 		return
 	}
 
-	if resErr := r.doDownload(w, cfg, db, activeRemoteRequests); resErr != nil {
+	if resErr := r.doDownload(w, cfg, db, activeRemoteRequests, activeThumbnailGeneration); resErr != nil {
 		r.jsonErrorResponse(w, *resErr)
 		return
 	}
@@ -118,10 +145,29 @@ func (r *downloadRequest) Validate() *util.JSONResponse {
 			JSON: jsonerror.NotFound("serverName must be a non-empty string"),
 		}
 	}
+
+	if r.IsThumbnailRequest {
+		if r.ThumbnailSize.Width <= 0 || r.ThumbnailSize.Height <= 0 {
+			return &util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.Unknown("width and height must be greater than 0"),
+			}
+		}
+		// Default method to scale if not set
+		if r.ThumbnailSize.ResizeMethod == "" {
+			r.ThumbnailSize.ResizeMethod = "scale"
+		}
+		if r.ThumbnailSize.ResizeMethod != "crop" && r.ThumbnailSize.ResizeMethod != "scale" {
+			return &util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.Unknown("method must be one of crop or scale"),
+			}
+		}
+	}
 	return nil
 }
 
-func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests) *util.JSONResponse {
+func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse {
 	// check if we have a record of the media in our database
 	mediaMetadata, err := db.GetMediaMetadata(r.MediaMetadata.MediaID, r.MediaMetadata.Origin)
 	if err != nil {
@@ -138,7 +184,7 @@ func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI
 			}
 		}
 		// If we do not have a record and the origin is remote, we need to fetch it and respond with that file
-		resErr := r.getRemoteFile(cfg, db, activeRemoteRequests)
+		resErr := r.getRemoteFile(cfg, db, activeRemoteRequests, activeThumbnailGeneration)
 		if resErr != nil {
 			return resErr
 		}
@@ -146,12 +192,12 @@ func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI
 		// If we have a record, we can respond from the local file
 		r.MediaMetadata = mediaMetadata
 	}
-	return r.respondFromLocalFile(w, cfg.AbsBasePath)
+	return r.respondFromLocalFile(w, cfg.AbsBasePath, activeThumbnailGeneration, cfg.MaxThumbnailGenerators, db, cfg.DynamicThumbnails, cfg.ThumbnailSizes)
 }
 
 // respondFromLocalFile reads a file from local storage and writes it to the http.ResponseWriter
 // Returns a util.JSONResponse error in case of error
-func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePath types.Path) *util.JSONResponse {
+func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) *util.JSONResponse {
 	filePath, err := fileutils.GetPathFromBase64Hash(r.MediaMetadata.Base64Hash, absBasePath)
 	if err != nil {
 		r.Logger.WithError(err).Error("Failed to get file path from metadata")
@@ -181,15 +227,43 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat
 		return &resErr
 	}
 
-	r.Logger.WithFields(log.Fields{
-		"UploadName":    r.MediaMetadata.UploadName,
-		"Base64Hash":    r.MediaMetadata.Base64Hash,
-		"FileSizeBytes": r.MediaMetadata.FileSizeBytes,
-		"Content-Type":  r.MediaMetadata.ContentType,
-	}).Info("Responding with file")
+	var responseFile *os.File
+	var responseMetadata *types.MediaMetadata
+	if r.IsThumbnailRequest {
+		thumbFile, thumbMetadata, resErr := r.getThumbnailFile(types.Path(filePath), activeThumbnailGeneration, maxThumbnailGenerators, db, dynamicThumbnails, thumbnailSizes)
+		if thumbFile != nil {
+			defer thumbFile.Close()
+		}
+		if resErr != nil {
+			return resErr
+		}
+		if thumbFile == nil {
+			r.Logger.WithFields(log.Fields{
+				"UploadName":    r.MediaMetadata.UploadName,
+				"Base64Hash":    r.MediaMetadata.Base64Hash,
+				"FileSizeBytes": r.MediaMetadata.FileSizeBytes,
+				"ContentType":   r.MediaMetadata.ContentType,
+			}).Info("No good thumbnail found. Responding with original file.")
+			responseFile = file
+			responseMetadata = r.MediaMetadata
+		} else {
+			r.Logger.Info("Responding with thumbnail")
+			responseFile = thumbFile
+			responseMetadata = thumbMetadata.MediaMetadata
+		}
+	} else {
+		r.Logger.WithFields(log.Fields{
+			"UploadName":    r.MediaMetadata.UploadName,
+			"Base64Hash":    r.MediaMetadata.Base64Hash,
+			"FileSizeBytes": r.MediaMetadata.FileSizeBytes,
+			"ContentType":   r.MediaMetadata.ContentType,
+		}).Info("Responding with file")
+		responseFile = file
+		responseMetadata = r.MediaMetadata
+	}
 
-	w.Header().Set("Content-Type", string(r.MediaMetadata.ContentType))
-	w.Header().Set("Content-Length", strconv.FormatInt(int64(r.MediaMetadata.FileSizeBytes), 10))
+	w.Header().Set("Content-Type", string(responseMetadata.ContentType))
+	w.Header().Set("Content-Length", strconv.FormatInt(int64(responseMetadata.FileSizeBytes), 10))
 	contentSecurityPolicy := "default-src 'none';" +
 		" script-src 'none';" +
 		" plugin-types application/pdf;" +
@@ -197,7 +271,7 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat
 		" object-src 'self';"
 	w.Header().Set("Content-Security-Policy", contentSecurityPolicy)
 
-	if bytesResponded, err := io.Copy(w, file); err != nil {
+	if bytesResponded, err := io.Copy(w, responseFile); err != nil {
 		r.Logger.WithError(err).Warn("Failed to copy from cache")
 		if bytesResponded == 0 {
 			resErr := jsonerror.InternalServerError()
@@ -209,12 +283,107 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat
 	return nil
 }
 
+// Note: Thumbnail generation may be ongoing asynchronously.
+func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) (*os.File, *types.ThumbnailMetadata, *util.JSONResponse) {
+	var thumbnail *types.ThumbnailMetadata
+	var resErr *util.JSONResponse
+
+	if dynamicThumbnails {
+		thumbnail, resErr = r.generateThumbnail(filePath, r.ThumbnailSize, activeThumbnailGeneration, maxThumbnailGenerators, db)
+		if resErr != nil {
+			return nil, nil, resErr
+		}
+	}
+	// If dynamicThumbnails is true but there are too many thumbnails being actively generated, we can fall back
+	// to trying to use a pre-generated thumbnail
+	if thumbnail == nil {
+		thumbnails, err := db.GetThumbnails(r.MediaMetadata.MediaID, r.MediaMetadata.Origin)
+		if err != nil {
+			r.Logger.WithError(err).Error("Error looking up thumbnails")
+			resErr := jsonerror.InternalServerError()
+			return nil, nil, &resErr
+		}
+
+		// If we get a thumbnailSize, a pre-generated thumbnail would be best but it is not yet generated.
+		// If we get a thumbnail, we're done.
+		var thumbnailSize *types.ThumbnailSize
+		thumbnail, thumbnailSize = thumbnailer.SelectThumbnail(r.ThumbnailSize, thumbnails, thumbnailSizes)
+		// If dynamicThumbnails is true and we are not over-loaded then we would have generated what was requested above.
+		// So we don't try to generate a pre-generated thumbnail here.
+		if thumbnailSize != nil && dynamicThumbnails == false {
+			r.Logger.WithFields(log.Fields{
+				"Width":        thumbnailSize.Width,
+				"Height":       thumbnailSize.Height,
+				"ResizeMethod": thumbnailSize.ResizeMethod,
+			}).Info("Pre-generating thumbnail for immediate response.")
+			thumbnail, resErr = r.generateThumbnail(filePath, *thumbnailSize, activeThumbnailGeneration, maxThumbnailGenerators, db)
+			if resErr != nil {
+				return nil, nil, resErr
+			}
+		}
+	}
+	if thumbnail == nil {
+		return nil, nil, nil
+	}
+	r.Logger = r.Logger.WithFields(log.Fields{
+		"Width":         thumbnail.ThumbnailSize.Width,
+		"Height":        thumbnail.ThumbnailSize.Height,
+		"ResizeMethod":  thumbnail.ThumbnailSize.ResizeMethod,
+		"FileSizeBytes": thumbnail.MediaMetadata.FileSizeBytes,
+		"ContentType":   thumbnail.MediaMetadata.ContentType,
+	})
+	thumbPath := string(thumbnailer.GetThumbnailPath(types.Path(filePath), thumbnail.ThumbnailSize))
+	thumbFile, err := os.Open(string(thumbPath))
+	if err != nil {
+		r.Logger.WithError(err).Warn("Failed to open file")
+		resErr := jsonerror.InternalServerError()
+		return thumbFile, nil, &resErr
+	}
+	thumbStat, err := thumbFile.Stat()
+	if err != nil {
+		r.Logger.WithError(err).Warn("Failed to stat file")
+		resErr := jsonerror.InternalServerError()
+		return thumbFile, nil, &resErr
+	}
+	if types.FileSizeBytes(thumbStat.Size()) != thumbnail.MediaMetadata.FileSizeBytes {
+		r.Logger.WithError(err).Warn("Thumbnail file sizes on disk and in database differ")
+		resErr := jsonerror.InternalServerError()
+		return thumbFile, nil, &resErr
+	}
+	return thumbFile, thumbnail, nil
+}
+
+func (r *downloadRequest) generateThumbnail(filePath types.Path, thumbnailSize types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database) (*types.ThumbnailMetadata, *util.JSONResponse) {
+	logger := r.Logger.WithFields(log.Fields{
+		"Width":        thumbnailSize.Width,
+		"Height":       thumbnailSize.Height,
+		"ResizeMethod": thumbnailSize.ResizeMethod,
+	})
+	busy, err := thumbnailer.GenerateThumbnail(filePath, thumbnailSize, r.MediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, r.Logger)
+	if err != nil {
+		logger.WithError(err).Error("Error creating thumbnail")
+		resErr := jsonerror.InternalServerError()
+		return nil, &resErr
+	}
+	if busy {
+		return nil, nil
+	}
+	var thumbnail *types.ThumbnailMetadata
+	thumbnail, err = db.GetThumbnail(r.MediaMetadata.MediaID, r.MediaMetadata.Origin, thumbnailSize.Width, thumbnailSize.Height, thumbnailSize.ResizeMethod)
+	if err != nil {
+		logger.WithError(err).Error("Error looking up thumbnails")
+		resErr := jsonerror.InternalServerError()
+		return nil, &resErr
+	}
+	return thumbnail, nil
+}
+
 // getRemoteFile fetches the remote file and caches it locally
 // A hash map of active remote requests to a struct containing a sync.Cond is used to only download remote files once,
 // regardless of how many download requests are received.
 // Note: The named errorResponse return variable is used in a deferred broadcast of the metadata and error response to waiting goroutines.
 // Returns a util.JSONResponse error in case of error
-func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests) (errorResponse *util.JSONResponse) {
+func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) (errorResponse *util.JSONResponse) {
 	// Note: getMediaMetadataFromActiveRequest uses mutexes and conditions from activeRemoteRequests
 	mediaMetadata, resErr := r.getMediaMetadataFromActiveRequest(activeRemoteRequests)
 	if resErr != nil {
@@ -245,7 +414,7 @@ func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Databa
 
 		if mediaMetadata == nil {
 			// If we do not have a record, we need to fetch the remote file first and then respond from the local file
-			resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, cfg.MaxFileSizeBytes, db)
+			resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators)
 			if resErr != nil {
 				return resErr
 			}
@@ -307,7 +476,7 @@ func (r *downloadRequest) broadcastMediaMetadata(activeRemoteRequests *types.Act
 }
 
 // fetchRemoteFileAndStoreMetadata fetches the file from the remote server and stores its metadata in the database
-func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes, db *storage.Database) *util.JSONResponse {
+func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse {
 	finalPath, duplicate, resErr := r.fetchRemoteFile(absBasePath, maxFileSizeBytes)
 	if resErr != nil {
 		return resErr
@@ -317,7 +486,7 @@ func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path
 		"Base64Hash":    r.MediaMetadata.Base64Hash,
 		"UploadName":    r.MediaMetadata.UploadName,
 		"FileSizeBytes": r.MediaMetadata.FileSizeBytes,
-		"Content-Type":  r.MediaMetadata.ContentType,
+		"ContentType":   r.MediaMetadata.ContentType,
 	}).Info("Storing file metadata to media repository database")
 
 	// FIXME: timeout db request
@@ -335,13 +504,21 @@ func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path
 		return &resErr
 	}
 
-	// TODO: generate thumbnails
+	go func() {
+		busy, err := thumbnailer.GenerateThumbnails(finalPath, thumbnailSizes, r.MediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, r.Logger)
+		if err != nil {
+			r.Logger.WithError(err).Warn("Error generating thumbnails")
+		}
+		if busy {
+			r.Logger.Warn("Maximum number of active thumbnail generators reached. Skipping pre-generation.")
+		}
+	}()
 
 	r.Logger.WithFields(log.Fields{
 		"UploadName":    r.MediaMetadata.UploadName,
 		"Base64Hash":    r.MediaMetadata.Base64Hash,
 		"FileSizeBytes": r.MediaMetadata.FileSizeBytes,
-		"Content-Type":  r.MediaMetadata.ContentType,
+		"ContentType":   r.MediaMetadata.ContentType,
 	}).Infof("Remote file cached")
 
 	return nil

src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go

@@ -27,6 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/mediaapi/config"
 	"github.com/matrix-org/dendrite/mediaapi/fileutils"
 	"github.com/matrix-org/dendrite/mediaapi/storage"
+	"github.com/matrix-org/dendrite/mediaapi/thumbnailer"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/util"
 )
@@ -50,13 +51,13 @@ type uploadResponse struct {
 // This implementation supports a configurable maximum file size limit in bytes. If a user tries to upload more than this, they will receive an error that their upload is too large.
 // Uploaded files are processed piece-wise to avoid DoS attacks which would starve the server of memory.
 // TODO: We should time out requests if they have not received any data within a configured timeout period.
-func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database) util.JSONResponse {
+func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse {
 	r, resErr := parseAndValidateRequest(req, cfg)
 	if resErr != nil {
 		return *resErr
 	}
 
-	if resErr = r.doUpload(req.Body, cfg, db); resErr != nil {
+	if resErr = r.doUpload(req.Body, cfg, db, activeThumbnailGeneration); resErr != nil {
 		return *resErr
 	}
 
@@ -89,14 +90,14 @@ func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRe
 		Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.ServerName),
 	}
 
-	if resErr := r.Validate(cfg.MaxFileSizeBytes); resErr != nil {
+	if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil {
 		return nil, resErr
 	}
 
 	return r, nil
 }
 
-func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *storage.Database) *util.JSONResponse {
+func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse {
 	r.Logger.WithFields(log.Fields{
 		"UploadName":    r.MediaMetadata.UploadName,
 		"FileSizeBytes": r.MediaMetadata.FileSizeBytes,
@@ -107,10 +108,10 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *
 	// method of deduplicating files to save storage, as well as a way to conduct
 	// integrity checks on the file data in the repository.
 	// Data is truncated to maxFileSizeBytes. Content-Length was reported as 0 < Content-Length <= maxFileSizeBytes so this is OK.
-	hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, cfg.MaxFileSizeBytes, cfg.AbsBasePath)
+	hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.MaxFileSizeBytes, cfg.AbsBasePath)
 	if err != nil {
 		r.Logger.WithError(err).WithFields(log.Fields{
-			"MaxFileSizeBytes": cfg.MaxFileSizeBytes,
+			"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
 		}).Warn("Error while transferring file")
 		fileutils.RemoveDir(tmpDir, r.Logger)
 		return &util.JSONResponse{
@@ -151,9 +152,7 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *
 		}
 	}
 
-	// TODO: generate thumbnails
-
-	if resErr := r.storeFileAndMetadata(tmpDir, cfg.AbsBasePath, db); resErr != nil {
+	if resErr := r.storeFileAndMetadata(tmpDir, cfg.AbsBasePath, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators); resErr != nil {
 		return resErr
 	}
 
@@ -216,7 +215,7 @@ func (r *uploadRequest) Validate(maxFileSizeBytes types.FileSizeBytes) *util.JSO
 // The order of operations is important as it avoids metadata entering the database before the file
 // is ready, and if we fail to move the file, it never gets added to the database.
 // Returns a util.JSONResponse error and cleans up directories in case of error.
-func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath types.Path, db *storage.Database) *util.JSONResponse {
+func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath types.Path, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse {
 	finalPath, duplicate, err := fileutils.MoveFileWithHashCheck(tmpDir, r.MediaMetadata, absBasePath, r.Logger)
 	if err != nil {
 		r.Logger.WithError(err).Error("Failed to move file.")
@@ -243,5 +242,15 @@ func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath type
 		}
 	}
 
+	go func() {
+		busy, err := thumbnailer.GenerateThumbnails(finalPath, thumbnailSizes, r.MediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, r.Logger)
+		if err != nil {
+			r.Logger.WithError(err).Warn("Error generating thumbnails")
+		}
+		if busy {
+			r.Logger.Warn("Maximum number of active thumbnail generators reached. Skipping pre-generation.")
+		}
+	}()
+
 	return nil
 }