Merge branch 'master' into nats

.gitignore

@@ -60,3 +60,6 @@ cmd/dendrite-demo-yggdrasil/embed/fs*.go
 
 # Test dependencies
 test/wasm/node_modules
+
+media_store/
+

appservice/appservice.go

@@ -135,6 +135,7 @@ func generateAppServiceAccount(
 		AccessToken:        as.ASToken,
 		DeviceID:           &as.SenderLocalpart,
 		DeviceDisplayName:  &as.SenderLocalpart,
+		NoDeviceListUpdate: true,
 	}, &devRes)
 	return err
 }

build/docker/docker-compose.deps.yml

@@ -3,7 +3,7 @@ services:
   # PostgreSQL is needed for both polylith and monolith modes.
   postgres:
     hostname: postgres
-    image: postgres:11
+    image: postgres:14
     restart: always
     volumes:
       - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh

build/gobind-pinecone/monolith.go

@@ -13,6 +13,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -87,15 +88,15 @@ func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
 		m.PineconeMulticast.Start()
 	} else {
 		m.PineconeMulticast.Stop()
-		m.DisconnectType(pineconeRouter.PeerTypeMulticast)
+		m.DisconnectType(int(pineconeRouter.PeerTypeMulticast))
 	}
 }
 
 func (m *DendriteMonolith) SetStaticPeer(uri string) {
 	m.staticPeerMutex.Lock()
-	m.staticPeerURI = uri
+	m.staticPeerURI = strings.TrimSpace(uri)
 	m.staticPeerMutex.Unlock()
-	m.DisconnectType(pineconeRouter.PeerTypeRemote)
+	m.DisconnectType(int(pineconeRouter.PeerTypeRemote))
 	if uri != "" {
 		go func() {
 			m.staticPeerAttempt <- struct{}{}
@@ -105,7 +106,7 @@ func (m *DendriteMonolith) SetStaticPeer(uri string) {
 
 func (m *DendriteMonolith) DisconnectType(peertype int) {
 	for _, p := range m.PineconeRouter.Peers() {
-		if peertype == p.PeerType {
+		if int(peertype) == p.PeerType {
 			m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil)
 		}
 	}
@@ -133,7 +134,11 @@ func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error)
 		for i := 1; i <= 10; i++ {
 			logrus.Errorf("Attempting authenticated connect (attempt %d)", i)
 			var err error
-			conduit.port, err = m.PineconeRouter.AuthenticatedConnect(l, zone, peertype, true)
+			conduit.port, err = m.PineconeRouter.Connect(
+				l,
+				pineconeRouter.ConnectionZone(zone),
+				pineconeRouter.ConnectionPeerType(peertype),
+			)
 			switch err {
 			case io.ErrClosedPipe:
 				logrus.Errorf("Authenticated connect failed due to closed pipe (attempt %d)", i)
@@ -195,19 +200,31 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e
 }
 
 func (m *DendriteMonolith) staticPeerConnect() {
+	connected := map[string]bool{} // URI -> connected?
 	attempt := func() {
-		if m.PineconeRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 {
 		m.staticPeerMutex.RLock()
 		uri := m.staticPeerURI
 		m.staticPeerMutex.RUnlock()
 		if uri == "" {
 			return
 		}
-			if err := conn.ConnectToPeer(m.PineconeRouter, uri); err != nil {
+		for k := range connected {
+			delete(connected, k)
+		}
+		for _, uri := range strings.Split(uri, ",") {
+			connected[strings.TrimSpace(uri)] = false
+		}
+		for _, info := range m.PineconeRouter.Peers() {
+			connected[info.URI] = true
+		}
+		for k, online := range connected {
+			if !online {
+				if err := conn.ConnectToPeer(m.PineconeRouter, k); err != nil {
 					logrus.WithError(err).Error("Failed to connect to static peer")
 				}
 			}
 		}
+	}
 	for {
 		select {
 		case <-m.processContext.Context().Done():
@@ -271,7 +288,7 @@ func (m *DendriteMonolith) Start() {
 	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-syncapi.db", m.StorageDirectory, prefix))
 	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-roomserver.db", m.StorageDirectory, prefix))
 	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-keyserver.db", m.StorageDirectory, prefix))
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationapi.db", m.StorageDirectory, prefix))
+	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationsender.db", m.StorageDirectory, prefix))
 	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-appservice.db", m.StorageDirectory, prefix))
 	cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
 	cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))

build/gobind-yggdrasil/monolith.go

@@ -93,7 +93,7 @@ func (m *DendriteMonolith) Start() {
 	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-syncapi.db", m.StorageDirectory))
 	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-roomserver.db", m.StorageDirectory))
 	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-keyserver.db", m.StorageDirectory))
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-federationapi.db", m.StorageDirectory))
+	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-federationsender.db", m.StorageDirectory))
 	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-appservice.db", m.StorageDirectory))
 	cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory))
 	cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory))

clientapi/auth/auth.go

@@ -70,11 +70,11 @@ func VerifyUserFromRequest(
 		jsonErr := jsonerror.InternalServerError()
 		return nil, &jsonErr
 	}
-	if res.Err != nil {
+	if res.Err != "" {
-		if forbidden, ok := res.Err.(*api.ErrorForbidden); ok {
+		if strings.HasPrefix(strings.ToLower(res.Err), "forbidden:") { // TODO: use actual error and no string comparison
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
-				JSON: jsonerror.Forbidden(forbidden.Message),
+				JSON: jsonerror.Forbidden(res.Err),
 			}
 		}
 	}

clientapi/auth/password.go

@@ -17,6 +17,7 @@ package auth
 import (
 	"context"
 	"net/http"
+	"strings"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
@@ -48,7 +49,8 @@ func (t *LoginTypePassword) Request() interface{} {
 
 func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
 	r := req.(*PasswordRequest)
-	username := r.Username()
+	// Squash username to all lowercase letters
+	username := strings.ToLower(r.Username())
 	if username == "" {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,

clientapi/routing/key_backup.go

@@ -62,12 +62,14 @@ func CreateKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI,
 		return *resErr
 	}
 	var performKeyBackupResp userapi.PerformKeyBackupResponse
-	userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
+	if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
 		UserID:    device.UserID,
 		Version:   "",
 		AuthData:  kb.AuthData,
 		Algorithm: kb.Algorithm,
-	}, &performKeyBackupResp)
+	}, &performKeyBackupResp); err != nil {
+		return jsonerror.InternalServerError()
+	}
 	if performKeyBackupResp.Error != "" {
 		if performKeyBackupResp.BadInput {
 			return util.JSONResponse{
@@ -123,12 +125,14 @@ func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.UserInter
 		return *resErr
 	}
 	var performKeyBackupResp userapi.PerformKeyBackupResponse
-	userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
+	if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
 		UserID:    device.UserID,
 		Version:   version,
 		AuthData:  kb.AuthData,
 		Algorithm: kb.Algorithm,
-	}, &performKeyBackupResp)
+	}, &performKeyBackupResp); err != nil {
+		return jsonerror.InternalServerError()
+	}
 	if performKeyBackupResp.Error != "" {
 		if performKeyBackupResp.BadInput {
 			return util.JSONResponse{
@@ -157,11 +161,13 @@ func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.UserInter
 // Implements DELETE  /_matrix/client/r0/room_keys/version/{version}
 func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse {
 	var performKeyBackupResp userapi.PerformKeyBackupResponse
-	userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
+	if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
 		UserID:       device.UserID,
 		Version:      version,
 		DeleteBackup: true,
-	}, &performKeyBackupResp)
+	}, &performKeyBackupResp); err != nil {
+		return jsonerror.InternalServerError()
+	}
 	if performKeyBackupResp.Error != "" {
 		if performKeyBackupResp.BadInput {
 			return util.JSONResponse{
@@ -191,11 +197,13 @@ func UploadBackupKeys(
 	req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string, keys *keyBackupSessionRequest,
 ) util.JSONResponse {
 	var performKeyBackupResp userapi.PerformKeyBackupResponse
-	userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
+	if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
 		UserID:  device.UserID,
 		Version: version,
 		Keys:    *keys,
-	}, &performKeyBackupResp)
+	}, &performKeyBackupResp); err != nil && performKeyBackupResp.Error == "" {
+		return jsonerror.InternalServerError()
+	}
 	if performKeyBackupResp.Error != "" {
 		if performKeyBackupResp.BadInput {
 			return util.JSONResponse{

clientapi/routing/routing.go

@@ -61,7 +61,7 @@ func Setup(
 	extRoomsProvider api.ExtraPublicRoomsProvider,
 	mscCfg *config.MSCs,
 ) {
-	rateLimits := newRateLimits(&cfg.RateLimiting)
+	rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
 	userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg)
 
 	unstableFeatures := map[string]bool{
@@ -127,7 +127,7 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/join/{roomIDOrAlias}",
 		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -143,7 +143,7 @@ func Setup(
 	if mscCfg.Enabled("msc2753") {
 		r0mux.Handle("/peek/{roomIDOrAlias}",
 			httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-				if r := rateLimits.rateLimit(req); r != nil {
+				if r := rateLimits.Limit(req); r != nil {
 					return *r
 				}
 				vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -163,7 +163,7 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/join",
 		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -177,7 +177,7 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/leave",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -211,7 +211,7 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/invite",
 		httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -329,14 +329,14 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
-		if r := rateLimits.rateLimit(req); r != nil {
+		if r := rateLimits.Limit(req); r != nil {
 			return *r
 		}
 		return Register(req, userAPI, accountDB, cfg)
 	})).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
-		if r := rateLimits.rateLimit(req); r != nil {
+		if r := rateLimits.Limit(req); r != nil {
 			return *r
 		}
 		return RegisterAvailable(req, cfg, accountDB)
@@ -410,7 +410,7 @@ func Setup(
 
 	r0mux.Handle("/rooms/{roomID}/typing/{userID}",
 		httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -466,7 +466,7 @@ func Setup(
 
 	r0mux.Handle("/account/whoami",
 		httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			return Whoami(req, device)
@@ -475,7 +475,7 @@ func Setup(
 
 	r0mux.Handle("/account/password",
 		httputil.MakeAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			return Password(req, userAPI, accountDB, device, cfg)
@@ -484,7 +484,7 @@ func Setup(
 
 	r0mux.Handle("/account/deactivate",
 		httputil.MakeAuthAPI("deactivate", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			return Deactivate(req, userInteractiveAuth, userAPI, device)
@@ -495,7 +495,7 @@ func Setup(
 
 	r0mux.Handle("/login",
 		httputil.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			return Login(req, accountDB, userAPI, cfg)
@@ -552,7 +552,7 @@ func Setup(
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
 		httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -577,7 +577,7 @@ func Setup(
 
 	r0mux.Handle("/profile/{userID}/displayname",
 		httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -617,7 +617,7 @@ func Setup(
 	// Element logs get flooded unless this is handled
 	r0mux.Handle("/presence/{userID}/status",
 		httputil.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			// TODO: Set presence (probably the responsibility of a presence server not clientapi)
@@ -630,7 +630,7 @@ func Setup(
 
 	r0mux.Handle("/voip/turnServer",
 		httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			return RequestTurnServer(req, device, cfg)
@@ -709,7 +709,7 @@ func Setup(
 
 	r0mux.Handle("/user/{userID}/openid/request_token",
 		httputil.MakeAuthAPI("openid_request_token", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -722,7 +722,7 @@ func Setup(
 
 	r0mux.Handle("/user_directory/search",
 		httputil.MakeAuthAPI("userdirectory_search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			postContent := struct {
@@ -767,7 +767,7 @@ func Setup(
 
 	r0mux.Handle("/rooms/{roomID}/read_markers",
 		httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -780,7 +780,7 @@ func Setup(
 
 	r0mux.Handle("/rooms/{roomID}/forget",
 		httputil.MakeAuthAPI("rooms_forget", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@@ -884,7 +884,7 @@ func Setup(
 
 	r0mux.Handle("/capabilities",
 		httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			return GetCapabilities(req, rsAPI)
@@ -1100,7 +1100,7 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",
 		httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-			if r := rateLimits.rateLimit(req); r != nil {
+			if r := rateLimits.Limit(req); r != nil {
 				return *r
 			}
 			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))

cmd/dendrite-demo-pinecone/conn/client.go

@@ -34,7 +34,12 @@ func ConnectToPeer(pRouter *pineconeRouter.Router, peer string) error {
 	if parent == nil {
 		return fmt.Errorf("failed to wrap connection")
 	}
-	_, err := pRouter.AuthenticatedConnect(parent, "static", pineconeRouter.PeerTypeRemote, true)
+	_, err := pRouter.Connect(
+		parent,
+		pineconeRouter.ConnectionZone("static"),
+		pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote),
+		pineconeRouter.ConnectionURI(peer),
+	)
 	return err
 }
 

cmd/dendrite-demo-pinecone/main.go

@@ -26,6 +26,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -61,7 +62,7 @@ import (
 var (
 	instanceName   = flag.String("name", "dendrite-p2p-pinecone", "the name of this P2P demo instance")
 	instancePort   = flag.Int("port", 8008, "the port that the client API will listen on")
-	instancePeer   = flag.String("peer", "", "the static Pinecone peer to connect to")
+	instancePeer   = flag.String("peer", "", "the static Pinecone peers to connect to, comma separated-list")
 	instanceListen = flag.String("listen", ":0", "the port Pinecone peers can connect to")
 )
 
@@ -109,9 +110,12 @@ func main() {
 				continue
 			}
 
-			port, err := pRouter.AuthenticatedConnect(conn, "", pineconeRouter.PeerTypeRemote, true)
+			port, err := pRouter.Connect(
+				conn,
+				pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote),
+			)
 			if err != nil {
-				logrus.WithError(err).Error("pSwitch.AuthenticatedConnect failed")
+				logrus.WithError(err).Error("pSwitch.Connect failed")
 				continue
 			}
 
@@ -124,17 +128,25 @@ func main() {
 	pMulticast.Start()
 
 	connectToStaticPeer := func() {
-		attempt := func() {
+		connected := map[string]bool{} // URI -> connected?
-			if pRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 {
+		for _, uri := range strings.Split(*instancePeer, ",") {
-				uri := *instancePeer
+			connected[strings.TrimSpace(uri)] = false
-				if uri == "" {
-					return
 		}
-				if err := conn.ConnectToPeer(pRouter, uri); err != nil {
+		attempt := func() {
+			for k := range connected {
+				connected[k] = false
+			}
+			for _, info := range pRouter.Peers() {
+				connected[info.URI] = true
+			}
+			for k, online := range connected {
+				if !online {
+					if err := conn.ConnectToPeer(pRouter, k); err != nil {
 						logrus.WithError(err).Error("Failed to connect to static peer")
 					}
 				}
 			}
+		}
 		for {
 			attempt()
 			time.Sleep(time.Second * 5)
@@ -229,7 +241,11 @@ func main() {
 			return
 		}
 		conn := conn.WrapWebSocketConn(c)
-		if _, err = pRouter.AuthenticatedConnect(conn, "websocket", pineconeRouter.PeerTypeRemote, true); err != nil {
+		if _, err = pRouter.Connect(
+			conn,
+			pineconeRouter.ConnectionZone("websocket"),
+			pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote),
+		); err != nil {
 			logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch")
 		}
 	})

cmd/dendrite-monolith-server/main.go

@@ -67,6 +67,7 @@ func main() {
 		cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr
 		cfg.RoomServer.InternalAPI.Connect = httpAPIAddr
 		cfg.SyncAPI.InternalAPI.Connect = httpAPIAddr
+		cfg.UserAPI.InternalAPI.Connect = httpAPIAddr
 		options = append(options, basepkg.UseHTTPAPIs)
 	}
 
@@ -102,20 +103,41 @@ func main() {
 	// This is different to rsAPI which can be the http client which doesn't need this dependency
 	rsImpl.SetFederationAPI(fsAPI)
 
-	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
+	keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
-	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
+	keyAPI := keyImpl
-	keyAPI.SetUserAPI(userAPI)
+	if base.UseHTTPAPIs {
+		keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI)
+		keyAPI = base.KeyServerHTTPClient()
+	}
+
+	userImpl := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
+	userAPI := userImpl
+	if base.UseHTTPAPIs {
+		userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
+		userAPI = base.UserAPIClient()
+	}
 	if traceInternal {
 		userAPI = &uapi.UserInternalAPITrace{
 			Impl: userAPI,
 		}
 	}
-	// needs to be after the SetUserAPI call above
+
+	// TODO: This should use userAPI, not userImpl, but the appservice setup races with
+	// the listeners and panics at startup if it tries to create appservice accounts
+	// before the listeners are up.
+	asAPI := appservice.NewInternalAPI(base, userImpl, rsAPI)
 	if base.UseHTTPAPIs {
-		keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI)
+		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
-		keyAPI = base.KeyServerHTTPClient()
+		asAPI = base.AppserviceHTTPClient()
 	}
 
+	// The underlying roomserver implementation needs to be able to call the fedsender.
+	// This is different to rsAPI which can be the http client which doesn't need this
+	// dependency. Other components also need updating after their dependencies are up.
+	rsImpl.SetFederationAPI(fsAPI)
+	rsImpl.SetAppserviceAPI(asAPI)
+	keyImpl.SetUserAPI(userAPI)
+
 	eduInputAPI := eduserver.NewInternalAPI(
 		base, cache.New(), userAPI,
 	)
@@ -124,13 +146,6 @@ func main() {
 		eduInputAPI = base.EDUServerClient()
 	}
 
-	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-	if base.UseHTTPAPIs {
-		appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
-		asAPI = base.AppserviceHTTPClient()
-	}
-	rsAPI.SetAppserviceAPI(asAPI)
-
 	monolith := setup.Monolith{
 		Config:    base.Cfg,
 		AccountDB: accountDB,

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -35,6 +35,9 @@ func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 		&base.Cfg.MSCs, nil,
 	)
 
+	intAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, true)
+	federationapi.AddInternalRoutes(base.InternalAPIMux, intAPI)
+
 	base.SetupAndServeHTTP(
 		base.Cfg.FederationAPI.InternalAPI.Listen,
 		base.Cfg.FederationAPI.ExternalAPI.Listen,

cmd/dendrite-polylith-multi/personalities/mediaapi.go

@@ -24,7 +24,7 @@ func MediaAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
 	userAPI := base.UserAPIClient()
 	client := base.CreateClient()
 
-	mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, userAPI, client)
+	mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, &base.Cfg.ClientAPI.RateLimiting, userAPI, client)
 
 	base.SetupAndServeHTTP(
 		base.Cfg.MediaAPI.InternalAPI.Listen,

go.mod

@@ -40,7 +40,7 @@ require (
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2
-	github.com/matrix-org/pinecone v0.0.0-20211116111603-febf3501584d
+	github.com/matrix-org/pinecone v0.0.0-20211129130654-b0bf9ad6f5c7
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.8
 	github.com/morikuni/aec v1.0.0 // indirect

go.sum

@@ -990,8 +990,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2 h1:RFsBN3509Ql6NJ7TDVkcKoN3bb/tmqUqzur5c0AwIHQ=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc=
-github.com/matrix-org/pinecone v0.0.0-20211116111603-febf3501584d h1:V1b6GZVvL95qTkjYSEWH9Pja6c0WcJKBt2MlAILlw+Q=
+github.com/matrix-org/pinecone v0.0.0-20211129130654-b0bf9ad6f5c7 h1:HC1TdU79ly+sxtyOuNDIpG2YBHQyGmvLux4VPQbT72I=
-github.com/matrix-org/pinecone v0.0.0-20211116111603-febf3501584d/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk=
+github.com/matrix-org/pinecone v0.0.0-20211129130654-b0bf9ad6f5c7/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

internal/httputil/http.go

@@ -23,6 +23,7 @@ import (
 	"net/url"
 	"strings"
 
+	"github.com/matrix-org/dendrite/userapi/api"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
 )
@@ -72,6 +73,9 @@ func PostJSON(
 		var errorBody struct {
 			Message string `json:"message"`
 		}
+		if _, ok := response.(*api.PerformKeyBackupResponse); ok { // TODO: remove this, once cross-boundary errors are a thing
+			return nil
+		}
 		if msgerr := json.NewDecoder(res.Body).Decode(&errorBody); msgerr == nil {
 			return fmt.Errorf("internal API: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message)
 		}

internal/httputil/rate_limiting.go

@@ -1,4 +1,4 @@
-package routing
+package httputil
 
 import (
 	"net/http"
@@ -10,7 +10,7 @@ import (
 	"github.com/matrix-org/util"
 )
 
-type rateLimits struct {
+type RateLimits struct {
 	limits           map[string]chan struct{}
 	limitsMutex      sync.RWMutex
 	cleanMutex       sync.RWMutex
@@ -19,8 +19,8 @@ type rateLimits struct {
 	cooloffDuration  time.Duration
 }
 
-func newRateLimits(cfg *config.RateLimiting) *rateLimits {
+func NewRateLimits(cfg *config.RateLimiting) *RateLimits {
-	l := &rateLimits{
+	l := &RateLimits{
 		limits:           make(map[string]chan struct{}),
 		enabled:          cfg.Enabled,
 		requestThreshold: cfg.Threshold,
@@ -32,7 +32,7 @@ func newRateLimits(cfg *config.RateLimiting) *rateLimits {
 	return l
 }
 
-func (l *rateLimits) clean() {
+func (l *RateLimits) clean() {
 	for {
 		// On a 30 second interval, we'll take an exclusive write
 		// lock of the entire map and see if any of the channels are
@@ -52,7 +52,7 @@ func (l *rateLimits) clean() {
 	}
 }
 
-func (l *rateLimits) rateLimit(req *http.Request) *util.JSONResponse {
+func (l *RateLimits) Limit(req *http.Request) *util.JSONResponse {
 	// If rate limiting is disabled then do nothing.
 	if !l.enabled {
 		return nil

mediaapi/mediaapi.go

@@ -26,7 +26,9 @@ import (
 
 // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component.
 func AddPublicRoutes(
-	router *mux.Router, cfg *config.MediaAPI,
+	router *mux.Router,
+	cfg *config.MediaAPI,
+	rateLimit *config.RateLimiting,
 	userAPI userapi.UserInternalAPI,
 	client *gomatrixserverlib.Client,
 ) {
@@ -36,6 +38,6 @@ func AddPublicRoutes(
 	}
 
 	routing.Setup(
-		router, cfg, mediaDB, userAPI, client,
+		router, cfg, rateLimit, mediaDB, userAPI, client,
 	)
 }

mediaapi/routing/routing.go

@@ -15,16 +15,16 @@
 package routing
 
 import (
+	"encoding/json"
 	"net/http"
 	"strings"
 
-	userapi "github.com/matrix-org/dendrite/userapi/api"
-
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/dendrite/mediaapi/storage"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/dendrite/setup/config"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/prometheus/client_golang/prometheus"
@@ -32,6 +32,12 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
+// configResponse is the response to GET /_matrix/media/r0/config
+// https://matrix.org/docs/spec/client_server/latest#get-matrix-media-r0-config
+type configResponse struct {
+	UploadSize config.FileSizeBytes `json:"m.upload.size"`
+}
+
 // Setup registers the media API HTTP handlers
 //
 // Due to Setup being used to call many other functions, a gocyclo nolint is
@@ -40,10 +46,13 @@ import (
 func Setup(
 	publicAPIMux *mux.Router,
 	cfg *config.MediaAPI,
+	rateLimit *config.RateLimiting,
 	db storage.Database,
 	userAPI userapi.UserInternalAPI,
 	client *gomatrixserverlib.Client,
 ) {
+	rateLimits := httputil.NewRateLimits(rateLimit)
+
 	r0mux := publicAPIMux.PathPrefix("/r0").Subrouter()
 	v1mux := publicAPIMux.PathPrefix("/v1").Subrouter()
 
@@ -54,31 +63,46 @@ func Setup(
 	uploadHandler := httputil.MakeAuthAPI(
 		"upload", userAPI,
 		func(req *http.Request, dev *userapi.Device) util.JSONResponse {
+			if r := rateLimits.Limit(req); r != nil {
+				return *r
+			}
 			return Upload(req, cfg, dev, db, activeThumbnailGeneration)
 		},
 	)
 
+	configHandler := httputil.MakeAuthAPI("config", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		if r := rateLimits.Limit(req); r != nil {
+			return *r
+		}
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: configResponse{UploadSize: *cfg.MaxFileSizeBytes},
+		}
+	})
+
 	r0mux.Handle("/upload", uploadHandler).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/config", configHandler).Methods(http.MethodGet, http.MethodOptions)
 	v1mux.Handle("/upload", uploadHandler).Methods(http.MethodPost, http.MethodOptions)
 
 	activeRemoteRequests := &types.ActiveRemoteRequests{
 		MXCToResult: map[string]*types.RemoteRequestResult{},
 	}
 
-	downloadHandler := makeDownloadAPI("download", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration)
+	downloadHandler := makeDownloadAPI("download", cfg, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration)
 	r0mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions)
 	r0mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions)
 	v1mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions)                // TODO: remove when synapse is fixed
 	v1mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed
 
 	r0mux.Handle("/thumbnail/{serverName}/{mediaId}",
-		makeDownloadAPI("thumbnail", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration),
+		makeDownloadAPI("thumbnail", cfg, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration),
 	).Methods(http.MethodGet, http.MethodOptions)
 }
 
 func makeDownloadAPI(
 	name string,
 	cfg *config.MediaAPI,
+	rateLimits *httputil.RateLimits,
 	db storage.Database,
 	client *gomatrixserverlib.Client,
 	activeRemoteRequests *types.ActiveRemoteRequests,
@@ -99,6 +123,16 @@ func makeDownloadAPI(
 		// Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors
 		w.Header().Set("Content-Type", "application/json")
 
+		// Ratelimit requests
+		if r := rateLimits.Limit(req); r != nil {
+			if err := json.NewEncoder(w).Encode(r); err != nil {
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+			w.WriteHeader(http.StatusTooManyRequests)
+			return
+		}
+
 		vars, _ := httputil.URLDecodeMapValues(mux.Vars(req))
 		serverName := gomatrixserverlib.ServerName(vars["serverName"])
 

setup/base/base.go

@@ -21,7 +21,6 @@ import (
 	"io"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
@@ -79,7 +78,6 @@ type BaseDendrite struct {
 	SynapseAdminMux        *mux.Router
 	UseHTTPAPIs            bool
 	apiHttpClient          *http.Client
-	httpClient             *http.Client
 	Cfg                    *config.Dendrite
 	Caches                 *caching.Caches
 	DNSCache               *gomatrixserverlib.DNSCache
@@ -181,13 +179,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 			},
 		},
 	}
-	client := http.Client{Timeout: HTTPClientTimeout}
-	if cfg.FederationAPI.Proxy.Enabled {
-		client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{
-			Scheme: cfg.FederationAPI.Proxy.Protocol,
-			Host:   fmt.Sprintf("%s:%d", cfg.FederationAPI.Proxy.Host, cfg.FederationAPI.Proxy.Port),
-		})}
-	}
 
 	// Ideally we would only use SkipClean on routes which we know can allow '/' but due to
 	// https://github.com/gorilla/mux/issues/460 we have to attach this at the top router.
@@ -217,7 +208,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
 		InternalAPIMux:         mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(),
 		SynapseAdminMux:        mux.NewRouter().SkipClean(true).PathPrefix("/_synapse/").Subrouter().UseEncodedPath(),
 		apiHttpClient:          &apiClient,
-		httpClient:             &client,
 	}
 }
 

setup/monolith.go

@@ -68,7 +68,7 @@ func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ss
 		m.KeyRing, m.RoomserverAPI, m.FederationAPI,
 		m.EDUInternalAPI, m.KeyAPI, &m.Config.MSCs, nil,
 	)
-	mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, m.UserAPI, m.Client)
+	mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, &m.Config.ClientAPI.RateLimiting, m.UserAPI, m.Client)
 	syncapi.AddPublicRoutes(
 		process, csMux, m.UserAPI, m.RoomserverAPI,
 		m.KeyAPI, m.FedClient, &m.Config.SyncAPI,

setup/mscs/msc2836/msc2836_test.go

@@ -7,7 +7,6 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"sort"
@@ -504,7 +503,7 @@ type testUserAPI struct {
 func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error {
 	dev, ok := u.accessTokens[req.AccessToken]
 	if !ok {
-		res.Err = fmt.Errorf("unknown token")
+		res.Err = "unknown token"
 		return nil
 	}
 	res.Device = &dev

setup/mscs/msc2946/msc2946_test.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"crypto/ed25519"
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -347,7 +346,7 @@ type testUserAPI struct {
 func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error {
 	dev, ok := u.accessTokens[req.AccessToken]
 	if !ok {
-		res.Err = fmt.Errorf("unknown token")
+		res.Err = "unknown token"
 		return nil
 	}
 	res.Device = &dev

sytest-whitelist

@@ -556,6 +556,7 @@ can fetch self-signing keys over federation
 Changing master key notifies local users
 Changing user-signing key notifies local users
 Inbound federation correctly handles soft failed events as extremities
+Can read configuration endpoint
 User can create and send/receive messages in a room with version 7
 local user can join room with version 7
 User can invite local user to room with version 7

userapi/api/api.go

@@ -33,7 +33,7 @@ type UserInternalAPI interface {
 	PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error
 	PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error
 	PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error
-	PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse)
+	PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error
 	QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse)
 	QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error
 	QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error
@@ -181,7 +181,7 @@ type QueryAccessTokenRequest struct {
 // QueryAccessTokenResponse is the response for QueryAccessToken
 type QueryAccessTokenResponse struct {
 	Device *Device
-	Err    error // e.g ErrorForbidden
+	Err    string // e.g ErrorForbidden
 }
 
 // QueryAccountDataRequest is the request for QueryAccountData
@@ -290,6 +290,10 @@ type PerformDeviceCreationRequest struct {
 	IPAddr string
 	// Useragent for this device
 	UserAgent string
+	// NoDeviceListUpdate determines whether we should avoid sending a device list
+	// update for this account. Generally the only reason to do this is if the account
+	// is an appservice account.
+	NoDeviceListUpdate bool
 }
 
 // PerformDeviceCreationResponse is the response for PerformDeviceCreation

userapi/api/api_trace.go

@@ -74,11 +74,14 @@ func (t *UserInternalAPITrace) PerformOpenIDTokenCreation(ctx context.Context, r
 	util.GetLogger(ctx).Infof("PerformOpenIDTokenCreation req=%+v res=%+v", js(req), js(res))
 	return err
 }
-func (t *UserInternalAPITrace) PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) {
+func (t *UserInternalAPITrace) PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error {
-	t.Impl.PerformKeyBackup(ctx, req, res)
+	err := t.Impl.PerformKeyBackup(ctx, req, res)
+	util.GetLogger(ctx).Infof("PerformKeyBackup req=%+v res=%+v", js(req), js(res))
+	return err
 }
 func (t *UserInternalAPITrace) QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse) {
 	t.Impl.QueryKeyBackup(ctx, req, res)
+	util.GetLogger(ctx).Infof("QueryKeyBackup req=%+v res=%+v", js(req), js(res))
 }
 func (t *UserInternalAPITrace) QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error {
 	err := t.Impl.QueryProfile(ctx, req, res)

userapi/internal/api.go

@@ -119,6 +119,9 @@ func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.Pe
 	}
 	res.DeviceCreated = true
 	res.Device = dev
+	if req.NoDeviceListUpdate {
+		return nil
+	}
 	// create empty device keys and upload them to trigger device list changes
 	return a.deviceListUpdate(dev.UserID, []string{dev.ID})
 }
@@ -358,8 +361,11 @@ func (a *UserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAc
 func (a *UserInternalAPI) QueryAccessToken(ctx context.Context, req *api.QueryAccessTokenRequest, res *api.QueryAccessTokenResponse) error {
 	if req.AppServiceUserID != "" {
 		appServiceDevice, err := a.queryAppServiceToken(ctx, req.AccessToken, req.AppServiceUserID)
+		if err != nil {
+			res.Err = err.Error()
+		}
 		res.Device = appServiceDevice
-		res.Err = err
+
 		return nil
 	}
 	device, err := a.DeviceDB.GetDeviceByAccessToken(ctx, req.AccessToken)
@@ -455,13 +461,16 @@ func (a *UserInternalAPI) QueryOpenIDToken(ctx context.Context, req *api.QueryOp
 	return nil
 }
 
-func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) {
+func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) error {
 	// Delete metadata
 	if req.DeleteBackup {
 		if req.Version == "" {
 			res.BadInput = true
 			res.Error = "must specify a version to delete"
-			return
+			if res.Error != "" {
+				return fmt.Errorf(res.Error)
+			}
+			return nil
 		}
 		exists, err := a.AccountDB.DeleteKeyBackup(ctx, req.UserID, req.Version)
 		if err != nil {
@@ -469,7 +478,10 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform
 		}
 		res.Exists = exists
 		res.Version = req.Version
-		return
+		if res.Error != "" {
+			return fmt.Errorf(res.Error)
+		}
+		return nil
 	}
 	// Create metadata
 	if req.Version == "" {
@@ -479,7 +491,10 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform
 		}
 		res.Exists = err == nil
 		res.Version = version
-		return
+		if res.Error != "" {
+			return fmt.Errorf(res.Error)
+		}
+		return nil
 	}
 	// Update metadata
 	if len(req.Keys.Rooms) == 0 {
@@ -489,10 +504,17 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform
 		}
 		res.Exists = err == nil
 		res.Version = req.Version
-		return
+		if res.Error != "" {
+			return fmt.Errorf(res.Error)
+		}
+		return nil
 	}
 	// Upload Keys for a specific version metadata
 	a.uploadBackupKeys(ctx, req, res)
+	if res.Error != "" {
+		return fmt.Errorf(res.Error)
+	}
+	return nil
 }
 
 func (a *UserInternalAPI) uploadBackupKeys(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) {

userapi/inthttp/client.go

@@ -228,7 +228,7 @@ func (h *httpUserInternalAPI) QueryOpenIDToken(ctx context.Context, req *api.Que
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
 }
 
-func (h *httpUserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) {
+func (h *httpUserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) error {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformKeyBackup")
 	defer span.Finish()
 
@@ -237,6 +237,7 @@ func (h *httpUserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Per
 	if err != nil {
 		res.Error = err.Error()
 	}
+	return nil
 }
 func (h *httpUserInternalAPI) QueryKeyBackup(ctx context.Context, req *api.QueryKeyBackupRequest, res *api.QueryKeyBackupResponse) {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryKeyBackup")

userapi/inthttp/server.go

@@ -16,6 +16,7 @@ package inthttp
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -234,4 +235,32 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(QueryKeyBackupPath,
+		httputil.MakeInternalAPI("queryKeyBackup", func(req *http.Request) util.JSONResponse {
+			request := api.QueryKeyBackupRequest{}
+			response := api.QueryKeyBackupResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			s.QueryKeyBackup(req.Context(), &request, &response)
+			if response.Error != "" {
+				return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", response.Error))
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	internalAPIMux.Handle(PerformKeyBackupPath,
+		httputil.MakeInternalAPI("performKeyBackup", func(req *http.Request) util.JSONResponse {
+			request := api.PerformKeyBackupRequest{}
+			response := api.PerformKeyBackupResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			err := s.PerformKeyBackup(req.Context(), &request, &response)
+			if err != nil {
+				return util.JSONResponse{Code: http.StatusBadRequest, JSON: &response}
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }