Send-to-device support (#1072)

* Groundwork for send-to-device messaging

* Update sample config

* Add unstable routing for now

* Send to device consumer in sync API

* Start the send-to-device consumer

* fix indentation in dendrite-config.yaml

* Create send-to-device database tables, other tweaks

* Add some logic for send-to-device messages, add them into sync stream

* Handle incoming send-to-device messages, count them with EDU stream pos

* Undo changes to test

* pq.Array

* Fix sync

* Logging

* Fix a couple of transaction things, fix client API

* Add send-to-device test, hopefully fix bugs

* Comments

* Refactor a bit

* Fix schema

* Fix queries

* Debug logging

* Fix storing and retrieving of send-to-device messages

* Try to avoid database locks

* Update sync position

* Use latest sync position

* Jiggle about sync a bit

* Fix tests

* Break out the retrieval from the update/delete behaviour

* Comments

* nolint on getResponseWithPDUsForCompleteSync

* Try to line up sync tokens again

* Implement wildcard

* Add all send-to-device tests to whitelist, what could possibly go wrong?

* Only care about wildcard when targeted locally

* Deduplicate transactions

* Handle tokens properly, return immediately if waiting send-to-device messages

* Fix sync

* Update sytest-whitelist

* Fix copyright notice (need to do more of this)

* Comments, copyrights

* Return errors from Do, fix dendritejs

* Review comments

* Comments

* Constructor for TransactionWriter

* defletions

* Update gomatrixserverlib, sytest-blacklist

clientapi/producers/eduserver.go

@@ -14,6 +14,7 @@ package producers
 
 import (
 	"context"
+	"encoding/json"
 	"time"
 
 	"github.com/matrix-org/dendrite/eduserver/api"
@@ -52,3 +53,28 @@ func (p *EDUServerProducer) SendTyping(
 
 	return err
 }
+
+// SendToDevice sends a typing event to EDU server
+func (p *EDUServerProducer) SendToDevice(
+	ctx context.Context, sender, userID, deviceID, eventType string,
+	message interface{},
+) error {
+	js, err := json.Marshal(message)
+	if err != nil {
+		return err
+	}
+	requestData := api.InputSendToDeviceEvent{
+		UserID:   userID,
+		DeviceID: deviceID,
+		SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{
+			Sender:  sender,
+			Type:    eventType,
+			Content: js,
+		},
+	}
+	request := api.InputSendToDeviceEventRequest{
+		InputSendToDeviceEvent: requestData,
+	}
+	response := api.InputSendToDeviceEventResponse{}
+	return p.InputAPI.InputSendToDeviceEvent(ctx, &request, &response)
+}

clientapi/routing/routing.go

@@ -274,6 +274,31 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
+	r0mux.Handle("/sendToDevice/{eventType}/{txnID}",
+		internal.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			txnID := vars["txnID"]
+			return SendToDevice(req, device, eduProducer, transactionsCache, vars["eventType"], &txnID)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	// This is only here because sytest refers to /unstable for this endpoint
+	// rather than r0. It's an exact duplicate of the above handler.
+	// TODO: Remove this if/when sytest is fixed!
+	unstableMux.Handle("/sendToDevice/{eventType}/{txnID}",
+		internal.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars, err := internal.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			txnID := vars["txnID"]
+			return SendToDevice(req, device, eduProducer, transactionsCache, vars["eventType"], &txnID)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
 	r0mux.Handle("/account/whoami",
 		internal.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
 			return Whoami(req, device)

clientapi/routing/sendtodevice.go

@@ -0,0 +1,70 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/internal/transactions"
+	"github.com/matrix-org/util"
+)
+
+// SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId}
+// sends the device events to the EDU Server
+func SendToDevice(
+	req *http.Request, device *authtypes.Device,
+	eduProducer *producers.EDUServerProducer,
+	txnCache *transactions.Cache,
+	eventType string, txnID *string,
+) util.JSONResponse {
+	if txnID != nil {
+		if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok {
+			return *res
+		}
+	}
+
+	var httpReq struct {
+		Messages map[string]map[string]json.RawMessage `json:"messages"`
+	}
+	resErr := httputil.UnmarshalJSONRequest(req, &httpReq)
+	if resErr != nil {
+		return *resErr
+	}
+
+	for userID, byUser := range httpReq.Messages {
+		for deviceID, message := range byUser {
+			if err := eduProducer.SendToDevice(
+				req.Context(), device.UserID, userID, deviceID, eventType, message,
+			); err != nil {
+				util.GetLogger(req.Context()).WithError(err).Error("eduProducer.SendToDevice failed")
+				return jsonerror.InternalServerError()
+			}
+		}
+	}
+
+	res := util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+
+	if txnID != nil {
+		txnCache.AddTransaction(device.AccessToken, *txnID, &res)
+	}
+
+	return res
+}