More invite support (#979)

* Update gomatixserverlib

* Try to build invite stripped state if not given to us

* SendInvite improvements

* Transpose invite_room_state into invite_state.events for sync API

* Remove syncapi debugging output

* Use RespInviteV2

* Update gomatrixserverlib

* Send the invite event as a normal roomserver event too, for incorporating into room (should this be done by the roomserver automatically for invite inputs?)

* Federation sender use invite_room_state, room server try to insert membership state

* Check supported room versions on the invite endpoint

* Prevent roomserver query API from trying to handle requests for stub rooms

* Adding a nolint

* Replace IsRoomStub with RoomNIDExcludingStubs, fix query API to use that instead

* Review comments

clientapi/producers/roomserver.go

@@ -106,12 +106,15 @@ func (c *RoomserverProducer) SendInputRoomEvents(
 func (c *RoomserverProducer) SendInvite(
 	ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent,
 	inviteRoomState []gomatrixserverlib.InviteV2StrippedState,
+	sendAsServer gomatrixserverlib.ServerName, txnID *api.TransactionID,
 ) error {
 	request := api.InputRoomEventsRequest{
 		InputInviteEvents: []api.InputInviteEvent{{
 			Event:           inviteEvent,
 			InviteRoomState: inviteRoomState,
 			RoomVersion:     inviteEvent.RoomVersion,
+			SendAsServer:    string(sendAsServer),
+			TransactionID:   txnID,
 		}},
 	}
 	var response api.InputRoomEventsResponse

clientapi/routing/membership.go

@@ -40,6 +40,8 @@ var errMissingUserID = errors.New("'user_id' must be supplied")
 
 // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite)
 // by building a m.room.member event then sending it to the room server
+// TODO: Can we improve the cyclo count here? Separate code paths for invites?
+// nolint:gocyclo
 func SendMembership(
 	req *http.Request, accountDB accounts.Database, device *authtypes.Device,
 	roomID string, membership string, cfg *config.Dendrite,
@@ -104,23 +106,39 @@ func SendMembership(
 		return jsonerror.InternalServerError()
 	}
 
-	if _, err := producer.SendEvents(
-		req.Context(),
-		[]gomatrixserverlib.HeaderedEvent{(*event).Headered(verRes.RoomVersion)},
-		cfg.Matrix.ServerName,
-		nil,
-	); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed")
-		return jsonerror.InternalServerError()
-	}
-
 	var returnData interface{} = struct{}{}
 
-	// The join membership requires the room id to be sent in the response
+	switch membership {
-	if membership == gomatrixserverlib.Join {
+	case gomatrixserverlib.Invite:
+		// Invites need to be handled specially
+		err = producer.SendInvite(
+			req.Context(),
+			event.Headered(verRes.RoomVersion),
+			nil, // ask the roomserver to draw up invite room state for us
+			cfg.Matrix.ServerName,
+			nil,
+		)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("producer.SendInvite failed")
+			return jsonerror.InternalServerError()
+		}
+	case gomatrixserverlib.Join:
+		// The join membership requires the room id to be sent in the response
 		returnData = struct {
 			RoomID string `json:"room_id"`
 		}{roomID}
+	default:
+	}
+
+	_, err = producer.SendEvents(
+		req.Context(),
+		[]gomatrixserverlib.HeaderedEvent{event.Headered(verRes.RoomVersion)},
+		cfg.Matrix.ServerName,
+		nil,
+	)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed")
+		return jsonerror.InternalServerError()
 	}
 
 	return util.JSONResponse{

federationapi/routing/invite.go

@@ -16,11 +16,13 @@ package routing
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/common/config"
+	roomserverVersion "github.com/matrix-org/dendrite/roomserver/version"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -44,6 +46,16 @@ func Invite(
 	}
 	event := inviteReq.Event()
 
+	// Check that we can accept invites for this room version.
+	if _, err := roomserverVersion.SupportedRoomVersion(inviteReq.RoomVersion()); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.UnsupportedRoomVersion(
+				fmt.Sprintf("Room version %q is not supported by this server.", inviteReq.RoomVersion()),
+			),
+		}
+	}
+
 	// Check that the room ID is correct.
 	if event.RoomID() != roomID {
 		return util.JSONResponse{
@@ -90,6 +102,8 @@ func Invite(
 		httpReq.Context(),
 		signedEvent.Headered(inviteReq.RoomVersion()),
 		inviteReq.InviteRoomState(),
+		event.Origin(),
+		nil,
 	); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendInvite failed")
 		return jsonerror.InternalServerError()
@@ -99,6 +113,6 @@ func Invite(
 	// the other servers in the room that we have been invited.
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: gomatrixserverlib.RespInvite{Event: signedEvent},
+		JSON: gomatrixserverlib.RespInviteV2{Event: signedEvent},
 	}
 }

federationsender/consumers/roomserver.go

@@ -28,6 +28,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	log "github.com/sirupsen/logrus"
+	"github.com/tidwall/gjson"
 )
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
@@ -187,49 +188,12 @@ func (s *OutputRoomEventConsumer) processInvite(oie api.OutputNewInviteEvent) er
 		return nil
 	}
 
-	// When sending a v2 invite, the inviting server should try and include
+	// Try to extract the room invite state. The roomserver will have stashed
-	// a "stripped down" version of the room state. This is pretty much just
+	// this for us in invite_room_state if it didn't already exist.
-	// enough information for the remote side to show something useful to the
-	// user, like the room name, aliases etc.
 	strippedState := []gomatrixserverlib.InviteV2StrippedState{}
-	stateWanted := []string{
+	if inviteRoomState := gjson.GetBytes(oie.Event.Unsigned(), "invite_room_state"); inviteRoomState.Exists() {
-		gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias,
+		if err := json.Unmarshal([]byte(inviteRoomState.Raw), &strippedState); err != nil {
-		gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules,
+			log.WithError(err).Warn("failed to extract invite_room_state from event unsigned")
-	}
-
-	// For each of the state keys that we want to try and send, ask the
-	// roomserver if we have a state event for that room that matches the
-	// state key.
-	for _, wanted := range stateWanted {
-		queryReq := api.QueryLatestEventsAndStateRequest{
-			RoomID: oie.Event.RoomID(),
-			StateToFetch: []gomatrixserverlib.StateKeyTuple{
-				gomatrixserverlib.StateKeyTuple{
-					EventType: wanted,
-					StateKey:  "",
-				},
-			},
-		}
-		// If this fails then we just move onto the next event - we don't
-		// actually know at this point whether the room even has that type
-		// of state.
-		queryRes := api.QueryLatestEventsAndStateResponse{}
-		if err := s.query.QueryLatestEventsAndState(context.TODO(), &queryReq, &queryRes); err != nil {
-			log.WithFields(log.Fields{
-				"room_id":    queryReq.RoomID,
-				"event_type": wanted,
-			}).WithError(err).Info("couldn't find state to strip")
-			continue
-		}
-		// Append the stripped down copy of the state to our list.
-		for _, headeredEvent := range queryRes.StateEvents {
-			event := headeredEvent.Unwrap()
-			strippedState = append(strippedState, gomatrixserverlib.NewInviteV2StrippedState(&event))
-
-			log.WithFields(log.Fields{
-				"room_id":    queryReq.RoomID,
-				"event_type": event.Type(),
-			}).Info("adding stripped state")
 		}
 	}
 

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200424101831-2f10e8068538
 	github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
 	github.com/mattn/go-sqlite3 v2.0.2+incompatible
@@ -27,6 +27,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.4.1
 	github.com/sirupsen/logrus v1.4.2
+	github.com/tidwall/gjson v1.6.0
 	github.com/uber/jaeger-client-go v2.15.0+incompatible
 	github.com/uber/jaeger-lib v1.5.0
 	go.uber.org/atomic v1.4.0

go.sum

@@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3 h1:xis1ojN99vjygwqudzB9VQq3cM2SJ7aCAMlXj/YN+88=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200424101831-2f10e8068538 h1:kj2LdNOdg2+vydS9HrPdbECEVeusRg9VTSOkYm61reA=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200424101831-2f10e8068538/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk=
 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A=
 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=

roomserver/api/input.go

@@ -89,6 +89,8 @@ type InputInviteEvent struct {
 	RoomVersion     gomatrixserverlib.RoomVersion             `json:"room_version"`
 	Event           gomatrixserverlib.HeaderedEvent           `json:"event"`
 	InviteRoomState []gomatrixserverlib.InviteV2StrippedState `json:"invite_room_state"`
+	SendAsServer    string                                    `json:"send_as_server"`
+	TransactionID   *TransactionID                            `json:"transaction_id"`
 }
 
 // InputRoomEventsRequest is a request to InputRoomEvents

roomserver/input/events.go

@@ -196,8 +196,23 @@ func processInviteEvent(
 
 	event := input.Event.Unwrap()
 
-	if err = event.SetUnsignedField("invite_room_state", input.InviteRoomState); err != nil {
+	if len(input.InviteRoomState) > 0 {
-		return err
+		// If we were supplied with some invite room state already (which is
+		// most likely to be if the event came in over federation) then use
+		// that.
+		if err = event.SetUnsignedField("invite_room_state", input.InviteRoomState); err != nil {
+			return err
+		}
+	} else {
+		// There's no invite room state, so let's have a go at building it
+		// up from local data (which is most likely to be if the event came
+		// from the CS API). If we know about the room then we can insert
+		// the invite room state, if we don't then we just fail quietly.
+		if irs, ierr := buildInviteStrippedState(ctx, db, input); ierr == nil {
+			if err = event.SetUnsignedField("invite_room_state", irs); err != nil {
+				return err
+			}
+		}
 	}
 
 	outputUpdates, err := updateToInviteMembership(updater, &event, nil, input.Event.RoomVersion)
@@ -212,3 +227,50 @@ func processInviteEvent(
 	succeeded = true
 	return nil
 }
+
+func buildInviteStrippedState(
+	ctx context.Context,
+	db storage.Database,
+	input api.InputInviteEvent,
+) ([]gomatrixserverlib.InviteV2StrippedState, error) {
+	roomNID, err := db.RoomNID(ctx, input.Event.RoomID())
+	if err != nil || roomNID == 0 {
+		return nil, fmt.Errorf("room %q unknown", input.Event.RoomID())
+	}
+	stateWanted := []gomatrixserverlib.StateKeyTuple{}
+	for _, t := range []string{
+		gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias,
+		gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules,
+	} {
+		stateWanted = append(stateWanted, gomatrixserverlib.StateKeyTuple{
+			EventType: t,
+			StateKey:  "",
+		})
+	}
+	_, currentStateSnapshotNID, _, err := db.LatestEventIDs(ctx, roomNID)
+	if err != nil {
+		return nil, err
+	}
+	roomState := state.NewStateResolution(db)
+	stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples(
+		ctx, currentStateSnapshotNID, stateWanted,
+	)
+	if err != nil {
+		return nil, err
+	}
+	stateNIDs := []types.EventNID{}
+	for _, stateNID := range stateEntries {
+		stateNIDs = append(stateNIDs, stateNID.EventNID)
+	}
+	stateEvents, err := db.Events(ctx, stateNIDs)
+	if err != nil {
+		return nil, err
+	}
+	inviteState := []gomatrixserverlib.InviteV2StrippedState{
+		gomatrixserverlib.NewInviteV2StrippedState(&input.Event.Event),
+	}
+	for _, event := range stateEvents {
+		inviteState = append(inviteState, gomatrixserverlib.NewInviteV2StrippedState(&event.Event))
+	}
+	return inviteState, nil
+}

roomserver/input/membership.go

@@ -144,7 +144,7 @@ func updateToInviteMembership(
 		// consider a single stream of events when determining whether a user
 		// is invited, rather than having to combine multiple streams themselves.
 		onie := api.OutputNewInviteEvent{
-			Event:       (*add).Headered(roomVersion),
+			Event:       add.Headered(roomVersion),
 			RoomVersion: roomVersion,
 		}
 		updates = append(updates, api.OutputEvent{

roomserver/query/query.go

@@ -54,7 +54,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState(
 	roomState := state.NewStateResolution(r.DB)
 
 	response.QueryLatestEventsAndStateRequest = *request
-	roomNID, err := r.DB.RoomNID(ctx, request.RoomID)
+	roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID)
 	if err != nil {
 		return err
 	}
@@ -114,7 +114,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents(
 	roomState := state.NewStateResolution(r.DB)
 
 	response.QueryStateAfterEventsRequest = *request
-	roomNID, err := r.DB.RoomNID(ctx, request.RoomID)
+	roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID)
 	if err != nil {
 		return err
 	}
@@ -649,7 +649,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain(
 	response *api.QueryStateAndAuthChainResponse,
 ) error {
 	response.QueryStateAndAuthChainRequest = *request
-	roomNID, err := r.DB.RoomNID(ctx, request.RoomID)
+	roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID)
 	if err != nil {
 		return err
 	}

roomserver/storage/interface.go

@@ -71,6 +71,10 @@ type Database interface {
 	GetLatestEventsForUpdate(ctx context.Context, roomNID types.RoomNID) (types.RoomRecentEventsUpdater, error)
 	GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error)
 	RoomNID(ctx context.Context, roomID string) (types.RoomNID, error)
+	// RoomNIDExcludingStubs is a special variation of RoomNID that will return 0 as if the room
+	// does not exist if the room has no latest events. This can happen when we've received an
+	// invite over federation for a room that we don't know anything else about yet.
+	RoomNIDExcludingStubs(ctx context.Context, roomID string) (types.RoomNID, error)
 	LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error)
 	GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error)
 	SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error

roomserver/storage/postgres/storage.go

@@ -471,6 +471,23 @@ func (d *Database) RoomNID(ctx context.Context, roomID string) (types.RoomNID, e
 	return roomNID, err
 }
 
+// RoomNIDExcludingStubs implements query.RoomserverQueryAPIDB
+func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) {
+	roomNID, err = d.RoomNID(ctx, roomID)
+	if err != nil {
+		return
+	}
+	latestEvents, _, err := d.statements.selectLatestEventNIDs(ctx, roomNID)
+	if err != nil {
+		return
+	}
+	if len(latestEvents) == 0 {
+		roomNID = 0
+		return
+	}
+	return
+}
+
 // LatestEventIDs implements query.RoomserverQueryAPIDatabase
 func (d *Database) LatestEventIDs(
 	ctx context.Context, roomNID types.RoomNID,

roomserver/storage/sqlite3/storage.go

@@ -590,6 +590,23 @@ func (d *Database) RoomNID(ctx context.Context, roomID string) (roomNID types.Ro
 	return
 }
 
+// RoomNIDExcludingStubs implements query.RoomserverQueryAPIDB
+func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) {
+	roomNID, err = d.RoomNID(ctx, roomID)
+	if err != nil {
+		return
+	}
+	latestEvents, _, err := d.statements.selectLatestEventNIDs(ctx, nil, roomNID)
+	if err != nil {
+		return
+	}
+	if len(latestEvents) == 0 {
+		roomNID = 0
+		return
+	}
+	return
+}
+
 // LatestEventIDs implements query.RoomserverQueryAPIDatabase
 func (d *Database) LatestEventIDs(
 	ctx context.Context, roomNID types.RoomNID,

syncapi/storage/postgres/syncserver.go

@@ -752,11 +752,7 @@ func (d *SyncServerDatasource) addInvitesToResponse(
 		return err
 	}
 	for roomID, inviteEvent := range invites {
-		ir := types.NewInviteResponse()
+		ir := types.NewInviteResponse(inviteEvent)
-		ir.InviteState.Events = gomatrixserverlib.ToClientEvents(
-			[]gomatrixserverlib.Event{inviteEvent.Event}, gomatrixserverlib.FormatSync,
-		)
-		// TODO: add the invite state from the invite event.
 		res.Rooms.Invite[roomID] = *ir
 	}
 	return nil

syncapi/storage/sqlite3/syncserver.go

@@ -799,11 +799,7 @@ func (d *SyncServerDatasource) addInvitesToResponse(
 		return err
 	}
 	for roomID, inviteEvent := range invites {
-		ir := types.NewInviteResponse()
+		ir := types.NewInviteResponse(inviteEvent)
-		ir.InviteState.Events = gomatrixserverlib.HeaderedToClientEvents(
-			[]gomatrixserverlib.HeaderedEvent{inviteEvent}, gomatrixserverlib.FormatSync,
-		)
-		// TODO: add the invite state from the invite event.
 		res.Rooms.Invite[roomID] = *ir
 	}
 	return nil

syncapi/types/types.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/tidwall/gjson"
 )
 
 var (
@@ -247,14 +248,17 @@ func NewJoinResponse() *JoinResponse {
 // InviteResponse represents a /sync response for a room which is under the 'invite' key.
 type InviteResponse struct {
 	InviteState struct {
-		Events []gomatrixserverlib.ClientEvent `json:"events"`
+		Events json.RawMessage `json:"events"`
 	} `json:"invite_state"`
 }
 
 // NewInviteResponse creates an empty response with initialised arrays.
-func NewInviteResponse() *InviteResponse {
+func NewInviteResponse(event gomatrixserverlib.HeaderedEvent) *InviteResponse {
 	res := InviteResponse{}
-	res.InviteState.Events = make([]gomatrixserverlib.ClientEvent, 0)
+	res.InviteState.Events = json.RawMessage{'[', ']'}
+	if inviteRoomState := gjson.GetBytes(event.Unsigned(), "invite_room_state"); inviteRoomState.Exists() {
+		res.InviteState.Events = json.RawMessage(inviteRoomState.Raw)
+	}
 	return &res
 }