Optimize /sync and history visibility (#2961)

Should fix the following issues or make a lot less worse when using
Postgres:

The main issue behind #2911: The client gives up after a certain time,
causing a cascade of context errors, because the response couldn't be
built up fast enough. This mostly happens on accounts with many rooms,
due to the inefficient way we're getting recent events and current state

For #2777: The queries for getting the membership events for history
visibility were being executed for each room (I think 185?), resulting
in a whooping 2k queries for membership events. (Getting the
statesnapshot -> block nids -> actual wanted membership event)

Both should now be better by:
- Using a LATERAL join to get all recent events for all joined rooms in
one go (TODO: maybe do the same for room summary and current state etc)
- If we're lazy loading on initial syncs, we're now not getting the
whole current state, just to drop the majority of it because we're lazy
loading members - we add a filter to exclude membership events on the
first call to `CurrentState`.
- Using an optimized query to get the membership events needed to
calculate history visibility

---------

Co-authored-by: kegsay <kegan@matrix.org>

syncapi/routing/context.go

@@ -67,6 +67,8 @@ func Context(
 			errMsg = "unable to parse filter"
 		case *strconv.NumError:
 			errMsg = "unable to parse limit"
+		default:
+			errMsg = err.Error()
 		}
 		return util.JSONResponse{
 			Code:    http.StatusBadRequest,
@@ -167,7 +169,18 @@ func Context(
 
 	eventsBeforeClient := gomatrixserverlib.HeaderedToClientEvents(eventsBeforeFiltered, gomatrixserverlib.FormatAll)
 	eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfterFiltered, gomatrixserverlib.FormatAll)
-	newState := applyLazyLoadMembers(device, filter, eventsAfterClient, eventsBeforeClient, state, lazyLoadCache)
+
+	newState := state
+	if filter.LazyLoadMembers {
+		allEvents := append(eventsBeforeFiltered, eventsAfterFiltered...)
+		allEvents = append(allEvents, &requestedEvent)
+		evs := gomatrixserverlib.HeaderedToClientEvents(allEvents, gomatrixserverlib.FormatAll)
+		newState, err = applyLazyLoadMembers(ctx, device, snapshot, roomID, evs, lazyLoadCache)
+		if err != nil {
+			logrus.WithError(err).Error("unable to load membership events")
+			return jsonerror.InternalServerError()
+		}
+	}
 
 	ev := gomatrixserverlib.HeaderedToClientEvent(&requestedEvent, gomatrixserverlib.FormatAll)
 	response := ContextRespsonse{
@@ -244,41 +257,43 @@ func getStartEnd(ctx context.Context, snapshot storage.DatabaseTransaction, star
 }
 
 func applyLazyLoadMembers(
+	ctx context.Context,
 	device *userapi.Device,
-	filter *gomatrixserverlib.RoomEventFilter,
-	eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent,
-	state []*gomatrixserverlib.HeaderedEvent,
+	snapshot storage.DatabaseTransaction,
+	roomID string,
+	events []gomatrixserverlib.ClientEvent,
 	lazyLoadCache caching.LazyLoadCache,
-) []*gomatrixserverlib.HeaderedEvent {
-	if filter == nil || !filter.LazyLoadMembers {
-		return state
-	}
-	allEvents := append(eventsBefore, eventsAfter...)
-	x := make(map[string]struct{})
+) ([]*gomatrixserverlib.HeaderedEvent, error) {
+	eventSenders := make(map[string]struct{})
 	// get members who actually send an event
-	for _, e := range allEvents {
+	for _, e := range events {
 		// Don't add membership events the client should already know about
 		if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, e.RoomID, e.Sender); cached {
 			continue
 		}
-		x[e.Sender] = struct{}{}
+		eventSenders[e.Sender] = struct{}{}
 	}
 
-	newState := []*gomatrixserverlib.HeaderedEvent{}
-	membershipEvents := []*gomatrixserverlib.HeaderedEvent{}
-	for _, event := range state {
-		if event.Type() != gomatrixserverlib.MRoomMember {
-			newState = append(newState, event)
-		} else {
-			// did the user send an event?
-			if _, ok := x[event.Sender()]; ok {
-				membershipEvents = append(membershipEvents, event)
-				lazyLoadCache.StoreLazyLoadedUser(device, event.RoomID(), event.Sender(), event.EventID())
-			}
-		}
+	wantUsers := make([]string, 0, len(eventSenders))
+	for userID := range eventSenders {
+		wantUsers = append(wantUsers, userID)
 	}
-	// Add the membershipEvents to the end of the list, to make Sytest happy
-	return append(newState, membershipEvents...)
+
+	// Query missing membership events
+	filter := gomatrixserverlib.DefaultStateFilter()
+	filter.Senders = &wantUsers
+	filter.Types = &[]string{gomatrixserverlib.MRoomMember}
+	memberships, err := snapshot.GetStateEventsForRoom(ctx, roomID, &filter)
+	if err != nil {
+		return nil, err
+	}
+
+	// cache the membership events
+	for _, membership := range memberships {
+		lazyLoadCache.StoreLazyLoadedUser(device, roomID, *membership.StateKey(), membership.EventID())
+	}
+
+	return memberships, nil
 }
 
 func parseRoomEventFilter(req *http.Request) (*gomatrixserverlib.RoomEventFilter, error) {

syncapi/routing/messages.go

@@ -64,6 +64,7 @@ type messagesResp struct {
 // OnIncomingMessagesRequest implements the /messages endpoint from the
 // client-server API.
 // See: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-client-r0-rooms-roomid-messages
+// nolint:gocyclo
 func OnIncomingMessagesRequest(
 	req *http.Request, db storage.Database, roomID string, device *userapi.Device,
 	rsAPI api.SyncRoomserverAPI,
@@ -246,7 +247,14 @@ func OnIncomingMessagesRequest(
 		Start: start.String(),
 		End:   end.String(),
 	}
-	res.applyLazyLoadMembers(req.Context(), snapshot, roomID, device, filter.LazyLoadMembers, lazyLoadCache)
+	if filter.LazyLoadMembers {
+		membershipEvents, err := applyLazyLoadMembers(req.Context(), device, snapshot, roomID, clientEvents, lazyLoadCache)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("failed to apply lazy loading")
+			return jsonerror.InternalServerError()
+		}
+		res.State = append(res.State, gomatrixserverlib.HeaderedToClientEvents(membershipEvents, gomatrixserverlib.FormatAll)...)
+	}
 
 	// If we didn't return any events, set the end to an empty string, so it will be omitted
 	// in the response JSON.
@@ -265,40 +273,6 @@ func OnIncomingMessagesRequest(
 	}
 }
 
-// applyLazyLoadMembers loads membership events for users returned in Chunk, if the filter has
-// LazyLoadMembers enabled.
-func (m *messagesResp) applyLazyLoadMembers(
-	ctx context.Context,
-	db storage.DatabaseTransaction,
-	roomID string,
-	device *userapi.Device,
-	lazyLoad bool,
-	lazyLoadCache caching.LazyLoadCache,
-) {
-	if !lazyLoad {
-		return
-	}
-	membershipToUser := make(map[string]*gomatrixserverlib.HeaderedEvent)
-	for _, evt := range m.Chunk {
-		// Don't add membership events the client should already know about
-		if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, roomID, evt.Sender); cached {
-			continue
-		}
-		membership, err := db.GetStateEvent(ctx, roomID, gomatrixserverlib.MRoomMember, evt.Sender)
-		if err != nil {
-			util.GetLogger(ctx).WithError(err).Error("failed to get membership event for user")
-			continue
-		}
-		if membership != nil {
-			membershipToUser[evt.Sender] = membership
-			lazyLoadCache.StoreLazyLoadedUser(device, roomID, evt.Sender, membership.EventID())
-		}
-	}
-	for _, evt := range membershipToUser {
-		m.State = append(m.State, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatAll))
-	}
-}
-
 func getMembershipForUser(ctx context.Context, roomID, userID string, rsAPI api.SyncRoomserverAPI) (resp api.QueryMembershipForUserResponse, err error) {
 	req := api.QueryMembershipForUserRequest{
 		RoomID: roomID,