Optimize /sync and history visibility (#2961)

Should fix the following issues or make a lot less worse when using
Postgres:

The main issue behind #2911: The client gives up after a certain time,
causing a cascade of context errors, because the response couldn't be
built up fast enough. This mostly happens on accounts with many rooms,
due to the inefficient way we're getting recent events and current state

For #2777: The queries for getting the membership events for history
visibility were being executed for each room (I think 185?), resulting
in a whooping 2k queries for membership events. (Getting the
statesnapshot -> block nids -> actual wanted membership event)

Both should now be better by:
- Using a LATERAL join to get all recent events for all joined rooms in
one go (TODO: maybe do the same for room summary and current state etc)
- If we're lazy loading on initial syncs, we're now not getting the
whole current state, just to drop the majority of it because we're lazy
loading members - we add a filter to exclude membership events on the
first call to `CurrentState`.
- Using an optimized query to get the membership events needed to
calculate history visibility

---------

Co-authored-by: kegsay <kegan@matrix.org>

syncapi/storage/shared/storage_sync.go

@@ -151,8 +151,8 @@ func (d *DatabaseTransaction) GetRoomSummary(ctx context.Context, roomID, userID
 	return summary, nil
 }
 
-func (d *DatabaseTransaction) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) {
-	return d.OutputEvents.SelectRecentEvents(ctx, d.txn, roomID, r, eventFilter, chronologicalOrder, onlySyncEvents)
+func (d *DatabaseTransaction) RecentEvents(ctx context.Context, roomIDs []string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) (map[string]types.RecentEvents, error) {
+	return d.OutputEvents.SelectRecentEvents(ctx, d.txn, roomIDs, r, eventFilter, chronologicalOrder, onlySyncEvents)
 }
 
 func (d *DatabaseTransaction) PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error) {
@@ -370,19 +370,25 @@ func (d *DatabaseTransaction) GetStateDeltas(
 	}
 
 	// get all the state events ever (i.e. for all available rooms) between these two positions
-	stateNeededFiltered, eventMapFiltered, err := d.OutputEvents.SelectStateInRange(ctx, d.txn, r, stateFilter, allRoomIDs)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, nil, nil
+	stateFiltered := state
+	// avoid hitting the database if the result would be the same as above
+	if !isStatefilterEmpty(stateFilter) {
+		var stateNeededFiltered map[string]map[string]bool
+		var eventMapFiltered map[string]types.StreamEvent
+		stateNeededFiltered, eventMapFiltered, err = d.OutputEvents.SelectStateInRange(ctx, d.txn, r, stateFilter, allRoomIDs)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, nil, nil
+			}
+			return nil, nil, err
 		}
-		return nil, nil, err
-	}
-	stateFiltered, err := d.fetchStateEvents(ctx, d.txn, stateNeededFiltered, eventMapFiltered)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil, nil, nil
+		stateFiltered, err = d.fetchStateEvents(ctx, d.txn, stateNeededFiltered, eventMapFiltered)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, nil, nil
+			}
+			return nil, nil, err
 		}
-		return nil, nil, err
 	}
 
 	// find out which rooms this user is peeking, if any.
@@ -701,6 +707,28 @@ func (d *DatabaseTransaction) MaxStreamPositionForRelations(ctx context.Context)
 	return types.StreamPosition(id), err
 }
 
+func isStatefilterEmpty(filter *gomatrixserverlib.StateFilter) bool {
+	if filter == nil {
+		return true
+	}
+	switch {
+	case filter.NotTypes != nil && len(*filter.NotTypes) > 0:
+		return false
+	case filter.Types != nil && len(*filter.Types) > 0:
+		return false
+	case filter.Senders != nil && len(*filter.Senders) > 0:
+		return false
+	case filter.NotSenders != nil && len(*filter.NotSenders) > 0:
+		return false
+	case filter.NotRooms != nil && len(*filter.NotRooms) > 0:
+		return false
+	case filter.ContainsURL != nil:
+		return false
+	default:
+		return true
+	}
+}
+
 func (d *DatabaseTransaction) RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) (
 	events []types.StreamEvent, prevBatch, nextBatch string, err error,
 ) {

syncapi/storage/shared/storage_sync_test.go

@@ -0,0 +1,72 @@
+package shared
+
+import (
+	"testing"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+func Test_isStatefilterEmpty(t *testing.T) {
+	filterSet := []string{"a"}
+	boolValue := false
+
+	tests := []struct {
+		name   string
+		filter *gomatrixserverlib.StateFilter
+		want   bool
+	}{
+		{
+			name:   "nil filter is empty",
+			filter: nil,
+			want:   true,
+		},
+		{
+			name:   "Empty filter is empty",
+			filter: &gomatrixserverlib.StateFilter{},
+			want:   true,
+		},
+		{
+			name: "NotTypes is set",
+			filter: &gomatrixserverlib.StateFilter{
+				NotTypes: &filterSet,
+			},
+		},
+		{
+			name: "Types is set",
+			filter: &gomatrixserverlib.StateFilter{
+				Types: &filterSet,
+			},
+		},
+		{
+			name: "Senders is set",
+			filter: &gomatrixserverlib.StateFilter{
+				Senders: &filterSet,
+			},
+		},
+		{
+			name: "NotSenders is set",
+			filter: &gomatrixserverlib.StateFilter{
+				NotSenders: &filterSet,
+			},
+		},
+		{
+			name: "NotRooms is set",
+			filter: &gomatrixserverlib.StateFilter{
+				NotRooms: &filterSet,
+			},
+		},
+		{
+			name: "ContainsURL is set",
+			filter: &gomatrixserverlib.StateFilter{
+				ContainsURL: &boolValue,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := isStatefilterEmpty(tt.filter); got != tt.want {
+				t.Errorf("isStatefilterEmpty() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}