AWSY missing federation tests (#2943)

In an attempt to fix the missing AWSY tests and to get to 100%
server-server compliance.

are-we-synapse-yet.list

@@ -936,4 +936,12 @@ fst Room state after a rejected message event is the same as before
 fst Room state after a rejected state event is the same as before
 fpb Federation publicRoom Name/topic keys are correct
 fed New federated private chats get full presence information (SYN-115) (10 subtests)
-dvk Rejects invalid device keys
+dvk Rejects invalid device keys
+rmv User can create and send/receive messages in a room with version 10
+rmv local user can join room with version 10
+rmv User can invite local user to room with version 10
+rmv remote user can join room with version 10
+rmv User can invite remote user to room with version 10
+rmv Remote user can backfill in a room with version 10
+rmv Can reject invites over federation for rooms with version 10
+rmv Can receive redactions from regular users over federation in room version 10

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20230105074811-965b10ae73ab
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20230119151039-d8748f6d5dc8
 	github.com/matrix-org/pinecone v0.0.0-20221118192051-fef26631b847
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15

go.sum

@@ -350,6 +350,10 @@ github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20230105074811-965b10ae73ab h1:ChaQdT2mpxMm3GRXNOZzLDQ/wOnlKZ8o60LmZGOjdj8=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20230105074811-965b10ae73ab/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230119135050-7da03ab58f45 h1:zGrmcm2M4F4f+zk5JXAkw3oHa/zXhOh5XVGBdl7GdPo=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230119135050-7da03ab58f45/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230119151039-d8748f6d5dc8 h1:P7me2oCmksST9B4+1I1nA+XrnDQwIqAWmy6ntQrXwc8=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230119151039-d8748f6d5dc8/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
 github.com/matrix-org/pinecone v0.0.0-20221118192051-fef26631b847 h1:auIBCi7gfZuvztD0aPr1G/J5Ya5vWr79M/+TJqwD/JM=
 github.com/matrix-org/pinecone v0.0.0-20221118192051-fef26631b847/go.mod h1:F3GHppRuHCTDeoOmmgjZMeJdbql91+RSGGsATWfC7oc=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=

roomserver/internal/input/input_events.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
 	"github.com/tidwall/gjson"
 
 	"github.com/matrix-org/gomatrixserverlib"
@@ -40,7 +41,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/hooks"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
-	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/types"
 )
@@ -166,6 +166,7 @@ func (r *Inputer) processRoomEvent(
 		missingPrev = !input.HasState && len(missingPrevIDs) > 0
 	}
 
+	// If we have missing events (auth or prev), we build a list of servers to ask
 	if missingAuth || missingPrev {
 		serverReq := &fedapi.QueryJoinedHostServerNamesInRoomRequest{
 			RoomID:             event.RoomID(),
@@ -200,59 +201,8 @@ func (r *Inputer) processRoomEvent(
 		}
 	}
 
-	// First of all, check that the auth events of the event are known.
-	// If they aren't then we will ask the federation API for them.
 	isRejected := false
-	authEvents := gomatrixserverlib.NewAuthEvents(nil)
-	knownEvents := map[string]*types.Event{}
-	if err = r.fetchAuthEvents(ctx, logger, roomInfo, virtualHost, headered, &authEvents, knownEvents, serverRes.ServerNames); err != nil {
-		return fmt.Errorf("r.fetchAuthEvents: %w", err)
-	}
-
-	// Check if the event is allowed by its auth events. If it isn't then
-	// we consider the event to be "rejected" — it will still be persisted.
 	var rejectionErr error
-	if rejectionErr = gomatrixserverlib.Allowed(event, &authEvents); rejectionErr != nil {
-		isRejected = true
-		logger.WithError(rejectionErr).Warnf("Event %s not allowed by auth events", event.EventID())
-	}
-
-	// Accumulate the auth event NIDs.
-	authEventIDs := event.AuthEventIDs()
-	authEventNIDs := make([]types.EventNID, 0, len(authEventIDs))
-	for _, authEventID := range authEventIDs {
-		if _, ok := knownEvents[authEventID]; !ok {
-			// Unknown auth events only really matter if the event actually failed
-			// auth. If it passed auth then we can assume that everything that was
-			// known was sufficient, even if extraneous auth events were specified
-			// but weren't found.
-			if isRejected {
-				if event.StateKey() != nil {
-					return fmt.Errorf(
-						"missing auth event %s for state event %s (type %q, state key %q)",
-						authEventID, event.EventID(), event.Type(), *event.StateKey(),
-					)
-				} else {
-					return fmt.Errorf(
-						"missing auth event %s for timeline event %s (type %q)",
-						authEventID, event.EventID(), event.Type(),
-					)
-				}
-			}
-		} else {
-			authEventNIDs = append(authEventNIDs, knownEvents[authEventID].EventNID)
-		}
-	}
-
-	var softfail bool
-	if input.Kind == api.KindNew {
-		// Check that the event passes authentication checks based on the
-		// current room state.
-		softfail, err = helpers.CheckForSoftFail(ctx, r.DB, headered, input.StateEventIDs)
-		if err != nil {
-			logger.WithError(err).Warn("Error authing soft-failed event")
-		}
-	}
 
 	// At this point we are checking whether we know all of the prev events, and
 	// if we know the state before the prev events. This is necessary before we
@@ -314,6 +264,59 @@ func (r *Inputer) processRoomEvent(
 		}
 	}
 
+	// Check that the auth events of the event are known.
+	// If they aren't then we will ask the federation API for them.
+	authEvents := gomatrixserverlib.NewAuthEvents(nil)
+	knownEvents := map[string]*types.Event{}
+	if err = r.fetchAuthEvents(ctx, logger, roomInfo, virtualHost, headered, &authEvents, knownEvents, serverRes.ServerNames); err != nil {
+		return fmt.Errorf("r.fetchAuthEvents: %w", err)
+	}
+
+	// Check if the event is allowed by its auth events. If it isn't then
+	// we consider the event to be "rejected" — it will still be persisted.
+	if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil {
+		isRejected = true
+		rejectionErr = err
+		logger.WithError(rejectionErr).Warnf("Event %s not allowed by auth events", event.EventID())
+	}
+
+	// Accumulate the auth event NIDs.
+	authEventIDs := event.AuthEventIDs()
+	authEventNIDs := make([]types.EventNID, 0, len(authEventIDs))
+	for _, authEventID := range authEventIDs {
+		if _, ok := knownEvents[authEventID]; !ok {
+			// Unknown auth events only really matter if the event actually failed
+			// auth. If it passed auth then we can assume that everything that was
+			// known was sufficient, even if extraneous auth events were specified
+			// but weren't found.
+			if isRejected {
+				if event.StateKey() != nil {
+					return fmt.Errorf(
+						"missing auth event %s for state event %s (type %q, state key %q)",
+						authEventID, event.EventID(), event.Type(), *event.StateKey(),
+					)
+				} else {
+					return fmt.Errorf(
+						"missing auth event %s for timeline event %s (type %q)",
+						authEventID, event.EventID(), event.Type(),
+					)
+				}
+			}
+		} else {
+			authEventNIDs = append(authEventNIDs, knownEvents[authEventID].EventNID)
+		}
+	}
+
+	var softfail bool
+	if input.Kind == api.KindNew {
+		// Check that the event passes authentication checks based on the
+		// current room state.
+		softfail, err = helpers.CheckForSoftFail(ctx, r.DB, headered, input.StateEventIDs)
+		if err != nil {
+			logger.WithError(err).Warn("Error authing soft-failed event")
+		}
+	}
+
 	// Get the state before the event so that we can work out if the event was
 	// allowed at the time, and also to get the history visibility. We won't
 	// bother doing this if the event was already rejected as it just ends up

sytest-blacklist

@@ -1,54 +1,18 @@
-# Relies on a rejected PL event which will never be accepted into the DAG
+# Blacklisted due to https://github.com/matrix-org/matrix-spec/issues/942
-
-# Caused by <https://github.com/matrix-org/sytest/pull/911>
-
-Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state
-
-# We don't implement lazy membership loading yet
-
 The only membership state included in a gapped incremental sync is for senders in the timeline
 
-# Blacklisted out of flakiness after #1479
-
-Invited user can reject local invite after originator leaves
-Invited user can reject invite for empty room
-If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes
-
-# Blacklisted due to flakiness
-
-Forgotten room messages cannot be paginated
-
-# Blacklisted due to flakiness after #1774
-
-Local device key changes get to remote servers with correct prev_id
-
-# we don't support groups
-
-Remove group category
-Remove group role
-
 # Flakey
-
 AS-ghosted users can use rooms themselves
 AS-ghosted users can use rooms via AS
 Events in rooms with AS-hosted room aliases are sent to AS server
 Inviting an AS-hosted user asks the AS server
 Accesing an AS-hosted room alias asks the AS server
 
-# More flakey
-
-Guest users can join guest_access rooms
-
 # This will fail in HTTP API mode, so blacklisted for now
-
 If a device list update goes missing, the server resyncs on the next one
 
 # Might be a bug in the test because leaves do appear :-(
-
 Leaves are present in non-gapped incremental syncs
 
-# Below test was passing for the wrong reason, failing correctly since #2858
-New federated private chats get full presence information (SYN-115)
-
 # We don't have any state to calculate m.room.guest_access when accepting invites
 Guest users can accept invites to private rooms over federation

sytest-whitelist

@@ -766,4 +766,16 @@ remote user has tags copied to the new room
 Local and remote users' homeservers remove a room from their public directory on upgrade
 Guest users denied access over federation if guest access prohibited
 Guest users are kicked from guest_access rooms on revocation of guest_access
-Guest users are kicked from guest_access rooms on revocation of guest_access over federation
+Guest users are kicked from guest_access rooms on revocation of guest_access over federation
+User can create and send/receive messages in a room with version 10
+local user can join room with version 10
+User can invite local user to room with version 10
+remote user can join room with version 10
+User can invite remote user to room with version 10
+Remote user can backfill in a room with version 10
+Can reject invites over federation for rooms with version 10
+Can receive redactions from regular users over federation in room version 10
+New federated private chats get full presence information (SYN-115)
+/state returns M_NOT_FOUND for an outlier
+/state_ids returns M_NOT_FOUND for an outlier
+Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state