Fetch missing auth events, implement QueryMissingAuthPrevEvents, try other servers in room for /event and /get_missing_events (#1450)

* Try to ask other servers in the room for missing events if the origin won't provide them

* Logging

* More logging

* Implement QueryMissingAuthPrevEvents

* Try to get missing auth events badly

* Use processEvent

* Logging

* Update QueryMissingAuthPrevEvents

* Try to find missing auth events

* Patchy fix for test

* Logging tweaks

* Send auth events as outliers

* Update check in QueryMissingAuthPrevEvents

* Error responses

* More return codes

* Don't return error on reject/soft-fail since it was ultimately handled

* More tweaks

* More error tweaks

roomserver/api/api.go

@@ -68,6 +68,13 @@ type RoomserverInternalAPI interface {
 		response *QueryStateAfterEventsResponse,
 	) error
 
+	// Query whether the roomserver is missing any auth or prev events.
+	QueryMissingAuthPrevEvents(
+		ctx context.Context,
+		request *QueryMissingAuthPrevEventsRequest,
+		response *QueryMissingAuthPrevEventsResponse,
+	) error
+
 	// Query a list of events by event ID.
 	QueryEventsByID(
 		ctx context.Context,

roomserver/api/api_trace.go

@@ -104,6 +104,16 @@ func (t *RoomserverInternalAPITrace) QueryStateAfterEvents(
 	return err
 }
 
+func (t *RoomserverInternalAPITrace) QueryMissingAuthPrevEvents(
+	ctx context.Context,
+	req *QueryMissingAuthPrevEventsRequest,
+	res *QueryMissingAuthPrevEventsResponse,
+) error {
+	err := t.Impl.QueryMissingAuthPrevEvents(ctx, req, res)
+	util.GetLogger(ctx).WithError(err).Infof("QueryMissingAuthPrevEvents req=%+v res=%+v", js(req), js(res))
+	return err
+}
+
 func (t *RoomserverInternalAPITrace) QueryEventsByID(
 	ctx context.Context,
 	req *QueryEventsByIDRequest,

roomserver/api/query.go

@@ -82,6 +82,27 @@ type QueryStateAfterEventsResponse struct {
 	StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"`
 }
 
+type QueryMissingAuthPrevEventsRequest struct {
+	// The room ID to query the state in.
+	RoomID string `json:"room_id"`
+	// The list of auth events to check the existence of.
+	AuthEventIDs []string `json:"auth_event_ids"`
+	// The list of previous events to check the existence of.
+	PrevEventIDs []string `json:"prev_event_ids"`
+}
+
+type QueryMissingAuthPrevEventsResponse struct {
+	// Does the room exist on this roomserver?
+	// If the room doesn't exist all other fields will be empty.
+	RoomExists bool `json:"room_exists"`
+	// The room version of the room.
+	RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"`
+	// The event IDs of the auth events that we don't know locally.
+	MissingAuthEventIDs []string `json:"missing_auth_event_ids"`
+	// The event IDs of the previous events that we don't know locally.
+	MissingPrevEventIDs []string `json:"missing_prev_event_ids"`
+}
+
 // QueryEventsByIDRequest is a request to QueryEventsByID
 type QueryEventsByIDRequest struct {
 	// The event IDs to look up.
@@ -154,6 +175,8 @@ type QueryServerJoinedToRoomResponse struct {
 	RoomExists bool `json:"room_exists"`
 	// True if we still believe that we are participating in the room
 	IsInRoom bool `json:"is_in_room"`
+	// List of servers that are also in the room
+	ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
 }
 
 // QueryServerAllowedToSeeEventRequest is a request to QueryServerAllowedToSeeEvent

roomserver/internal/helpers/auth.go

@@ -83,7 +83,7 @@ func CheckForSoftFail(
 	// Check if the event is allowed.
 	if err = gomatrixserverlib.Allowed(event.Event, &authEvents); err != nil {
 		// return true, nil
-		return true, fmt.Errorf("gomatrixserverlib.Allowed: %w", err)
+		return true, err
 	}
 	return false, nil
 }
@@ -99,7 +99,7 @@ func CheckAuthEvents(
 	// Grab the numeric IDs for the supplied auth state events from the database.
 	authStateEntries, err := db.StateEntriesForEventIDs(ctx, authEventIDs)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("db.StateEntriesForEventIDs: %w", err)
 	}
 	authStateEntries = types.DeduplicateStateEntries(authStateEntries)
 
@@ -109,7 +109,7 @@ func CheckAuthEvents(
 	// Load the actual auth events from the database.
 	authEvents, err := loadAuthEvents(ctx, db, stateNeeded, authStateEntries)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("loadAuthEvents: %w", err)
 	}
 
 	// Check if the event is allowed.

roomserver/internal/input/input_events.go

@@ -49,7 +49,7 @@ func (r *Inputer) processRoomEvent(
 	isRejected := false
 	authEventNIDs, rejectionErr := helpers.CheckAuthEvents(ctx, r.DB, headered, input.AuthEventIDs)
 	if rejectionErr != nil {
-		logrus.WithError(rejectionErr).WithField("event_id", event.EventID()).WithField("auth_event_ids", input.AuthEventIDs).Error("processRoomEvent.checkAuthEvents failed for event, rejecting event")
+		logrus.WithError(rejectionErr).WithField("event_id", event.EventID()).WithField("auth_event_ids", input.AuthEventIDs).Error("helpers.CheckAuthEvents failed for event, rejecting event")
 		isRejected = true
 	}
 

roomserver/internal/perform/perform_invite.go

@@ -136,14 +136,10 @@ func (r *Inviter) PerformInvite(
 			log.WithError(err).WithField("event_id", event.EventID()).WithField("auth_event_ids", event.AuthEventIDs()).Error(
 				"processInviteEvent.checkAuthEvents failed for event",
 			)
-			if _, ok := err.(*gomatrixserverlib.NotAllowed); ok {
-				res.Error = &api.PerformError{
-					Msg:  err.Error(),
-					Code: api.PerformErrorNotAllowed,
-				}
-				return nil, nil
+			res.Error = &api.PerformError{
+				Msg:  err.Error(),
+				Code: api.PerformErrorNotAllowed,
 			}
-			return nil, fmt.Errorf("checkAuthEvents: %w", err)
 		}
 
 		// If the invite originated from us and the target isn't local then we
@@ -160,7 +156,7 @@ func (r *Inviter) PerformInvite(
 			if err = r.FSAPI.PerformInvite(ctx, fsReq, fsRes); err != nil {
 				res.Error = &api.PerformError{
 					Msg:  err.Error(),
-					Code: api.PerformErrorNoOperation,
+					Code: api.PerformErrorNotAllowed,
 				}
 				log.WithError(err).WithField("event_id", event.EventID()).Error("r.FSAPI.PerformInvite failed")
 				return nil, nil
@@ -185,7 +181,12 @@ func (r *Inviter) PerformInvite(
 		inputRes := &api.InputRoomEventsResponse{}
 		r.Inputer.InputRoomEvents(context.Background(), inputReq, inputRes)
 		if err = inputRes.Err(); err != nil {
-			return nil, fmt.Errorf("r.InputRoomEvents: %w", err)
+			res.Error = &api.PerformError{
+				Msg:  fmt.Sprintf("r.InputRoomEvents: %s", err.Error()),
+				Code: api.PerformErrorNotAllowed,
+			}
+			log.WithError(err).WithField("event_id", event.EventID()).Error("r.InputRoomEvents failed")
+			return nil, nil
 		}
 	} else {
 		// The invite originated over federation. Process the membership

roomserver/internal/perform/perform_join.go

@@ -249,14 +249,10 @@ func (r *Joiner) performJoinRoomByID(
 			inputRes := api.InputRoomEventsResponse{}
 			r.Inputer.InputRoomEvents(ctx, &inputReq, &inputRes)
 			if err = inputRes.Err(); err != nil {
-				var notAllowed *gomatrixserverlib.NotAllowed
-				if errors.As(err, &notAllowed) {
-					return "", &api.PerformError{
-						Code: api.PerformErrorNotAllowed,
-						Msg:  fmt.Sprintf("InputRoomEvents auth failed: %s", err),
-					}
+				return "", &api.PerformError{
+					Code: api.PerformErrorNotAllowed,
+					Msg:  fmt.Sprintf("InputRoomEvents auth failed: %s", err),
 				}
-				return "", fmt.Errorf("r.InputRoomEvents: %w", err)
 			}
 		}
 

roomserver/internal/query/query.go

@@ -98,6 +98,38 @@ func (r *Queryer) QueryStateAfterEvents(
 	return nil
 }
 
+// QueryMissingAuthPrevEvents implements api.RoomserverInternalAPI
+func (r *Queryer) QueryMissingAuthPrevEvents(
+	ctx context.Context,
+	request *api.QueryMissingAuthPrevEventsRequest,
+	response *api.QueryMissingAuthPrevEventsResponse,
+) error {
+	info, err := r.DB.RoomInfo(ctx, request.RoomID)
+	if err != nil {
+		return err
+	}
+	if info == nil {
+		return errors.New("room doesn't exist")
+	}
+
+	response.RoomExists = !info.IsStub
+	response.RoomVersion = info.RoomVersion
+
+	for _, authEventID := range request.AuthEventIDs {
+		if nids, err := r.DB.EventNIDs(ctx, []string{authEventID}); err != nil || len(nids) == 0 {
+			response.MissingAuthEventIDs = append(response.MissingAuthEventIDs, authEventID)
+		}
+	}
+
+	for _, prevEventID := range request.PrevEventIDs {
+		if nids, err := r.DB.EventNIDs(ctx, []string{prevEventID}); err != nil || len(nids) == 0 {
+			response.MissingPrevEventIDs = append(response.MissingPrevEventIDs, prevEventID)
+		}
+	}
+
+	return nil
+}
+
 // QueryEventsByID implements api.RoomserverInternalAPI
 func (r *Queryer) QueryEventsByID(
 	ctx context.Context,
@@ -255,19 +287,24 @@ func (r *Queryer) QueryServerJoinedToRoom(
 		return fmt.Errorf("r.DB.Events: %w", err)
 	}
 
+	servers := map[gomatrixserverlib.ServerName]struct{}{}
 	for _, e := range events {
 		if e.Type() == gomatrixserverlib.MRoomMember && e.StateKey() != nil {
 			_, serverName, err := gomatrixserverlib.SplitID('@', *e.StateKey())
 			if err != nil {
 				continue
 			}
+			servers[serverName] = struct{}{}
 			if serverName == request.ServerName {
 				response.IsInRoom = true
-				break
 			}
 		}
 	}
 
+	for server := range servers {
+		response.ServerNames = append(response.ServerNames, server)
+	}
+
 	return nil
 }
 

roomserver/inthttp/client.go

@@ -35,6 +35,7 @@ const (
 	// Query operations
 	RoomserverQueryLatestEventsAndStatePath    = "/roomserver/queryLatestEventsAndState"
 	RoomserverQueryStateAfterEventsPath        = "/roomserver/queryStateAfterEvents"
+	RoomserverQueryMissingAuthPrevEventsPath   = "/roomserver/queryMissingAuthPrevEvents"
 	RoomserverQueryEventsByIDPath              = "/roomserver/queryEventsByID"
 	RoomserverQueryMembershipForUserPath       = "/roomserver/queryMembershipForUser"
 	RoomserverQueryMembershipsForRoomPath      = "/roomserver/queryMembershipsForRoom"
@@ -262,6 +263,19 @@ func (h *httpRoomserverInternalAPI) QueryStateAfterEvents(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
+// QueryStateAfterEvents implements RoomserverQueryAPI
+func (h *httpRoomserverInternalAPI) QueryMissingAuthPrevEvents(
+	ctx context.Context,
+	request *api.QueryMissingAuthPrevEventsRequest,
+	response *api.QueryMissingAuthPrevEventsResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMissingAuthPrevEvents")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverQueryMissingAuthPrevEventsPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
 // QueryEventsByID implements RoomserverQueryAPI
 func (h *httpRoomserverInternalAPI) QueryEventsByID(
 	ctx context.Context,

roomserver/inthttp/server.go

@@ -125,6 +125,20 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(
+		RoomserverQueryMissingAuthPrevEventsPath,
+		httputil.MakeInternalAPI("queryMissingAuthPrevEvents", func(req *http.Request) util.JSONResponse {
+			var request api.QueryMissingAuthPrevEventsRequest
+			var response api.QueryMissingAuthPrevEventsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryMissingAuthPrevEvents(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 	internalAPIMux.Handle(
 		RoomserverQueryEventsByIDPath,
 		httputil.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse {