mirror of
https://github.com/hoernschen/dendrite.git
synced 2025-07-31 21:32:46 +00:00
Roomserver/federation input refactor (#2104)
* Put federation client functions into their own file
* Look for missing auth events in RS input
* Remove retrieveMissingAuthEvents from federation API
* Logging
* Sorta transplanted the code over
* Use event origin failing all else
* Don't get stuck on mutexes:
* Add verifier
* Don't mark state events with zero snapshot NID as not existing
* Check missing state if not an outlier before storing the event
* Reject instead of soft-fail, don't copy roominfo so much
* Use synchronous contexts, limit time to fetch missing events
* Clean up some commented out bits
* Simplify `/send` endpoint significantly
* Submit async
* Report errors on sending to RS input
* Set max payload in NATS to 16MB
* Tweak metrics
* Add `workerForRoom` for tidiness
* Try skipping unmarshalling errors for RespMissingEvents
* Track missing prev events separately to avoid calculating state when not possible
* Tweak logic around checking missing state
* Care about state when checking missing prev events
* Don't check missing state for create events
* Try that again
* Handle create events better
* Send create room events as new
* Use given event kind when sending auth/state events
* Revert "Use given event kind when sending auth/state events"
This reverts commit 089d64d271.
* Only search for missing prev events or state for new events
* Tweaks
* We only have missing prev if we don't supply state
* Room version tweaks
* Allow async inputs again
* Apply backpressure to consumers/synchronous requests to hopefully stop things being overwhelmed
* Set timeouts on roomserver input tasks (need to decide what timeout makes sense)
* Use work queue policy, deliver all on restart
* Reduce chance of duplicates being sent by NATS
* Limit the number of servers we attempt to reduce backpressure
* Some review comment fixes
* Tidy up a couple things
* Don't limit servers, randomise order using map
* Some context refactoring
* Update gmsl
* Don't resend create events
* Set stateIDs length correctly or else the roomserver thinks there are missing events when there aren't
* Exclude our own servername
* Try backing off servers
* Make excluding self behaviour optional
* Exclude self from g_m_e
* Update sytest-whitelist
* Update consumers for the roomserver output stream
* Remember to send outliers for state returned from /gme
* Make full HTTP tests less upsetti
* Remove 'If a device list update goes missing, the server resyncs on the next one' from the sytest blacklist
* Remove debugging test
* Fix blacklist again, remove unnecessary duplicate context
* Clearer contexts, don't use background in case there's something happening there
* Don't queue up events more than once in memory
* Correctly identify create events when checking for state
* Fill in gaps again in /gme code
* Remove `AuthEventIDs` from `InputRoomEvent`
* Remove stray field
Co-authored-by: Kegan Dougal <kegan@matrix.org>
This commit is contained in:
Neil Alexander
GitHub
parent
5b4999afa9
commit
a763cbb0e1
46 changed files with 1549 additions and 1285 deletions
|
|
@ -26,16 +26,18 @@ const (
|
|||
FederationAPIPerformServersAlivePath = "/federationapi/performServersAlive"
|
||||
FederationAPIPerformBroadcastEDUPath = "/federationapi/performBroadcastEDU"
|
||||
|
||||
FederationAPIGetUserDevicesPath = "/federationapi/client/getUserDevices"
|
||||
FederationAPIClaimKeysPath = "/federationapi/client/claimKeys"
|
||||
FederationAPIQueryKeysPath = "/federationapi/client/queryKeys"
|
||||
FederationAPIBackfillPath = "/federationapi/client/backfill"
|
||||
FederationAPILookupStatePath = "/federationapi/client/lookupState"
|
||||
FederationAPILookupStateIDsPath = "/federationapi/client/lookupStateIDs"
|
||||
FederationAPIGetEventPath = "/federationapi/client/getEvent"
|
||||
FederationAPILookupServerKeysPath = "/federationapi/client/lookupServerKeys"
|
||||
FederationAPIEventRelationshipsPath = "/federationapi/client/msc2836eventRelationships"
|
||||
FederationAPISpacesSummaryPath = "/federationapi/client/msc2946spacesSummary"
|
||||
FederationAPIGetUserDevicesPath = "/federationapi/client/getUserDevices"
|
||||
FederationAPIClaimKeysPath = "/federationapi/client/claimKeys"
|
||||
FederationAPIQueryKeysPath = "/federationapi/client/queryKeys"
|
||||
FederationAPIBackfillPath = "/federationapi/client/backfill"
|
||||
FederationAPILookupStatePath = "/federationapi/client/lookupState"
|
||||
FederationAPILookupStateIDsPath = "/federationapi/client/lookupStateIDs"
|
||||
FederationAPILookupMissingEventsPath = "/federationapi/client/lookupMissingEvents"
|
||||
FederationAPIGetEventPath = "/federationapi/client/getEvent"
|
||||
FederationAPILookupServerKeysPath = "/federationapi/client/lookupServerKeys"
|
||||
FederationAPIEventRelationshipsPath = "/federationapi/client/msc2836eventRelationships"
|
||||
FederationAPISpacesSummaryPath = "/federationapi/client/msc2946spacesSummary"
|
||||
FederationAPIGetEventAuthPath = "/federationapi/client/getEventAuth"
|
||||
|
||||
FederationAPIInputPublicKeyPath = "/federationapi/inputPublicKey"
|
||||
FederationAPIQueryPublicKeyPath = "/federationapi/queryPublicKey"
|
||||
|
|
@ -353,6 +355,49 @@ func (h *httpFederationInternalAPI) LookupStateIDs(
|
|||
return *response.Res, nil
|
||||
}
|
||||
|
||||
type lookupMissingEvents struct {
|
||||
S gomatrixserverlib.ServerName
|
||||
RoomID string
|
||||
Missing gomatrixserverlib.MissingEvents
|
||||
RoomVersion gomatrixserverlib.RoomVersion
|
||||
Res struct {
|
||||
Events []gomatrixserverlib.RawJSON `json:"events"`
|
||||
}
|
||||
Err *api.FederationClientError
|
||||
}
|
||||
|
||||
func (h *httpFederationInternalAPI) LookupMissingEvents(
|
||||
ctx context.Context, s gomatrixserverlib.ServerName, roomID string,
|
||||
missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion,
|
||||
) (res gomatrixserverlib.RespMissingEvents, err error) {
|
||||
span, ctx := opentracing.StartSpanFromContext(ctx, "LookupMissingEvents")
|
||||
defer span.Finish()
|
||||
|
||||
request := lookupMissingEvents{
|
||||
S: s,
|
||||
RoomID: roomID,
|
||||
Missing: missing,
|
||||
RoomVersion: roomVersion,
|
||||
}
|
||||
apiURL := h.federationAPIURL + FederationAPILookupMissingEventsPath
|
||||
err = httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &request)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
if request.Err != nil {
|
||||
return res, request.Err
|
||||
}
|
||||
res.Events = make([]*gomatrixserverlib.Event, 0, len(request.Res.Events))
|
||||
for _, js := range request.Res.Events {
|
||||
ev, err := gomatrixserverlib.NewEventFromUntrustedJSON(js, roomVersion)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Events = append(res.Events, ev)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
type getEvent struct {
|
||||
S gomatrixserverlib.ServerName
|
||||
EventID string
|
||||
|
|
@ -382,6 +427,40 @@ func (h *httpFederationInternalAPI) GetEvent(
|
|||
return *response.Res, nil
|
||||
}
|
||||
|
||||
type getEventAuth struct {
|
||||
S gomatrixserverlib.ServerName
|
||||
RoomVersion gomatrixserverlib.RoomVersion
|
||||
RoomID string
|
||||
EventID string
|
||||
Res *gomatrixserverlib.RespEventAuth
|
||||
Err *api.FederationClientError
|
||||
}
|
||||
|
||||
func (h *httpFederationInternalAPI) GetEventAuth(
|
||||
ctx context.Context, s gomatrixserverlib.ServerName,
|
||||
roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string,
|
||||
) (gomatrixserverlib.RespEventAuth, error) {
|
||||
span, ctx := opentracing.StartSpanFromContext(ctx, "GetEventAuth")
|
||||
defer span.Finish()
|
||||
|
||||
request := getEventAuth{
|
||||
S: s,
|
||||
RoomVersion: roomVersion,
|
||||
RoomID: roomID,
|
||||
EventID: eventID,
|
||||
}
|
||||
var response getEventAuth
|
||||
apiURL := h.federationAPIURL + FederationAPIGetEventAuthPath
|
||||
err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response)
|
||||
if err != nil {
|
||||
return gomatrixserverlib.RespEventAuth{}, err
|
||||
}
|
||||
if response.Err != nil {
|
||||
return gomatrixserverlib.RespEventAuth{}, response.Err
|
||||
}
|
||||
return *response.Res, nil
|
||||
}
|
||||
|
||||
func (h *httpFederationInternalAPI) QueryServerKeys(
|
||||
ctx context.Context, req *api.QueryServerKeysRequest, res *api.QueryServerKeysResponse,
|
||||
) error {
|
||||
|
|
|
|||
|
|
@ -241,6 +241,34 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
|
|||
return util.JSONResponse{Code: http.StatusOK, JSON: request}
|
||||
}),
|
||||
)
|
||||
internalAPIMux.Handle(
|
||||
FederationAPILookupMissingEventsPath,
|
||||
httputil.MakeInternalAPI("LookupMissingEvents", func(req *http.Request) util.JSONResponse {
|
||||
var request lookupMissingEvents
|
||||
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
|
||||
return util.MessageResponse(http.StatusBadRequest, err.Error())
|
||||
}
|
||||
res, err := intAPI.LookupMissingEvents(req.Context(), request.S, request.RoomID, request.Missing, request.RoomVersion)
|
||||
if err != nil {
|
||||
ferr, ok := err.(*api.FederationClientError)
|
||||
if ok {
|
||||
request.Err = ferr
|
||||
} else {
|
||||
request.Err = &api.FederationClientError{
|
||||
Err: err.Error(),
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, event := range res.Events {
|
||||
js, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
return util.MessageResponse(http.StatusInternalServerError, err.Error())
|
||||
}
|
||||
request.Res.Events = append(request.Res.Events, js)
|
||||
}
|
||||
return util.JSONResponse{Code: http.StatusOK, JSON: request}
|
||||
}),
|
||||
)
|
||||
internalAPIMux.Handle(
|
||||
FederationAPIGetEventPath,
|
||||
httputil.MakeInternalAPI("GetEvent", func(req *http.Request) util.JSONResponse {
|
||||
|
|
@ -263,6 +291,28 @@ func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) {
|
|||
return util.JSONResponse{Code: http.StatusOK, JSON: request}
|
||||
}),
|
||||
)
|
||||
internalAPIMux.Handle(
|
||||
FederationAPIGetEventAuthPath,
|
||||
httputil.MakeInternalAPI("GetEventAuth", func(req *http.Request) util.JSONResponse {
|
||||
var request getEventAuth
|
||||
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
|
||||
return util.MessageResponse(http.StatusBadRequest, err.Error())
|
||||
}
|
||||
res, err := intAPI.GetEventAuth(req.Context(), request.S, request.RoomVersion, request.RoomID, request.EventID)
|
||||
if err != nil {
|
||||
ferr, ok := err.(*api.FederationClientError)
|
||||
if ok {
|
||||
request.Err = ferr
|
||||
} else {
|
||||
request.Err = &api.FederationClientError{
|
||||
Err: err.Error(),
|
||||
}
|
||||
}
|
||||
}
|
||||
request.Res = &res
|
||||
return util.JSONResponse{Code: http.StatusOK, JSON: request}
|
||||
}),
|
||||
)
|
||||
internalAPIMux.Handle(
|
||||
FederationAPIQueryServerKeysPath,
|
||||
httputil.MakeInternalAPI("QueryServerKeys", func(req *http.Request) util.JSONResponse {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue