Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)

* Improve federation sender performance and behaviour, add backoff

* Tweaks

* Tweaks

* Tweaks

* Take copies of events before passing to destination queues

* Don't accidentally drop queued messages

* Don't take copies again

* Tidy up a bit

* Break out statistics (tracked component-wide), report success and failures from Perform actions

* Fix comment, use atomic add

* Improve logic a bit, don't block on wakeup, move idle check

* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc

* Dedupe destinations, fix other bug hopefully

* Dispatch sends again

* Federation sender to ignore invites that are destined locally

* Loopback invite events

* Remodel a bit with channels

* Linter

* Only loopback invite event if we know the room

* We should tell other resident servers about the invite if we know about the room

* Correct invite signing

* Fix invite loopback

* Check HTTP response codes, push new invites to front of queue

* Review comments
This commit is contained in:
Neil Alexander 2020-05-07 12:42:06 +01:00 committed by GitHub
parent 3b98535dc5
commit a16db1c408
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 474 additions and 142 deletions

View file

@ -18,12 +18,13 @@ import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
"github.com/matrix-org/dendrite/federationsender/producers"
"github.com/matrix-org/dendrite/federationsender/types"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"
"go.uber.org/atomic"
)
@ -33,92 +34,190 @@ import (
// ensures that only one request is in flight to a given destination
// at a time.
type destinationQueue struct {
rsProducer *producers.RoomserverProducer
client *gomatrixserverlib.FederationClient
origin gomatrixserverlib.ServerName
destination gomatrixserverlib.ServerName
running atomic.Bool
// The running mutex protects sentCounter, lastTransactionIDs and
// pendingEvents, pendingEDUs.
runningMutex sync.Mutex
sentCounter int
lastTransactionIDs []gomatrixserverlib.TransactionID
pendingEvents []*gomatrixserverlib.HeaderedEvent
pendingEDUs []*gomatrixserverlib.EDU
pendingInvites []*gomatrixserverlib.InviteV2Request
rsProducer *producers.RoomserverProducer // roomserver producer
client *gomatrixserverlib.FederationClient // federation client
origin gomatrixserverlib.ServerName // origin of requests
destination gomatrixserverlib.ServerName // destination of requests
running atomic.Bool // is the queue worker running?
statistics *types.ServerStatistics // statistics about this remote server
incomingPDUs chan *gomatrixserverlib.HeaderedEvent // PDUs to send
incomingEDUs chan *gomatrixserverlib.EDU // EDUs to send
incomingInvites chan *gomatrixserverlib.InviteV2Request // invites to send
lastTransactionIDs []gomatrixserverlib.TransactionID // last transaction ID
pendingPDUs []*gomatrixserverlib.HeaderedEvent // owned by backgroundSend
pendingEDUs []*gomatrixserverlib.EDU // owned by backgroundSend
pendingInvites []*gomatrixserverlib.InviteV2Request // owned by backgroundSend
}
// Send event adds the event to the pending queue for the destination.
// If the queue is empty then it starts a background goroutine to
// start sending events to that destination.
func (oq *destinationQueue) sendEvent(ev *gomatrixserverlib.HeaderedEvent) {
oq.runningMutex.Lock()
defer oq.runningMutex.Unlock()
oq.pendingEvents = append(oq.pendingEvents, ev)
if oq.statistics.Blacklisted() {
// If the destination is blacklisted then drop the event.
return
}
if !oq.running.Load() {
go oq.backgroundSend()
}
oq.incomingPDUs <- ev
}
// sendEDU adds the EDU event to the pending queue for the destination.
// If the queue is empty then it starts a background goroutine to
// start sending events to that destination.
func (oq *destinationQueue) sendEDU(e *gomatrixserverlib.EDU) {
oq.runningMutex.Lock()
defer oq.runningMutex.Unlock()
oq.pendingEDUs = append(oq.pendingEDUs, e)
func (oq *destinationQueue) sendEDU(ev *gomatrixserverlib.EDU) {
if oq.statistics.Blacklisted() {
// If the destination is blacklisted then drop the event.
return
}
if !oq.running.Load() {
go oq.backgroundSend()
}
oq.incomingEDUs <- ev
}
// sendInvite adds the invite event to the pending queue for the
// destination. If the queue is empty then it starts a background
// goroutine to start sending events to that destination.
func (oq *destinationQueue) sendInvite(ev *gomatrixserverlib.InviteV2Request) {
oq.runningMutex.Lock()
defer oq.runningMutex.Unlock()
oq.pendingInvites = append(oq.pendingInvites, ev)
if oq.statistics.Blacklisted() {
// If the destination is blacklisted then drop the event.
return
}
if !oq.running.Load() {
go oq.backgroundSend()
}
oq.incomingInvites <- ev
}
// backgroundSend is the worker goroutine for sending events.
// nolint:gocyclo
func (oq *destinationQueue) backgroundSend() {
oq.running.Store(true)
// Check if a worker is already running, and if it isn't, then
// mark it as started.
if !oq.running.CAS(false, true) {
return
}
defer oq.running.Store(false)
for {
transaction, invites := oq.nextTransaction(), oq.nextInvites()
if !transaction && !invites {
// If the queue is empty then stop processing for this destination.
// TODO: Remove this destination from the queue map.
// Wait either for incoming events, or until we hit an
// idle timeout.
select {
case pdu := <-oq.incomingPDUs:
// Ordering of PDUs is important so we add them to the end
// of the queue and they will all be added to transactions
// in order.
oq.pendingPDUs = append(oq.pendingPDUs, pdu)
case edu := <-oq.incomingEDUs:
// Likewise for EDUs, although we should probably not try
// too hard with some EDUs (like typing notifications) after
// a certain amount of time has passed.
// TODO: think about EDU expiry some more
oq.pendingEDUs = append(oq.pendingEDUs, edu)
case invite := <-oq.incomingInvites:
// There's no strict ordering requirement for invites like
// there is for transactions, so we put the invite onto the
// front of the queue. This means that if an invite that is
// stuck failing already, that it won't block our new invite
// from being sent.
oq.pendingInvites = append(
[]*gomatrixserverlib.InviteV2Request{invite},
oq.pendingInvites...,
)
case <-time.After(time.Second * 30):
// The worker is idle so stop the goroutine. It'll
// get restarted automatically the next time we
// get an event.
return
}
// TODO: handle retries.
// TODO: blacklist uncooperative servers.
// If we are backing off this server then wait for the
// backoff duration to complete first.
if backoff, duration := oq.statistics.BackoffDuration(); backoff {
<-time.After(duration)
}
// How many things do we have waiting?
numPDUs := len(oq.pendingPDUs)
numEDUs := len(oq.pendingEDUs)
numInvites := len(oq.pendingInvites)
// If we have pending PDUs or EDUs then construct a transaction.
if numPDUs > 0 || numEDUs > 0 {
// Try sending the next transaction and see what happens.
transaction, terr := oq.nextTransaction(oq.pendingPDUs, oq.pendingEDUs, oq.statistics.SuccessCount())
if terr != nil {
// We failed to send the transaction.
if giveUp := oq.statistics.Failure(); giveUp {
// It's been suggested that we should give up because
// the backoff has exceeded a maximum allowable value.
return
}
} else if transaction {
// If we successfully sent the transaction then clear out
// the pending events and EDUs.
oq.statistics.Success()
// Reallocate so that the underlying arrays can be GC'd, as
// opposed to growing forever.
for i := 0; i < numPDUs; i++ {
oq.pendingPDUs[i] = nil
}
for i := 0; i < numEDUs; i++ {
oq.pendingEDUs[i] = nil
}
oq.pendingPDUs = append(
[]*gomatrixserverlib.HeaderedEvent{},
oq.pendingPDUs[numPDUs:]...,
)
oq.pendingEDUs = append(
[]*gomatrixserverlib.EDU{},
oq.pendingEDUs[numEDUs:]...,
)
}
}
// Try sending the next invite and see what happens.
if numInvites > 0 {
sent, ierr := oq.nextInvites(oq.pendingInvites)
if ierr != nil {
// We failed to send the transaction so increase the
// backoff and give it another go shortly.
if giveUp := oq.statistics.Failure(); giveUp {
// It's been suggested that we should give up because
// the backoff has exceeded a maximum allowable value.
return
}
} else if sent > 0 {
// If we successfully sent the invites then clear out
// the pending invites.
oq.statistics.Success()
// Reallocate so that the underlying array can be GC'd, as
// opposed to growing forever.
oq.pendingInvites = append(
[]*gomatrixserverlib.InviteV2Request{},
oq.pendingInvites[sent:]...,
)
}
}
}
}
// nextTransaction creates a new transaction from the pending event
// queue and sends it. Returns true if a transaction was sent or
// false otherwise.
func (oq *destinationQueue) nextTransaction() bool {
oq.runningMutex.Lock()
defer oq.runningMutex.Unlock()
if len(oq.pendingEvents) == 0 && len(oq.pendingEDUs) == 0 {
return false
}
func (oq *destinationQueue) nextTransaction(
pendingPDUs []*gomatrixserverlib.HeaderedEvent,
pendingEDUs []*gomatrixserverlib.EDU,
sentCounter uint32,
) (bool, error) {
t := gomatrixserverlib.Transaction{
PDUs: []json.RawMessage{},
EDUs: []gomatrixserverlib.EDU{},
}
now := gomatrixserverlib.AsTimestamp(time.Now())
t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.sentCounter))
t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, sentCounter))
t.Origin = oq.origin
t.Destination = oq.destination
t.OriginServerTS = now
@ -129,44 +228,54 @@ func (oq *destinationQueue) nextTransaction() bool {
oq.lastTransactionIDs = []gomatrixserverlib.TransactionID{t.TransactionID}
for _, pdu := range oq.pendingEvents {
for _, pdu := range pendingPDUs {
// Append the JSON of the event, since this is a json.RawMessage type in the
// gomatrixserverlib.Transaction struct
t.PDUs = append(t.PDUs, (*pdu).JSON())
}
oq.pendingEvents = nil
oq.sentCounter += len(t.PDUs)
for _, edu := range oq.pendingEDUs {
for _, edu := range pendingEDUs {
t.EDUs = append(t.EDUs, *edu)
}
oq.pendingEDUs = nil
oq.sentCounter += len(t.EDUs)
util.GetLogger(context.TODO()).Infof("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs))
logrus.WithField("server_name", oq.destination).Infof("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs))
// TODO: we should check for 500-ish fails vs 400-ish here,
// since we shouldn't queue things indefinitely in response
// to a 400-ish error
_, err := oq.client.SendTransaction(context.TODO(), t)
if err != nil {
switch e := err.(type) {
case nil:
// No error was returned so the transaction looks to have
// been successfully sent.
return true, nil
case gomatrix.HTTPError:
// We received a HTTP error back. In this instance we only
// should report an error if
if e.Code >= 400 && e.Code <= 499 {
// We tried but the remote side has sent back a client error.
// It's no use retrying because it will happen again.
return true, nil
}
// Otherwise, report that we failed to send the transaction
// and we will retry again.
return false, err
default:
log.WithFields(log.Fields{
"destination": oq.destination,
log.ErrorKey: err,
}).Info("problem sending transaction")
return false, err
}
return true
}
// nextInvite takes pending invite events from the queue and sends
// them. Returns true if a transaction was sent or false otherwise.
func (oq *destinationQueue) nextInvites() bool {
oq.runningMutex.Lock()
defer oq.runningMutex.Unlock()
if len(oq.pendingInvites) == 0 {
return false
}
for _, inviteReq := range oq.pendingInvites {
func (oq *destinationQueue) nextInvites(
pendingInvites []*gomatrixserverlib.InviteV2Request,
) (int, error) {
done := 0
for _, inviteReq := range pendingInvites {
ev, roomVersion := inviteReq.Event(), inviteReq.RoomVersion()
log.WithFields(log.Fields{
@ -180,13 +289,32 @@ func (oq *destinationQueue) nextInvites() bool {
oq.destination,
*inviteReq,
)
if err != nil {
switch e := err.(type) {
case nil:
done++
case gomatrix.HTTPError:
log.WithFields(log.Fields{
"event_id": ev.EventID(),
"state_key": ev.StateKey(),
"destination": oq.destination,
"status_code": e.Code,
}).WithError(err).Error("failed to send invite due to HTTP error")
// Check whether we should do something about the error or
// just accept it as unavoidable.
if e.Code >= 400 && e.Code <= 499 {
// We tried but the remote side has sent back a client error.
// It's no use retrying because it will happen again.
done++
continue
}
return done, err
default:
log.WithFields(log.Fields{
"event_id": ev.EventID(),
"state_key": ev.StateKey(),
"destination": oq.destination,
}).WithError(err).Error("failed to send invite")
continue
return done, err
}
if _, err = oq.rsProducer.SendInviteResponse(
@ -199,10 +327,9 @@ func (oq *destinationQueue) nextInvites() bool {
"state_key": ev.StateKey(),
"destination": oq.destination,
}).WithError(err).Error("failed to return signed invite to roomserver")
return done, err
}
}
oq.pendingInvites = nil
return true
return done, nil
}

View file

@ -19,18 +19,20 @@ import (
"sync"
"github.com/matrix-org/dendrite/federationsender/producers"
"github.com/matrix-org/dendrite/federationsender/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
log "github.com/sirupsen/logrus"
)
// OutgoingQueues is a collection of queues for sending transactions to other
// matrix servers
type OutgoingQueues struct {
rsProducer *producers.RoomserverProducer
origin gomatrixserverlib.ServerName
client *gomatrixserverlib.FederationClient
// The queuesMutex protects queues
queuesMutex sync.Mutex
rsProducer *producers.RoomserverProducer
origin gomatrixserverlib.ServerName
client *gomatrixserverlib.FederationClient
statistics *types.Statistics
queuesMutex sync.Mutex // protects the below
queues map[gomatrixserverlib.ServerName]*destinationQueue
}
@ -39,15 +41,37 @@ func NewOutgoingQueues(
origin gomatrixserverlib.ServerName,
client *gomatrixserverlib.FederationClient,
rsProducer *producers.RoomserverProducer,
statistics *types.Statistics,
) *OutgoingQueues {
return &OutgoingQueues{
rsProducer: rsProducer,
origin: origin,
client: client,
statistics: statistics,
queues: map[gomatrixserverlib.ServerName]*destinationQueue{},
}
}
func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *destinationQueue {
oqs.queuesMutex.Lock()
defer oqs.queuesMutex.Unlock()
oq := oqs.queues[destination]
if oq == nil {
oq = &destinationQueue{
rsProducer: oqs.rsProducer,
origin: oqs.origin,
destination: destination,
client: oqs.client,
statistics: oqs.statistics.ForServer(destination),
incomingPDUs: make(chan *gomatrixserverlib.HeaderedEvent, 128),
incomingEDUs: make(chan *gomatrixserverlib.EDU, 128),
incomingInvites: make(chan *gomatrixserverlib.InviteV2Request, 128),
}
oqs.queues[destination] = oq
}
return oq
}
// SendEvent sends an event to the destinations
func (oqs *OutgoingQueues) SendEvent(
ev *gomatrixserverlib.HeaderedEvent, origin gomatrixserverlib.ServerName,
@ -62,27 +86,14 @@ func (oqs *OutgoingQueues) SendEvent(
}
// Remove our own server from the list of destinations.
destinations = filterDestinations(oqs.origin, destinations)
destinations = filterAndDedupeDests(oqs.origin, destinations)
log.WithFields(log.Fields{
"destinations": destinations, "event": ev.EventID(),
}).Info("Sending event")
oqs.queuesMutex.Lock()
defer oqs.queuesMutex.Unlock()
for _, destination := range destinations {
oq := oqs.queues[destination]
if oq == nil {
oq = &destinationQueue{
rsProducer: oqs.rsProducer,
origin: oqs.origin,
destination: destination,
client: oqs.client,
}
oqs.queues[destination] = oq
}
oq.sendEvent(ev)
oqs.getQueue(destination).sendEvent(ev)
}
return nil
@ -111,23 +122,11 @@ func (oqs *OutgoingQueues) SendInvite(
}
log.WithFields(log.Fields{
"event_id": ev.EventID(),
"event_id": ev.EventID(),
"server_name": destination,
}).Info("Sending invite")
oqs.queuesMutex.Lock()
defer oqs.queuesMutex.Unlock()
oq := oqs.queues[destination]
if oq == nil {
oq = &destinationQueue{
rsProducer: oqs.rsProducer,
origin: oqs.origin,
destination: destination,
client: oqs.client,
}
oqs.queues[destination] = oq
}
oq.sendInvite(inviteReq)
oqs.getQueue(destination).sendInvite(inviteReq)
return nil
}
@ -146,7 +145,7 @@ func (oqs *OutgoingQueues) SendEDU(
}
// Remove our own server from the list of destinations.
destinations = filterDestinations(oqs.origin, destinations)
destinations = filterAndDedupeDests(oqs.origin, destinations)
if len(destinations) > 0 {
log.WithFields(log.Fields{
@ -154,35 +153,27 @@ func (oqs *OutgoingQueues) SendEDU(
}).Info("Sending EDU event")
}
oqs.queuesMutex.Lock()
defer oqs.queuesMutex.Unlock()
for _, destination := range destinations {
oq := oqs.queues[destination]
if oq == nil {
oq = &destinationQueue{
rsProducer: oqs.rsProducer,
origin: oqs.origin,
destination: destination,
client: oqs.client,
}
oqs.queues[destination] = oq
}
oq.sendEDU(e)
oqs.getQueue(destination).sendEDU(e)
}
return nil
}
// filterDestinations removes our own server from the list of destinations.
// Otherwise we could end up trying to talk to ourselves.
func filterDestinations(origin gomatrixserverlib.ServerName, destinations []gomatrixserverlib.ServerName) []gomatrixserverlib.ServerName {
var result []gomatrixserverlib.ServerName
for _, destination := range destinations {
if destination == origin {
// filterAndDedupeDests removes our own server from the list of destinations
// and deduplicates any servers in the list that may appear more than once.
func filterAndDedupeDests(origin gomatrixserverlib.ServerName, destinations []gomatrixserverlib.ServerName) (
result []gomatrixserverlib.ServerName,
) {
strs := make([]string, len(destinations))
for i, d := range destinations {
strs[i] = string(d)
}
for _, destination := range util.UniqueStrings(strs) {
if gomatrixserverlib.ServerName(destination) == origin {
continue
}
result = append(result, destination)
result = append(result, gomatrixserverlib.ServerName(destination))
}
return result
}