Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007)

* Improve federation sender performance and behaviour, add backoff

* Tweaks

* Tweaks

* Tweaks

* Take copies of events before passing to destination queues

* Don't accidentally drop queued messages

* Don't take copies again

* Tidy up a bit

* Break out statistics (tracked component-wide), report success and failures from Perform actions

* Fix comment, use atomic add

* Improve logic a bit, don't block on wakeup, move idle check

* Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc

* Dedupe destinations, fix other bug hopefully

* Dispatch sends again

* Federation sender to ignore invites that are destined locally

* Loopback invite events

* Remodel a bit with channels

* Linter

* Only loopback invite event if we know the room

* We should tell other resident servers about the invite if we know about the room

* Correct invite signing

* Fix invite loopback

* Check HTTP response codes, push new invites to front of queue

* Review comments

federationsender/internal/perform.go

@@ -25,10 +25,12 @@ func (r *FederationSenderInternalAPI) PerformDirectoryLookup(
 		request.RoomAlias,
 	)
 	if err != nil {
+		r.statistics.ForServer(request.ServerName).Failure()
 		return err
 	}
 	response.RoomID = dir.RoomID
 	response.ServerNames = dir.Servers
+	r.statistics.ForServer(request.ServerName).Success()
 	return nil
 }
 
@@ -61,6 +63,7 @@ func (r *FederationSenderInternalAPI) PerformJoin(
 		)
 		if err != nil {
 			// TODO: Check if the user was not allowed to join the room.
+			r.statistics.ForServer(serverName).Failure()
 			return fmt.Errorf("r.federation.MakeJoin: %w", err)
 		}
 
@@ -112,6 +115,7 @@ func (r *FederationSenderInternalAPI) PerformJoin(
 		)
 		if err != nil {
 			logrus.WithError(err).Warnf("r.federation.SendJoin failed")
+			r.statistics.ForServer(serverName).Failure()
 			continue
 		}
 
@@ -137,6 +141,7 @@ func (r *FederationSenderInternalAPI) PerformJoin(
 		}
 
 		// We're all good.
+		r.statistics.ForServer(serverName).Success()
 		return nil
 	}
 
@@ -170,6 +175,7 @@ func (r *FederationSenderInternalAPI) PerformLeave(
 		if err != nil {
 			// TODO: Check if the user was not allowed to leave the room.
 			logrus.WithError(err).Warnf("r.federation.MakeLeave failed")
+			r.statistics.ForServer(serverName).Failure()
 			continue
 		}
 
@@ -221,9 +227,11 @@ func (r *FederationSenderInternalAPI) PerformLeave(
 		)
 		if err != nil {
 			logrus.WithError(err).Warnf("r.federation.SendLeave failed")
+			r.statistics.ForServer(serverName).Failure()
 			continue
 		}
 
+		r.statistics.ForServer(serverName).Success()
 		return nil
 	}