Hydrate destination queues at startup (#1179)

* Hydrate destination queues at startup

* Review comments

federationsender/queue/destinationqueue.go

@@ -250,6 +250,12 @@ func (oq *destinationQueue) backgroundSend() {
 					oq.cleanPendingEDUs()
 					oq.cleanPendingInvites()
 					return
+				} else {
+					// We haven't been told to give up terminally yet but we still have
+					// PDUs waiting to be sent. By sending a message into the wake chan,
+					// the next loop iteration will try processing these PDUs again,
+					// subject to the backoff.
+					oq.wakeServerCh <- true
 				}
 			} else if transaction {
 				// If we successfully sent the transaction then clear out

federationsender/queue/queue.go

@@ -51,7 +51,7 @@ func NewOutgoingQueues(
 	statistics *types.Statistics,
 	signing *SigningInfo,
 ) *OutgoingQueues {
-	return &OutgoingQueues{
+	queues := &OutgoingQueues{
 		db:         db,
 		rsAPI:      rsAPI,
 		origin:     origin,
@@ -60,6 +60,15 @@ func NewOutgoingQueues(
 		signing:    signing,
 		queues:     map[gomatrixserverlib.ServerName]*destinationQueue{},
 	}
+	// Look up which servers we have pending items for and then rehydrate those queues.
+	if serverNames, err := db.GetPendingServerNames(context.Background()); err == nil {
+		for _, serverName := range serverNames {
+			queues.getQueue(serverName).wakeQueueIfNeeded()
+		}
+	} else {
+		log.WithError(err).Error("Failed to get server names for destination queue hydration")
+	}
+	return queues
 }
 
 // TODO: Move this somewhere useful for other components as we often need to ferry these 3 variables