Slower federation warm-up (#2320)

* Wake destination queues gradually, rather than all at once

* Delay device list updates too

* Maximum two minute warmup period

federationapi/queue/queue.go

@@ -104,7 +104,6 @@ func NewOutgoingQueues(
 	}
 	// Look up which servers we have pending items for and then rehydrate those queues.
 	if !disabled {
-		time.AfterFunc(time.Second*5, func() {
 		serverNames := map[gomatrixserverlib.ServerName]struct{}{}
 		if names, err := db.GetPendingPDUServerNames(context.Background()); err == nil {
 			for _, serverName := range names {
@@ -120,12 +119,16 @@ func NewOutgoingQueues(
 		} else {
 			log.WithError(err).Error("Failed to get EDU server names for destination queue hydration")
 		}
+		offset, step := time.Second*5, time.Second
+		if max := len(serverNames); max > 120 {
+			step = (time.Second * 120) / time.Duration(max)
+		}
 		for serverName := range serverNames {
 			if queue := queues.getQueue(serverName); queue != nil {
-					queue.wakeQueueIfNeeded()
+				time.AfterFunc(offset, queue.wakeQueueIfNeeded)
+				offset += step
 			}
 		}
-		})
 	}
 	return queues
 }

keyserver/internal/device_list_update.go

@@ -157,8 +157,15 @@ func (u *DeviceListUpdater) Start() error {
 	if err != nil {
 		return err
 	}
+	offset, step := time.Second*10, time.Second
+	if max := len(staleLists); max > 120 {
+		step = (time.Second * 120) / time.Duration(max)
+	}
 	for _, userID := range staleLists {
+		time.AfterFunc(offset, func() {
 			u.notifyWorkers(userID)
+		})
+		offset += step
 	}
 	return nil
 }