Don't hold destination queues in memory forever (#1769)

* Don't hold destination queues in memory forever

* Close channels

* Fix ordering

* Clear more aggressively

* clearQueue only called by defer so should be safe to delete queue in any case

* Wake queue when created, otherwise cleanup doesn't get called in all cases

* Clean up periodically, we hit a race condition otherwise

* Tweaks

* Don't create queues for blacklisted hosts

* Check blacklist properly
This commit is contained in:
Neil Alexander 2021-02-17 15:16:35 +00:00 committed by GitHub
parent da797c7998
commit 8b5cd256cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 27 additions and 9 deletions

View file

@ -46,6 +46,7 @@ const (
// ensures that only one request is in flight to a given destination
// at a time.
type destinationQueue struct {
queues *OutgoingQueues
db storage.Database
process *process.ProcessContext
signing *SigningInfo
@ -246,6 +247,7 @@ func (oq *destinationQueue) backgroundSend() {
}
destinationQueueRunning.Inc()
defer destinationQueueRunning.Dec()
defer oq.queues.clearQueue(oq)
defer oq.running.Store(false)
// Mark the queue as overflowed, so we will consult the database