hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2025-08-02 06:12:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Persistent federation sender queues (PDUs) (#1173)

Browse source 
* Initial work on persistent queues

* Update index for event ID and server name

* Put things into database (postgres for now)

* Duplicate postgres code into sqlite for now just to stop build errors, will fix SQLite soon

* Fix table name

* Fix index

* Fix table name

* Use RETURNING because LastInsertID is not supported by postgres

* Use functions

* Marshal headered event

* Don't error on now rows

* Don't block if there are PDUs waiting

* Try to tidy up JSON

* Debug logging

* Fix query, use transactions in postgres

* Clean up

* Rehydrate more opportunistically

* Fix SQLite

* remove unused types

* Review comments

* Shuffle things around a bit

* Clean up transaction properly

* Don't send empty transactions

* Reduce unnecessary retries

* Count PDUs to make more resilient

* Don't stop when there is work to be done

* Try to limit wakeups

* well this is tedious

* Fix race in incomplete transactions

* Thread safety on transaction ID/count
This commit is contained in:
Neil Alexander 2020-07-01 11:46:38 +01:00 • committed by GitHub
parent 6f49758b90
commit 42dd962425
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 1049 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

21
federationsender/queue/queue.go
View file

@ -15,10 +15,13 @@
package queue
import (
"context"
"crypto/ed25519"
"encoding/json"
"fmt"
"sync"
"github.com/matrix-org/dendrite/federationsender/storage"
"github.com/matrix-org/dendrite/federationsender/types"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
@ -29,6 +32,7 @@ import (
// OutgoingQueues is a collection of queues for sending transactions to other
// matrix servers
type OutgoingQueues struct {
db storage.Database
rsAPI api.RoomserverInternalAPI
origin gomatrixserverlib.ServerName
client *gomatrixserverlib.FederationClient
@ -40,6 +44,7 @@ type OutgoingQueues struct {
// NewOutgoingQueues makes a new OutgoingQueues
func NewOutgoingQueues(
db storage.Database,
origin gomatrixserverlib.ServerName,
client *gomatrixserverlib.FederationClient,
rsAPI api.RoomserverInternalAPI,
@ -47,6 +52,7 @@ func NewOutgoingQueues(
signing *SigningInfo,
) *OutgoingQueues {
return &OutgoingQueues{
db: db,
rsAPI: rsAPI,
origin: origin,
client: client,
@ -76,14 +82,15 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d
oq := oqs.queues[destination]
if oq == nil {
oq = &destinationQueue{
db: oqs.db,
rsAPI: oqs.rsAPI,
origin: oqs.origin,
destination: destination,
client: oqs.client,
statistics: oqs.statistics.ForServer(destination),
incomingPDUs: make(chan *gomatrixserverlib.HeaderedEvent, 128),
incomingEDUs: make(chan *gomatrixserverlib.EDU, 128),
incomingInvites: make(chan *gomatrixserverlib.InviteV2Request, 128),
wakeServerCh: make(chan bool, 128),
retryServerCh: make(chan bool),
signing: oqs.signing,
}
@ -115,8 +122,18 @@ func (oqs *OutgoingQueues) SendEvent(
"destinations": destinations, "event": ev.EventID(),
}).Info("Sending event")
headeredJSON, err := json.Marshal(ev)
if err != nil {
return fmt.Errorf("json.Marshal: %w", err)
}
nid, err := oqs.db.StoreJSON(context.TODO(), string(headeredJSON))
if err != nil {
return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err)
}
for _, destination := range destinations {
oqs.getQueue(destination).sendEvent(ev)
oqs.getQueue(destination).sendEvent(nid)
}
return nil