Initial Store & Forward Implementation (#2917)

This adds store & forward relays into dendrite for p2p.
A few things have changed:
- new relay api serves new http endpoints for s&f federation
- updated outbound federation queueing which will attempt to forward
using s&f if appropriate
- database entries to track s&f relays for other nodes

relayapi/storage/interface.go

@@ -0,0 +1,47 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type Database interface {
+	// Adds a new transaction to the queue json table.
+	// Adding a duplicate transaction will result in a new row being added and a new unique nid.
+	// return: unique nid representing this entry.
+	StoreTransaction(ctx context.Context, txn gomatrixserverlib.Transaction) (*receipt.Receipt, error)
+
+	// Adds a new transaction_id: server_name mapping with associated json table nid to the queue
+	// entry table for each provided destination.
+	AssociateTransactionWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.UserID]struct{}, transactionID gomatrixserverlib.TransactionID, dbReceipt *receipt.Receipt) error
+
+	// Removes every server_name: receipt pair provided from the queue entries table.
+	// Will then remove every entry for each receipt provided from the queue json table.
+	// If any of the entries don't exist in either table, nothing will happen for that entry and
+	// an error will not be generated.
+	CleanTransactions(ctx context.Context, userID gomatrixserverlib.UserID, receipts []*receipt.Receipt) error
+
+	// Gets the oldest transaction for the provided server_name.
+	// If no transactions exist, returns nil and no error.
+	GetTransaction(ctx context.Context, userID gomatrixserverlib.UserID) (*gomatrixserverlib.Transaction, *receipt.Receipt, error)
+
+	// Gets the number of transactions being stored for the provided server_name.
+	// If the server doesn't exist in the database then 0 is returned with no error.
+	GetTransactionCount(ctx context.Context, userID gomatrixserverlib.UserID) (int64, error)
+}

relayapi/storage/postgres/relay_queue_json_table.go

@@ -0,0 +1,113 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+const relayQueueJSONSchema = `
+-- The relayapi_queue_json table contains event contents that
+-- we are storing for future forwarding. 
+CREATE TABLE IF NOT EXISTS relayapi_queue_json (
+	-- The JSON NID. This allows cross-referencing to find the JSON blob.
+	json_nid BIGSERIAL,
+	-- The JSON body. Text so that we preserve UTF-8.
+	json_body TEXT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS relayapi_queue_json_json_nid_idx
+	ON relayapi_queue_json (json_nid);
+`
+
+const insertQueueJSONSQL = "" +
+	"INSERT INTO relayapi_queue_json (json_body)" +
+	" VALUES ($1)" +
+	" RETURNING json_nid"
+
+const deleteQueueJSONSQL = "" +
+	"DELETE FROM relayapi_queue_json WHERE json_nid = ANY($1)"
+
+const selectQueueJSONSQL = "" +
+	"SELECT json_nid, json_body FROM relayapi_queue_json" +
+	" WHERE json_nid = ANY($1)"
+
+type relayQueueJSONStatements struct {
+	db             *sql.DB
+	insertJSONStmt *sql.Stmt
+	deleteJSONStmt *sql.Stmt
+	selectJSONStmt *sql.Stmt
+}
+
+func NewPostgresRelayQueueJSONTable(db *sql.DB) (s *relayQueueJSONStatements, err error) {
+	s = &relayQueueJSONStatements{
+		db: db,
+	}
+	_, err = s.db.Exec(relayQueueJSONSchema)
+	if err != nil {
+		return
+	}
+
+	return s, sqlutil.StatementList{
+		{&s.insertJSONStmt, insertQueueJSONSQL},
+		{&s.deleteJSONStmt, deleteQueueJSONSQL},
+		{&s.selectJSONStmt, selectQueueJSONSQL},
+	}.Prepare(db)
+}
+
+func (s *relayQueueJSONStatements) InsertQueueJSON(
+	ctx context.Context, txn *sql.Tx, json string,
+) (int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.insertJSONStmt)
+	var lastid int64
+	if err := stmt.QueryRowContext(ctx, json).Scan(&lastid); err != nil {
+		return 0, err
+	}
+	return lastid, nil
+}
+
+func (s *relayQueueJSONStatements) DeleteQueueJSON(
+	ctx context.Context, txn *sql.Tx, nids []int64,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteJSONStmt)
+	_, err := stmt.ExecContext(ctx, pq.Int64Array(nids))
+	return err
+}
+
+func (s *relayQueueJSONStatements) SelectQueueJSON(
+	ctx context.Context, txn *sql.Tx, jsonNIDs []int64,
+) (map[int64][]byte, error) {
+	blobs := map[int64][]byte{}
+	stmt := sqlutil.TxStmt(txn, s.selectJSONStmt)
+	rows, err := stmt.QueryContext(ctx, pq.Int64Array(jsonNIDs))
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectJSON: rows.close() failed")
+	for rows.Next() {
+		var nid int64
+		var blob []byte
+		if err = rows.Scan(&nid, &blob); err != nil {
+			return nil, err
+		}
+		blobs[nid] = blob
+	}
+	return blobs, err
+}

relayapi/storage/postgres/relay_queue_table.go

@@ -0,0 +1,156 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const relayQueueSchema = `
+CREATE TABLE IF NOT EXISTS relayapi_queue (
+	-- The transaction ID that was generated before persisting the event.
+	transaction_id TEXT NOT NULL,
+	-- The destination server that we will send the event to.
+	server_name TEXT NOT NULL,
+	-- The JSON NID from the relayapi_queue_json table.
+	json_nid BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS relayapi_queue_queue_json_nid_idx
+	ON relayapi_queue (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS relayapi_queue_json_nid_idx
+	ON relayapi_queue (json_nid);
+CREATE INDEX IF NOT EXISTS relayapi_queue_server_name_idx
+	ON relayapi_queue (server_name);
+`
+
+const insertQueueEntrySQL = "" +
+	"INSERT INTO relayapi_queue (transaction_id, server_name, json_nid)" +
+	" VALUES ($1, $2, $3)"
+
+const deleteQueueEntriesSQL = "" +
+	"DELETE FROM relayapi_queue WHERE server_name = $1 AND json_nid = ANY($2)"
+
+const selectQueueEntriesSQL = "" +
+	"SELECT json_nid FROM relayapi_queue" +
+	" WHERE server_name = $1" +
+	" ORDER BY json_nid" +
+	" LIMIT $2"
+
+const selectQueueEntryCountSQL = "" +
+	"SELECT COUNT(*) FROM relayapi_queue" +
+	" WHERE server_name = $1"
+
+type relayQueueStatements struct {
+	db                        *sql.DB
+	insertQueueEntryStmt      *sql.Stmt
+	deleteQueueEntriesStmt    *sql.Stmt
+	selectQueueEntriesStmt    *sql.Stmt
+	selectQueueEntryCountStmt *sql.Stmt
+}
+
+func NewPostgresRelayQueueTable(
+	db *sql.DB,
+) (s *relayQueueStatements, err error) {
+	s = &relayQueueStatements{
+		db: db,
+	}
+	_, err = s.db.Exec(relayQueueSchema)
+	if err != nil {
+		return
+	}
+
+	return s, sqlutil.StatementList{
+		{&s.insertQueueEntryStmt, insertQueueEntrySQL},
+		{&s.deleteQueueEntriesStmt, deleteQueueEntriesSQL},
+		{&s.selectQueueEntriesStmt, selectQueueEntriesSQL},
+		{&s.selectQueueEntryCountStmt, selectQueueEntryCountSQL},
+	}.Prepare(db)
+}
+
+func (s *relayQueueStatements) InsertQueueEntry(
+	ctx context.Context,
+	txn *sql.Tx,
+	transactionID gomatrixserverlib.TransactionID,
+	serverName gomatrixserverlib.ServerName,
+	nid int64,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.insertQueueEntryStmt)
+	_, err := stmt.ExecContext(
+		ctx,
+		transactionID, // the transaction ID that we initially attempted
+		serverName,    // destination server name
+		nid,           // JSON blob NID
+	)
+	return err
+}
+
+func (s *relayQueueStatements) DeleteQueueEntries(
+	ctx context.Context,
+	txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+	jsonNIDs []int64,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteQueueEntriesStmt)
+	_, err := stmt.ExecContext(ctx, serverName, pq.Int64Array(jsonNIDs))
+	return err
+}
+
+func (s *relayQueueStatements) SelectQueueEntries(
+	ctx context.Context,
+	txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+	limit int,
+) ([]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectQueueEntriesStmt)
+	rows, err := stmt.QueryContext(ctx, serverName, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "queueFromStmt: rows.close() failed")
+	var result []int64
+	for rows.Next() {
+		var nid int64
+		if err = rows.Scan(&nid); err != nil {
+			return nil, err
+		}
+		result = append(result, nid)
+	}
+
+	return result, rows.Err()
+}
+
+func (s *relayQueueStatements) SelectQueueEntryCount(
+	ctx context.Context,
+	txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+) (int64, error) {
+	var count int64
+	stmt := sqlutil.TxStmt(txn, s.selectQueueEntryCountStmt)
+	err := stmt.QueryRowContext(ctx, serverName).Scan(&count)
+	if err == sql.ErrNoRows {
+		// It's acceptable for there to be no rows referencing a given
+		// JSON NID but it's not an error condition. Just return as if
+		// there's a zero count.
+		return 0, nil
+	}
+	return count, err
+}

relayapi/storage/postgres/storage.go

@@ -0,0 +1,64 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/relayapi/storage/shared"
+	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// Database stores information needed by the relayapi
+type Database struct {
+	shared.Database
+	db     *sql.DB
+	writer sqlutil.Writer
+}
+
+// NewDatabase opens a new database
+func NewDatabase(
+	base *base.BaseDendrite,
+	dbProperties *config.DatabaseOptions,
+	cache caching.FederationCache,
+	isLocalServerName func(gomatrixserverlib.ServerName) bool,
+) (*Database, error) {
+	var d Database
+	var err error
+	if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil {
+		return nil, err
+	}
+	queue, err := NewPostgresRelayQueueTable(d.db)
+	if err != nil {
+		return nil, err
+	}
+	queueJSON, err := NewPostgresRelayQueueJSONTable(d.db)
+	if err != nil {
+		return nil, err
+	}
+	d.Database = shared.Database{
+		DB:                d.db,
+		IsLocalServerName: isLocalServerName,
+		Cache:             cache,
+		Writer:            d.writer,
+		RelayQueue:        queue,
+		RelayQueueJSON:    queueJSON,
+	}
+	return &d, nil
+}

relayapi/storage/shared/storage.go

@@ -0,0 +1,170 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shared
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/federationapi/storage/shared/receipt"
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/relayapi/storage/tables"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type Database struct {
+	DB                *sql.DB
+	IsLocalServerName func(gomatrixserverlib.ServerName) bool
+	Cache             caching.FederationCache
+	Writer            sqlutil.Writer
+	RelayQueue        tables.RelayQueue
+	RelayQueueJSON    tables.RelayQueueJSON
+}
+
+func (d *Database) StoreTransaction(
+	ctx context.Context,
+	transaction gomatrixserverlib.Transaction,
+) (*receipt.Receipt, error) {
+	var err error
+	jsonTransaction, err := json.Marshal(transaction)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal: %w", err)
+	}
+
+	var nid int64
+	_ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		nid, err = d.RelayQueueJSON.InsertQueueJSON(ctx, txn, string(jsonTransaction))
+		return err
+	})
+	if err != nil {
+		return nil, fmt.Errorf("d.insertQueueJSON: %w", err)
+	}
+
+	newReceipt := receipt.NewReceipt(nid)
+	return &newReceipt, nil
+}
+
+func (d *Database) AssociateTransactionWithDestinations(
+	ctx context.Context,
+	destinations map[gomatrixserverlib.UserID]struct{},
+	transactionID gomatrixserverlib.TransactionID,
+	dbReceipt *receipt.Receipt,
+) error {
+	err := d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		var lastErr error
+		for destination := range destinations {
+			destination := destination
+			err := d.RelayQueue.InsertQueueEntry(
+				ctx,
+				txn,
+				transactionID,
+				destination.Domain(),
+				dbReceipt.GetNID(),
+			)
+			if err != nil {
+				lastErr = fmt.Errorf("d.insertQueueEntry: %w", err)
+			}
+		}
+		return lastErr
+	})
+
+	return err
+}
+
+func (d *Database) CleanTransactions(
+	ctx context.Context,
+	userID gomatrixserverlib.UserID,
+	receipts []*receipt.Receipt,
+) error {
+	nids := make([]int64, len(receipts))
+	for i, dbReceipt := range receipts {
+		nids[i] = dbReceipt.GetNID()
+	}
+
+	err := d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		deleteEntryErr := d.RelayQueue.DeleteQueueEntries(ctx, txn, userID.Domain(), nids)
+		// TODO : If there are still queue entries for any of these nids for other destinations
+		// then we shouldn't delete the json entries.
+		// But this can't happen with the current api design.
+		// There will only ever be one server entry for each nid since each call to send_relay
+		// only accepts a single server name and inside there we create a new json entry.
+		// So for multiple destinations we would call send_relay multiple times and have multiple
+		// json entries of the same transaction.
+		//
+		// TLDR; this works as expected right now but can easily be optimised in the future.
+		deleteJSONErr := d.RelayQueueJSON.DeleteQueueJSON(ctx, txn, nids)
+
+		if deleteEntryErr != nil {
+			return fmt.Errorf("d.deleteQueueEntries: %w", deleteEntryErr)
+		}
+		if deleteJSONErr != nil {
+			return fmt.Errorf("d.deleteQueueJSON: %w", deleteJSONErr)
+		}
+		return nil
+	})
+
+	return err
+}
+
+func (d *Database) GetTransaction(
+	ctx context.Context,
+	userID gomatrixserverlib.UserID,
+) (*gomatrixserverlib.Transaction, *receipt.Receipt, error) {
+	entriesRequested := 1
+	nids, err := d.RelayQueue.SelectQueueEntries(ctx, nil, userID.Domain(), entriesRequested)
+	if err != nil {
+		return nil, nil, fmt.Errorf("d.SelectQueueEntries: %w", err)
+	}
+	if len(nids) == 0 {
+		return nil, nil, nil
+	}
+	firstNID := nids[0]
+
+	txns := map[int64][]byte{}
+	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
+		txns, err = d.RelayQueueJSON.SelectQueueJSON(ctx, txn, nids)
+		return err
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("d.SelectQueueJSON: %w", err)
+	}
+
+	transaction := &gomatrixserverlib.Transaction{}
+	if _, ok := txns[firstNID]; !ok {
+		return nil, nil, fmt.Errorf("Failed retrieving json blob for transaction: %d", firstNID)
+	}
+
+	err = json.Unmarshal(txns[firstNID], transaction)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Unmarshal transaction: %w", err)
+	}
+
+	newReceipt := receipt.NewReceipt(firstNID)
+	return transaction, &newReceipt, nil
+}
+
+func (d *Database) GetTransactionCount(
+	ctx context.Context,
+	userID gomatrixserverlib.UserID,
+) (int64, error) {
+	count, err := d.RelayQueue.SelectQueueEntryCount(ctx, nil, userID.Domain())
+	if err != nil {
+		return 0, fmt.Errorf("d.SelectQueueEntryCount: %w", err)
+	}
+	return count, nil
+}

relayapi/storage/sqlite3/relay_queue_json_table.go

@@ -0,0 +1,137 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+const relayQueueJSONSchema = `
+-- The relayapi_queue_json table contains event contents that
+-- we are storing for future forwarding. 
+CREATE TABLE IF NOT EXISTS relayapi_queue_json (
+	-- The JSON NID. This allows cross-referencing to find the JSON blob.
+	json_nid INTEGER PRIMARY KEY AUTOINCREMENT,
+	-- The JSON body. Text so that we preserve UTF-8.
+	json_body TEXT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS relayapi_queue_json_json_nid_idx
+	ON relayapi_queue_json (json_nid);
+`
+
+const insertQueueJSONSQL = "" +
+	"INSERT INTO relayapi_queue_json (json_body)" +
+	" VALUES ($1)"
+
+const deleteQueueJSONSQL = "" +
+	"DELETE FROM relayapi_queue_json WHERE json_nid IN ($1)"
+
+const selectQueueJSONSQL = "" +
+	"SELECT json_nid, json_body FROM relayapi_queue_json" +
+	" WHERE json_nid IN ($1)"
+
+type relayQueueJSONStatements struct {
+	db             *sql.DB
+	insertJSONStmt *sql.Stmt
+	//deleteJSONStmt *sql.Stmt - prepared at runtime due to variadic
+	//selectJSONStmt *sql.Stmt - prepared at runtime due to variadic
+}
+
+func NewSQLiteRelayQueueJSONTable(db *sql.DB) (s *relayQueueJSONStatements, err error) {
+	s = &relayQueueJSONStatements{
+		db: db,
+	}
+	_, err = db.Exec(relayQueueJSONSchema)
+	if err != nil {
+		return
+	}
+
+	return s, sqlutil.StatementList{
+		{&s.insertJSONStmt, insertQueueJSONSQL},
+	}.Prepare(db)
+}
+
+func (s *relayQueueJSONStatements) InsertQueueJSON(
+	ctx context.Context, txn *sql.Tx, json string,
+) (lastid int64, err error) {
+	stmt := sqlutil.TxStmt(txn, s.insertJSONStmt)
+	res, err := stmt.ExecContext(ctx, json)
+	if err != nil {
+		return 0, fmt.Errorf("stmt.QueryContext: %w", err)
+	}
+	lastid, err = res.LastInsertId()
+	if err != nil {
+		return 0, fmt.Errorf("res.LastInsertId: %w", err)
+	}
+	return
+}
+
+func (s *relayQueueJSONStatements) DeleteQueueJSON(
+	ctx context.Context, txn *sql.Tx, nids []int64,
+) error {
+	deleteSQL := strings.Replace(deleteQueueJSONSQL, "($1)", sqlutil.QueryVariadic(len(nids)), 1)
+	deleteStmt, err := txn.Prepare(deleteSQL)
+	if err != nil {
+		return fmt.Errorf("s.deleteQueueJSON s.db.Prepare: %w", err)
+	}
+
+	iNIDs := make([]interface{}, len(nids))
+	for k, v := range nids {
+		iNIDs[k] = v
+	}
+
+	stmt := sqlutil.TxStmt(txn, deleteStmt)
+	_, err = stmt.ExecContext(ctx, iNIDs...)
+	return err
+}
+
+func (s *relayQueueJSONStatements) SelectQueueJSON(
+	ctx context.Context, txn *sql.Tx, jsonNIDs []int64,
+) (map[int64][]byte, error) {
+	selectSQL := strings.Replace(selectQueueJSONSQL, "($1)", sqlutil.QueryVariadic(len(jsonNIDs)), 1)
+	selectStmt, err := txn.Prepare(selectSQL)
+	if err != nil {
+		return nil, fmt.Errorf("s.selectQueueJSON s.db.Prepare: %w", err)
+	}
+
+	iNIDs := make([]interface{}, len(jsonNIDs))
+	for k, v := range jsonNIDs {
+		iNIDs[k] = v
+	}
+
+	blobs := map[int64][]byte{}
+	stmt := sqlutil.TxStmt(txn, selectStmt)
+	rows, err := stmt.QueryContext(ctx, iNIDs...)
+	if err != nil {
+		return nil, fmt.Errorf("s.selectQueueJSON stmt.QueryContext: %w", err)
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectQueueJSON: rows.close() failed")
+	for rows.Next() {
+		var nid int64
+		var blob []byte
+		if err = rows.Scan(&nid, &blob); err != nil {
+			return nil, fmt.Errorf("s.selectQueueJSON rows.Scan: %w", err)
+		}
+		blobs[nid] = blob
+	}
+	return blobs, err
+}

relayapi/storage/sqlite3/relay_queue_table.go

@@ -0,0 +1,168 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const relayQueueSchema = `
+CREATE TABLE IF NOT EXISTS relayapi_queue (
+	-- The transaction ID that was generated before persisting the event.
+	transaction_id TEXT NOT NULL,
+	-- The domain part of the user ID the m.room.member event is for.
+	server_name TEXT NOT NULL,
+	-- The JSON NID from the relayapi_queue_json table.
+	json_nid BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS relayapi_queue_queue_json_nid_idx
+	ON relayapi_queue (json_nid, server_name);
+CREATE INDEX IF NOT EXISTS relayapi_queue_json_nid_idx
+	ON relayapi_queue (json_nid);
+CREATE INDEX IF NOT EXISTS relayapi_queue_server_name_idx
+	ON relayapi_queue (server_name);
+`
+
+const insertQueueEntrySQL = "" +
+	"INSERT INTO relayapi_queue (transaction_id, server_name, json_nid)" +
+	" VALUES ($1, $2, $3)"
+
+const deleteQueueEntriesSQL = "" +
+	"DELETE FROM relayapi_queue WHERE server_name = $1 AND json_nid IN ($2)"
+
+const selectQueueEntriesSQL = "" +
+	"SELECT json_nid FROM relayapi_queue" +
+	" WHERE server_name = $1" +
+	" ORDER BY json_nid" +
+	" LIMIT $2"
+
+const selectQueueEntryCountSQL = "" +
+	"SELECT COUNT(*) FROM relayapi_queue" +
+	" WHERE server_name = $1"
+
+type relayQueueStatements struct {
+	db                        *sql.DB
+	insertQueueEntryStmt      *sql.Stmt
+	selectQueueEntriesStmt    *sql.Stmt
+	selectQueueEntryCountStmt *sql.Stmt
+	// deleteQueueEntriesStmt *sql.Stmt - prepared at runtime due to variadic
+}
+
+func NewSQLiteRelayQueueTable(
+	db *sql.DB,
+) (s *relayQueueStatements, err error) {
+	s = &relayQueueStatements{
+		db: db,
+	}
+	_, err = db.Exec(relayQueueSchema)
+	if err != nil {
+		return
+	}
+
+	return s, sqlutil.StatementList{
+		{&s.insertQueueEntryStmt, insertQueueEntrySQL},
+		{&s.selectQueueEntriesStmt, selectQueueEntriesSQL},
+		{&s.selectQueueEntryCountStmt, selectQueueEntryCountSQL},
+	}.Prepare(db)
+}
+
+func (s *relayQueueStatements) InsertQueueEntry(
+	ctx context.Context,
+	txn *sql.Tx,
+	transactionID gomatrixserverlib.TransactionID,
+	serverName gomatrixserverlib.ServerName,
+	nid int64,
+) error {
+	stmt := sqlutil.TxStmt(txn, s.insertQueueEntryStmt)
+	_, err := stmt.ExecContext(
+		ctx,
+		transactionID, // the transaction ID that we initially attempted
+		serverName,    // destination server name
+		nid,           // JSON blob NID
+	)
+	return err
+}
+
+func (s *relayQueueStatements) DeleteQueueEntries(
+	ctx context.Context,
+	txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+	jsonNIDs []int64,
+) error {
+	deleteSQL := strings.Replace(deleteQueueEntriesSQL, "($2)", sqlutil.QueryVariadicOffset(len(jsonNIDs), 1), 1)
+	deleteStmt, err := txn.Prepare(deleteSQL)
+	if err != nil {
+		return fmt.Errorf("s.deleteQueueEntries s.db.Prepare: %w", err)
+	}
+
+	params := make([]interface{}, len(jsonNIDs)+1)
+	params[0] = serverName
+	for k, v := range jsonNIDs {
+		params[k+1] = v
+	}
+
+	stmt := sqlutil.TxStmt(txn, deleteStmt)
+	_, err = stmt.ExecContext(ctx, params...)
+	return err
+}
+
+func (s *relayQueueStatements) SelectQueueEntries(
+	ctx context.Context,
+	txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+	limit int,
+) ([]int64, error) {
+	stmt := sqlutil.TxStmt(txn, s.selectQueueEntriesStmt)
+	rows, err := stmt.QueryContext(ctx, serverName, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "queueFromStmt: rows.close() failed")
+	var result []int64
+	for rows.Next() {
+		var nid int64
+		if err = rows.Scan(&nid); err != nil {
+			return nil, err
+		}
+		result = append(result, nid)
+	}
+
+	return result, rows.Err()
+}
+
+func (s *relayQueueStatements) SelectQueueEntryCount(
+	ctx context.Context,
+	txn *sql.Tx,
+	serverName gomatrixserverlib.ServerName,
+) (int64, error) {
+	var count int64
+	stmt := sqlutil.TxStmt(txn, s.selectQueueEntryCountStmt)
+	err := stmt.QueryRowContext(ctx, serverName).Scan(&count)
+	if err == sql.ErrNoRows {
+		// It's acceptable for there to be no rows referencing a given
+		// JSON NID but it's not an error condition. Just return as if
+		// there's a zero count.
+		return 0, nil
+	}
+	return count, err
+}

relayapi/storage/sqlite3/storage.go

@@ -0,0 +1,64 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/relayapi/storage/shared"
+	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// Database stores information needed by the federation sender
+type Database struct {
+	shared.Database
+	db     *sql.DB
+	writer sqlutil.Writer
+}
+
+// NewDatabase opens a new database
+func NewDatabase(
+	base *base.BaseDendrite,
+	dbProperties *config.DatabaseOptions,
+	cache caching.FederationCache,
+	isLocalServerName func(gomatrixserverlib.ServerName) bool,
+) (*Database, error) {
+	var d Database
+	var err error
+	if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil {
+		return nil, err
+	}
+	queue, err := NewSQLiteRelayQueueTable(d.db)
+	if err != nil {
+		return nil, err
+	}
+	queueJSON, err := NewSQLiteRelayQueueJSONTable(d.db)
+	if err != nil {
+		return nil, err
+	}
+	d.Database = shared.Database{
+		DB:                d.db,
+		IsLocalServerName: isLocalServerName,
+		Cache:             cache,
+		Writer:            d.writer,
+		RelayQueue:        queue,
+		RelayQueueJSON:    queueJSON,
+	}
+	return &d, nil
+}

relayapi/storage/storage.go

@@ -0,0 +1,46 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !wasm
+// +build !wasm
+
+package storage
+
+import (
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/caching"
+	"github.com/matrix-org/dendrite/relayapi/storage/postgres"
+	"github.com/matrix-org/dendrite/relayapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/setup/base"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// NewDatabase opens a new database
+func NewDatabase(
+	base *base.BaseDendrite,
+	dbProperties *config.DatabaseOptions,
+	cache caching.FederationCache,
+	isLocalServerName func(gomatrixserverlib.ServerName) bool,
+) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(base, dbProperties, cache, isLocalServerName)
+	case dbProperties.ConnectionString.IsPostgres():
+		return postgres.NewDatabase(base, dbProperties, cache, isLocalServerName)
+	default:
+		return nil, fmt.Errorf("unexpected database type")
+	}
+}

relayapi/storage/tables/interface.go

@@ -0,0 +1,66 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tables
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// RelayQueue table contains a mapping of server name to transaction id and the corresponding nid.
+// These are the transactions being stored for the given destination server.
+// The nids correspond to entries in the RelayQueueJSON table.
+type RelayQueue interface {
+	// Adds a new transaction_id: server_name mapping with associated json table nid to the table.
+	// Will ensure only one transaction id is present for each server_name: nid mapping.
+	// Adding duplicates will silently do nothing.
+	InsertQueueEntry(ctx context.Context, txn *sql.Tx, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, nid int64) error
+
+	// Removes multiple entries from the table corresponding the the list of nids provided.
+	// If any of the provided nids don't match a row in the table, that deletion is considered
+	// successful.
+	DeleteQueueEntries(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, jsonNIDs []int64) error
+
+	// Get a list of nids associated with the provided server name.
+	// Returns up to `limit` nids. The entries are returned oldest first.
+	// Will return an empty list if no matches were found.
+	SelectQueueEntries(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, limit int) ([]int64, error)
+
+	// Get the number of entries in the table associated with the provided server name.
+	// If there are no matching rows, a count of 0 is returned with err set to nil.
+	SelectQueueEntryCount(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) (int64, error)
+}
+
+// RelayQueueJSON table contains a map of nid to the raw transaction json.
+type RelayQueueJSON interface {
+	// Adds a new transaction to the table.
+	// Adding a duplicate transaction will result in a new row being added and a new unique nid.
+	// return: unique nid representing this entry.
+	InsertQueueJSON(ctx context.Context, txn *sql.Tx, json string) (int64, error)
+
+	// Removes multiple nids from the table.
+	// If any of the provided nids don't match a row in the table, that deletion is considered
+	// successful.
+	DeleteQueueJSON(ctx context.Context, txn *sql.Tx, nids []int64) error
+
+	// Get the transaction json corresponding to the provided nids.
+	// Will return a partial result containing any matching nid from the table.
+	// Will return an empty map if no matches were found.
+	// It is the caller's responsibility to deal with the results appropriately.
+	// return: map indexed by nid of each matching transaction json.
+	SelectQueueJSON(ctx context.Context, txn *sql.Tx, jsonNIDs []int64) (map[int64][]byte, error)
+}

relayapi/storage/tables/relay_queue_json_table_test.go

@@ -0,0 +1,173 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tables_test
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/relayapi/storage/postgres"
+	"github.com/matrix-org/dendrite/relayapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/relayapi/storage/tables"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testOrigin = gomatrixserverlib.ServerName("kaer.morhen")
+)
+
+func mustCreateTransaction() gomatrixserverlib.Transaction {
+	txn := gomatrixserverlib.Transaction{}
+	txn.PDUs = []json.RawMessage{
+		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":5,"event_id":"$gl2T9l3qm0kUbiIJ:kaer.morhen","hashes":{"sha256":"Qx3nRMHLDPSL5hBAzuX84FiSSP0K0Kju2iFoBWH4Za8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$UKNe10XzYzG0TeA9:kaer.morhen",{"sha256":"KtSRyMjt0ZSjsv2koixTRCxIRCGoOp6QrKscsW97XRo"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sqDgv3EG7ml5VREzmT9aZeBpS4gAPNIaIeJOwqjDhY0GPU/BcpX5wY4R7hYLrNe5cChgV+eFy/GWm1Zfg5FfDg"}},"type":"m.room.message"}`),
+	}
+	txn.Origin = testOrigin
+
+	return txn
+}
+
+type RelayQueueJSONDatabase struct {
+	DB     *sql.DB
+	Writer sqlutil.Writer
+	Table  tables.RelayQueueJSON
+}
+
+func mustCreateQueueJSONTable(
+	t *testing.T,
+	dbType test.DBType,
+) (database RelayQueueJSONDatabase, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	var tab tables.RelayQueueJSON
+	switch dbType {
+	case test.DBTypePostgres:
+		tab, err = postgres.NewPostgresRelayQueueJSONTable(db)
+		assert.NoError(t, err)
+	case test.DBTypeSQLite:
+		tab, err = sqlite3.NewSQLiteRelayQueueJSONTable(db)
+		assert.NoError(t, err)
+	}
+	assert.NoError(t, err)
+
+	database = RelayQueueJSONDatabase{
+		DB:     db,
+		Writer: sqlutil.NewDummyWriter(),
+		Table:  tab,
+	}
+	return database, close
+}
+
+func TestShoudInsertTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueJSONTable(t, dbType)
+		defer close()
+
+		transaction := mustCreateTransaction()
+		tx, err := json.Marshal(transaction)
+		if err != nil {
+			t.Fatalf("Invalid transaction: %s", err.Error())
+		}
+
+		_, err = db.Table.InsertQueueJSON(ctx, nil, string(tx))
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+	})
+}
+
+func TestShouldRetrieveInsertedTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueJSONTable(t, dbType)
+		defer close()
+
+		transaction := mustCreateTransaction()
+		tx, err := json.Marshal(transaction)
+		if err != nil {
+			t.Fatalf("Invalid transaction: %s", err.Error())
+		}
+
+		nid, err := db.Table.InsertQueueJSON(ctx, nil, string(tx))
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		var storedJSON map[int64][]byte
+		_ = db.Writer.Do(db.DB, nil, func(txn *sql.Tx) error {
+			storedJSON, err = db.Table.SelectQueueJSON(ctx, txn, []int64{nid})
+			return err
+		})
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction: %s", err.Error())
+		}
+
+		assert.Equal(t, 1, len(storedJSON))
+
+		var storedTx gomatrixserverlib.Transaction
+		json.Unmarshal(storedJSON[1], &storedTx)
+
+		assert.Equal(t, transaction, storedTx)
+	})
+}
+
+func TestShouldDeleteTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueJSONTable(t, dbType)
+		defer close()
+
+		transaction := mustCreateTransaction()
+		tx, err := json.Marshal(transaction)
+		if err != nil {
+			t.Fatalf("Invalid transaction: %s", err.Error())
+		}
+
+		nid, err := db.Table.InsertQueueJSON(ctx, nil, string(tx))
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		storedJSON := map[int64][]byte{}
+		_ = db.Writer.Do(db.DB, nil, func(txn *sql.Tx) error {
+			err = db.Table.DeleteQueueJSON(ctx, txn, []int64{nid})
+			return err
+		})
+		if err != nil {
+			t.Fatalf("Failed deleting transaction: %s", err.Error())
+		}
+
+		storedJSON = map[int64][]byte{}
+		_ = db.Writer.Do(db.DB, nil, func(txn *sql.Tx) error {
+			storedJSON, err = db.Table.SelectQueueJSON(ctx, txn, []int64{nid})
+			return err
+		})
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction: %s", err.Error())
+		}
+
+		assert.Equal(t, 0, len(storedJSON))
+	})
+}

relayapi/storage/tables/relay_queue_table_test.go

@@ -0,0 +1,229 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tables_test
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/relayapi/storage/postgres"
+	"github.com/matrix-org/dendrite/relayapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/relayapi/storage/tables"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+type RelayQueueDatabase struct {
+	DB     *sql.DB
+	Writer sqlutil.Writer
+	Table  tables.RelayQueue
+}
+
+func mustCreateQueueTable(
+	t *testing.T,
+	dbType test.DBType,
+) (database RelayQueueDatabase, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	var tab tables.RelayQueue
+	switch dbType {
+	case test.DBTypePostgres:
+		tab, err = postgres.NewPostgresRelayQueueTable(db)
+		assert.NoError(t, err)
+	case test.DBTypeSQLite:
+		tab, err = sqlite3.NewSQLiteRelayQueueTable(db)
+		assert.NoError(t, err)
+	}
+	assert.NoError(t, err)
+
+	database = RelayQueueDatabase{
+		DB:     db,
+		Writer: sqlutil.NewDummyWriter(),
+		Table:  tab,
+	}
+	return database, close
+}
+
+func TestShoudInsertQueueTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueTable(t, dbType)
+		defer close()
+
+		transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
+		serverName := gomatrixserverlib.ServerName("domain")
+		nid := int64(1)
+		err := db.Table.InsertQueueEntry(ctx, nil, transactionID, serverName, nid)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+	})
+}
+
+func TestShouldRetrieveInsertedQueueTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueTable(t, dbType)
+		defer close()
+
+		transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
+		serverName := gomatrixserverlib.ServerName("domain")
+		nid := int64(1)
+
+		err := db.Table.InsertQueueEntry(ctx, nil, transactionID, serverName, nid)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		retrievedNids, err := db.Table.SelectQueueEntries(ctx, nil, serverName, 10)
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction: %s", err.Error())
+		}
+
+		assert.Equal(t, nid, retrievedNids[0])
+		assert.Equal(t, 1, len(retrievedNids))
+	})
+}
+
+func TestShouldRetrieveOldestInsertedQueueTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueTable(t, dbType)
+		defer close()
+
+		transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
+		serverName := gomatrixserverlib.ServerName("domain")
+		nid := int64(2)
+		err := db.Table.InsertQueueEntry(ctx, nil, transactionID, serverName, nid)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		transactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
+		serverName = gomatrixserverlib.ServerName("domain")
+		oldestNID := int64(1)
+		err = db.Table.InsertQueueEntry(ctx, nil, transactionID, serverName, oldestNID)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		retrievedNids, err := db.Table.SelectQueueEntries(ctx, nil, serverName, 1)
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction: %s", err.Error())
+		}
+
+		assert.Equal(t, oldestNID, retrievedNids[0])
+		assert.Equal(t, 1, len(retrievedNids))
+
+		retrievedNids, err = db.Table.SelectQueueEntries(ctx, nil, serverName, 10)
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction: %s", err.Error())
+		}
+
+		assert.Equal(t, oldestNID, retrievedNids[0])
+		assert.Equal(t, nid, retrievedNids[1])
+		assert.Equal(t, 2, len(retrievedNids))
+	})
+}
+
+func TestShouldDeleteQueueTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueTable(t, dbType)
+		defer close()
+
+		transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
+		serverName := gomatrixserverlib.ServerName("domain")
+		nid := int64(1)
+
+		err := db.Table.InsertQueueEntry(ctx, nil, transactionID, serverName, nid)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		_ = db.Writer.Do(db.DB, nil, func(txn *sql.Tx) error {
+			err = db.Table.DeleteQueueEntries(ctx, txn, serverName, []int64{nid})
+			return err
+		})
+		if err != nil {
+			t.Fatalf("Failed deleting transaction: %s", err.Error())
+		}
+
+		count, err := db.Table.SelectQueueEntryCount(ctx, nil, serverName)
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction count: %s", err.Error())
+		}
+		assert.Equal(t, int64(0), count)
+	})
+}
+
+func TestShouldDeleteOnlySpecifiedQueueTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueTable(t, dbType)
+		defer close()
+
+		transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
+		serverName := gomatrixserverlib.ServerName("domain")
+		nid := int64(1)
+		transactionID2 := gomatrixserverlib.TransactionID(fmt.Sprintf("%d2", time.Now().UnixNano()))
+		serverName2 := gomatrixserverlib.ServerName("domain2")
+		nid2 := int64(2)
+		transactionID3 := gomatrixserverlib.TransactionID(fmt.Sprintf("%d3", time.Now().UnixNano()))
+
+		err := db.Table.InsertQueueEntry(ctx, nil, transactionID, serverName, nid)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+		err = db.Table.InsertQueueEntry(ctx, nil, transactionID2, serverName2, nid)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+		err = db.Table.InsertQueueEntry(ctx, nil, transactionID3, serverName, nid2)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		_ = db.Writer.Do(db.DB, nil, func(txn *sql.Tx) error {
+			err = db.Table.DeleteQueueEntries(ctx, txn, serverName, []int64{nid})
+			return err
+		})
+		if err != nil {
+			t.Fatalf("Failed deleting transaction: %s", err.Error())
+		}
+
+		count, err := db.Table.SelectQueueEntryCount(ctx, nil, serverName)
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction count: %s", err.Error())
+		}
+		assert.Equal(t, int64(1), count)
+
+		count, err = db.Table.SelectQueueEntryCount(ctx, nil, serverName2)
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction count: %s", err.Error())
+		}
+		assert.Equal(t, int64(1), count)
+	})
+}