Initial Store & Forward Implementation (#2917)

This adds store & forward relays into dendrite for p2p. A few things have changed: - new relay api serves new http endpoints for s&f federation - updated outbound federation queueing which will attempt to forward using s&f if appropriate - database entries to track s&f relays for other nodes
2025-08-01 13:52:46 +00:00 · 2023-01-23 17:55:12 +00:00 · 2023-01-23 17:55:12 +00:00 · 5b73592f5a
commit 5b73592f5a
parent 48fa869fa3
77 changed files with 7646 additions and 1373 deletions
--- a/relayapi/storage/tables/interface.go
+++ b/relayapi/storage/tables/interface.go
@ -0,0 +1,66 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tables
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// RelayQueue table contains a mapping of server name to transaction id and the corresponding nid.
+// These are the transactions being stored for the given destination server.
+// The nids correspond to entries in the RelayQueueJSON table.
+type RelayQueue interface {
+	// Adds a new transaction_id: server_name mapping with associated json table nid to the table.
+	// Will ensure only one transaction id is present for each server_name: nid mapping.
+	// Adding duplicates will silently do nothing.
+	InsertQueueEntry(ctx context.Context, txn *sql.Tx, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, nid int64) error
+
+	// Removes multiple entries from the table corresponding the the list of nids provided.
+	// If any of the provided nids don't match a row in the table, that deletion is considered
+	// successful.
+	DeleteQueueEntries(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, jsonNIDs []int64) error
+
+	// Get a list of nids associated with the provided server name.
+	// Returns up to `limit` nids. The entries are returned oldest first.
+	// Will return an empty list if no matches were found.
+	SelectQueueEntries(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, limit int) ([]int64, error)
+
+	// Get the number of entries in the table associated with the provided server name.
+	// If there are no matching rows, a count of 0 is returned with err set to nil.
+	SelectQueueEntryCount(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) (int64, error)
+}
+
+// RelayQueueJSON table contains a map of nid to the raw transaction json.
+type RelayQueueJSON interface {
+	// Adds a new transaction to the table.
+	// Adding a duplicate transaction will result in a new row being added and a new unique nid.
+	// return: unique nid representing this entry.
+	InsertQueueJSON(ctx context.Context, txn *sql.Tx, json string) (int64, error)
+
+	// Removes multiple nids from the table.
+	// If any of the provided nids don't match a row in the table, that deletion is considered
+	// successful.
+	DeleteQueueJSON(ctx context.Context, txn *sql.Tx, nids []int64) error
+
+	// Get the transaction json corresponding to the provided nids.
+	// Will return a partial result containing any matching nid from the table.
+	// Will return an empty map if no matches were found.
+	// It is the caller's responsibility to deal with the results appropriately.
+	// return: map indexed by nid of each matching transaction json.
+	SelectQueueJSON(ctx context.Context, txn *sql.Tx, jsonNIDs []int64) (map[int64][]byte, error)
+}
--- a/relayapi/storage/tables/relay_queue_json_table_test.go
+++ b/relayapi/storage/tables/relay_queue_json_table_test.go
@ -0,0 +1,173 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tables_test
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"testing"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/relayapi/storage/postgres"
+	"github.com/matrix-org/dendrite/relayapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/relayapi/storage/tables"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testOrigin = gomatrixserverlib.ServerName("kaer.morhen")
+)
+
+func mustCreateTransaction() gomatrixserverlib.Transaction {
+	txn := gomatrixserverlib.Transaction{}
+	txn.PDUs = []json.RawMessage{
+		[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":5,"event_id":"$gl2T9l3qm0kUbiIJ:kaer.morhen","hashes":{"sha256":"Qx3nRMHLDPSL5hBAzuX84FiSSP0K0Kju2iFoBWH4Za8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$UKNe10XzYzG0TeA9:kaer.morhen",{"sha256":"KtSRyMjt0ZSjsv2koixTRCxIRCGoOp6QrKscsW97XRo"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sqDgv3EG7ml5VREzmT9aZeBpS4gAPNIaIeJOwqjDhY0GPU/BcpX5wY4R7hYLrNe5cChgV+eFy/GWm1Zfg5FfDg"}},"type":"m.room.message"}`),
+	}
+	txn.Origin = testOrigin
+
+	return txn
+}
+
+type RelayQueueJSONDatabase struct {
+	DB     *sql.DB
+	Writer sqlutil.Writer
+	Table  tables.RelayQueueJSON
+}
+
+func mustCreateQueueJSONTable(
+	t *testing.T,
+	dbType test.DBType,
+) (database RelayQueueJSONDatabase, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	var tab tables.RelayQueueJSON
+	switch dbType {
+	case test.DBTypePostgres:
+		tab, err = postgres.NewPostgresRelayQueueJSONTable(db)
+		assert.NoError(t, err)
+	case test.DBTypeSQLite:
+		tab, err = sqlite3.NewSQLiteRelayQueueJSONTable(db)
+		assert.NoError(t, err)
+	}
+	assert.NoError(t, err)
+
+	database = RelayQueueJSONDatabase{
+		DB:     db,
+		Writer: sqlutil.NewDummyWriter(),
+		Table:  tab,
+	}
+	return database, close
+}
+
+func TestShoudInsertTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueJSONTable(t, dbType)
+		defer close()
+
+		transaction := mustCreateTransaction()
+		tx, err := json.Marshal(transaction)
+		if err != nil {
+			t.Fatalf("Invalid transaction: %s", err.Error())
+		}
+
+		_, err = db.Table.InsertQueueJSON(ctx, nil, string(tx))
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+	})
+}
+
+func TestShouldRetrieveInsertedTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueJSONTable(t, dbType)
+		defer close()
+
+		transaction := mustCreateTransaction()
+		tx, err := json.Marshal(transaction)
+		if err != nil {
+			t.Fatalf("Invalid transaction: %s", err.Error())
+		}
+
+		nid, err := db.Table.InsertQueueJSON(ctx, nil, string(tx))
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		var storedJSON map[int64][]byte
+		_ = db.Writer.Do(db.DB, nil, func(txn *sql.Tx) error {
+			storedJSON, err = db.Table.SelectQueueJSON(ctx, txn, []int64{nid})
+			return err
+		})
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction: %s", err.Error())
+		}
+
+		assert.Equal(t, 1, len(storedJSON))
+
+		var storedTx gomatrixserverlib.Transaction
+		json.Unmarshal(storedJSON[1], &storedTx)
+
+		assert.Equal(t, transaction, storedTx)
+	})
+}
+
+func TestShouldDeleteTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueJSONTable(t, dbType)
+		defer close()
+
+		transaction := mustCreateTransaction()
+		tx, err := json.Marshal(transaction)
+		if err != nil {
+			t.Fatalf("Invalid transaction: %s", err.Error())
+		}
+
+		nid, err := db.Table.InsertQueueJSON(ctx, nil, string(tx))
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		storedJSON := map[int64][]byte{}
+		_ = db.Writer.Do(db.DB, nil, func(txn *sql.Tx) error {
+			err = db.Table.DeleteQueueJSON(ctx, txn, []int64{nid})
+			return err
+		})
+		if err != nil {
+			t.Fatalf("Failed deleting transaction: %s", err.Error())
+		}
+
+		storedJSON = map[int64][]byte{}
+		_ = db.Writer.Do(db.DB, nil, func(txn *sql.Tx) error {
+			storedJSON, err = db.Table.SelectQueueJSON(ctx, txn, []int64{nid})
+			return err
+		})
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction: %s", err.Error())
+		}
+
+		assert.Equal(t, 0, len(storedJSON))
+	})
+}
--- a/relayapi/storage/tables/relay_queue_table_test.go
+++ b/relayapi/storage/tables/relay_queue_table_test.go
@ -0,0 +1,229 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tables_test
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/relayapi/storage/postgres"
+	"github.com/matrix-org/dendrite/relayapi/storage/sqlite3"
+	"github.com/matrix-org/dendrite/relayapi/storage/tables"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+type RelayQueueDatabase struct {
+	DB     *sql.DB
+	Writer sqlutil.Writer
+	Table  tables.RelayQueue
+}
+
+func mustCreateQueueTable(
+	t *testing.T,
+	dbType test.DBType,
+) (database RelayQueueDatabase, close func()) {
+	t.Helper()
+	connStr, close := test.PrepareDBConnectionString(t, dbType)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(connStr),
+	}, sqlutil.NewExclusiveWriter())
+	assert.NoError(t, err)
+	var tab tables.RelayQueue
+	switch dbType {
+	case test.DBTypePostgres:
+		tab, err = postgres.NewPostgresRelayQueueTable(db)
+		assert.NoError(t, err)
+	case test.DBTypeSQLite:
+		tab, err = sqlite3.NewSQLiteRelayQueueTable(db)
+		assert.NoError(t, err)
+	}
+	assert.NoError(t, err)
+
+	database = RelayQueueDatabase{
+		DB:     db,
+		Writer: sqlutil.NewDummyWriter(),
+		Table:  tab,
+	}
+	return database, close
+}
+
+func TestShoudInsertQueueTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueTable(t, dbType)
+		defer close()
+
+		transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
+		serverName := gomatrixserverlib.ServerName("domain")
+		nid := int64(1)
+		err := db.Table.InsertQueueEntry(ctx, nil, transactionID, serverName, nid)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+	})
+}
+
+func TestShouldRetrieveInsertedQueueTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueTable(t, dbType)
+		defer close()
+
+		transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
+		serverName := gomatrixserverlib.ServerName("domain")
+		nid := int64(1)
+
+		err := db.Table.InsertQueueEntry(ctx, nil, transactionID, serverName, nid)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		retrievedNids, err := db.Table.SelectQueueEntries(ctx, nil, serverName, 10)
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction: %s", err.Error())
+		}
+
+		assert.Equal(t, nid, retrievedNids[0])
+		assert.Equal(t, 1, len(retrievedNids))
+	})
+}
+
+func TestShouldRetrieveOldestInsertedQueueTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueTable(t, dbType)
+		defer close()
+
+		transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
+		serverName := gomatrixserverlib.ServerName("domain")
+		nid := int64(2)
+		err := db.Table.InsertQueueEntry(ctx, nil, transactionID, serverName, nid)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		transactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
+		serverName = gomatrixserverlib.ServerName("domain")
+		oldestNID := int64(1)
+		err = db.Table.InsertQueueEntry(ctx, nil, transactionID, serverName, oldestNID)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		retrievedNids, err := db.Table.SelectQueueEntries(ctx, nil, serverName, 1)
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction: %s", err.Error())
+		}
+
+		assert.Equal(t, oldestNID, retrievedNids[0])
+		assert.Equal(t, 1, len(retrievedNids))
+
+		retrievedNids, err = db.Table.SelectQueueEntries(ctx, nil, serverName, 10)
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction: %s", err.Error())
+		}
+
+		assert.Equal(t, oldestNID, retrievedNids[0])
+		assert.Equal(t, nid, retrievedNids[1])
+		assert.Equal(t, 2, len(retrievedNids))
+	})
+}
+
+func TestShouldDeleteQueueTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueTable(t, dbType)
+		defer close()
+
+		transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
+		serverName := gomatrixserverlib.ServerName("domain")
+		nid := int64(1)
+
+		err := db.Table.InsertQueueEntry(ctx, nil, transactionID, serverName, nid)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		_ = db.Writer.Do(db.DB, nil, func(txn *sql.Tx) error {
+			err = db.Table.DeleteQueueEntries(ctx, txn, serverName, []int64{nid})
+			return err
+		})
+		if err != nil {
+			t.Fatalf("Failed deleting transaction: %s", err.Error())
+		}
+
+		count, err := db.Table.SelectQueueEntryCount(ctx, nil, serverName)
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction count: %s", err.Error())
+		}
+		assert.Equal(t, int64(0), count)
+	})
+}
+
+func TestShouldDeleteOnlySpecifiedQueueTransaction(t *testing.T) {
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		db, close := mustCreateQueueTable(t, dbType)
+		defer close()
+
+		transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano()))
+		serverName := gomatrixserverlib.ServerName("domain")
+		nid := int64(1)
+		transactionID2 := gomatrixserverlib.TransactionID(fmt.Sprintf("%d2", time.Now().UnixNano()))
+		serverName2 := gomatrixserverlib.ServerName("domain2")
+		nid2 := int64(2)
+		transactionID3 := gomatrixserverlib.TransactionID(fmt.Sprintf("%d3", time.Now().UnixNano()))
+
+		err := db.Table.InsertQueueEntry(ctx, nil, transactionID, serverName, nid)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+		err = db.Table.InsertQueueEntry(ctx, nil, transactionID2, serverName2, nid)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+		err = db.Table.InsertQueueEntry(ctx, nil, transactionID3, serverName, nid2)
+		if err != nil {
+			t.Fatalf("Failed inserting transaction: %s", err.Error())
+		}
+
+		_ = db.Writer.Do(db.DB, nil, func(txn *sql.Tx) error {
+			err = db.Table.DeleteQueueEntries(ctx, txn, serverName, []int64{nid})
+			return err
+		})
+		if err != nil {
+			t.Fatalf("Failed deleting transaction: %s", err.Error())
+		}
+
+		count, err := db.Table.SelectQueueEntryCount(ctx, nil, serverName)
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction count: %s", err.Error())
+		}
+		assert.Equal(t, int64(1), count)
+
+		count, err = db.Table.SelectQueueEntryCount(ctx, nil, serverName2)
+		if err != nil {
+			t.Fatalf("Failed retrieving transaction count: %s", err.Error())
+		}
+		assert.Equal(t, int64(1), count)
+	})
+}