Initial Store & Forward Implementation (#2917)

This adds store & forward relays into dendrite for p2p.
A few things have changed:
- new relay api serves new http endpoints for s&f federation
- updated outbound federation queueing which will attempt to forward
using s&f if appropriate
- database entries to track s&f relays for other nodes

federationapi/internal/api.go

@@ -109,13 +109,14 @@ func NewFederationInternalAPI(
 
 func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s gomatrixserverlib.ServerName) (*statistics.ServerStatistics, error) {
 	stats := a.statistics.ForServer(s)
-	until, blacklisted := stats.BackoffInfo()
-	if blacklisted {
+	if stats.Blacklisted() {
 		return stats, &api.FederationClientError{
 			Blacklisted: true,
 		}
 	}
+
 	now := time.Now()
+	until := stats.BackoffInfo()
 	if until != nil && now.Before(*until) {
 		return stats, &api.FederationClientError{
 			RetryAfter: time.Until(*until),
@@ -163,7 +164,7 @@ func (a *FederationInternalAPI) doRequestIfNotBackingOffOrBlacklisted(
 			RetryAfter:  retryAfter,
 		}
 	}
-	stats.Success()
+	stats.Success(statistics.SendDirect)
 	return res, nil
 }
 
@@ -171,7 +172,7 @@ func (a *FederationInternalAPI) doRequestIfNotBlacklisted(
 	s gomatrixserverlib.ServerName, request func() (interface{}, error),
 ) (interface{}, error) {
 	stats := a.statistics.ForServer(s)
-	if _, blacklisted := stats.BackoffInfo(); blacklisted {
+	if blacklisted := stats.Blacklisted(); blacklisted {
 		return stats, &api.FederationClientError{
 			Err:         fmt.Sprintf("server %q is blacklisted", s),
 			Blacklisted: true,

federationapi/internal/federationclient_test.go

@@ -0,0 +1,202 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/matrix-org/dendrite/federationapi/queue"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	FailuresUntilAssumedOffline = 3
+	FailuresUntilBlacklist      = 8
+)
+
+func (t *testFedClient) QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error) {
+	t.queryKeysCalled = true
+	if t.shouldFail {
+		return gomatrixserverlib.RespQueryKeys{}, fmt.Errorf("Failure")
+	}
+	return gomatrixserverlib.RespQueryKeys{}, nil
+}
+
+func (t *testFedClient) ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error) {
+	t.claimKeysCalled = true
+	if t.shouldFail {
+		return gomatrixserverlib.RespClaimKeys{}, fmt.Errorf("Failure")
+	}
+	return gomatrixserverlib.RespClaimKeys{}, nil
+}
+
+func TestFederationClientQueryKeys(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.QueryKeys(context.Background(), "origin", "server", nil)
+	assert.Nil(t, err)
+	assert.True(t, fedClient.queryKeysCalled)
+}
+
+func TestFederationClientQueryKeysBlacklisted(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+	testDB.AddServerToBlacklist("server")
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.QueryKeys(context.Background(), "origin", "server", nil)
+	assert.NotNil(t, err)
+	assert.False(t, fedClient.queryKeysCalled)
+}
+
+func TestFederationClientQueryKeysFailure(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{shouldFail: true}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.QueryKeys(context.Background(), "origin", "server", nil)
+	assert.NotNil(t, err)
+	assert.True(t, fedClient.queryKeysCalled)
+}
+
+func TestFederationClientClaimKeys(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.ClaimKeys(context.Background(), "origin", "server", nil)
+	assert.Nil(t, err)
+	assert.True(t, fedClient.claimKeysCalled)
+}
+
+func TestFederationClientClaimKeysBlacklisted(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+	testDB.AddServerToBlacklist("server")
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "server",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedapi := FederationInternalAPI{
+		db:         testDB,
+		cfg:        &cfg,
+		statistics: &stats,
+		federation: fedClient,
+		queues:     queues,
+	}
+	_, err := fedapi.ClaimKeys(context.Background(), "origin", "server", nil)
+	assert.NotNil(t, err)
+	assert.False(t, fedClient.claimKeysCalled)
+}

federationapi/internal/perform.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/matrix-org/dendrite/federationapi/api"
 	"github.com/matrix-org/dendrite/federationapi/consumers"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/version"
 )
@@ -24,6 +25,10 @@ func (r *FederationInternalAPI) PerformDirectoryLookup(
 	request *api.PerformDirectoryLookupRequest,
 	response *api.PerformDirectoryLookupResponse,
 ) (err error) {
+	if !r.shouldAttemptDirectFederation(request.ServerName) {
+		return fmt.Errorf("relay servers have no meaningful response for directory lookup.")
+	}
+
 	dir, err := r.federation.LookupRoomAlias(
 		ctx,
 		r.cfg.Matrix.ServerName,
@@ -36,7 +41,7 @@ func (r *FederationInternalAPI) PerformDirectoryLookup(
 	}
 	response.RoomID = dir.RoomID
 	response.ServerNames = dir.Servers
-	r.statistics.ForServer(request.ServerName).Success()
+	r.statistics.ForServer(request.ServerName).Success(statistics.SendDirect)
 	return nil
 }
 
@@ -144,6 +149,10 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 	supportedVersions []gomatrixserverlib.RoomVersion,
 	unsigned map[string]interface{},
 ) error {
+	if !r.shouldAttemptDirectFederation(serverName) {
+		return fmt.Errorf("relay servers have no meaningful response for join.")
+	}
+
 	_, origin, err := r.cfg.Matrix.SplitLocalID('@', userID)
 	if err != nil {
 		return err
@@ -164,7 +173,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		r.statistics.ForServer(serverName).Failure()
 		return fmt.Errorf("r.federation.MakeJoin: %w", err)
 	}
-	r.statistics.ForServer(serverName).Success()
+	r.statistics.ForServer(serverName).Success(statistics.SendDirect)
 
 	// Set all the fields to be what they should be, this should be a no-op
 	// but it's possible that the remote server returned us something "odd"
@@ -219,7 +228,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		r.statistics.ForServer(serverName).Failure()
 		return fmt.Errorf("r.federation.SendJoin: %w", err)
 	}
-	r.statistics.ForServer(serverName).Success()
+	r.statistics.ForServer(serverName).Success(statistics.SendDirect)
 
 	// If the remote server returned an event in the "event" key of
 	// the send_join request then we should use that instead. It may
@@ -407,6 +416,10 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 	serverName gomatrixserverlib.ServerName,
 	supportedVersions []gomatrixserverlib.RoomVersion,
 ) error {
+	if !r.shouldAttemptDirectFederation(serverName) {
+		return fmt.Errorf("relay servers have no meaningful response for outbound peek.")
+	}
+
 	// create a unique ID for this peek.
 	// for now we just use the room ID again. In future, if we ever
 	// support concurrent peeks to the same room with different filters
@@ -446,7 +459,7 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
 		r.statistics.ForServer(serverName).Failure()
 		return fmt.Errorf("r.federation.Peek: %w", err)
 	}
-	r.statistics.ForServer(serverName).Success()
+	r.statistics.ForServer(serverName).Success(statistics.SendDirect)
 
 	// Work out if we support the room version that has been supplied in
 	// the peek response.
@@ -516,6 +529,10 @@ func (r *FederationInternalAPI) PerformLeave(
 	// Try each server that we were provided until we land on one that
 	// successfully completes the make-leave send-leave dance.
 	for _, serverName := range request.ServerNames {
+		if !r.shouldAttemptDirectFederation(serverName) {
+			continue
+		}
+
 		// Try to perform a make_leave using the information supplied in the
 		// request.
 		respMakeLeave, err := r.federation.MakeLeave(
@@ -585,7 +602,7 @@ func (r *FederationInternalAPI) PerformLeave(
 			continue
 		}
 
-		r.statistics.ForServer(serverName).Success()
+		r.statistics.ForServer(serverName).Success(statistics.SendDirect)
 		return nil
 	}
 
@@ -616,6 +633,12 @@ func (r *FederationInternalAPI) PerformInvite(
 		return fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
 	}
 
+	// TODO (devon): This should be allowed via a relay. Currently only transactions
+	// can be sent to relays. Would need to extend relays to handle invites.
+	if !r.shouldAttemptDirectFederation(destination) {
+		return fmt.Errorf("relay servers have no meaningful response for invite.")
+	}
+
 	logrus.WithFields(logrus.Fields{
 		"event_id":     request.Event.EventID(),
 		"user_id":      *request.Event.StateKey(),
@@ -682,12 +705,8 @@ func (r *FederationInternalAPI) PerformWakeupServers(
 
 func (r *FederationInternalAPI) MarkServersAlive(destinations []gomatrixserverlib.ServerName) {
 	for _, srv := range destinations {
-		// Check the statistics cache for the blacklist status to prevent hitting
-		// the database unnecessarily.
-		if r.queues.IsServerBlacklisted(srv) {
-			_ = r.db.RemoveServerFromBlacklist(srv)
-		}
-		r.queues.RetryServer(srv)
+		wasBlacklisted := r.statistics.ForServer(srv).MarkServerAlive()
+		r.queues.RetryServer(srv, wasBlacklisted)
 	}
 }
 
@@ -719,7 +738,9 @@ func sanityCheckAuthChain(authChain []*gomatrixserverlib.Event) error {
 	return fmt.Errorf("auth chain response is missing m.room.create event")
 }
 
-func setDefaultRoomVersionFromJoinEvent(joinEvent gomatrixserverlib.EventBuilder) gomatrixserverlib.RoomVersion {
+func setDefaultRoomVersionFromJoinEvent(
+	joinEvent gomatrixserverlib.EventBuilder,
+) gomatrixserverlib.RoomVersion {
 	// if auth events are not event references we know it must be v3+
 	// we have to do these shenanigans to satisfy sytest, specifically for:
 	// "Outbound federation rejects m.room.create events with an unknown room version"
@@ -802,3 +823,31 @@ func federatedAuthProvider(
 		return returning, nil
 	}
 }
+
+// P2PQueryRelayServers implements api.FederationInternalAPI
+func (r *FederationInternalAPI) P2PQueryRelayServers(
+	ctx context.Context,
+	request *api.P2PQueryRelayServersRequest,
+	response *api.P2PQueryRelayServersResponse,
+) error {
+	logrus.Infof("Getting relay servers for: %s", request.Server)
+	relayServers, err := r.db.P2PGetRelayServersForServer(ctx, request.Server)
+	if err != nil {
+		return err
+	}
+
+	response.RelayServers = relayServers
+	return nil
+}
+
+func (r *FederationInternalAPI) shouldAttemptDirectFederation(
+	destination gomatrixserverlib.ServerName,
+) bool {
+	var shouldRelay bool
+	stats := r.statistics.ForServer(destination)
+	if stats.AssumedOffline() && len(stats.KnownRelayServers()) > 0 {
+		shouldRelay = true
+	}
+
+	return !shouldRelay
+}

federationapi/internal/perform_test.go

@@ -0,0 +1,190 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"context"
+	"testing"
+
+	"github.com/matrix-org/dendrite/federationapi/api"
+	"github.com/matrix-org/dendrite/federationapi/queue"
+	"github.com/matrix-org/dendrite/federationapi/statistics"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/stretchr/testify/assert"
+)
+
+type testFedClient struct {
+	api.FederationClient
+	queryKeysCalled bool
+	claimKeysCalled bool
+	shouldFail      bool
+}
+
+func (t *testFedClient) LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error) {
+	return gomatrixserverlib.RespDirectory{}, nil
+}
+
+func TestPerformWakeupServers(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	server := gomatrixserverlib.ServerName("wakeup")
+	testDB.AddServerToBlacklist(server)
+	testDB.SetServerAssumedOffline(context.Background(), server)
+	blacklisted, err := testDB.IsServerBlacklisted(server)
+	assert.NoError(t, err)
+	assert.True(t, blacklisted)
+	offline, err := testDB.IsServerAssumedOffline(context.Background(), server)
+	assert.NoError(t, err)
+	assert.True(t, offline)
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "relay",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.PerformWakeupServersRequest{
+		ServerNames: []gomatrixserverlib.ServerName{server},
+	}
+	res := api.PerformWakeupServersResponse{}
+	err = fedAPI.PerformWakeupServers(context.Background(), &req, &res)
+	assert.NoError(t, err)
+
+	blacklisted, err = testDB.IsServerBlacklisted(server)
+	assert.NoError(t, err)
+	assert.False(t, blacklisted)
+	offline, err = testDB.IsServerAssumedOffline(context.Background(), server)
+	assert.NoError(t, err)
+	assert.False(t, offline)
+}
+
+func TestQueryRelayServers(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	server := gomatrixserverlib.ServerName("wakeup")
+	relayServers := []gomatrixserverlib.ServerName{"relay1", "relay2"}
+	err := testDB.P2PAddRelayServersForServer(context.Background(), server, relayServers)
+	assert.NoError(t, err)
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "relay",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.P2PQueryRelayServersRequest{
+		Server: server,
+	}
+	res := api.P2PQueryRelayServersResponse{}
+	err = fedAPI.P2PQueryRelayServers(context.Background(), &req, &res)
+	assert.NoError(t, err)
+
+	assert.Equal(t, len(relayServers), len(res.RelayServers))
+}
+
+func TestPerformDirectoryLookup(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: "relay",
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.PerformDirectoryLookupRequest{
+		RoomAlias:  "room",
+		ServerName: "server",
+	}
+	res := api.PerformDirectoryLookupResponse{}
+	err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
+	assert.NoError(t, err)
+}
+
+func TestPerformDirectoryLookupRelaying(t *testing.T) {
+	testDB := test.NewInMemoryFederationDatabase()
+
+	server := gomatrixserverlib.ServerName("wakeup")
+	testDB.SetServerAssumedOffline(context.Background(), server)
+	testDB.P2PAddRelayServersForServer(context.Background(), server, []gomatrixserverlib.ServerName{"relay"})
+
+	cfg := config.FederationAPI{
+		Matrix: &config.Global{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				ServerName: server,
+			},
+		},
+	}
+	fedClient := &testFedClient{}
+	stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
+	queues := queue.NewOutgoingQueues(
+		testDB, process.NewProcessContext(),
+		false,
+		cfg.Matrix.ServerName, fedClient, nil, &stats,
+		nil,
+	)
+	fedAPI := NewFederationInternalAPI(
+		testDB, &cfg, nil, fedClient, &stats, nil, queues, nil,
+	)
+
+	req := api.PerformDirectoryLookupRequest{
+		RoomAlias:  "room",
+		ServerName: server,
+	}
+	res := api.PerformDirectoryLookupResponse{}
+	err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
+	assert.Error(t, err)
+}