Ristretto cache (#2563)

* Try Ristretto cache

* Tweak

* It's beautiful

* Update GMSL

* More strict keyable interface

* Fix that some more

* Make less panicky

* Don't enforce mutability checks for now

* Determine mutability using deep equality

* Tweaks

* Namespace keys

* Make federation caches mutable

* Update cost estimation, add metric

* Update GMSL

* Estimate cost for metrics better

* Reduce counters a bit

* Try caching events

* Some guards

* Try again

* Try this

* Use separate caches for hopefully better hash distribution

* Fix bug with admitting events into cache

* Try to fix bugs

* Check nil

* Try that again

* Preserve order jeezo this is messy

* thanks VS Code for doing exactly the wrong thing

* Try this again

* Be more specific

* aaaaargh

* One more time

* That might be better

* Stronger sorting

* Cache expiries, async publishing of EDUs

* Put it back

* Use a shared cache again

* Cost estimation fixes

* Update ristretto

* Reduce counters a bit

* Clean up a bit

* Update GMSL

* 1GB

* Configurable cache sizees

* Tweaks

* Add `config.DataUnit` for specifying friendly cache sizes

* Various tweaks

* Update GMSL

* Add back some lazy loading caching

* Include key in cost

* Include key in cost

* Tweak max age handling, config key name

* Only register prometheus metrics if requested

* Review comments @S7evinK

* Don't return errors when creating caches (it is better just to crash since otherwise we'll `nil`-pointer exception everywhere)

* Review comments

* Update sample configs

* Update GHA Workflow

* Update Complement images to Go 1.18

* Remove the cache test from the federation API as we no longer guarantee immediate cache admission

* Don't check the caches in the renewal test

* Possibly fix the upgrade tests

* Update to matrix-org/gomatrixserverlib#322

* Update documentation to refer to Go 1.18

federationapi/federationapi_keys_test.go

@@ -9,7 +9,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"reflect"
 	"testing"
 	"time"
 
@@ -64,10 +63,7 @@ func TestMain(m *testing.M) {
 			}
 
 			// Create a new cache but don't enable prometheus!
-			s.cache, err = caching.NewInMemoryLRUCache(false)
-			if err != nil {
-				panic("can't create cache: " + err.Error())
-			}
+			s.cache = caching.NewRistrettoCache(8*1024*1024, time.Hour, false)
 
 			// Create a temporary directory for JetStream.
 			d, err := ioutil.TempDir("./", "jetstream*")
@@ -170,72 +166,6 @@ func TestServersRequestOwnKeys(t *testing.T) {
 	}
 }
 
-func TestCachingBehaviour(t *testing.T) {
-	// Server A will request Server B's key, which has a validity
-	// period of an hour from now. We should retrieve the key and
-	// it should make it into the cache automatically.
-
-	req := gomatrixserverlib.PublicKeyLookupRequest{
-		ServerName: serverB.name,
-		KeyID:      serverKeyID,
-	}
-	ts := gomatrixserverlib.AsTimestamp(time.Now())
-
-	res, err := serverA.api.FetchKeys(
-		context.Background(),
-		map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{
-			req: ts,
-		},
-	)
-	if err != nil {
-		t.Fatalf("server A failed to retrieve server B key: %s", err)
-	}
-	if len(res) != 1 {
-		t.Fatalf("server B should have returned one key but instead returned %d keys", len(res))
-	}
-	if _, ok := res[req]; !ok {
-		t.Fatalf("server B isn't included in the key fetch response")
-	}
-
-	// At this point, if the previous key request was a success,
-	// then the cache should now contain the key. Check if that's
-	// the case - if it isn't then there's something wrong with
-	// the cache implementation or we failed to get the key.
-
-	cres, ok := serverA.cache.GetServerKey(req, ts)
-	if !ok {
-		t.Fatalf("server B key should be in cache but isn't")
-	}
-	if !reflect.DeepEqual(cres, res[req]) {
-		t.Fatalf("the cached result from server B wasn't what server B gave us")
-	}
-
-	// If we ask the cache for the same key but this time for an event
-	// that happened in +30 minutes. Since the validity period is for
-	// another hour, then we should get a response back from the cache.
-
-	_, ok = serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*30)),
-	)
-	if !ok {
-		t.Fatalf("server B key isn't in cache when it should be (+30 minutes)")
-	}
-
-	// If we ask the cache for the same key but this time for an event
-	// that happened in +90 minutes then we should expect to get no
-	// cache result. This is because the cache shouldn't return a result
-	// that is obviously past the validity of the event.
-
-	_, ok = serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*90)),
-	)
-	if ok {
-		t.Fatalf("server B key is in cache when it shouldn't be (+90 minutes)")
-	}
-}
-
 func TestRenewalBehaviour(t *testing.T) {
 	// Server A will request Server C's key but their validity period
 	// is an hour in the past. We'll retrieve the key as, even though it's
@@ -262,32 +192,7 @@ func TestRenewalBehaviour(t *testing.T) {
 		t.Fatalf("server C isn't included in the key fetch response")
 	}
 
-	// If we ask the cache for the server key for an event that happened
-	// 90 minutes ago then we should get a cache result, as the key hadn't
-	// passed its validity by that point. The fact that the key is now in
-	// the cache is, in itself, proof that we successfully retrieved the
-	// key before.
-
-	oldcached, ok := serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*90)),
-	)
-	if !ok {
-		t.Fatalf("server C key isn't in cache when it should be (-90 minutes)")
-	}
-
-	// If we now ask the cache for the same key but this time for an event
-	// that only happened 30 minutes ago then we shouldn't get a cached
-	// result, as the event happened after the key validity expired. This
-	// is really just for sanity checking.
-
-	_, ok = serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)),
-	)
-	if ok {
-		t.Fatalf("server B key is in cache when it shouldn't be (-30 minutes)")
-	}
+	originalValidity := res[req].ValidUntilTS
 
 	// We're now going to kick server C into renewing its key. Since we're
 	// happy at this point that the key that we already have is from the past
@@ -308,24 +213,13 @@ func TestRenewalBehaviour(t *testing.T) {
 	if len(res) != 1 {
 		t.Fatalf("server C should have returned one key but instead returned %d keys", len(res))
 	}
-	if _, ok = res[req]; !ok {
+	if _, ok := res[req]; !ok {
 		t.Fatalf("server C isn't included in the key fetch response")
 	}
 
-	// We're now going to ask the cache what the new key validity is. If
-	// it is still the same as the previous validity then we've failed to
-	// retrieve the renewed key. If it's newer then we've successfully got
-	// the renewed key.
+	currentValidity := res[req].ValidUntilTS
 
-	newcached, ok := serverA.cache.GetServerKey(
-		req,
-		gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)),
-	)
-	if !ok {
-		t.Fatalf("server B key isn't in cache when it shouldn't be (post-renewal)")
+	if originalValidity == currentValidity {
+		t.Fatalf("server C key should have renewed but didn't")
 	}
-	if oldcached.ValidUntilTS >= newcached.ValidUntilTS {
-		t.Fatalf("the server B key should have been renewed but wasn't")
-	}
-	t.Log(res)
 }

federationapi/routing/invite.go

@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // InviteV2 implements /_matrix/federation/v2/invite/{roomID}/{eventID}
@@ -141,10 +142,17 @@ func processInvite(
 	}
 
 	// Check that the event is signed by the server sending the request.
-	redacted := event.Redact()
+	redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version())
+	if err != nil {
+		logrus.WithError(err).Errorf("XXX: invite.go")
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The event JSON could not be redacted"),
+		}
+	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
 		ServerName:             event.Origin(),
-		Message:                redacted.JSON(),
+		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
 	}}

federationapi/routing/join.go

@@ -266,10 +266,17 @@ func SendJoin(
 	}
 
 	// Check that the event is signed by the server sending the request.
-	redacted := event.Redact()
+	redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version())
+	if err != nil {
+		logrus.WithError(err).Errorf("XXX: join.go")
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The event JSON could not be redacted"),
+		}
+	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
 		ServerName:             event.Origin(),
-		Message:                redacted.JSON(),
+		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
 	}}

federationapi/routing/leave.go

@@ -231,10 +231,17 @@ func SendLeave(
 	}
 
 	// Check that the event is signed by the server sending the request.
-	redacted := event.Redact()
+	redacted, err := gomatrixserverlib.RedactEventJSON(event.JSON(), event.Version())
+	if err != nil {
+		logrus.WithError(err).Errorf("XXX: leave.go")
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The event JSON could not be redacted"),
+		}
+	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
 		ServerName:             event.Origin(),
-		Message:                redacted.JSON(),
+		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,
 	}}