hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2025-07-29 12:42:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Ristretto cache (#2563)

Browse source 
* Try Ristretto cache

* Tweak

* It's beautiful

* Update GMSL

* More strict keyable interface

* Fix that some more

* Make less panicky

* Don't enforce mutability checks for now

* Determine mutability using deep equality

* Tweaks

* Namespace keys

* Make federation caches mutable

* Update cost estimation, add metric

* Update GMSL

* Estimate cost for metrics better

* Reduce counters a bit

* Try caching events

* Some guards

* Try again

* Try this

* Use separate caches for hopefully better hash distribution

* Fix bug with admitting events into cache

* Try to fix bugs

* Check nil

* Try that again

* Preserve order jeezo this is messy

* thanks VS Code for doing exactly the wrong thing

* Try this again

* Be more specific

* aaaaargh

* One more time

* That might be better

* Stronger sorting

* Cache expiries, async publishing of EDUs

* Put it back

* Use a shared cache again

* Cost estimation fixes

* Update ristretto

* Reduce counters a bit

* Clean up a bit

* Update GMSL

* 1GB

* Configurable cache sizees

* Tweaks

* Add `config.DataUnit` for specifying friendly cache sizes

* Various tweaks

* Update GMSL

* Add back some lazy loading caching

* Include key in cost

* Include key in cost

* Tweak max age handling, config key name

* Only register prometheus metrics if requested

* Review comments @S7evinK

* Don't return errors when creating caches (it is better just to crash since otherwise we'll `nil`-pointer exception everywhere)

* Review comments

* Update sample configs

* Update GHA Workflow

* Update Complement images to Go 1.18

* Remove the cache test from the federation API as we no longer guarantee immediate cache admission

* Don't check the caches in the renewal test

* Possibly fix the upgrade tests

* Update to matrix-org/gomatrixserverlib#322

* Update documentation to refer to Go 1.18
This commit is contained in:
Neil Alexander 2022-07-11 14:31:31 +01:00 • committed by GitHub
parent eb8dc50a97
commit 3ea21273bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 603 additions and 764 deletions
Download patch file Download diff file Expand all files Collapse all files

46
setup/config/config_global.go
View file

@ -2,6 +2,8 @@ package config
import (
"math/rand"
"strconv"
"strings"
"time"
"github.com/matrix-org/gomatrixserverlib"
@ -73,6 +75,9 @@ type Global struct {
// ReportStats configures opt-in anonymous stats reporting.
ReportStats ReportStats `yaml:"report_stats"`
// Configuration for the caches.
Cache Cache `yaml:"cache"`
}
func (c *Global) Defaults(generate bool) {
@ -90,6 +95,7 @@ func (c *Global) Defaults(generate bool) {
c.Sentry.Defaults()
c.ServerNotices.Defaults(generate)
c.ReportStats.Defaults()
c.Cache.Defaults(generate)
}
func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
@ -102,6 +108,7 @@ func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
c.DNSCache.Verify(configErrs, isMonolith)
c.ServerNotices.Verify(configErrs, isMonolith)
c.ReportStats.Verify(configErrs, isMonolith)
c.Cache.Verify(configErrs, isMonolith)
}
type OldVerifyKeys struct {
@ -168,6 +175,20 @@ func (c *ServerNotices) Defaults(generate bool) {
func (c *ServerNotices) Verify(errors *ConfigErrors, isMonolith bool) {}
type Cache struct {
EstimatedMaxSize DataUnit `yaml:"max_size_estimated"`
MaxAge time.Duration `yaml:"max_age"`
}
func (c *Cache) Defaults(generate bool) {
c.EstimatedMaxSize = 1024 * 1024 * 1024 // 1GB
c.MaxAge = time.Hour
}
func (c *Cache) Verify(errors *ConfigErrors, isMonolith bool) {
checkPositive(errors, "max_size_estimated", int64(c.EstimatedMaxSize))
}
// ReportStats configures opt-in anonymous stats reporting.
type ReportStats struct {
// Enabled configures anonymous usage stats of the server
@ -268,3 +289,28 @@ type PresenceOptions struct {
// Whether outbound presence events are allowed
EnableOutbound bool `yaml:"enable_outbound"`
}
type DataUnit int64
func (d *DataUnit) UnmarshalText(text []byte) error {
var magnitude float64
s := strings.ToLower(string(text))
switch {
case strings.HasSuffix(s, "tb"):
s, magnitude = s[:len(s)-2], 1024*1024*1024*1024
case strings.HasSuffix(s, "gb"):
s, magnitude = s[:len(s)-2], 1024*1024*1024
case strings.HasSuffix(s, "mb"):
s, magnitude = s[:len(s)-2], 1024*1024
case strings.HasSuffix(s, "kb"):
s, magnitude = s[:len(s)-2], 1024
default:
magnitude = 1
}
v, err := strconv.ParseFloat(s, 64)
if err != nil {
return err
}
*d = DataUnit(v * magnitude)
return nil
}

21
setup/config/config_test.go
View file

@ -17,6 +17,8 @@ package config
import (
"fmt"
"testing"
"gopkg.in/yaml.v2"
)
func TestLoadConfigRelative(t *testing.T) {
@ -268,3 +270,22 @@ n0Xq64k7fc42HXJpF8CGBkSaIhtlzcruO+vqR80B9r62+D0V7VmHOnP135MT6noU
ANAf5kxmMsM0zlN2hkxl0H6o7wKlBSw3RI3cjfilXiMWRPJrzlc4
-----END CERTIFICATE-----
`
func TestUnmarshalDataUnit(t *testing.T) {
target := struct {
Got DataUnit `yaml:"value"`
}{}
for input, expect := range map[string]DataUnit{
"value: 0.6tb": 659706976665,
"value: 1.2gb": 1288490188,
"value: 256mb": 268435456,
"value: 128kb": 131072,
"value: 128": 128,
} {
if err := yaml.Unmarshal([]byte(input), &target); err != nil {
t.Fatal(err)
} else if target.Got != expect {
t.Fatalf("expected value %d but got %d", expect, target.Got)
}
}
}