DNS caching (#1728)

* Allow configuring DNS cache

* Update sample configs

* Fix build errors

* Fix time resolution

* Default 5m

* In seconds

* Use WithDNScache

* Correct field name

* Update go.mod/go.sum to matrix-org/gomatrixserverlib#251

build/docker/config/dendrite-config.yaml

@@ -91,6 +91,17 @@ global:
       username: metrics
       password: metrics
 
+  # DNS cache options. The DNS cache may reduce the load on DNS servers
+  # if there is no local caching resolver available for use.
+  dns_cache:
+    # Whether or not the DNS cache is enabled.
+    enabled: false
+
+    # Maximum number of entries to hold in the DNS cache, and
+    # for how long those items should be considered valid in seconds.
+    cache_size: 256
+    cache_lifetime: 300
+
 # Configuration for the Appservice API.
 app_service_api:
   internal_api:

dendrite-config.yaml

@@ -103,6 +103,17 @@ global:
       username: metrics
       password: metrics
 
+  # DNS cache options. The DNS cache may reduce the load on DNS servers
+  # if there is no local caching resolver available for use.
+  dns_cache:
+    # Whether or not the DNS cache is enabled.
+    enabled: false
+
+    # Maximum number of entries to hold in the DNS cache, and
+    # for how long those items should be considered valid in seconds.
+    cache_size: 256
+    cache_lifetime: 300
+
 # Configuration for the Appservice API.
 app_service_api:
   internal_api:

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20210121135706-63c66d16c96a
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20210122141547-5c2c047bfdcd
 	github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.2

go.sum

@@ -567,8 +567,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20210121135706-63c66d16c96a h1:cEuy0TSs4F6tYzgsi77cjx3FMBADsU7WWl+USUztD0k=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20210122141547-5c2c047bfdcd h1:SbWP7aHgACfAQpI2te6LzxK3l0+UqPOb8NOCOxzwV1c=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20210121135706-63c66d16c96a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20210122141547-5c2c047bfdcd/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=

setup/base.go

@@ -73,6 +73,7 @@ type BaseDendrite struct {
 	httpClient             *http.Client
 	Cfg                    *config.Dendrite
 	Caches                 *caching.Caches
+	DNSCache               *gomatrixserverlib.DNSCache
 	//	KafkaConsumer          sarama.Consumer
 	//	KafkaProducer          sarama.SyncProducer
 }
@@ -111,6 +112,20 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
 		logrus.WithError(err).Warnf("Failed to create cache")
 	}
 
+	var dnsCache *gomatrixserverlib.DNSCache
+	if cfg.Global.DNSCache.Enabled {
+		lifetime := time.Second * cfg.Global.DNSCache.CacheLifetime
+		dnsCache = gomatrixserverlib.NewDNSCache(
+			cfg.Global.DNSCache.CacheSize,
+			lifetime,
+		)
+		logrus.Infof(
+			"DNS cache enabled (size %d, lifetime %s)",
+			cfg.Global.DNSCache.CacheSize,
+			lifetime,
+		)
+	}
+
 	apiClient := http.Client{
 		Timeout: time.Minute * 10,
 		Transport: &http2.Transport{
@@ -152,6 +167,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
 		tracerCloser:           closer,
 		Cfg:                    cfg,
 		Caches:                 cache,
+		DNSCache:               dnsCache,
 		PublicClientAPIMux:     mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicClientPathPrefix).Subrouter().UseEncodedPath(),
 		PublicFederationAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath(),
 		PublicKeyAPIMux:        mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath(),
@@ -252,8 +268,12 @@ func (b *BaseDendrite) CreateClient() *gomatrixserverlib.Client {
 	if b.Cfg.Global.DisableFederation {
 		return gomatrixserverlib.NewClientWithTransport(noOpHTTPTransport)
 	}
+	opts := []interface{}{}
+	if b.Cfg.Global.DNSCache.Enabled {
+		opts = append(opts, gomatrixserverlib.WithDNSCache{DNSCache: b.DNSCache})
+	}
 	client := gomatrixserverlib.NewClient(
-		b.Cfg.FederationSender.DisableTLSValidation,
+		b.Cfg.FederationSender.DisableTLSValidation, opts...,
 	)
 	client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString()))
 	return client
@@ -268,9 +288,13 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli
 			b.Cfg.FederationSender.DisableTLSValidation, noOpHTTPTransport,
 		)
 	}
+	opts := []interface{}{}
+	if b.Cfg.Global.DNSCache.Enabled {
+		opts = append(opts, gomatrixserverlib.WithDNSCache{DNSCache: b.DNSCache})
+	}
 	client := gomatrixserverlib.NewFederationClientWithTimeout(
 		b.Cfg.Global.ServerName, b.Cfg.Global.KeyID, b.Cfg.Global.PrivateKey,
-		b.Cfg.FederationSender.DisableTLSValidation, time.Minute*5,
+		b.Cfg.FederationSender.DisableTLSValidation, time.Minute*5, opts...,
 	)
 	client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString()))
 	return client

setup/config/config_global.go

@@ -48,6 +48,9 @@ type Global struct {
 
 	// Metrics configuration
 	Metrics Metrics `yaml:"metrics"`
+
+	// DNS caching options for all outbound HTTP requests
+	DNSCache DNSCacheOptions `yaml:"dns_cache"`
 }
 
 func (c *Global) Defaults() {
@@ -59,6 +62,7 @@ func (c *Global) Defaults() {
 
 	c.Kafka.Defaults()
 	c.Metrics.Defaults()
+	c.DNSCache.Defaults()
 }
 
 func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
@@ -67,6 +71,7 @@ func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
 
 	c.Kafka.Verify(configErrs, isMonolith)
 	c.Metrics.Verify(configErrs, isMonolith)
+	c.DNSCache.Verify(configErrs, isMonolith)
 }
 
 type OldVerifyKeys struct {
@@ -140,3 +145,23 @@ func (c DatabaseOptions) MaxOpenConns() int {
 func (c DatabaseOptions) ConnMaxLifetime() time.Duration {
 	return time.Duration(c.ConnMaxLifetimeSeconds) * time.Second
 }
+
+type DNSCacheOptions struct {
+	// Whether the DNS cache is enabled or not
+	Enabled bool `yaml:"enabled"`
+	// How many entries to store in the DNS cache at a given time
+	CacheSize int `yaml:"cache_size"`
+	// How long a cache entry should be considered valid for
+	CacheLifetime time.Duration `yaml:"cache_lifetime"`
+}
+
+func (c *DNSCacheOptions) Defaults() {
+	c.Enabled = false
+	c.CacheSize = 256
+	c.CacheLifetime = time.Minute * 5
+}
+
+func (c *DNSCacheOptions) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkPositive(configErrs, "cache_size", int64(c.CacheSize))
+	checkPositive(configErrs, "cache_lifetime", int64(c.CacheLifetime))
+}