Merge branch 'master' into add-nats-support

setup/base.go

@@ -27,6 +27,8 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/getsentry/sentry-go"
+	sentryhttp "github.com/getsentry/sentry-go/http"
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/httputil"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -114,6 +116,21 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
 		logrus.WithError(err).Panicf("failed to start opentracing")
 	}
 
+	if cfg.Global.Sentry.Enabled {
+		logrus.Info("Setting up Sentry for debugging...")
+		err = sentry.Init(sentry.ClientOptions{
+			Dsn:              cfg.Global.Sentry.DSN,
+			Environment:      cfg.Global.Sentry.Environment,
+			Debug:            true,
+			ServerName:       string(cfg.Global.ServerName),
+			Release:          "dendrite@" + internal.VersionString(),
+			AttachStacktrace: true,
+		})
+		if err != nil {
+			logrus.WithError(err).Panic("failed to start Sentry")
+		}
+	}
+
 	cache, err := caching.NewInMemoryLRUCache(true)
 	if err != nil {
 		logrus.WithError(err).Warnf("Failed to create cache")
@@ -263,7 +280,7 @@ func (b *BaseDendrite) KeyServerHTTPClient() keyserverAPI.KeyInternalAPI {
 // CreateAccountsDB creates a new instance of the accounts database. Should only
 // be called once per component.
 func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
-	db, err := accounts.NewDatabase(&b.Cfg.UserAPI.AccountDatabase, b.Cfg.Global.ServerName)
+	db, err := accounts.NewDatabase(&b.Cfg.UserAPI.AccountDatabase, b.Cfg.Global.ServerName, b.Cfg.UserAPI.BCryptCost, b.Cfg.UserAPI.OpenIDTokenLifetimeMS)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to accounts db")
 	}
@@ -316,7 +333,6 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli
 
 // SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on
 // ApiMux under /api/ and adds a prometheus handler under /metrics.
-// nolint:gocyclo
 func (b *BaseDendrite) SetupAndServeHTTP(
 	internalHTTPAddr, externalHTTPAddr config.HTTPAddress,
 	certFile, keyFile *string,
@@ -354,10 +370,26 @@ func (b *BaseDendrite) SetupAndServeHTTP(
 		internalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), b.Cfg.Global.Metrics.BasicAuth))
 	}
 
-	externalRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(b.PublicClientAPIMux)
+	var clientHandler http.Handler
+	clientHandler = b.PublicClientAPIMux
+	if b.Cfg.Global.Sentry.Enabled {
+		sentryHandler := sentryhttp.New(sentryhttp.Options{
+			Repanic: true,
+		})
+		clientHandler = sentryHandler.Handle(b.PublicClientAPIMux)
+	}
+	var federationHandler http.Handler
+	federationHandler = b.PublicFederationAPIMux
+	if b.Cfg.Global.Sentry.Enabled {
+		sentryHandler := sentryhttp.New(sentryhttp.Options{
+			Repanic: true,
+		})
+		federationHandler = sentryHandler.Handle(b.PublicFederationAPIMux)
+	}
+	externalRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(clientHandler)
 	if !b.Cfg.Global.DisableFederation {
 		externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(b.PublicKeyAPIMux)
-		externalRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(b.PublicFederationAPIMux)
+		externalRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(federationHandler)
 	}
 	externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(b.PublicMediaAPIMux)
 
@@ -437,6 +469,11 @@ func (b *BaseDendrite) WaitForShutdown() {
 
 	b.ProcessContext.ShutdownDendrite()
 	b.ProcessContext.WaitForComponentsToFinish()
+	if b.Cfg.Global.Sentry.Enabled {
+		if !sentry.Flush(time.Second * 5) {
+			logrus.Warnf("failed to flush all Sentry events!")
+		}
+	}
 
 	logrus.Warnf("Dendrite is exiting now")
 }

setup/config/config_appservice.go

@@ -33,13 +33,17 @@ type AppServiceAPI struct {
 
 	Database DatabaseOptions `yaml:"database"`
 
+	// DisableTLSValidation disables the validation of X.509 TLS certs
+	// on appservice endpoints. This is not recommended in production!
+	DisableTLSValidation bool `yaml:"disable_tls_validation"`
+
 	ConfigFiles []string `yaml:"config_files"`
 }
 
 func (c *AppServiceAPI) Defaults() {
 	c.InternalAPI.Listen = "http://localhost:7777"
 	c.InternalAPI.Connect = "http://localhost:7777"
-	c.Database.Defaults()
+	c.Database.Defaults(5)
 	c.Database.ConnectionString = "file:appservice.db"
 }
 
@@ -193,7 +197,7 @@ func loadAppServices(config *AppServiceAPI, derived *Derived) error {
 // setupRegexps will create regex objects for exclusive and non-exclusive
 // usernames, aliases and rooms of all application services, so that other
 // methods can quickly check if a particular string matches any of them.
-func setupRegexps(_ *AppServiceAPI, derived *Derived) (err error) {
+func setupRegexps(asAPI *AppServiceAPI, derived *Derived) (err error) {
 	// Combine all exclusive namespaces for later string checking
 	var exclusiveUsernameStrings, exclusiveAliasStrings []string
 
@@ -201,6 +205,16 @@ func setupRegexps(_ *AppServiceAPI, derived *Derived) (err error) {
 	// its contents to the overall exlusive regex string. Room regex
 	// not necessary as we aren't denying exclusive room ID creation
 	for _, appservice := range derived.ApplicationServices {
+		// The sender_localpart can be considered an exclusive regex for a single user, so let's do that
+		// to simplify the code
+		var senderUserIDSlice = []string{fmt.Sprintf("@%s:%s", appservice.SenderLocalpart, asAPI.Matrix.ServerName)}
+		usersSlice, found := appservice.NamespaceMap["users"]
+		if !found {
+			usersSlice = []ApplicationServiceNamespace{}
+			appservice.NamespaceMap["users"] = usersSlice
+		}
+		appendExclusiveNamespaceRegexs(&senderUserIDSlice, usersSlice)
+
 		for key, namespaceSlice := range appservice.NamespaceMap {
 			switch key {
 			case "users":

setup/config/config_federationsender.go

@@ -25,7 +25,7 @@ type FederationSender struct {
 func (c *FederationSender) Defaults() {
 	c.InternalAPI.Listen = "http://localhost:7775"
 	c.InternalAPI.Connect = "http://localhost:7775"
-	c.Database.Defaults()
+	c.Database.Defaults(10)
 	c.Database.ConnectionString = "file:federationsender.db"
 
 	c.FederationMaxRetries = 16

setup/config/config_global.go

@@ -49,6 +49,9 @@ type Global struct {
 	// Metrics configuration
 	Metrics Metrics `yaml:"metrics"`
 
+	// Sentry configuration
+	Sentry Sentry `yaml:"sentry"`
+
 	// DNS caching options for all outbound HTTP requests
 	DNSCache DNSCacheOptions `yaml:"dns_cache"`
 }
@@ -63,6 +66,7 @@ func (c *Global) Defaults() {
 	c.Kafka.Defaults()
 	c.Metrics.Defaults()
 	c.DNSCache.Defaults()
+	c.Sentry.Defaults()
 }
 
 func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
@@ -71,6 +75,7 @@ func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
 
 	c.Kafka.Verify(configErrs, isMonolith)
 	c.Metrics.Verify(configErrs, isMonolith)
+	c.Sentry.Verify(configErrs, isMonolith)
 	c.DNSCache.Verify(configErrs, isMonolith)
 }
 
@@ -111,6 +116,24 @@ func (c *Metrics) Defaults() {
 func (c *Metrics) Verify(configErrs *ConfigErrors, isMonolith bool) {
 }
 
+// The configuration to use for Sentry error reporting
+type Sentry struct {
+	Enabled bool `yaml:"enabled"`
+	// The DSN to connect to e.g "https://examplePublicKey@o0.ingest.sentry.io/0"
+	// See https://docs.sentry.io/platforms/go/configuration/options/
+	DSN string `yaml:"dsn"`
+	// The environment e.g "production"
+	// See https://docs.sentry.io/platforms/go/configuration/environments/
+	Environment string `yaml:"environment"`
+}
+
+func (c *Sentry) Defaults() {
+	c.Enabled = false
+}
+
+func (c *Sentry) Verify(configErrs *ConfigErrors, isMonolith bool) {
+}
+
 type DatabaseOptions struct {
 	// The connection string, file:filename.db or postgres://server....
 	ConnectionString DataSource `yaml:"connection_string"`
@@ -122,8 +145,8 @@ type DatabaseOptions struct {
 	ConnMaxLifetimeSeconds int `yaml:"conn_max_lifetime"`
 }
 
-func (c *DatabaseOptions) Defaults() {
-	c.MaxOpenConnections = 100
+func (c *DatabaseOptions) Defaults(conns int) {
+	c.MaxOpenConnections = conns
 	c.MaxIdleConnections = 2
 	c.ConnMaxLifetimeSeconds = -1
 }

setup/config/config_kafka.go

@@ -48,7 +48,7 @@ func (k *Kafka) TopicFor(name string) string {
 
 func (c *Kafka) Defaults() {
 	c.UseNaffka = true
-	c.Database.Defaults()
+	c.Database.Defaults(10)
 	c.Addresses = []string{"localhost:2181"}
 	c.Database.ConnectionString = DataSource("file:naffka.db")
 	c.TopicPrefix = "Dendrite"

setup/config/config_keyserver.go

@@ -11,7 +11,7 @@ type KeyServer struct {
 func (c *KeyServer) Defaults() {
 	c.InternalAPI.Listen = "http://localhost:7779"
 	c.InternalAPI.Connect = "http://localhost:7779"
-	c.Database.Defaults()
+	c.Database.Defaults(10)
 	c.Database.ConnectionString = "file:keyserver.db"
 }
 

setup/config/config_mediaapi.go

@@ -39,7 +39,7 @@ func (c *MediaAPI) Defaults() {
 	c.InternalAPI.Listen = "http://localhost:7774"
 	c.InternalAPI.Connect = "http://localhost:7774"
 	c.ExternalAPI.Listen = "http://[::]:8074"
-	c.Database.Defaults()
+	c.Database.Defaults(5)
 	c.Database.ConnectionString = "file:mediaapi.db"
 
 	defaultMaxFileSizeBytes := FileSizeBytes(10485760)

setup/config/config_mscs.go

@@ -14,7 +14,7 @@ type MSCs struct {
 }
 
 func (c *MSCs) Defaults() {
-	c.Database.Defaults()
+	c.Database.Defaults(5)
 	c.Database.ConnectionString = "file:mscs.db"
 }
 

setup/config/config_roomserver.go

@@ -11,7 +11,7 @@ type RoomServer struct {
 func (c *RoomServer) Defaults() {
 	c.InternalAPI.Listen = "http://localhost:7770"
 	c.InternalAPI.Connect = "http://localhost:7770"
-	c.Database.Defaults()
+	c.Database.Defaults(10)
 	c.Database.ConnectionString = "file:roomserver.db"
 }
 

setup/config/config_signingkeyserver.go

@@ -22,7 +22,7 @@ type SigningKeyServer struct {
 func (c *SigningKeyServer) Defaults() {
 	c.InternalAPI.Listen = "http://localhost:7780"
 	c.InternalAPI.Connect = "http://localhost:7780"
-	c.Database.Defaults()
+	c.Database.Defaults(10)
 	c.Database.ConnectionString = "file:signingkeyserver.db"
 }
 

setup/config/config_syncapi.go

@@ -15,7 +15,7 @@ func (c *SyncAPI) Defaults() {
 	c.InternalAPI.Listen = "http://localhost:7773"
 	c.InternalAPI.Connect = "http://localhost:7773"
 	c.ExternalAPI.Listen = "http://localhost:8073"
-	c.Database.Defaults()
+	c.Database.Defaults(10)
 	c.Database.ConnectionString = "file:syncapi.db"
 }
 

setup/config/config_userapi.go

@@ -1,10 +1,18 @@
 package config
 
+import "golang.org/x/crypto/bcrypt"
+
 type UserAPI struct {
 	Matrix *Global `yaml:"-"`
 
 	InternalAPI InternalAPIOptions `yaml:"internal_api"`
 
+	// The cost when hashing passwords.
+	BCryptCost int `yaml:"bcrypt_cost"`
+
+	// The length of time an OpenID token is condidered valid in milliseconds
+	OpenIDTokenLifetimeMS int64 `yaml:"openid_token_lifetime_ms"`
+
 	// The Account database stores the login details and account information
 	// for local users. It is accessed by the UserAPI.
 	AccountDatabase DatabaseOptions `yaml:"account_database"`
@@ -13,13 +21,17 @@ type UserAPI struct {
 	DeviceDatabase DatabaseOptions `yaml:"device_database"`
 }
 
+const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes
+
 func (c *UserAPI) Defaults() {
 	c.InternalAPI.Listen = "http://localhost:7781"
 	c.InternalAPI.Connect = "http://localhost:7781"
-	c.AccountDatabase.Defaults()
-	c.DeviceDatabase.Defaults()
+	c.AccountDatabase.Defaults(10)
+	c.DeviceDatabase.Defaults(10)
 	c.AccountDatabase.ConnectionString = "file:userapi_accounts.db"
 	c.DeviceDatabase.ConnectionString = "file:userapi_devices.db"
+	c.BCryptCost = bcrypt.DefaultCost
+	c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS
 }
 
 func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
@@ -27,4 +39,5 @@ func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect))
 	checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))
 	checkNotEmpty(configErrs, "user_api.device_database.connection_string", string(c.DeviceDatabase.ConnectionString))
+	checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS)
 }

setup/mscs/msc2836/msc2836.go

@@ -238,7 +238,6 @@ func federatedEventRelationship(
 	}
 }
 
-// nolint:gocyclo
 func (rc *reqCtx) process() (*gomatrixserverlib.MSC2836EventRelationshipsResponse, *util.JSONResponse) {
 	var res gomatrixserverlib.MSC2836EventRelationshipsResponse
 	var returnEvents []*gomatrixserverlib.HeaderedEvent

setup/mscs/msc2836/msc2836_test.go

@@ -524,6 +524,9 @@ func (u *testUserAPI) PerformLastSeenUpdate(ctx context.Context, req *userapi.Pe
 func (u *testUserAPI) PerformAccountDeactivation(ctx context.Context, req *userapi.PerformAccountDeactivationRequest, res *userapi.PerformAccountDeactivationResponse) error {
 	return nil
 }
+func (u *testUserAPI) PerformOpenIDTokenCreation(ctx context.Context, req *userapi.PerformOpenIDTokenCreationRequest, res *userapi.PerformOpenIDTokenCreationResponse) error {
+	return nil
+}
 func (u *testUserAPI) QueryProfile(ctx context.Context, req *userapi.QueryProfileRequest, res *userapi.QueryProfileResponse) error {
 	return nil
 }
@@ -548,6 +551,9 @@ func (u *testUserAPI) QueryDeviceInfos(ctx context.Context, req *userapi.QueryDe
 func (u *testUserAPI) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error {
 	return nil
 }
+func (u *testUserAPI) QueryOpenIDToken(ctx context.Context, req *userapi.QueryOpenIDTokenRequest, res *userapi.QueryOpenIDTokenResponse) error {
+	return nil
+}
 
 type testRoomserverAPI struct {
 	// use a trace API as it implements method stubs so we don't need to have them here.

setup/mscs/msc2946/msc2946.go

@@ -70,11 +70,11 @@ func Enable(
 		}
 	})
 
-	base.PublicClientAPIMux.Handle("/unstable/rooms/{roomID}/spaces",
+	base.PublicClientAPIMux.Handle("/unstable/org.matrix.msc2946/rooms/{roomID}/spaces",
 		httputil.MakeAuthAPI("spaces", userAPI, spacesHandler(db, rsAPI, fsAPI, base.Cfg.Global.ServerName)),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	base.PublicFederationAPIMux.Handle("/unstable/spaces/{roomID}", httputil.MakeExternalAPI(
+	base.PublicFederationAPIMux.Handle("/unstable/org.matrix.msc2946/spaces/{roomID}", httputil.MakeExternalAPI(
 		"msc2946_fed_spaces", func(req *http.Request) util.JSONResponse {
 			fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
 				req, time.Now(), base.Cfg.Global.ServerName, keyRing,
@@ -217,7 +217,6 @@ func (w *walker) markSent(id string) {
 	w.inMemoryBatchCache[w.callerID()] = m
 }
 
-// nolint:gocyclo
 func (w *walker) walk() *gomatrixserverlib.MSC2946SpacesResponse {
 	var res gomatrixserverlib.MSC2946SpacesResponse
 	// Begin walking the graph starting with the room ID in the request in a queue of unvisited rooms

setup/mscs/msc2946/msc2946_test.go

@@ -309,7 +309,7 @@ func postSpaces(t *testing.T, expectCode int, accessToken, roomID string, req *g
 		t.Fatalf("failed to marshal request: %s", err)
 	}
 	httpReq, err := http.NewRequest(
-		"POST", "http://localhost:8010/_matrix/client/unstable/rooms/"+url.PathEscape(roomID)+"/spaces",
+		"POST", "http://localhost:8010/_matrix/client/unstable/org.matrix.msc2946/rooms/"+url.PathEscape(roomID)+"/spaces",
 		bytes.NewBuffer(data),
 	)
 	httpReq.Header.Set("Authorization", "Bearer "+accessToken)
@@ -367,6 +367,9 @@ func (u *testUserAPI) PerformLastSeenUpdate(ctx context.Context, req *userapi.Pe
 func (u *testUserAPI) PerformAccountDeactivation(ctx context.Context, req *userapi.PerformAccountDeactivationRequest, res *userapi.PerformAccountDeactivationResponse) error {
 	return nil
 }
+func (u *testUserAPI) PerformOpenIDTokenCreation(ctx context.Context, req *userapi.PerformOpenIDTokenCreationRequest, res *userapi.PerformOpenIDTokenCreationResponse) error {
+	return nil
+}
 func (u *testUserAPI) QueryProfile(ctx context.Context, req *userapi.QueryProfileRequest, res *userapi.QueryProfileResponse) error {
 	return nil
 }
@@ -391,6 +394,9 @@ func (u *testUserAPI) QueryDeviceInfos(ctx context.Context, req *userapi.QueryDe
 func (u *testUserAPI) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error {
 	return nil
 }
+func (u *testUserAPI) QueryOpenIDToken(ctx context.Context, req *userapi.QueryOpenIDTokenRequest, res *userapi.QueryOpenIDTokenResponse) error {
+	return nil
+}
 
 type testRoomserverAPI struct {
 	// use a trace API as it implements method stubs so we don't need to have them here.