hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2025-07-29 12:42:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

API setup refactoring (#1266)

Browse source 
* Start HTTP endpoint refactoring

* Update SetupAndServeHTTP

* Fix builds

* Don't set up external listener if no address configured

* TLS HTTP setup

* Break apart client/federation/key/media muxes

* Tweaks

* Fix P2P demos

* Fix media API routing

* Review comments @Kegsay

* Update sample config

* Fix gobind build

* Fix External -> Public in federation API test
This commit is contained in:
Neil Alexander 2020-08-13 12:16:37 +01:00 • committed by GitHub
parent 820c56c165
commit 9677a95afc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
49 changed files with 542 additions and 383 deletions
Download patch file Download diff file Expand all files Collapse all files

57
internal/config/config.go
View file

@ -21,6 +21,7 @@ import (
"fmt"
"io"
"io/ioutil"
"net/url"
"path/filepath"
"regexp"
"strings"
@ -110,6 +111,15 @@ type Derived struct {
// servers from creating RoomIDs in exclusive application service namespaces
}
type InternalAPIOptions struct {
Listen HTTPAddress `yaml:"listen"`
Connect HTTPAddress `yaml:"connect"`
}
type ExternalAPIOptions struct {
Listen HTTPAddress `yaml:"listen"`
}
// A Path on the filesystem.
type Path string
@ -132,6 +142,17 @@ type Topic string
// An Address to listen on.
type Address string
// An HTTPAddress to listen on, starting with either http:// or https://.
type HTTPAddress string
func (h HTTPAddress) Address() (Address, error) {
url, err := url.Parse(string(h))
if err != nil {
return "", err
}
return Address(url.Host), nil
}
// FileSizeBytes is a file size in bytes
type FileSizeBytes int64
@ -360,6 +381,26 @@ func checkPositive(configErrs *ConfigErrors, key string, value int64) {
}
}
// checkURL verifies that the parameter is a valid URL
func checkURL(configErrs *ConfigErrors, key, value string) {
if value == "" {
configErrs.Add(fmt.Sprintf("missing config key %q", key))
return
}
url, err := url.Parse(value)
if err != nil {
configErrs.Add(fmt.Sprintf("config key %q contains invalid URL (%s)", key, err.Error()))
return
}
switch url.Scheme {
case "http":
case "https":
default:
configErrs.Add(fmt.Sprintf("config key %q URL should be http:// or https://", key))
return
}
}
// checkLogging verifies the parameters logging.* are valid.
func (config *Dendrite) checkLogging(configErrs *ConfigErrors) {
for _, logrusHook := range config.Logging {
@ -450,7 +491,7 @@ func (config *Dendrite) AppServiceURL() string {
// If we support HTTPS we need to think of a practical way to do certificate validation.
// People setting up servers shouldn't need to get a certificate valid for the public
// internet for an internal API.
return "http://" + string(config.AppServiceAPI.Listen)
return string(config.AppServiceAPI.InternalAPI.Connect)
}
// RoomServerURL returns an HTTP URL for where the roomserver is listening.
@ -459,7 +500,7 @@ func (config *Dendrite) RoomServerURL() string {
// If we support HTTPS we need to think of a practical way to do certificate validation.
// People setting up servers shouldn't need to get a certificate valid for the public
// internet for an internal API.
return "http://" + string(config.RoomServer.Listen)
return string(config.RoomServer.InternalAPI.Connect)
}
// UserAPIURL returns an HTTP URL for where the userapi is listening.
@ -468,7 +509,7 @@ func (config *Dendrite) UserAPIURL() string {
// If we support HTTPS we need to think of a practical way to do certificate validation.
// People setting up servers shouldn't need to get a certificate valid for the public
// internet for an internal API.
return "http://" + string(config.UserAPI.Listen)
return string(config.UserAPI.InternalAPI.Connect)
}
// CurrentStateAPIURL returns an HTTP URL for where the currentstateserver is listening.
@ -477,7 +518,7 @@ func (config *Dendrite) CurrentStateAPIURL() string {
// If we support HTTPS we need to think of a practical way to do certificate validation.
// People setting up servers shouldn't need to get a certificate valid for the public
// internet for an internal API.
return "http://" + string(config.CurrentStateServer.Listen)
return string(config.CurrentStateServer.InternalAPI.Connect)
}
// EDUServerURL returns an HTTP URL for where the EDU server is listening.
@ -486,7 +527,7 @@ func (config *Dendrite) EDUServerURL() string {
// If we support HTTPS we need to think of a practical way to do certificate validation.
// People setting up servers shouldn't need to get a certificate valid for the public
// internet for an internal API.
return "http://" + string(config.EDUServer.Listen)
return string(config.EDUServer.InternalAPI.Connect)
}
// FederationSenderURL returns an HTTP URL for where the federation sender is listening.
@ -495,7 +536,7 @@ func (config *Dendrite) FederationSenderURL() string {
// If we support HTTPS we need to think of a practical way to do certificate validation.
// People setting up servers shouldn't need to get a certificate valid for the public
// internet for an internal API.
return "http://" + string(config.FederationSender.Listen)
return string(config.FederationSender.InternalAPI.Connect)
}
// ServerKeyAPIURL returns an HTTP URL for where the server key API is listening.
@ -504,7 +545,7 @@ func (config *Dendrite) ServerKeyAPIURL() string {
// If we support HTTPS we need to think of a practical way to do certificate validation.
// People setting up servers shouldn't need to get a certificate valid for the public
// internet for an internal API.
return "http://" + string(config.ServerKeyAPI.Listen)
return string(config.ServerKeyAPI.InternalAPI.Connect)
}
// KeyServerURL returns an HTTP URL for where the key server is listening.
@ -513,7 +554,7 @@ func (config *Dendrite) KeyServerURL() string {
// If we support HTTPS we need to think of a practical way to do certificate validation.
// People setting up servers shouldn't need to get a certificate valid for the public
// internet for an internal API.
return "http://" + string(config.KeyServer.Listen)
return string(config.KeyServer.InternalAPI.Connect)
}
// SetupTracing configures the opentracing using the supplied configuration.

11
internal/config/config_appservice.go
View file

@ -29,8 +29,7 @@ type AppServiceAPI struct {
Matrix *Global `yaml:"-"`
Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit
Listen Address `yaml:"listen"`
Bind Address `yaml:"bind"`
InternalAPI InternalAPIOptions `yaml:"internal_api"`
Database DatabaseOptions `yaml:"database"`
@ -38,15 +37,15 @@ type AppServiceAPI struct {
}
func (c *AppServiceAPI) Defaults() {
c.Listen = "localhost:7777"
c.Bind = "localhost:7777"
c.InternalAPI.Listen = "http://localhost:7777"
c.InternalAPI.Connect = "http://localhost:7777"
c.Database.Defaults()
c.Database.ConnectionString = "file:appservice.db"
}
func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "app_service_api.listen", string(c.Listen))
checkNotEmpty(configErrs, "app_service_api.bind", string(c.Bind))
checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "app_service_api.internal_api.bind", string(c.InternalAPI.Connect))
checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString))
}

16
internal/config/config_clientapi.go
View file

@ -9,8 +9,8 @@ type ClientAPI struct {
Matrix *Global `yaml:"-"`
Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit
Listen Address `yaml:"listen"`
Bind Address `yaml:"bind"`
InternalAPI InternalAPIOptions `yaml:"internal_api"`
ExternalAPI ExternalAPIOptions `yaml:"external_api"`
// If set disables new users from registering (except via shared
// secrets)
@ -37,8 +37,9 @@ type ClientAPI struct {
}
func (c *ClientAPI) Defaults() {
c.Listen = "localhost:7771"
c.Bind = "localhost:7771"
c.InternalAPI.Listen = "http://localhost:7771"
c.InternalAPI.Connect = "http://localhost:7771"
c.ExternalAPI.Listen = "http://[::]:8071"
c.RegistrationSharedSecret = ""
c.RecaptchaPublicKey = ""
c.RecaptchaPrivateKey = ""
@ -49,8 +50,11 @@ func (c *ClientAPI) Defaults() {
}
func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "client_api.listen", string(c.Listen))
checkNotEmpty(configErrs, "client_api.bind", string(c.Bind))
checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect))
if !isMonolith {
checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen))
}
if c.RecaptchaEnabled {
checkNotEmpty(configErrs, "client_api.recaptcha_public_key", string(c.RecaptchaPublicKey))
checkNotEmpty(configErrs, "client_api.recaptcha_private_key", string(c.RecaptchaPrivateKey))

11
internal/config/config_currentstate.go
View file

@ -3,8 +3,7 @@ package config
type CurrentStateServer struct {
Matrix *Global `yaml:"-"`
Listen Address `yaml:"listen"`
Bind Address `yaml:"bind"`
InternalAPI InternalAPIOptions `yaml:"internal_api"`
// The CurrentState database stores the current state of all rooms.
// It is accessed by the CurrentStateServer.
@ -12,14 +11,14 @@ type CurrentStateServer struct {
}
func (c *CurrentStateServer) Defaults() {
c.Listen = "localhost:7782"
c.Bind = "localhost:7782"
c.InternalAPI.Listen = "http://localhost:7782"
c.InternalAPI.Connect = "http://localhost:7782"
c.Database.Defaults()
c.Database.ConnectionString = "file:currentstate.db"
}
func (c *CurrentStateServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "current_state_server.listen", string(c.Listen))
checkNotEmpty(configErrs, "current_state_server.bind", string(c.Bind))
checkURL(configErrs, "current_state_server.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "current_state_server.internal_api.connect", string(c.InternalAPI.Connect))
checkNotEmpty(configErrs, "current_state_server.database.connection_string", string(c.Database.ConnectionString))
}

11
internal/config/config_eduserver.go
View file

@ -3,16 +3,15 @@ package config
type EDUServer struct {
Matrix *Global `yaml:"-"`
Listen Address `yaml:"listen"`
Bind Address `yaml:"bind"`
InternalAPI InternalAPIOptions `yaml:"internal_api"`
}
func (c *EDUServer) Defaults() {
c.Listen = "localhost:7778"
c.Bind = "localhost:7778"
c.InternalAPI.Listen = "http://localhost:7778"
c.InternalAPI.Connect = "http://localhost:7778"
}
func (c *EDUServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "edu_server.listen", string(c.Listen))
checkNotEmpty(configErrs, "edu_server.bind", string(c.Bind))
checkURL(configErrs, "edu_server.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "edu_server.internal_api.connect", string(c.InternalAPI.Connect))
}

16
internal/config/config_federationapi.go
View file

@ -5,8 +5,8 @@ import "github.com/matrix-org/gomatrixserverlib"
type FederationAPI struct {
Matrix *Global `yaml:"-"`
Listen Address `yaml:"listen"`
Bind Address `yaml:"bind"`
InternalAPI InternalAPIOptions `yaml:"internal_api"`
ExternalAPI ExternalAPIOptions `yaml:"external_api"`
// List of paths to X509 certificates used by the external federation listeners.
// These are used to calculate the TLS fingerprints to publish for this server.
@ -21,13 +21,17 @@ type FederationAPI struct {
}
func (c *FederationAPI) Defaults() {
c.Listen = "localhost:7772"
c.Bind = "localhost:7772"
c.InternalAPI.Listen = "http://localhost:7772"
c.InternalAPI.Connect = "http://localhost:7772"
c.ExternalAPI.Listen = "http://[::]:8072"
}
func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "federation_api.listen", string(c.Listen))
checkNotEmpty(configErrs, "federation_api.bind", string(c.Bind))
checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect))
if !isMonolith {
checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen))
}
// TODO: not applicable always, e.g. in demos
//checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths)))
}

11
internal/config/config_federationsender.go
View file

@ -3,8 +3,7 @@ package config
type FederationSender struct {
Matrix *Global `yaml:"-"`
Listen Address `yaml:"listen"`
Bind Address `yaml:"bind"`
InternalAPI InternalAPIOptions `yaml:"internal_api"`
// The FederationSender database stores information used by the FederationSender
// It is only accessed by the FederationSender.
@ -24,8 +23,8 @@ type FederationSender struct {
}
func (c *FederationSender) Defaults() {
c.Listen = "localhost:7775"
c.Bind = "localhost:7775"
c.InternalAPI.Listen = "http://localhost:7775"
c.InternalAPI.Connect = "http://localhost:7775"
c.Database.Defaults()
c.Database.ConnectionString = "file:federationsender.db"
@ -36,8 +35,8 @@ func (c *FederationSender) Defaults() {
}
func (c *FederationSender) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "federation_sender.listen", string(c.Listen))
checkNotEmpty(configErrs, "federation_sender.bind", string(c.Bind))
checkURL(configErrs, "federation_sender.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "federation_sender.internal_api.connect", string(c.InternalAPI.Connect))
checkNotEmpty(configErrs, "federation_sender.database.connection_string", string(c.Database.ConnectionString))
}

11
internal/config/config_keyserver.go
View file

@ -3,21 +3,20 @@ package config
type KeyServer struct {
Matrix *Global `yaml:"-"`
Listen Address `yaml:"listen"`
Bind Address `yaml:"bind"`
InternalAPI InternalAPIOptions `yaml:"internal_api"`
Database DatabaseOptions `yaml:"database"`
}
func (c *KeyServer) Defaults() {
c.Listen = "localhost:7779"
c.Bind = "localhost:7779"
c.InternalAPI.Listen = "http://localhost:7779"
c.InternalAPI.Connect = "http://localhost:7779"
c.Database.Defaults()
c.Database.ConnectionString = "file:keyserver.db"
}
func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "key_server.listen", string(c.Listen))
checkNotEmpty(configErrs, "key_server.bind", string(c.Bind))
checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "key_server.internal_api.bind", string(c.InternalAPI.Connect))
checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString))
}

16
internal/config/config_mediaapi.go
View file

@ -7,8 +7,8 @@ import (
type MediaAPI struct {
Matrix *Global `yaml:"-"`
Listen Address `yaml:"listen"`
Bind Address `yaml:"bind"`
InternalAPI InternalAPIOptions `yaml:"internal_api"`
ExternalAPI ExternalAPIOptions `yaml:"external_api"`
// The MediaAPI database stores information about files uploaded and downloaded
// by local users. It is only accessed by the MediaAPI.
@ -36,8 +36,9 @@ type MediaAPI struct {
}
func (c *MediaAPI) Defaults() {
c.Listen = "localhost:7774"
c.Bind = "localhost:7774"
c.InternalAPI.Listen = "http://localhost:7774"
c.InternalAPI.Connect = "http://localhost:7774"
c.ExternalAPI.Listen = "http://[::]:8074"
c.Database.Defaults()
c.Database.ConnectionString = "file:mediaapi.db"
@ -48,8 +49,11 @@ func (c *MediaAPI) Defaults() {
}
func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "media_api.listen", string(c.Listen))
checkNotEmpty(configErrs, "media_api.bind", string(c.Bind))
checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect))
if !isMonolith {
checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen))
}
checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath))

11
internal/config/config_roomserver.go
View file

@ -3,21 +3,20 @@ package config
type RoomServer struct {
Matrix *Global `yaml:"-"`
Listen Address `yaml:"listen"`
Bind Address `yaml:"bind"`
InternalAPI InternalAPIOptions `yaml:"internal_api"`
Database DatabaseOptions `yaml:"database"`
}
func (c *RoomServer) Defaults() {
c.Listen = "localhost:7770"
c.Bind = "localhost:7770"
c.InternalAPI.Listen = "http://localhost:7770"
c.InternalAPI.Connect = "http://localhost:7770"
c.Database.Defaults()
c.Database.ConnectionString = "file:roomserver.db"
}
func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "room_server.listen", string(c.Listen))
checkNotEmpty(configErrs, "room_server.bind", string(c.Bind))
checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "room_server.internal_ap.bind", string(c.InternalAPI.Connect))
checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString))
}

11
internal/config/config_serverkey.go
View file

@ -5,8 +5,7 @@ import "github.com/matrix-org/gomatrixserverlib"
type ServerKeyAPI struct {
Matrix *Global `yaml:"-"`
Listen Address `yaml:"listen"`
Bind Address `yaml:"bind"`
InternalAPI InternalAPIOptions `yaml:"internal_api"`
// The ServerKey database caches the public keys of remote servers.
// It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI.
@ -18,15 +17,15 @@ type ServerKeyAPI struct {
}
func (c *ServerKeyAPI) Defaults() {
c.Listen = "localhost:7780"
c.Bind = "localhost:7780"
c.InternalAPI.Listen = "http://localhost:7780"
c.InternalAPI.Connect = "http://localhost:7780"
c.Database.Defaults()
c.Database.ConnectionString = "file:serverkeyapi.db"
}
func (c *ServerKeyAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "server_key_api.listen", string(c.Listen))
checkNotEmpty(configErrs, "server_key_api.bind", string(c.Bind))
checkURL(configErrs, "server_key_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "server_key_api.internal_api.bind", string(c.InternalAPI.Connect))
checkNotEmpty(configErrs, "server_key_api.database.connection_string", string(c.Database.ConnectionString))
}

11
internal/config/config_syncapi.go
View file

@ -3,21 +3,20 @@ package config
type SyncAPI struct {
Matrix *Global `yaml:"-"`
Listen Address `yaml:"listen"`
Bind Address `yaml:"bind"`
InternalAPI InternalAPIOptions `yaml:"internal_api"`
Database DatabaseOptions `yaml:"database"`
}
func (c *SyncAPI) Defaults() {
c.Listen = "localhost:7773"
c.Bind = "localhost:7773"
c.InternalAPI.Listen = "http://localhost:7773"
c.InternalAPI.Connect = "http://localhost:7773"
c.Database.Defaults()
c.Database.ConnectionString = "file:syncapi.db"
}
func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "sync_api.listen", string(c.Listen))
checkNotEmpty(configErrs, "sync_api.bind", string(c.Bind))
checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "sync_api.internal_api.bind", string(c.InternalAPI.Connect))
checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))
}

69
internal/config/config_test.go
View file

@ -43,7 +43,8 @@ global:
- matrix.org
- vector.im
kafka:
addresses: []
addresses:
- localhost:2181
topic_prefix: Dendrite
use_naffka: true
naffka_database:
@ -57,8 +58,9 @@ global:
username: metrics
password: metrics
app_service_api:
listen: localhost:7777
bind: localhost:7777
internal_api:
listen: http://localhost:7777
connect: http://localhost:7777
database:
connection_string: file:appservice.db
max_open_conns: 100
@ -66,8 +68,11 @@ app_service_api:
conn_max_lifetime: -1
config_files: []
client_api:
listen: localhost:7771
bind: localhost:7771
internal_api:
listen: http://localhost:7771
connect: http://localhost:7771
external_api:
listen: http://[::]:8071
registration_disabled: false
registration_shared_secret: ""
enable_registration_captcha: false
@ -82,23 +87,29 @@ client_api:
turn_username: ""
turn_password: ""
current_state_server:
listen: localhost:7782
bind: localhost:7782
internal_api:
listen: http://localhost:7782
connect: http://localhost:7782
database:
connection_string: file:currentstate.db
max_open_conns: 100
max_idle_conns: 2
conn_max_lifetime: -1
edu_server:
listen: localhost:7778
bind: localhost:7778
internal_api:
listen: http://localhost:7778
connect: http://localhost:7778
federation_api:
listen: localhost:7772
bind: localhost:7772
internal_api:
listen: http://localhost:7772
connect: http://localhost:7772
external_api:
listen: http://[::]:8072
federation_certificates: []
federation_sender:
listen: localhost:7775
bind: localhost:7775
internal_api:
listen: http://localhost:7775
connect: http://localhost:7775
database:
connection_string: file:federationsender.db
max_open_conns: 100
@ -112,16 +123,20 @@ federation_sender:
host: localhost
port: 8080
key_server:
listen: localhost:7779
bind: localhost:7779
internal_api:
listen: http://localhost:7779
connect: http://localhost:7779
database:
connection_string: file:keyserver.db
max_open_conns: 100
max_idle_conns: 2
conn_max_lifetime: -1
media_api:
listen: localhost:7774
bind: localhost:7774
internal_api:
listen: http://localhost:7774
connect: http://localhost:7774
external_api:
listen: http://[::]:8074
database:
connection_string: file:mediaapi.db
max_open_conns: 100
@ -142,16 +157,18 @@ media_api:
height: 480
method: scale
room_server:
listen: localhost:7770
bind: localhost:7770
internal_api:
listen: http://localhost:7770
connect: http://localhost:7770
database:
connection_string: file:roomserver.db
max_open_conns: 100
max_idle_conns: 2
conn_max_lifetime: -1
server_key_api:
listen: localhost:7780
bind: localhost:7780
internal_api:
listen: http://localhost:7780
connect: http://localhost:7780
database:
connection_string: file:serverkeyapi.db
max_open_conns: 100
@ -165,16 +182,18 @@ server_key_api:
- key_id: ed25519:a_RXGa
public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
sync_api:
listen: localhost:7773
bind: localhost:7773
internal_api:
listen: http://localhost:7773
connect: http://localhost:7773
database:
connection_string: file:syncapi.db
max_open_conns: 100
max_idle_conns: 2
conn_max_lifetime: -1
user_api:
listen: localhost:7781
bind: localhost:7781
internal_api:
listen: http://localhost:7781
connect: http://localhost:7781
account_database:
connection_string: file:userapi_accounts.db
max_open_conns: 100

11
internal/config/config_userapi.go
View file

@ -3,8 +3,7 @@ package config
type UserAPI struct {
Matrix *Global `yaml:"-"`
Listen Address `yaml:"listen"`
Bind Address `yaml:"bind"`
InternalAPI InternalAPIOptions `yaml:"internal_api"`
// The Account database stores the login details and account information
// for local users. It is accessed by the UserAPI.
@ -15,8 +14,8 @@ type UserAPI struct {
}
func (c *UserAPI) Defaults() {
c.Listen = "localhost:7781"
c.Bind = "localhost:7781"
c.InternalAPI.Listen = "http://localhost:7781"
c.InternalAPI.Connect = "http://localhost:7781"
c.AccountDatabase.Defaults()
c.DeviceDatabase.Defaults()
c.AccountDatabase.ConnectionString = "file:userapi_accounts.db"
@ -24,8 +23,8 @@ func (c *UserAPI) Defaults() {
}
func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
checkNotEmpty(configErrs, "user_api.listen", string(c.Listen))
checkNotEmpty(configErrs, "user_api.bind", string(c.Bind))
checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen))
checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect))
checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))
checkNotEmpty(configErrs, "user_api.device_database.connection_string", string(c.DeviceDatabase.ConnectionString))
}

12
internal/httputil/httpapi.go
View file

@ -28,7 +28,6 @@ import (
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/clientapi/auth"
federationsenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
@ -233,17 +232,6 @@ func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib
}
}
// SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics listener
func SetupHTTPAPI(servMux, publicApiMux, internalApiMux *mux.Router, cfg *config.Global, enableHTTPAPIs bool) {
if cfg.Metrics.Enabled {
servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth))
}
if enableHTTPAPIs {
servMux.Handle(InternalPathPrefix, internalApiMux)
}
servMux.Handle(PublicPathPrefix, WrapHandlerInCORS(publicApiMux))
}
// WrapHandlerInBasicAuth adds basic auth to a handler. Only used for /metrics
func WrapHandlerInBasicAuth(h http.Handler, b BasicAuth) http.HandlerFunc {
if b.Username == "" || b.Password == "" {

7
internal/httputil/paths.go
View file

@ -15,6 +15,9 @@
package httputil
const (
PublicPathPrefix = "/_matrix/"
InternalPathPrefix = "/api/"
PublicClientPathPrefix = "/_matrix/client/"
PublicFederationPathPrefix = "/_matrix/federation/"
PublicKeyPathPrefix = "/_matrix/key/"
PublicMediaPathPrefix = "/_matrix/media/"
InternalPathPrefix = "/api/"
)

142
internal/setup/base.go
View file

@ -27,6 +27,7 @@ import (
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/naffka"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
@ -62,24 +63,26 @@ import (
// should only be used during start up.
// Must be closed when shutting down.
type BaseDendrite struct {
componentName string
tracerCloser io.Closer
// PublicAPIMux should be used to register new public matrix api endpoints
PublicAPIMux *mux.Router
InternalAPIMux *mux.Router
BaseMux *mux.Router // base router which created public/internal subrouters
UseHTTPAPIs bool
httpClient *http.Client
Cfg *config.Dendrite
Caches *caching.Caches
KafkaConsumer sarama.Consumer
KafkaProducer sarama.SyncProducer
componentName string
tracerCloser io.Closer
PublicClientAPIMux *mux.Router
PublicFederationAPIMux *mux.Router
PublicKeyAPIMux *mux.Router
PublicMediaAPIMux *mux.Router
InternalAPIMux *mux.Router
UseHTTPAPIs bool
httpClient *http.Client
Cfg *config.Dendrite
Caches *caching.Caches
KafkaConsumer sarama.Consumer
KafkaProducer sarama.SyncProducer
}
const HTTPServerTimeout = time.Minute * 5
const HTTPClientTimeout = time.Second * 30
const NoExternalListener = ""
// NewBaseDendrite creates a new instance to be used by a component.
// The componentName is used for logging purposes, and should be a friendly name
// of the compontent running, e.g. "SyncAPI"
@ -134,20 +137,20 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
// We need to be careful with media APIs if they read from a filesystem to make sure they
// are not inadvertently reading paths without cleaning, else this could introduce a
// directory traversal attack e.g /../../../etc/passwd
httpmux := mux.NewRouter().SkipClean(true)
return &BaseDendrite{
componentName: componentName,
UseHTTPAPIs: useHTTPAPIs,
tracerCloser: closer,
Cfg: cfg,
Caches: cache,
BaseMux: httpmux,
PublicAPIMux: httpmux.PathPrefix(httputil.PublicPathPrefix).Subrouter().UseEncodedPath(),
InternalAPIMux: httpmux.PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(),
httpClient: &client,
KafkaConsumer: kafkaConsumer,
KafkaProducer: kafkaProducer,
componentName: componentName,
UseHTTPAPIs: useHTTPAPIs,
tracerCloser: closer,
Cfg: cfg,
Caches: cache,
PublicClientAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicClientPathPrefix).Subrouter().UseEncodedPath(),
PublicFederationAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath(),
PublicKeyAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath(),
PublicMediaAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicMediaPathPrefix).Subrouter().UseEncodedPath(),
InternalAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(),
httpClient: &client,
KafkaConsumer: kafkaConsumer,
KafkaProducer: kafkaProducer,
}
}
@ -266,37 +269,74 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli
// SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on
// ApiMux under /api/ and adds a prometheus handler under /metrics.
func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) {
// If a separate bind address is defined, listen on that. Otherwise use
// the listen address
var addr string
if bindaddr != "" {
addr = bindaddr
} else {
addr = listenaddr
}
// nolint:gocyclo
func (b *BaseDendrite) SetupAndServeHTTP(
internalHTTPAddr, externalHTTPAddr config.HTTPAddress,
certFile, keyFile *string,
) {
internalAddr, _ := internalHTTPAddr.Address()
externalAddr, _ := externalHTTPAddr.Address()
serv := http.Server{
Addr: addr,
internalRouter := mux.NewRouter()
externalRouter := internalRouter
internalServ := &http.Server{
Addr: string(internalAddr),
WriteTimeout: HTTPServerTimeout,
Handler: internalRouter,
}
externalServ := internalServ
if externalAddr != "" && externalAddr != internalAddr {
externalRouter = mux.NewRouter()
externalServ = &http.Server{
Addr: string(externalAddr),
WriteTimeout: HTTPServerTimeout,
Handler: externalRouter,
}
}
httputil.SetupHTTPAPI(
b.BaseMux,
b.PublicAPIMux,
b.InternalAPIMux,
&b.Cfg.Global,
b.UseHTTPAPIs,
)
serv.Handler = b.BaseMux
logrus.Infof("Starting %s server on %s", b.componentName, serv.Addr)
err := serv.ListenAndServe()
if err != nil {
logrus.WithError(err).Fatal("failed to serve http")
internalRouter.PathPrefix(httputil.InternalPathPrefix).Handler(b.InternalAPIMux)
if b.Cfg.Global.Metrics.Enabled {
internalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), b.Cfg.Global.Metrics.BasicAuth))
}
logrus.Infof("Stopped %s server on %s", b.componentName, serv.Addr)
externalRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(b.PublicClientAPIMux)
externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(b.PublicKeyAPIMux)
externalRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(b.PublicFederationAPIMux)
externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(b.PublicMediaAPIMux)
go func() {
logrus.Infof("Starting %s listener on %s", b.componentName, externalServ.Addr)
if certFile != nil && keyFile != nil {
if err := externalServ.ListenAndServeTLS(*certFile, *keyFile); err != nil {
logrus.WithError(err).Fatal("failed to serve HTTPS")
}
} else {
if err := externalServ.ListenAndServe(); err != nil {
logrus.WithError(err).Fatal("failed to serve HTTP")
}
}
logrus.Infof("Stopped %s listener on %s", b.componentName, externalServ.Addr)
}()
if internalAddr != "" && internalAddr != externalAddr {
go func() {
logrus.Infof("Starting %s listener on %s", b.componentName, internalServ.Addr)
if certFile != nil && keyFile != nil {
if err := internalServ.ListenAndServeTLS(*certFile, *keyFile); err != nil {
logrus.WithError(err).Fatal("failed to serve HTTPS")
}
} else {
if err := internalServ.ListenAndServe(); err != nil {
logrus.WithError(err).Fatal("failed to serve HTTP")
}
}
logrus.Infof("Stopped %s listener on %s", b.componentName, internalServ.Addr)
}()
}
select {}
}
// setupKafka creates kafka consumer/producer pair from the config.

10
internal/setup/monolith.go
View file

@ -63,21 +63,21 @@ type Monolith struct {
}
// AddAllPublicRoutes attaches all public paths to the given router
func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) {
func (m *Monolith) AddAllPublicRoutes(csMux, ssMux, keyMux, mediaMux *mux.Router) {
clientapi.AddPublicRoutes(
publicMux, &m.Config.ClientAPI, m.KafkaProducer, m.DeviceDB, m.AccountDB,
csMux, &m.Config.ClientAPI, m.KafkaProducer, m.DeviceDB, m.AccountDB,
m.FedClient, m.RoomserverAPI,
m.EDUInternalAPI, m.AppserviceAPI, m.StateAPI, transactions.New(),
m.FederationSenderAPI, m.UserAPI, m.KeyAPI, m.ExtPublicRoomsProvider,
)
federationapi.AddPublicRoutes(
publicMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient,
ssMux, keyMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient,
m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI,
m.EDUInternalAPI, m.StateAPI, m.KeyAPI,
)
mediaapi.AddPublicRoutes(publicMux, &m.Config.MediaAPI, m.UserAPI, m.Client)
mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, m.UserAPI, m.Client)
syncapi.AddPublicRoutes(
publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI,
csMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI,
m.KeyAPI, m.StateAPI, m.FedClient, &m.Config.SyncAPI,
)
}

48
internal/test/config.go
View file

@ -52,8 +52,8 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
cfg.Defaults()
port := startPort
assignAddress := func() config.Address {
result := config.Address(fmt.Sprintf("%s:%d", host, port))
assignAddress := func() config.HTTPAddress {
result := config.HTTPAddress(fmt.Sprintf("http://%s:%d", host, port))
port++
return result
}
@ -97,29 +97,29 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database)
cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(database)
cfg.AppServiceAPI.Listen = assignAddress()
cfg.CurrentStateServer.Listen = assignAddress()
cfg.EDUServer.Listen = assignAddress()
cfg.FederationAPI.Listen = assignAddress()
cfg.FederationSender.Listen = assignAddress()
cfg.KeyServer.Listen = assignAddress()
cfg.MediaAPI.Listen = assignAddress()
cfg.RoomServer.Listen = assignAddress()
cfg.ServerKeyAPI.Listen = assignAddress()
cfg.SyncAPI.Listen = assignAddress()
cfg.UserAPI.Listen = assignAddress()
cfg.AppServiceAPI.InternalAPI.Listen = assignAddress()
cfg.CurrentStateServer.InternalAPI.Listen = assignAddress()
cfg.EDUServer.InternalAPI.Listen = assignAddress()
cfg.FederationAPI.InternalAPI.Listen = assignAddress()
cfg.FederationSender.InternalAPI.Listen = assignAddress()
cfg.KeyServer.InternalAPI.Listen = assignAddress()
cfg.MediaAPI.InternalAPI.Listen = assignAddress()
cfg.RoomServer.InternalAPI.Listen = assignAddress()
cfg.ServerKeyAPI.InternalAPI.Listen = assignAddress()
cfg.SyncAPI.InternalAPI.Listen = assignAddress()
cfg.UserAPI.InternalAPI.Listen = assignAddress()
cfg.AppServiceAPI.Bind = cfg.AppServiceAPI.Listen
cfg.CurrentStateServer.Bind = cfg.CurrentStateServer.Listen
cfg.EDUServer.Bind = cfg.EDUServer.Listen
cfg.FederationAPI.Bind = cfg.FederationAPI.Listen
cfg.FederationSender.Bind = cfg.FederationSender.Listen
cfg.KeyServer.Bind = cfg.KeyServer.Listen
cfg.MediaAPI.Bind = cfg.MediaAPI.Listen
cfg.RoomServer.Bind = cfg.RoomServer.Listen
cfg.ServerKeyAPI.Bind = cfg.ServerKeyAPI.Listen
cfg.SyncAPI.Bind = cfg.SyncAPI.Listen
cfg.UserAPI.Bind = cfg.UserAPI.Listen
cfg.AppServiceAPI.InternalAPI.Connect = cfg.AppServiceAPI.InternalAPI.Listen
cfg.CurrentStateServer.InternalAPI.Connect = cfg.CurrentStateServer.InternalAPI.Listen
cfg.EDUServer.InternalAPI.Connect = cfg.EDUServer.InternalAPI.Listen
cfg.FederationAPI.InternalAPI.Connect = cfg.FederationAPI.InternalAPI.Listen
cfg.FederationSender.InternalAPI.Connect = cfg.FederationSender.InternalAPI.Listen
cfg.KeyServer.InternalAPI.Connect = cfg.KeyServer.InternalAPI.Listen
cfg.MediaAPI.InternalAPI.Connect = cfg.MediaAPI.InternalAPI.Listen
cfg.RoomServer.InternalAPI.Connect = cfg.RoomServer.InternalAPI.Listen
cfg.ServerKeyAPI.InternalAPI.Connect = cfg.ServerKeyAPI.InternalAPI.Listen
cfg.SyncAPI.InternalAPI.Connect = cfg.SyncAPI.InternalAPI.Listen
cfg.UserAPI.InternalAPI.Connect = cfg.UserAPI.InternalAPI.Listen
return &cfg, port, nil
}

6
internal/test/server.go
View file

@ -96,9 +96,9 @@ func InitDatabase(postgresDatabase, postgresContainerName string, databases []st
func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) {
proxyArgs := []string{
"--bind-address", bindAddr,
"--sync-api-server-url", "http://" + string(cfg.SyncAPI.Listen),
"--client-api-server-url", "http://" + string(cfg.ClientAPI.Listen),
"--media-api-server-url", "http://" + string(cfg.MediaAPI.Listen),
"--sync-api-server-url", "http://" + string(cfg.SyncAPI.InternalAPI.Connect),
"--client-api-server-url", "http://" + string(cfg.ClientAPI.InternalAPI.Connect),
"--media-api-server-url", "http://" + string(cfg.MediaAPI.InternalAPI.Connect),
"--tls-cert", "server.crt",
"--tls-key", "server.key",
}