Remove polylith/API mode (#2967)

This removes most of the code used for polylith/API mode.

This removes the `/api` internal endpoints entirely. 

Binary size change roughly 5%: 
```
51437560 Feb 13 10:15 dendrite-monolith-server # old
48759008 Feb 13 10:15 dendrite-monolith-server # new
```
This commit is contained in:
Till 2023-02-14 12:47:47 +01:00 committed by GitHub
parent cc59879faa
commit 11d9b9db0e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
106 changed files with 374 additions and 5850 deletions

View file

@ -80,8 +80,8 @@ type P2PMonolith struct {
func GenerateDefaultConfig(sk ed25519.PrivateKey, storageDir string, cacheDir string, dbPrefix string) *config.Dendrite {
cfg := config.Dendrite{}
cfg.Defaults(config.DefaultOpts{
Generate: true,
Monolithic: true,
Generate: true,
SingleDatabase: true,
})
cfg.Global.PrivateKey = sk
cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", filepath.Join(cacheDir, dbPrefix)))
@ -121,9 +121,9 @@ func (p *P2PMonolith) SetupPinecone(sk ed25519.PrivateKey) {
func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelaying bool, enableMetrics bool, enableWebsockets bool) {
if enableMetrics {
p.BaseDendrite = base.NewBaseDendrite(cfg, "Monolith")
p.BaseDendrite = base.NewBaseDendrite(cfg)
} else {
p.BaseDendrite = base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
p.BaseDendrite = base.NewBaseDendrite(cfg, base.DisableMetrics)
}
p.port = port
p.BaseDendrite.ConfigureAdminEndpoints()
@ -236,7 +236,6 @@ func (p *P2PMonolith) Addr() string {
func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, enableWebsockets bool) {
p.httpMux = mux.NewRouter().SkipClean(true).UseEncodedPath()
p.httpMux.PathPrefix(httputil.InternalPathPrefix).Handler(p.BaseDendrite.InternalAPIMux)
p.httpMux.PathPrefix(httputil.PublicClientPathPrefix).Handler(p.BaseDendrite.PublicClientAPIMux)
p.httpMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.PublicMediaAPIMux)
p.httpMux.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(p.BaseDendrite.DendriteAdminMux)

View file

@ -117,8 +117,8 @@ func main() {
cfg = setup.ParseFlags(true)
} else {
cfg.Defaults(config.DefaultOpts{
Generate: true,
Monolithic: true,
Generate: true,
SingleDatabase: true,
})
cfg.Global.PrivateKey = sk
cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(*instanceDir, *instanceName))
@ -143,7 +143,7 @@ func main() {
cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
base := base.NewBaseDendrite(cfg, "Monolith")
base := base.NewBaseDendrite(cfg)
base.ConfigureAdminEndpoints()
defer base.Close() // nolint: errcheck
@ -157,13 +157,11 @@ func main() {
serverKeyAPI := &signing.YggdrasilKeys{}
keyRing := serverKeyAPI.KeyRing()
rsComponent := roomserver.NewInternalAPI(
rsAPI := roomserver.NewInternalAPI(
base,
)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsComponent)
rsAPI := rsComponent
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsAPI)
userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
keyAPI.SetUserAPI(userAPI)
@ -174,7 +172,7 @@ func main() {
base, federation, rsAPI, base.Caches, keyRing, true,
)
rsComponent.SetFederationAPI(fsAPI, keyRing)
rsAPI.SetFederationAPI(fsAPI, keyRing)
monolith := setup.Monolith{
Config: base.Cfg,
@ -197,7 +195,6 @@ func main() {
}
httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)

View file

@ -1,178 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
"os"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup"
basepkg "github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/mscs"
"github.com/matrix-org/dendrite/userapi"
uapi "github.com/matrix-org/dendrite/userapi/api"
)
var (
httpBindAddr = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server")
httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
apiBindAddr = flag.String("api-bind-address", "localhost:18008", "The HTTP listening port for the internal HTTP APIs (if -api is enabled)")
certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS")
enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)")
traceInternal = os.Getenv("DENDRITE_TRACE_INTERNAL") == "1"
)
func main() {
cfg := setup.ParseFlags(true)
httpAddr := config.HTTPAddress("http://" + *httpBindAddr)
httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr)
httpAPIAddr := httpAddr
options := []basepkg.BaseDendriteOptions{}
if *enableHTTPAPIs {
logrus.Warnf("DANGER! The -api option is enabled, exposing internal APIs on %q!", *apiBindAddr)
httpAPIAddr = config.HTTPAddress("http://" + *apiBindAddr)
// If the HTTP APIs are enabled then we need to update the Listen
// statements in the configuration so that we know where to find
// the API endpoints. They'll listen on the same port as the monolith
// itself.
cfg.AppServiceAPI.InternalAPI.Connect = httpAPIAddr
cfg.ClientAPI.InternalAPI.Connect = httpAPIAddr
cfg.FederationAPI.InternalAPI.Connect = httpAPIAddr
cfg.KeyServer.InternalAPI.Connect = httpAPIAddr
cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr
cfg.RoomServer.InternalAPI.Connect = httpAPIAddr
cfg.SyncAPI.InternalAPI.Connect = httpAPIAddr
cfg.UserAPI.InternalAPI.Connect = httpAPIAddr
options = append(options, basepkg.UseHTTPAPIs)
}
base := basepkg.NewBaseDendrite(cfg, "Monolith", options...)
defer base.Close() // nolint: errcheck
federation := base.CreateFederationClient()
rsImpl := roomserver.NewInternalAPI(base)
// call functions directly on the impl unless running in HTTP mode
rsAPI := rsImpl
if base.UseHTTPAPIs {
roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl, base.EnableMetrics)
rsAPI = base.RoomserverHTTPClient()
}
if traceInternal {
rsAPI = &api.RoomserverInternalAPITrace{
Impl: rsAPI,
}
}
fsAPI := federationapi.NewInternalAPI(
base, federation, rsAPI, base.Caches, nil, false,
)
fsImplAPI := fsAPI
if base.UseHTTPAPIs {
federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics)
fsAPI = base.FederationAPIHTTPClient()
}
keyRing := fsAPI.KeyRing()
keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI)
keyAPI := keyImpl
if base.UseHTTPAPIs {
keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI, base.EnableMetrics)
keyAPI = base.KeyServerHTTPClient()
}
pgClient := base.PushGatewayHTTPClient()
userImpl := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient)
userAPI := userImpl
if base.UseHTTPAPIs {
userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics)
userAPI = base.UserAPIClient()
}
if traceInternal {
userAPI = &uapi.UserInternalAPITrace{
Impl: userAPI,
}
}
// TODO: This should use userAPI, not userImpl, but the appservice setup races with
// the listeners and panics at startup if it tries to create appservice accounts
// before the listeners are up.
asAPI := appservice.NewInternalAPI(base, userImpl, rsAPI)
if base.UseHTTPAPIs {
appservice.AddInternalRoutes(base.InternalAPIMux, asAPI, base.EnableMetrics)
asAPI = base.AppserviceHTTPClient()
}
// The underlying roomserver implementation needs to be able to call the fedsender.
// This is different to rsAPI which can be the http client which doesn't need this
// dependency. Other components also need updating after their dependencies are up.
rsImpl.SetFederationAPI(fsAPI, keyRing)
rsImpl.SetAppserviceAPI(asAPI)
rsImpl.SetUserAPI(userAPI)
keyImpl.SetUserAPI(userAPI)
monolith := setup.Monolith{
Config: base.Cfg,
Client: base.CreateClient(),
FedClient: federation,
KeyRing: keyRing,
AppserviceAPI: asAPI,
// always use the concrete impl here even in -http mode because adding public routes
// must be done on the concrete impl not an HTTP client else fedapi will call itself
FederationAPI: fsImplAPI,
RoomserverAPI: rsAPI,
UserAPI: userAPI,
KeyAPI: keyAPI,
}
monolith.AddAllPublicRoutes(base)
if len(base.Cfg.MSCs.MSCs) > 0 {
if err := mscs.Enable(base, &monolith); err != nil {
logrus.WithError(err).Fatalf("Failed to enable MSCs")
}
}
// Expose the matrix APIs directly rather than putting them under a /api path.
go func() {
base.SetupAndServeHTTP(
httpAPIAddr, // internal API
httpAddr, // external API
nil, nil, // TLS settings
)
}()
// Handle HTTPS if certificate and key are provided
if *certFile != "" && *keyFile != "" {
go func() {
base.SetupAndServeHTTP(
basepkg.NoListener, // internal API
httpsAddr, // external API
certFile, keyFile, // TLS settings
)
}()
}
// We want to block forever to let the HTTP and HTTPS handler serve the APIs
base.WaitForShutdown()
}

View file

@ -1,77 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
"os"
"strings"
"github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/sirupsen/logrus"
)
type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite)
func main() {
cfg := setup.ParseFlags(false)
component := ""
if flag.NFlag() > 0 {
component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi
} else if len(os.Args) > 1 {
component = os.Args[1] // ./dendrite-polylith-multi clientapi
}
components := map[string]entrypoint{
"appservice": personalities.Appservice,
"clientapi": personalities.ClientAPI,
"federationapi": personalities.FederationAPI,
"keyserver": personalities.KeyServer,
"mediaapi": personalities.MediaAPI,
"roomserver": personalities.RoomServer,
"syncapi": personalities.SyncAPI,
"userapi": personalities.UserAPI,
}
start, ok := components[component]
if !ok {
if component == "" {
logrus.Errorf("No component specified")
logrus.Info("The first argument on the command line must be the name of the component to run")
} else {
logrus.Errorf("Unknown component %q specified", component)
}
var list []string
for c := range components {
list = append(list, c)
}
logrus.Infof("Valid components: %s", strings.Join(list, ", "))
os.Exit(1)
}
logrus.Infof("Starting %q component", component)
base := base.NewBaseDendrite(cfg, component, base.PolylithMode) // TODO
defer base.Close() // nolint: errcheck
go start(base, cfg)
base.WaitForShutdown()
}

View file

@ -1,36 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package personalities
import (
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/setup/base"
basepkg "github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
)
func Appservice(base *base.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
rsAPI := base.RoomserverHTTPClient()
intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
appservice.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener
basepkg.NoListener, // external listener
nil, nil,
)
}

View file

@ -1,44 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package personalities
import (
"github.com/matrix-org/dendrite/clientapi"
"github.com/matrix-org/dendrite/internal/transactions"
basepkg "github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
)
func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
federation := base.CreateFederationClient()
asQuery := base.AppserviceHTTPClient()
rsAPI := base.RoomserverHTTPClient()
fsAPI := base.FederationAPIHTTPClient()
userAPI := base.UserAPIClient()
keyAPI := base.KeyServerHTTPClient()
clientapi.AddPublicRoutes(
base, federation, rsAPI, asQuery,
transactions.New(), fsAPI, userAPI, userAPI,
keyAPI, nil,
)
base.SetupAndServeHTTP(
base.Cfg.ClientAPI.InternalAPI.Listen,
base.Cfg.ClientAPI.ExternalAPI.Listen,
nil, nil,
)
}

View file

@ -1,44 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package personalities
import (
"github.com/matrix-org/dendrite/federationapi"
basepkg "github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
)
func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
federation := base.CreateFederationClient()
rsAPI := base.RoomserverHTTPClient()
keyAPI := base.KeyServerHTTPClient()
fsAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, nil, true)
keyRing := fsAPI.KeyRing()
federationapi.AddPublicRoutes(
base,
userAPI, federation, keyRing,
rsAPI, fsAPI, keyAPI, nil,
)
federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.FederationAPI.InternalAPI.Listen,
base.Cfg.FederationAPI.ExternalAPI.Listen,
nil, nil,
)
}

View file

@ -1,36 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package personalities
import (
"github.com/matrix-org/dendrite/keyserver"
basepkg "github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
)
func KeyServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
fsAPI := base.FederationAPIHTTPClient()
rsAPI := base.RoomserverHTTPClient()
intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI)
intAPI.SetUserAPI(base.UserAPIClient())
keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.KeyServer.InternalAPI.Listen, // internal listener
basepkg.NoListener, // external listener
nil, nil,
)
}

View file

@ -1,36 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package personalities
import (
"github.com/matrix-org/dendrite/mediaapi"
basepkg "github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
)
func MediaAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
client := base.CreateClient()
mediaapi.AddPublicRoutes(
base, userAPI, client,
)
base.SetupAndServeHTTP(
base.Cfg.MediaAPI.InternalAPI.Listen,
base.Cfg.MediaAPI.ExternalAPI.Listen,
nil, nil,
)
}

View file

@ -1,36 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package personalities
import (
"github.com/matrix-org/dendrite/roomserver"
basepkg "github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
)
func RoomServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
asAPI := base.AppserviceHTTPClient()
fsAPI := base.FederationAPIHTTPClient()
rsAPI := roomserver.NewInternalAPI(base)
rsAPI.SetFederationAPI(fsAPI, fsAPI.KeyRing())
rsAPI.SetAppserviceAPI(asAPI)
roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.RoomServer.InternalAPI.Listen, // internal listener
basepkg.NoListener, // external listener
nil, nil,
)
}

View file

@ -1,39 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package personalities
import (
basepkg "github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/syncapi"
)
func SyncAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
rsAPI := base.RoomserverHTTPClient()
syncapi.AddPublicRoutes(
base,
userAPI, rsAPI,
base.KeyServerHTTPClient(),
)
base.SetupAndServeHTTP(
base.Cfg.SyncAPI.InternalAPI.Listen,
base.Cfg.SyncAPI.ExternalAPI.Listen,
nil, nil,
)
}

View file

@ -1,37 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package personalities
import (
basepkg "github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi"
)
func UserAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
userAPI := userapi.NewInternalAPI(
base, &cfg.UserAPI, cfg.Derived.ApplicationServices,
base.KeyServerHTTPClient(), base.RoomserverHTTPClient(),
base.PushGatewayHTTPClient(),
)
userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.UserAPI.InternalAPI.Listen, // internal listener
basepkg.NoListener, // external listener
nil, nil,
)
}

View file

@ -45,6 +45,10 @@ var (
const HEAD = "HEAD"
// The binary was renamed after v0.11.1, so everything after that should use the new name
var binaryChangeVersion, _ = semver.NewVersion("v0.11.1")
var latest, _ = semver.NewVersion("v6.6.6") // Dummy version, used as "HEAD"
// Embed the Dockerfile to use when building dendrite versions.
// We cannot use the dockerfile associated with the repo with each version sadly due to changes in
// Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients
@ -54,12 +58,13 @@ const HEAD = "HEAD"
const DockerfilePostgreSQL = `FROM golang:1.18-stretch as build
RUN apt-get update && apt-get install -y postgresql
WORKDIR /build
ARG BINARY
# Copy the build context to the repo as this is the right dendrite code. This is different to the
# Complement Dockerfile which wgets a branch.
COPY . .
RUN go build ./cmd/dendrite-monolith-server
RUN go build ./cmd/${BINARY}
RUN go build ./cmd/generate-keys
RUN go build ./cmd/generate-config
RUN go build ./cmd/create-account
@ -88,22 +93,24 @@ done \n\
\n\
sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\
PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\
./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\
./${BINARY} --really-enable-open-registration ${PARAMS} || ./${BINARY} ${PARAMS} \n\
' > run_dendrite.sh && chmod +x run_dendrite.sh
ENV SERVER_NAME=localhost
ENV BINARY=dendrite
EXPOSE 8008 8448
CMD /build/run_dendrite.sh `
CMD /build/run_dendrite.sh`
const DockerfileSQLite = `FROM golang:1.18-stretch as build
RUN apt-get update && apt-get install -y postgresql
WORKDIR /build
ARG BINARY
# Copy the build context to the repo as this is the right dendrite code. This is different to the
# Complement Dockerfile which wgets a branch.
COPY . .
RUN go build ./cmd/dendrite-monolith-server
RUN go build ./cmd/${BINARY}
RUN go build ./cmd/generate-keys
RUN go build ./cmd/generate-config
RUN go build ./cmd/create-account
@ -118,10 +125,11 @@ RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postg
RUN echo '\
sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\
PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\
./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\
./${BINARY} --really-enable-open-registration ${PARAMS} || ./${BINARY} ${PARAMS} \n\
' > run_dendrite.sh && chmod +x run_dendrite.sh
ENV SERVER_NAME=localhost
ENV BINARY=dendrite
EXPOSE 8008 8448
CMD /build/run_dendrite.sh `
@ -182,7 +190,7 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b
}
// buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error
func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, branchOrTagName string) (string, error) {
func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir string, branchOrTagName, binary string) (string, error) {
var tarball *bytes.Buffer
var err error
// If a custom HEAD location is given, use that, else pull from github. Mostly useful for CI
@ -216,6 +224,9 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir,
log.Printf("%s: Building version %s\n", branchOrTagName, branchOrTagName)
res, err := dockerClient.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{
Tags: []string{"dendrite-upgrade"},
BuildArgs: map[string]*string{
"BINARY": &binary,
},
})
if err != nil {
return "", fmt.Errorf("failed to start building image: %s", err)
@ -272,7 +283,7 @@ func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Ve
return semVers, nil
}
func calculateVersions(cli *http.Client, from, to string, direct bool) []string {
func calculateVersions(cli *http.Client, from, to string, direct bool) []*semver.Version {
semvers, err := getAndSortVersionsFromGithub(cli)
if err != nil {
log.Fatalf("failed to collect semvers from github: %s", err)
@ -320,28 +331,25 @@ func calculateVersions(cli *http.Client, from, to string, direct bool) []string
}
semvers = semvers[:i+1]
}
var versions []string
for _, sv := range semvers {
versions = append(versions, sv.Original())
}
if to == HEAD {
versions = append(versions, HEAD)
semvers = append(semvers, latest)
}
if direct {
versions = []string{versions[0], versions[len(versions)-1]}
semvers = []*semver.Version{semvers[0], semvers[len(semvers)-1]}
}
return versions
return semvers
}
func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, branchOrTagNames []string) map[string]string {
func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, versions []*semver.Version) map[string]string {
// concurrently build all versions, this can be done in any order. The mutex protects the map
branchToImageID := make(map[string]string)
var mu sync.Mutex
var wg sync.WaitGroup
wg.Add(concurrency)
ch := make(chan string, len(branchOrTagNames))
for _, branchName := range branchOrTagNames {
ch := make(chan *semver.Version, len(versions))
for _, branchName := range versions {
ch <- branchName
}
close(ch)
@ -349,11 +357,13 @@ func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, b
for i := 0; i < concurrency; i++ {
go func() {
defer wg.Done()
for branchName := range ch {
for version := range ch {
branchName, binary := versionToBranchAndBinary(version)
log.Printf("Building version %s with binary %s", branchName, binary)
tmpDir := baseTempDir + alphaNumerics.ReplaceAllString(branchName, "")
imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName)
imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName, binary)
if err != nil {
log.Fatalf("%s: failed to build dendrite image: %s", branchName, err)
log.Fatalf("%s: failed to build dendrite image: %s", version, err)
}
mu.Lock()
branchToImageID[branchName] = imgID
@ -365,13 +375,14 @@ func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, b
return branchToImageID
}
func runImage(dockerClient *client.Client, volumeName, version, imageID string) (csAPIURL, containerID string, err error) {
log.Printf("%s: running image %s\n", version, imageID)
func runImage(dockerClient *client.Client, volumeName string, branchNameToImageID map[string]string, version *semver.Version) (csAPIURL, containerID string, err error) {
branchName, binary := versionToBranchAndBinary(version)
imageID := branchNameToImageID[branchName]
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
body, err := dockerClient.ContainerCreate(ctx, &container.Config{
Image: imageID,
Env: []string{"SERVER_NAME=hs1"},
Env: []string{"SERVER_NAME=hs1", fmt.Sprintf("BINARY=%s", binary)},
Labels: map[string]string{
dendriteUpgradeTestLabel: "yes",
},
@ -384,7 +395,7 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string)
Target: "/var/lib/postgresql/9.6/main",
},
},
}, nil, nil, "dendrite_upgrade_test_"+version)
}, nil, nil, "dendrite_upgrade_test_"+branchName)
if err != nil {
return "", "", fmt.Errorf("failed to ContainerCreate: %s", err)
}
@ -451,8 +462,8 @@ func destroyContainer(dockerClient *client.Client, containerID string) {
}
}
func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchToImageID map[string]string) error {
csAPIURL, containerID, err := runImage(dockerClient, volumeName, v, branchToImageID[v])
func loadAndRunTests(dockerClient *client.Client, volumeName string, v *semver.Version, branchToImageID map[string]string) error {
csAPIURL, containerID, err := runImage(dockerClient, volumeName, branchToImageID, v)
if err != nil {
return fmt.Errorf("failed to run container for branch %v: %v", v, err)
}
@ -470,9 +481,10 @@ func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchTo
}
// test that create-account is working
func testCreateAccount(dockerClient *client.Client, v string, containerID string) error {
createUser := strings.ToLower("createaccountuser-" + v)
log.Printf("%s: Creating account %s with create-account\n", v, createUser)
func testCreateAccount(dockerClient *client.Client, version *semver.Version, containerID string) error {
branchName, _ := versionToBranchAndBinary(version)
createUser := strings.ToLower("createaccountuser-" + branchName)
log.Printf("%s: Creating account %s with create-account\n", branchName, createUser)
respID, err := dockerClient.ContainerExecCreate(context.Background(), containerID, types.ExecConfig{
AttachStderr: true,
@ -504,9 +516,21 @@ func testCreateAccount(dockerClient *client.Client, v string, containerID string
return nil
}
func verifyTests(dockerClient *client.Client, volumeName string, versions []string, branchToImageID map[string]string) error {
func versionToBranchAndBinary(version *semver.Version) (branchName, binary string) {
binary = "dendrite-monolith-server"
branchName = version.Original()
if version.GreaterThan(binaryChangeVersion) {
binary = "dendrite"
if version.Equal(latest) {
branchName = HEAD
}
}
return
}
func verifyTests(dockerClient *client.Client, volumeName string, versions []*semver.Version, branchToImageID map[string]string) error {
lastVer := versions[len(versions)-1]
csAPIURL, containerID, err := runImage(dockerClient, volumeName, lastVer, branchToImageID[lastVer])
csAPIURL, containerID, err := runImage(dockerClient, volumeName, branchToImageID, lastVer)
if err != nil {
return fmt.Errorf("failed to run container for branch %v: %v", lastVer, err)
}

View file

@ -6,6 +6,7 @@ import (
"strings"
"time"
"github.com/Masterminds/semver/v3"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
)
@ -22,7 +23,8 @@ type user struct {
// - register alice and bob with branch name muxed into the localpart
// - create a DM room for the 2 users and exchange messages
// - create/join a public #global room and exchange messages
func runTests(baseURL, branchName string) error {
func runTests(baseURL string, v *semver.Version) error {
branchName, _ := versionToBranchAndBinary(v)
// register 2 users
users := []user{
{
@ -164,15 +166,16 @@ func runTests(baseURL, branchName string) error {
}
// verifyTestsRan checks that the HS has the right rooms/messages
func verifyTestsRan(baseURL string, branchNames []string) error {
func verifyTestsRan(baseURL string, versions []*semver.Version) error {
log.Println("Verifying tests....")
// check we can login as all users
var resp *gomatrix.RespLogin
for _, branchName := range branchNames {
for _, version := range versions {
client, err := gomatrix.NewClient(baseURL, "", "")
if err != nil {
return err
}
branchName, _ := versionToBranchAndBinary(version)
userLocalparts := []string{
"alice" + branchName,
"bob" + branchName,
@ -224,7 +227,7 @@ func verifyTestsRan(baseURL string, branchNames []string) error {
msgCount += 1
}
}
wantMsgCount := len(branchNames) * 4
wantMsgCount := len(versions) * 4
if msgCount != wantMsgCount {
return fmt.Errorf("got %d messages in global room, want %d", msgCount, wantMsgCount)
}

109
cmd/dendrite/main.go Normal file
View file

@ -0,0 +1,109 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup"
basepkg "github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/mscs"
"github.com/matrix-org/dendrite/userapi"
)
var (
httpBindAddr = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server")
httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server")
certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS")
)
func main() {
cfg := setup.ParseFlags(true)
httpAddr := config.HTTPAddress("http://" + *httpBindAddr)
httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr)
options := []basepkg.BaseDendriteOptions{}
base := basepkg.NewBaseDendrite(cfg, options...)
defer base.Close() // nolint: errcheck
federation := base.CreateFederationClient()
rsAPI := roomserver.NewInternalAPI(base)
fsAPI := federationapi.NewInternalAPI(
base, federation, rsAPI, base.Caches, nil, false,
)
keyRing := fsAPI.KeyRing()
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI)
pgClient := base.PushGatewayHTTPClient()
userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient)
asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
// The underlying roomserver implementation needs to be able to call the fedsender.
// This is different to rsAPI which can be the http client which doesn't need this
// dependency. Other components also need updating after their dependencies are up.
rsAPI.SetFederationAPI(fsAPI, keyRing)
rsAPI.SetAppserviceAPI(asAPI)
rsAPI.SetUserAPI(userAPI)
keyAPI.SetUserAPI(userAPI)
monolith := setup.Monolith{
Config: base.Cfg,
Client: base.CreateClient(),
FedClient: federation,
KeyRing: keyRing,
AppserviceAPI: asAPI,
// always use the concrete impl here even in -http mode because adding public routes
// must be done on the concrete impl not an HTTP client else fedapi will call itself
FederationAPI: fsAPI,
RoomserverAPI: rsAPI,
UserAPI: userAPI,
KeyAPI: keyAPI,
}
monolith.AddAllPublicRoutes(base)
if len(base.Cfg.MSCs.MSCs) > 0 {
if err := mscs.Enable(base, &monolith); err != nil {
logrus.WithError(err).Fatalf("Failed to enable MSCs")
}
}
// Expose the matrix APIs directly rather than putting them under a /api path.
go func() {
base.SetupAndServeHTTP(httpAddr, nil, nil)
}()
// Handle HTTPS if certificate and key are provided
if *certFile != "" && *keyFile != "" {
go func() {
base.SetupAndServeHTTP(httpsAddr, certFile, keyFile)
}()
}
// We want to block forever to let the HTTP and HTTPS handler serve the APIs
base.WaitForShutdown()
}

View file

@ -18,7 +18,6 @@ func main() {
dbURI := flag.String("db", "", "The DB URI to use for all components (PostgreSQL only)")
dirPath := flag.String("dir", "./", "The folder to use for paths (like SQLite databases, media storage)")
normalise := flag.String("normalise", "", "Normalise an existing configuration file by adding new/missing options and defaults")
polylith := flag.Bool("polylith", false, "Generate a config that makes sense for polylith deployments")
flag.Parse()
var cfg *config.Dendrite
@ -27,14 +26,14 @@ func main() {
Version: config.Version,
}
cfg.Defaults(config.DefaultOpts{
Generate: true,
Monolithic: !*polylith,
Generate: true,
SingleDatabase: true,
})
if *serverName != "" {
cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName)
}
uri := config.DataSource(*dbURI)
if *polylith || uri.IsSQLite() || uri == "" {
if uri.IsSQLite() || uri == "" {
for name, db := range map[string]*config.DatabaseOptions{
"federationapi": &cfg.FederationAPI.Database,
"keyserver": &cfg.KeyServer.Database,
@ -43,6 +42,7 @@ func main() {
"roomserver": &cfg.RoomServer.Database,
"syncapi": &cfg.SyncAPI.Database,
"userapi": &cfg.UserAPI.AccountDatabase,
"relayapi": &cfg.RelayAPI.Database,
} {
if uri == "" {
path := filepath.Join(*dirPath, fmt.Sprintf("dendrite_%s.db", name))
@ -96,7 +96,7 @@ func main() {
}
} else {
var err error
if cfg, err = config.Load(*normalise, !*polylith); err != nil {
if cfg, err = config.Load(*normalise); err != nil {
panic(err)
}
}

View file

@ -40,7 +40,7 @@ func main() {
Level: "error",
})
cfg.ClientAPI.RegistrationDisabled = true
base := base.NewBaseDendrite(cfg, "ResolveState", base.DisableMetrics)
base := base.NewBaseDendrite(cfg, base.DisableMetrics)
args := flag.Args()
fmt.Println("Room version", *roomVersion)