Multi-personality polylith binary (#1552)

* Initial work oon multipersonality binary

* Remove old binaries

* Monolith and polylith binaries

* Better logging

* dendrite-poly-multi

* Fix path

* Copyright notices etc

* Tweaks

* Update Docker, INSTALL.md

* Take first argument if flags package doesn't find any args

* Postgres 9.6 or later, fix some more Docker stuff

* Don't create unnecessary e2ekey DB

* Run go mod tidy

cmd/dendrite-polylith-multi/main.go

@@ -0,0 +1,78 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"os"
+	"strings"
+
+	"github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/sirupsen/logrus"
+)
+
+type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite)
+
+func main() {
+	cfg := setup.ParseFlags(true)
+
+	component := ""
+	if flag.NFlag() > 0 {
+		component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi
+	} else if len(os.Args) > 1 {
+		component = os.Args[1] // ./dendrite-polylith-multi clientapi
+	}
+
+	components := map[string]entrypoint{
+		"appservice":       personalities.Appservice,
+		"clientapi":        personalities.ClientAPI,
+		"eduserver":        personalities.EDUServer,
+		"federationapi":    personalities.FederationAPI,
+		"federationsender": personalities.FederationSender,
+		"keyserver":        personalities.KeyServer,
+		"mediaapi":         personalities.MediaAPI,
+		"roomserver":       personalities.RoomServer,
+		"signingkeyserver": personalities.SigningKeyServer,
+		"syncapi":          personalities.SyncAPI,
+		"userapi":          personalities.UserAPI,
+	}
+
+	start, ok := components[component]
+	if !ok {
+		if component == "" {
+			logrus.Errorf("No component specified")
+			logrus.Info("The first argument on the command line must be the name of the component to run")
+		} else {
+			logrus.Errorf("Unknown component %q specified", component)
+		}
+
+		var list []string
+		for c := range components {
+			list = append(list, c)
+		}
+		logrus.Infof("Valid components: %s", strings.Join(list, ", "))
+
+		os.Exit(1)
+	}
+
+	logrus.Infof("Starting %q component", component)
+
+	base := setup.NewBaseDendrite(cfg, component, false) // TODO
+	defer base.Close()                                   // nolint: errcheck
+
+	start(base, cfg)
+}

cmd/dendrite-polylith-multi/personalities/appservice.go

@@ -0,0 +1,35 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+)
+
+func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	userAPI := base.UserAPIClient()
+	rsAPI := base.RoomserverHTTPClient()
+
+	intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener
+		setup.NoListener, // external listener
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/clientapi.go

@@ -0,0 +1,45 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/clientapi"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/internal/transactions"
+)
+
+func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	accountDB := base.CreateAccountsDB()
+	federation := base.CreateFederationClient()
+
+	asQuery := base.AppserviceHTTPClient()
+	rsAPI := base.RoomserverHTTPClient()
+	fsAPI := base.FederationSenderHTTPClient()
+	eduInputAPI := base.EDUServerClient()
+	userAPI := base.UserAPIClient()
+	keyAPI := base.KeyServerHTTPClient()
+
+	clientapi.AddPublicRoutes(
+		base.PublicClientAPIMux, &base.Cfg.ClientAPI, accountDB, federation,
+		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, keyAPI, nil,
+	)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.ClientAPI.InternalAPI.Listen,
+		base.Cfg.ClientAPI.ExternalAPI.Listen,
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/eduserver.go

@@ -0,0 +1,33 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/eduserver"
+	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+)
+
+func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
+	eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.EDUServer.InternalAPI.Listen, // internal listener
+		setup.NoListener,                      // external listener
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -0,0 +1,43 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+)
+
+func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	userAPI := base.UserAPIClient()
+	federation := base.CreateFederationClient()
+	serverKeyAPI := base.SigningKeyServerHTTPClient()
+	keyRing := serverKeyAPI.KeyRing()
+	fsAPI := base.FederationSenderHTTPClient()
+	rsAPI := base.RoomserverHTTPClient()
+	keyAPI := base.KeyServerHTTPClient()
+
+	federationapi.AddPublicRoutes(
+		base.PublicFederationAPIMux, base.PublicKeyAPIMux,
+		&base.Cfg.FederationAPI, userAPI, federation, keyRing,
+		rsAPI, fsAPI, base.EDUServerClient(), keyAPI,
+	)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.FederationAPI.InternalAPI.Listen,
+		base.Cfg.FederationAPI.ExternalAPI.Listen,
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/federationsender.go

@@ -0,0 +1,40 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+)
+
+func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	federation := base.CreateFederationClient()
+
+	serverKeyAPI := base.SigningKeyServerHTTPClient()
+	keyRing := serverKeyAPI.KeyRing()
+
+	rsAPI := base.RoomserverHTTPClient()
+	fsAPI := federationsender.NewInternalAPI(
+		base, federation, rsAPI, keyRing,
+	)
+	federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.FederationSender.InternalAPI.Listen, // internal listener
+		setup.NoListener, // external listener
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/keyserver.go

@@ -0,0 +1,34 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/keyserver"
+)
+
+func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient())
+	intAPI.SetUserAPI(base.UserAPIClient())
+
+	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.KeyServer.InternalAPI.Listen, // internal listener
+		setup.NoListener,                      // external listener
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/mediaapi.go

@@ -0,0 +1,34 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/mediaapi"
+)
+
+func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	userAPI := base.UserAPIClient()
+	client := base.CreateClient()
+
+	mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, userAPI, client)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.MediaAPI.InternalAPI.Listen,
+		base.Cfg.MediaAPI.ExternalAPI.Listen,
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/roomserver.go

@@ -0,0 +1,37 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/roomserver"
+)
+
+func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	serverKeyAPI := base.SigningKeyServerHTTPClient()
+	keyRing := serverKeyAPI.KeyRing()
+
+	fsAPI := base.FederationSenderHTTPClient()
+	rsAPI := roomserver.NewInternalAPI(base, keyRing)
+	rsAPI.SetFederationSenderAPI(fsAPI)
+	roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.RoomServer.InternalAPI.Listen, // internal listener
+		setup.NoListener,                       // external listener
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/signingkeyserver.go

@@ -0,0 +1,34 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/signingkeyserver"
+)
+
+func SigningKeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	federation := base.CreateFederationClient()
+
+	intAPI := signingkeyserver.NewInternalAPI(&base.Cfg.SigningKeyServer, federation, base.Caches)
+	signingkeyserver.AddInternalRoutes(base.InternalAPIMux, intAPI, base.Caches)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.SigningKeyServer.InternalAPI.Listen,
+		setup.NoListener,
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/syncapi.go

@@ -0,0 +1,40 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/syncapi"
+)
+
+func SyncAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	userAPI := base.UserAPIClient()
+	federation := base.CreateFederationClient()
+
+	rsAPI := base.RoomserverHTTPClient()
+
+	syncapi.AddPublicRoutes(
+		base.PublicClientAPIMux, userAPI, rsAPI,
+		base.KeyServerHTTPClient(),
+		federation, &cfg.SyncAPI,
+	)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.SyncAPI.InternalAPI.Listen,
+		base.Cfg.SyncAPI.ExternalAPI.Listen,
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/userapi.go

@@ -0,0 +1,35 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/userapi"
+)
+
+func UserAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	accountDB := base.CreateAccountsDB()
+
+	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient())
+
+	userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.UserAPI.InternalAPI.Listen, // internal listener
+		setup.NoListener,                    // external listener
+		nil, nil,
+	)
+}