Multi-personality polylith binary (#1552)

* Initial work oon multipersonality binary

* Remove old binaries

* Monolith and polylith binaries

* Better logging

* dendrite-poly-multi

* Fix path

* Copyright notices etc

* Tweaks

* Update Docker, INSTALL.md

* Take first argument if flags package doesn't find any args

* Postgres 9.6 or later, fix some more Docker stuff

* Don't create unnecessary e2ekey DB

* Run go mod tidy

cmd/dendrite-polylith-multi/main.go

@@ -0,0 +1,78 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"os"
+	"strings"
+
+	"github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/sirupsen/logrus"
+)
+
+type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite)
+
+func main() {
+	cfg := setup.ParseFlags(true)
+
+	component := ""
+	if flag.NFlag() > 0 {
+		component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi
+	} else if len(os.Args) > 1 {
+		component = os.Args[1] // ./dendrite-polylith-multi clientapi
+	}
+
+	components := map[string]entrypoint{
+		"appservice":       personalities.Appservice,
+		"clientapi":        personalities.ClientAPI,
+		"eduserver":        personalities.EDUServer,
+		"federationapi":    personalities.FederationAPI,
+		"federationsender": personalities.FederationSender,
+		"keyserver":        personalities.KeyServer,
+		"mediaapi":         personalities.MediaAPI,
+		"roomserver":       personalities.RoomServer,
+		"signingkeyserver": personalities.SigningKeyServer,
+		"syncapi":          personalities.SyncAPI,
+		"userapi":          personalities.UserAPI,
+	}
+
+	start, ok := components[component]
+	if !ok {
+		if component == "" {
+			logrus.Errorf("No component specified")
+			logrus.Info("The first argument on the command line must be the name of the component to run")
+		} else {
+			logrus.Errorf("Unknown component %q specified", component)
+		}
+
+		var list []string
+		for c := range components {
+			list = append(list, c)
+		}
+		logrus.Infof("Valid components: %s", strings.Join(list, ", "))
+
+		os.Exit(1)
+	}
+
+	logrus.Infof("Starting %q component", component)
+
+	base := setup.NewBaseDendrite(cfg, component, false) // TODO
+	defer base.Close()                                   // nolint: errcheck
+
+	start(base, cfg)
+}

cmd/dendrite-polylith-multi/personalities/appservice.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
 	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true)
-
-	defer base.Close() // nolint: errcheck
+func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	userAPI := base.UserAPIClient()
 	rsAPI := base.RoomserverHTTPClient()
 

cmd/dendrite-polylith-multi/personalities/clientapi.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,20 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
 	"github.com/matrix-org/dendrite/clientapi"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/internal/transactions"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-
-	base := setup.NewBaseDendrite(cfg, "ClientAPI", true)
-	defer base.Close() // nolint: errcheck
-
+func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	accountDB := base.CreateAccountsDB()
 	federation := base.CreateFederationClient()
 

cmd/dendrite-polylith-multi/personalities/eduserver.go

@@ -1,3 +1,5 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -10,26 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
-	_ "net/http/pprof"
-
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/sirupsen/logrus"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "EDUServerAPI", true)
-	defer func() {
-		if err := base.Close(); err != nil {
-			logrus.WithError(err).Warn("BaseDendrite close failed")
-		}
-	}()
-
+func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
 	eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
 

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
 	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "FederationAPI", true)
-	defer base.Close() // nolint: errcheck
-
+func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	userAPI := base.UserAPIClient()
 	federation := base.CreateFederationClient()
 	serverKeyAPI := base.SigningKeyServerHTTPClient()

cmd/dendrite-polylith-multi/personalities/federationsender.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
 	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "FederationSender", true)
-	defer base.Close() // nolint: errcheck
-
+func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	federation := base.CreateFederationClient()
 
 	serverKeyAPI := base.SigningKeyServerHTTPClient()

cmd/dendrite-polylith-multi/personalities/keyserver.go

@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/keyserver"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "KeyServer", true)
-	defer base.Close() // nolint: errcheck
-
+func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient())
 	intAPI.SetUserAPI(base.UserAPIClient())
 

cmd/dendrite-polylith-multi/personalities/mediaapi.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/mediaapi"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "MediaAPI", true)
-	defer base.Close() // nolint: errcheck
-
+func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	userAPI := base.UserAPIClient()
 	client := base.CreateClient()
 

cmd/dendrite-polylith-multi/personalities/roomserver.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/roomserver"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "RoomServerAPI", true)
-	defer base.Close() // nolint: errcheck
-
+func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	serverKeyAPI := base.SigningKeyServerHTTPClient()
 	keyRing := serverKeyAPI.KeyRing()
 

cmd/dendrite-polylith-multi/personalities/signingkeyserver.go

@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/signingkeyserver"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "SigningKeyServer", true)
-	defer base.Close() // nolint: errcheck
-
+func SigningKeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	federation := base.CreateFederationClient()
 
 	intAPI := signingkeyserver.NewInternalAPI(&base.Cfg.SigningKeyServer, federation, base.Caches)

cmd/dendrite-polylith-multi/personalities/syncapi.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,19 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/syncapi"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-
-	base := setup.NewBaseDendrite(cfg, "SyncAPI", true)
-	defer base.Close() // nolint: errcheck
-
+func SyncAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	userAPI := base.UserAPIClient()
 	federation := base.CreateFederationClient()
 

cmd/dendrite-polylith-multi/personalities/userapi.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/userapi"
 )
 
-func main() {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "UserAPI", true)
-	defer base.Close() // nolint: errcheck
-
+func UserAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	accountDB := base.CreateAccountsDB()
 
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient())