Multi-personality polylith binary (#1552)

* Initial work oon multipersonality binary

* Remove old binaries

* Monolith and polylith binaries

* Better logging

* dendrite-poly-multi

* Fix path

* Copyright notices etc

* Tweaks

* Update Docker, INSTALL.md

* Take first argument if flags package doesn't find any args

* Postgres 9.6 or later, fix some more Docker stuff

* Don't create unnecessary e2ekey DB

* Run go mod tidy

cmd/dendrite-polylith-multi/personalities/appservice.go

@@ -0,0 +1,35 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+)
+
+func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	userAPI := base.UserAPIClient()
+	rsAPI := base.RoomserverHTTPClient()
+
+	intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
+	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener
+		setup.NoListener, // external listener
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/clientapi.go

@@ -0,0 +1,45 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/clientapi"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/internal/transactions"
+)
+
+func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	accountDB := base.CreateAccountsDB()
+	federation := base.CreateFederationClient()
+
+	asQuery := base.AppserviceHTTPClient()
+	rsAPI := base.RoomserverHTTPClient()
+	fsAPI := base.FederationSenderHTTPClient()
+	eduInputAPI := base.EDUServerClient()
+	userAPI := base.UserAPIClient()
+	keyAPI := base.KeyServerHTTPClient()
+
+	clientapi.AddPublicRoutes(
+		base.PublicClientAPIMux, &base.Cfg.ClientAPI, accountDB, federation,
+		rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, keyAPI, nil,
+	)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.ClientAPI.InternalAPI.Listen,
+		base.Cfg.ClientAPI.ExternalAPI.Listen,
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/eduserver.go

@@ -0,0 +1,33 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/eduserver"
+	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+)
+
+func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
+	eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.EDUServer.InternalAPI.Listen, // internal listener
+		setup.NoListener,                      // external listener
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -0,0 +1,43 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+)
+
+func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	userAPI := base.UserAPIClient()
+	federation := base.CreateFederationClient()
+	serverKeyAPI := base.SigningKeyServerHTTPClient()
+	keyRing := serverKeyAPI.KeyRing()
+	fsAPI := base.FederationSenderHTTPClient()
+	rsAPI := base.RoomserverHTTPClient()
+	keyAPI := base.KeyServerHTTPClient()
+
+	federationapi.AddPublicRoutes(
+		base.PublicFederationAPIMux, base.PublicKeyAPIMux,
+		&base.Cfg.FederationAPI, userAPI, federation, keyRing,
+		rsAPI, fsAPI, base.EDUServerClient(), keyAPI,
+	)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.FederationAPI.InternalAPI.Listen,
+		base.Cfg.FederationAPI.ExternalAPI.Listen,
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/federationsender.go

@@ -0,0 +1,40 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+)
+
+func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	federation := base.CreateFederationClient()
+
+	serverKeyAPI := base.SigningKeyServerHTTPClient()
+	keyRing := serverKeyAPI.KeyRing()
+
+	rsAPI := base.RoomserverHTTPClient()
+	fsAPI := federationsender.NewInternalAPI(
+		base, federation, rsAPI, keyRing,
+	)
+	federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.FederationSender.InternalAPI.Listen, // internal listener
+		setup.NoListener, // external listener
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/keyserver.go

@@ -0,0 +1,34 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/keyserver"
+)
+
+func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient())
+	intAPI.SetUserAPI(base.UserAPIClient())
+
+	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.KeyServer.InternalAPI.Listen, // internal listener
+		setup.NoListener,                      // external listener
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/mediaapi.go

@@ -0,0 +1,34 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/mediaapi"
+)
+
+func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	userAPI := base.UserAPIClient()
+	client := base.CreateClient()
+
+	mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, userAPI, client)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.MediaAPI.InternalAPI.Listen,
+		base.Cfg.MediaAPI.ExternalAPI.Listen,
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/roomserver.go

@@ -0,0 +1,37 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/roomserver"
+)
+
+func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	serverKeyAPI := base.SigningKeyServerHTTPClient()
+	keyRing := serverKeyAPI.KeyRing()
+
+	fsAPI := base.FederationSenderHTTPClient()
+	rsAPI := roomserver.NewInternalAPI(base, keyRing)
+	rsAPI.SetFederationSenderAPI(fsAPI)
+	roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.RoomServer.InternalAPI.Listen, // internal listener
+		setup.NoListener,                       // external listener
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/signingkeyserver.go

@@ -0,0 +1,34 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/signingkeyserver"
+)
+
+func SigningKeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	federation := base.CreateFederationClient()
+
+	intAPI := signingkeyserver.NewInternalAPI(&base.Cfg.SigningKeyServer, federation, base.Caches)
+	signingkeyserver.AddInternalRoutes(base.InternalAPIMux, intAPI, base.Caches)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.SigningKeyServer.InternalAPI.Listen,
+		setup.NoListener,
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/syncapi.go

@@ -0,0 +1,40 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/syncapi"
+)
+
+func SyncAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	userAPI := base.UserAPIClient()
+	federation := base.CreateFederationClient()
+
+	rsAPI := base.RoomserverHTTPClient()
+
+	syncapi.AddPublicRoutes(
+		base.PublicClientAPIMux, userAPI, rsAPI,
+		base.KeyServerHTTPClient(),
+		federation, &cfg.SyncAPI,
+	)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.SyncAPI.InternalAPI.Listen,
+		base.Cfg.SyncAPI.ExternalAPI.Listen,
+		nil, nil,
+	)
+}

cmd/dendrite-polylith-multi/personalities/userapi.go

@@ -0,0 +1,35 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package personalities
+
+import (
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/matrix-org/dendrite/userapi"
+)
+
+func UserAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
+	accountDB := base.CreateAccountsDB()
+
+	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient())
+
+	userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
+
+	base.SetupAndServeHTTP(
+		base.Cfg.UserAPI.InternalAPI.Listen, // internal listener
+		setup.NoListener,                    // external listener
+		nil, nil,
+	)
+}