Merge branch 'master' into add-nats-support

cmd/create-account/main.go

@@ -22,7 +22,6 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
-	"syscall"
 
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -121,13 +120,13 @@ func getPassword(password, pwdFile *string, pwdStdin, askPass *bool, r io.Reader
 	// ask the user to provide the password
 	if *askPass {
 		fmt.Print("Enter Password: ")
-		bytePassword, err := term.ReadPassword(syscall.Stdin)
+		bytePassword, err := term.ReadPassword(int(os.Stdin.Fd()))
 		if err != nil {
 			logrus.Fatalln("Unable to read password:", err)
 		}
 		fmt.Println()
 		fmt.Print("Confirm Password: ")
-		bytePassword2, err := term.ReadPassword(syscall.Stdin)
+		bytePassword2, err := term.ReadPassword(int(os.Stdin.Fd()))
 		if err != nil {
 			logrus.Fatalln("Unable to read password:", err)
 		}

cmd/dendrite-demo-libp2p/main.go

@@ -146,7 +146,7 @@ func main() {
 
 	accountDB := base.Base.CreateAccountsDB()
 	federation := createFederationClient(base)
-	keyAPI := keyserver.NewInternalAPI(&base.Base.Cfg.KeyServer, federation)
+	keyAPI := keyserver.NewInternalAPI(&base.Base, &base.Base.Cfg.KeyServer, federation)
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
@@ -197,6 +197,7 @@ func main() {
 		base.Base.PublicClientAPIMux,
 		base.Base.PublicFederationAPIMux,
 		base.Base.PublicKeyAPIMux,
+		base.Base.PublicWellKnownAPIMux,
 		base.Base.PublicMediaAPIMux,
 		base.Base.SynapseAdminMux,
 	)

cmd/dendrite-demo-pinecone/conn/client.go

@@ -34,7 +34,7 @@ func ConnectToPeer(pRouter *pineconeRouter.Router, peer string) error {
 	if parent == nil {
 		return fmt.Errorf("failed to wrap connection")
 	}
-	_, err := pRouter.AuthenticatedConnect(parent, "static", pineconeRouter.PeerTypeRemote)
+	_, err := pRouter.AuthenticatedConnect(parent, "static", pineconeRouter.PeerTypeRemote, true)
 	return err
 }
 

cmd/dendrite-demo-pinecone/embed/embed_elementweb.go

@@ -1,3 +1,4 @@
+//go:build elementweb
 // +build elementweb
 
 package embed

cmd/dendrite-demo-pinecone/embed/embed_other.go

@@ -1,3 +1,4 @@
+//go:build !elementweb
 // +build !elementweb
 
 package embed

cmd/dendrite-demo-pinecone/main.go

@@ -49,7 +49,6 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	pineconeMulticast "github.com/matrix-org/pinecone/multicast"
-	"github.com/matrix-org/pinecone/router"
 	pineconeRouter "github.com/matrix-org/pinecone/router"
 	pineconeSessions "github.com/matrix-org/pinecone/sessions"
 
@@ -92,7 +91,7 @@ func main() {
 	}
 
 	logger := log.New(os.Stdout, "", 0)
-	pRouter := pineconeRouter.NewRouter(logger, "dendrite", sk, pk, nil)
+	pRouter := pineconeRouter.NewRouter(logger, sk, false)
 
 	go func() {
 		listener, err := net.Listen("tcp", *instanceListen)
@@ -109,7 +108,7 @@ func main() {
 				continue
 			}
 
-			port, err := pRouter.AuthenticatedConnect(conn, "", pineconeRouter.PeerTypeRemote)
+			port, err := pRouter.AuthenticatedConnect(conn, "", pineconeRouter.PeerTypeRemote, true)
 			if err != nil {
 				logrus.WithError(err).Error("pSwitch.AuthenticatedConnect failed")
 				continue
@@ -125,7 +124,7 @@ func main() {
 
 	connectToStaticPeer := func() {
 		attempt := func() {
-			if pRouter.PeerCount(router.PeerTypeRemote) == 0 {
+			if pRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 {
 				uri := *instancePeer
 				if uri == "" {
 					return
@@ -178,7 +177,7 @@ func main() {
 		base, federation, rsAPI, keyRing, true,
 	)
 
-	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
@@ -210,6 +209,7 @@ func main() {
 		base.PublicClientAPIMux,
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
+		base.PublicWellKnownAPIMux,
 		base.PublicMediaAPIMux,
 		base.SynapseAdminMux,
 	)
@@ -230,7 +230,7 @@ func main() {
 			return
 		}
 		conn := conn.WrapWebSocketConn(c)
-		if _, err = pRouter.AuthenticatedConnect(conn, "websocket", pineconeRouter.PeerTypeRemote); err != nil {
+		if _, err = pRouter.AuthenticatedConnect(conn, "websocket", pineconeRouter.PeerTypeRemote, true); err != nil {
 			logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch")
 		}
 	})

cmd/dendrite-demo-pinecone/rooms/rooms.go

@@ -50,11 +50,9 @@ func NewPineconeRoomProvider(
 }
 
 func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
-	known := p.r.KnownNodes()
-	//known = append(known, p.s.Sessions()...)
 	list := []gomatrixserverlib.ServerName{}
-	for _, k := range known {
-		list = append(list, gomatrixserverlib.ServerName(k.String()))
+	for _, k := range p.r.Peers() {
+		list = append(list, gomatrixserverlib.ServerName(k.PublicKey))
 	}
 	return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, list)
 }

cmd/dendrite-demo-yggdrasil/README.md

@@ -1,6 +1,6 @@
 # Yggdrasil Demo
 
-This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.14 or later.
+This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.15 or later.
 
 To run the homeserver, start at the root of the Dendrite repository and run:
 

cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go

@@ -1,3 +1,4 @@
+//go:build elementweb
 // +build elementweb
 
 package embed

cmd/dendrite-demo-yggdrasil/embed/embed_other.go

@@ -1,3 +1,4 @@
+//go:build !elementweb
 // +build !elementweb
 
 package embed

cmd/dendrite-demo-yggdrasil/main.go

@@ -101,7 +101,7 @@ func main() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
@@ -144,6 +144,7 @@ func main() {
 		base.PublicClientAPIMux,
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
+		base.PublicWellKnownAPIMux,
 		base.PublicMediaAPIMux,
 		base.SynapseAdminMux,
 	)

cmd/dendrite-monolith-server/main.go

@@ -30,6 +30,7 @@ import (
 	"github.com/matrix-org/dendrite/setup/mscs"
 	"github.com/matrix-org/dendrite/signingkeyserver"
 	"github.com/matrix-org/dendrite/userapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/sirupsen/logrus"
 
 	_ "github.com/mattn/go-sqlite3"
@@ -111,9 +112,19 @@ func main() {
 	// This is different to rsAPI which can be the http client which doesn't need this dependency
 	rsImpl.SetFederationSenderAPI(fsAPI)
 
-	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
+	if traceInternal {
+		userAPI = &uapi.UserInternalAPITrace{
+			Impl: userAPI,
+		}
+	}
+	// needs to be after the SetUserAPI call above
+	if base.UseHTTPAPIs {
+		keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI)
+		keyAPI = base.KeyServerHTTPClient()
+	}
 
 	eduInputAPI := eduserver.NewInternalAPI(
 		base, cache.New(), userAPI,
@@ -150,6 +161,7 @@ func main() {
 		base.PublicClientAPIMux,
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
+		base.PublicWellKnownAPIMux,
 		base.PublicMediaAPIMux,
 		base.SynapseAdminMux,
 	)

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -30,7 +30,7 @@ func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	keyAPI := base.KeyServerHTTPClient()
 
 	federationapi.AddPublicRoutes(
-		base.PublicFederationAPIMux, base.PublicKeyAPIMux,
+		base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux,
 		&base.Cfg.FederationAPI, userAPI, federation, keyRing,
 		rsAPI, fsAPI, base.EDUServerClient(), keyAPI,
 		&base.Cfg.MSCs, nil,

cmd/dendrite-polylith-multi/personalities/keyserver.go

@@ -22,7 +22,7 @@ import (
 
 func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
 	fsAPI := base.FederationSenderHTTPClient()
-	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI)
+	intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
 	intAPI.SetUserAPI(base.UserAPIClient())
 
 	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)

cmd/dendrite-upgrade-tests/main.go

@@ -148,7 +148,7 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir,
 		// add top level Dockerfile
 		err = ioutil.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm)
 		if err != nil {
-			return "", fmt.Errorf("Custom HEAD: failed to inject /Dockerfile: %w", err)
+			return "", fmt.Errorf("custom HEAD: failed to inject /Dockerfile: %w", err)
 		}
 		// now tarball it
 		var buffer bytes.Buffer

cmd/dendritejs-pinecone/jsServer.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main

cmd/dendritejs-pinecone/main.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main
@@ -155,7 +156,7 @@ func startup() {
 	pk := sk.Public().(ed25519.PublicKey)
 
 	logger := log.New(os.Stdout, "", 0)
-	pRouter := pineconeRouter.NewRouter(logger, "dendrite", sk, pk, nil)
+	pRouter := pineconeRouter.NewRouter(logger, sk, false)
 	pSessions := pineconeSessions.NewSessions(logger, pRouter)
 
 	cfg := &config.Dendrite{}
@@ -183,7 +184,7 @@ func startup() {
 
 	accountDB := base.CreateAccountsDB()
 	federation := conn.CreateFederationClient(base, pSessions)
-	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
@@ -220,6 +221,7 @@ func startup() {
 		base.PublicClientAPIMux,
 		base.PublicFederationAPIMux,
 		base.PublicKeyAPIMux,
+		base.PublicWellKnownAPIMux,
 		base.PublicMediaAPIMux,
 		base.SynapseAdminMux,
 	)

cmd/dendritejs-pinecone/main_noop.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package main

cmd/dendritejs-pinecone/main_test.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main

cmd/dendritejs/jsServer.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main

cmd/dendritejs/keyfetcher.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main

cmd/dendritejs/main.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main
@@ -191,7 +192,7 @@ func main() {
 
 	accountDB := base.CreateAccountsDB()
 	federation := createFederationClient(cfg, node)
-	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation)
+	keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 

cmd/dendritejs/main_noop.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package main

cmd/dendritejs/publicrooms.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main

cmd/generate-config/main.go

@@ -5,16 +5,35 @@ import (
 	"fmt"
 
 	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
 	"gopkg.in/yaml.v2"
 )
 
 func main() {
 	defaultsForCI := flag.Bool("ci", false, "sane defaults for CI testing")
+	serverName := flag.String("server", "", "The domain name of the server if not 'localhost'")
+	dbURI := flag.String("db", "", "The DB URI to use for all components if not SQLite files")
 	flag.Parse()
 
 	cfg := &config.Dendrite{}
 	cfg.Defaults()
+	if *serverName != "" {
+		cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName)
+	}
+	if *dbURI != "" {
+		cfg.Global.Kafka.Database.ConnectionString = config.DataSource(*dbURI)
+		cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(*dbURI)
+		cfg.FederationSender.Database.ConnectionString = config.DataSource(*dbURI)
+		cfg.KeyServer.Database.ConnectionString = config.DataSource(*dbURI)
+		cfg.MSCs.Database.ConnectionString = config.DataSource(*dbURI)
+		cfg.MediaAPI.Database.ConnectionString = config.DataSource(*dbURI)
+		cfg.RoomServer.Database.ConnectionString = config.DataSource(*dbURI)
+		cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(*dbURI)
+		cfg.SyncAPI.Database.ConnectionString = config.DataSource(*dbURI)
+		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(*dbURI)
+		cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(*dbURI)
+	}
 	cfg.Global.TrustedIDServers = []string{
 		"matrix.org",
 		"vector.im",