Merge branch 'master' of https://github.com/matrix-org/dendrite into add-nats-support

.github/workflows/wasm.yml

@@ -0,0 +1,49 @@
+name: WebAssembly
+
+on:
+    push:
+    pull_request:
+
+jobs:
+    test:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v2
+
+            - name: Install Go
+              uses: actions/setup-go@v2
+              with:
+                  go-version: 1.16.5
+
+            - uses: actions/cache@v2
+              with:
+                  path: |
+                      ~/.cache/go-build
+                      ~/go/pkg/mod
+                  key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+                  restore-keys: |
+                      ${{ runner.os }}-go-
+
+            - name: Install Node
+              uses: actions/setup-node@v2
+              with:
+                  node-version: 14
+
+            - uses: actions/cache@v2
+              with:
+                  path: ~/.npm
+                  key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+                  restore-keys: |
+                      ${{ runner.os }}-node-
+
+            - name: Reconfigure Git to use HTTPS auth for repo packages
+              run: >
+                  git config --global url."https://github.com/".insteadOf
+                  ssh://git@github.com/
+
+            - name: Install test dependencies
+              working-directory: ./test/wasm
+              run: npm ci
+
+            - name: Test
+              run: ./test-dendritejs.sh

.gitignore

@@ -3,6 +3,9 @@
 # Hidden files
 .*
 
+# Allow GitHub config
+!.github
+
 # Downloads
 /.downloads
 
@@ -36,6 +39,7 @@ _testmain.go
 *.exe
 *.test
 *.prof
+*.wasm
 
 # Generated keys
 *.pem
@@ -53,3 +57,6 @@ dendrite.yaml
 
 # Generated code
 cmd/dendrite-demo-yggdrasil/embed/fs*.go
+
+# Test dependencies
+test/wasm/node_modules

appservice/storage/sqlite3/storage.go

@@ -23,7 +23,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
-	_ "github.com/mattn/go-sqlite3"
 )
 
 // Database stores events intended to be later sent to application services

build-dendritejs.sh

@@ -1,4 +1,4 @@
 #!/bin/sh -eu
 
 export GIT_COMMIT=$(git rev-list -1 HEAD) && \
-GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o main.wasm ./cmd/dendritejs
+GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o bin/main.wasm ./cmd/dendritejs-pinecone

build.sh

@@ -21,4 +21,4 @@ mkdir -p bin
 
 CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
 
-CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs
+CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone

clientapi/routing/aliases.go

@@ -0,0 +1,96 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/util"
+)
+
+// GetAliases implements GET /_matrix/client/r0/rooms/{roomId}/aliases
+func GetAliases(
+	req *http.Request, rsAPI api.RoomserverInternalAPI, device *userapi.Device, roomID string,
+) util.JSONResponse {
+	stateTuple := gomatrixserverlib.StateKeyTuple{
+		EventType: gomatrixserverlib.MRoomHistoryVisibility,
+		StateKey:  "",
+	}
+	stateReq := &api.QueryCurrentStateRequest{
+		RoomID:      roomID,
+		StateTuples: []gomatrixserverlib.StateKeyTuple{stateTuple},
+	}
+	stateRes := &api.QueryCurrentStateResponse{}
+	if err := rsAPI.QueryCurrentState(req.Context(), stateReq, stateRes); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryCurrentState failed")
+		return util.ErrorResponse(fmt.Errorf("rsAPI.QueryCurrentState: %w", err))
+	}
+
+	visibility := "invite"
+	if historyVisEvent, ok := stateRes.StateEvents[stateTuple]; ok {
+		var err error
+		visibility, err = historyVisEvent.HistoryVisibility()
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("historyVisEvent.HistoryVisibility failed")
+			return util.ErrorResponse(fmt.Errorf("historyVisEvent.HistoryVisibility: %w", err))
+		}
+	}
+	if visibility != gomatrixserverlib.WorldReadable {
+		queryReq := api.QueryMembershipForUserRequest{
+			RoomID: roomID,
+			UserID: device.UserID,
+		}
+		var queryRes api.QueryMembershipForUserResponse
+		if err := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed")
+			return jsonerror.InternalServerError()
+		}
+		if !queryRes.IsInRoom {
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden("You aren't a member of this room."),
+			}
+		}
+	}
+
+	aliasesReq := api.GetAliasesForRoomIDRequest{
+		RoomID: roomID,
+	}
+	aliasesRes := api.GetAliasesForRoomIDResponse{}
+	if err := rsAPI.GetAliasesForRoomID(req.Context(), &aliasesReq, &aliasesRes); err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("rsAPI.GetAliasesForRoomID failed")
+		return util.ErrorResponse(fmt.Errorf("rsAPI.GetAliasesForRoomID: %w", err))
+	}
+
+	response := struct {
+		Aliases []string `json:"aliases"`
+	}{
+		Aliases: aliasesRes.Aliases,
+	}
+	if response.Aliases == nil {
+		response.Aliases = []string{} // pleases sytest
+	}
+
+	return util.JSONResponse{
+		Code: 200,
+		JSON: response,
+	}
+}

clientapi/routing/createroom.go

@@ -43,7 +43,7 @@ type createRoomRequest struct {
 	Visibility                string                        `json:"visibility"`
 	Topic                     string                        `json:"topic"`
 	Preset                    string                        `json:"preset"`
-	CreationContent           map[string]interface{}        `json:"creation_content"`
+	CreationContent           json.RawMessage               `json:"creation_content"`
 	InitialState              []fledglingEvent              `json:"initial_state"`
 	RoomAliasName             string                        `json:"room_alias_name"`
 	GuestCanJoin              bool                          `json:"guest_can_join"`
@@ -177,11 +177,6 @@ func createRoom(
 
 	// Clobber keys: creator, room_version
 
-	if r.CreationContent == nil {
-		r.CreationContent = make(map[string]interface{}, 2)
-	}
-
-	r.CreationContent["creator"] = userID
 	roomVersion := roomserverVersion.DefaultRoomVersion()
 	if r.RoomVersion != "" {
 		candidateVersion := gomatrixserverlib.RoomVersion(r.RoomVersion)
@@ -194,7 +189,6 @@ func createRoom(
 		}
 		roomVersion = candidateVersion
 	}
-	r.CreationContent["room_version"] = roomVersion
 
 	// TODO: visibility/presets/raw initial state
 	// TODO: Create room alias association
@@ -203,7 +197,7 @@ func createRoom(
 	logger.WithFields(log.Fields{
 		"userID":      userID,
 		"roomID":      roomID,
-		"roomVersion": r.CreationContent["room_version"],
+		"roomVersion": roomVersion,
 	}).Info("Creating new room")
 
 	profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB)
@@ -212,6 +206,109 @@ func createRoom(
 		return jsonerror.InternalServerError()
 	}
 
+	createContent := map[string]interface{}{}
+	if len(r.CreationContent) > 0 {
+		if err = json.Unmarshal(r.CreationContent, &createContent); err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal for creation_content failed")
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON("invalid create content"),
+			}
+		}
+	}
+	createContent["creator"] = userID
+	createContent["room_version"] = roomVersion
+	powerLevelContent := eventutil.InitialPowerLevelsContent(userID)
+	joinRuleContent := gomatrixserverlib.JoinRuleContent{
+		JoinRule: gomatrixserverlib.Invite,
+	}
+	historyVisibilityContent := gomatrixserverlib.HistoryVisibilityContent{
+		HistoryVisibility: historyVisibilityShared,
+	}
+
+	if r.PowerLevelContentOverride != nil {
+		// Merge powerLevelContentOverride fields by unmarshalling it atop the defaults
+		err = json.Unmarshal(r.PowerLevelContentOverride, &powerLevelContent)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal for power_level_content_override failed")
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON("malformed power_level_content_override"),
+			}
+		}
+	}
+
+	switch r.Preset {
+	case presetPrivateChat:
+		joinRuleContent.JoinRule = gomatrixserverlib.Invite
+		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
+	case presetTrustedPrivateChat:
+		joinRuleContent.JoinRule = gomatrixserverlib.Invite
+		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
+		// TODO If trusted_private_chat, all invitees are given the same power level as the room creator.
+	case presetPublicChat:
+		joinRuleContent.JoinRule = gomatrixserverlib.Public
+		historyVisibilityContent.HistoryVisibility = historyVisibilityShared
+	}
+
+	createEvent := fledglingEvent{
+		Type:    gomatrixserverlib.MRoomCreate,
+		Content: createContent,
+	}
+	powerLevelEvent := fledglingEvent{
+		Type:    gomatrixserverlib.MRoomPowerLevels,
+		Content: powerLevelContent,
+	}
+	joinRuleEvent := fledglingEvent{
+		Type:    gomatrixserverlib.MRoomJoinRules,
+		Content: joinRuleContent,
+	}
+	historyVisibilityEvent := fledglingEvent{
+		Type:    gomatrixserverlib.MRoomHistoryVisibility,
+		Content: historyVisibilityContent,
+	}
+	membershipEvent := fledglingEvent{
+		Type:     gomatrixserverlib.MRoomMember,
+		StateKey: userID,
+		Content: gomatrixserverlib.MemberContent{
+			Membership:  gomatrixserverlib.Join,
+			DisplayName: profile.DisplayName,
+			AvatarURL:   profile.AvatarURL,
+		},
+	}
+
+	var nameEvent *fledglingEvent
+	var topicEvent *fledglingEvent
+	var guestAccessEvent *fledglingEvent
+	var aliasEvent *fledglingEvent
+
+	if r.Name != "" {
+		nameEvent = &fledglingEvent{
+			Type: gomatrixserverlib.MRoomName,
+			Content: eventutil.NameContent{
+				Name: r.Name,
+			},
+		}
+	}
+
+	if r.Topic != "" {
+		topicEvent = &fledglingEvent{
+			Type: gomatrixserverlib.MRoomTopic,
+			Content: eventutil.TopicContent{
+				Topic: r.Topic,
+			},
+		}
+	}
+
+	if r.GuestCanJoin {
+		guestAccessEvent = &fledglingEvent{
+			Type: gomatrixserverlib.MRoomGuestAccess,
+			Content: eventutil.GuestAccessContent{
+				GuestAccess: "can_join",
+			},
+		}
+	}
+
 	var roomAlias string
 	if r.RoomAliasName != "" {
 		roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, cfg.Matrix.ServerName)
@@ -230,44 +327,46 @@ func createRoom(
 		if aliasResp.RoomID != "" {
 			return util.MessageResponse(400, "Alias already exists")
 		}
+
+		aliasEvent = &fledglingEvent{
+			Type: gomatrixserverlib.MRoomCanonicalAlias,
+			Content: eventutil.CanonicalAlias{
+				Alias: roomAlias,
+			},
+		}
 	}
 
-	membershipContent := gomatrixserverlib.MemberContent{
+	var initialStateEvents []fledglingEvent
-		Membership:  gomatrixserverlib.Join,
+	for i := range r.InitialState {
-		DisplayName: profile.DisplayName,
+		if r.InitialState[i].StateKey != "" {
-		AvatarURL:   profile.AvatarURL,
+			initialStateEvents = append(initialStateEvents, r.InitialState[i])
-	}
+			continue
+		}
 
-	var joinRules, historyVisibility string
+		switch r.InitialState[i].Type {
-	switch r.Preset {
+		case gomatrixserverlib.MRoomCreate:
-	case presetPrivateChat:
+			continue
-		joinRules = gomatrixserverlib.Invite
-		historyVisibility = historyVisibilityShared
-	case presetTrustedPrivateChat:
-		joinRules = gomatrixserverlib.Invite
-		historyVisibility = historyVisibilityShared
-		// TODO If trusted_private_chat, all invitees are given the same power level as the room creator.
-	case presetPublicChat:
-		joinRules = gomatrixserverlib.Public
-		historyVisibility = historyVisibilityShared
-	default:
-		// Default room rules, r.Preset was previously checked for valid values so
-		// only a request with no preset should end up here.
-		joinRules = gomatrixserverlib.Invite
-		historyVisibility = historyVisibilityShared
-	}
 
-	var builtEvents []*gomatrixserverlib.HeaderedEvent
+		case gomatrixserverlib.MRoomPowerLevels:
+			powerLevelEvent = r.InitialState[i]
 
-	powerLevelContent := eventutil.InitialPowerLevelsContent(userID)
+		case gomatrixserverlib.MRoomJoinRules:
-	if r.PowerLevelContentOverride != nil {
+			joinRuleEvent = r.InitialState[i]
-		// Merge powerLevelContentOverride fields by unmarshalling it atop the defaults
+
-		err = json.Unmarshal(r.PowerLevelContentOverride, &powerLevelContent)
+		case gomatrixserverlib.MRoomHistoryVisibility:
-		if err != nil {
+			historyVisibilityEvent = r.InitialState[i]
-			return util.JSONResponse{
+
-				Code: http.StatusBadRequest,
+		case gomatrixserverlib.MRoomGuestAccess:
-				JSON: jsonerror.BadJSON("malformed power_level_content_override"),
+			guestAccessEvent = &r.InitialState[i]
-			}
+
+		case gomatrixserverlib.MRoomName:
+			nameEvent = &r.InitialState[i]
+
+		case gomatrixserverlib.MRoomTopic:
+			topicEvent = &r.InitialState[i]
+
+		default:
+			initialStateEvents = append(initialStateEvents, r.InitialState[i])
 		}
 	}
 
@@ -284,37 +383,33 @@ func createRoom(
 	//  10- m.room.topic (opt)
 	//  11- invite events (opt) - with is_direct flag if applicable TODO
 	//  12- 3pid invite events (opt) TODO
-	//  13- m.room.aliases event for HS (if alias specified) TODO
 	// This differs from Synapse slightly. Synapse would vary the ordering of 3-7
 	// depending on if those events were in "initial_state" or not. This made it
 	// harder to reason about, hence sticking to a strict static ordering.
 	// TODO: Synapse has txn/token ID on each event. Do we need to do this here?
 	eventsToMake := []fledglingEvent{
-		{"m.room.create", "", r.CreationContent},
+		createEvent, membershipEvent, powerLevelEvent, joinRuleEvent, historyVisibilityEvent,
-		{"m.room.member", userID, membershipContent},
-		{"m.room.power_levels", "", powerLevelContent},
-		{"m.room.join_rules", "", gomatrixserverlib.JoinRuleContent{JoinRule: joinRules}},
-		{"m.room.history_visibility", "", eventutil.HistoryVisibilityContent{HistoryVisibility: historyVisibility}},
 	}
-	if roomAlias != "" {
+	if guestAccessEvent != nil {
+		eventsToMake = append(eventsToMake, *guestAccessEvent)
+	}
+	eventsToMake = append(eventsToMake, initialStateEvents...)
+	if nameEvent != nil {
+		eventsToMake = append(eventsToMake, *nameEvent)
+	}
+	if topicEvent != nil {
+		eventsToMake = append(eventsToMake, *topicEvent)
+	}
+	if aliasEvent != nil {
 		// TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room.
 		// This means we might fail creating the alias but say the canonical alias is something that doesn't exist.
-		// m.room.aliases is handled when we call roomserver.SetRoomAlias
+		eventsToMake = append(eventsToMake, *aliasEvent)
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", eventutil.CanonicalAlias{Alias: roomAlias}})
-	}
-	if r.GuestCanJoin {
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", eventutil.GuestAccessContent{GuestAccess: "can_join"}})
-	}
-	eventsToMake = append(eventsToMake, r.InitialState...)
-	if r.Name != "" {
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", eventutil.NameContent{Name: r.Name}})
-	}
-	if r.Topic != "" {
-		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", eventutil.TopicContent{Topic: r.Topic}})
 	}
+
 	// TODO: invite events
 	// TODO: 3pid invite events
 
+	var builtEvents []*gomatrixserverlib.HeaderedEvent
 	authEvents := gomatrixserverlib.NewAuthEvents(nil)
 	for i, e := range eventsToMake {
 		depth := i + 1 // depth starts at 1
@@ -403,7 +498,7 @@ func createRoom(
 				fallthrough
 			case gomatrixserverlib.MRoomCanonicalAlias:
 				fallthrough
-			case "m.room.encryption": // TODO: move this to gmsl
+			case gomatrixserverlib.MRoomEncryption:
 				fallthrough
 			case gomatrixserverlib.MRoomMember:
 				fallthrough

clientapi/routing/directory.go

@@ -113,13 +113,12 @@ func DirectoryRoom(
 }
 
 // SetLocalAlias implements PUT /directory/room/{roomAlias}
-// TODO: Check if the user has the power level to set an alias
 func SetLocalAlias(
 	req *http.Request,
 	device *api.Device,
 	alias string,
 	cfg *config.ClientAPI,
-	aliasAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', alias)
 	if err != nil {
@@ -172,7 +171,7 @@ func SetLocalAlias(
 		Alias:  alias,
 	}
 	var queryRes roomserverAPI.SetRoomAliasResponse
-	if err := aliasAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
+	if err := rsAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed")
 		return jsonerror.InternalServerError()
 	}
@@ -195,43 +194,32 @@ func RemoveLocalAlias(
 	req *http.Request,
 	device *api.Device,
 	alias string,
-	aliasAPI roomserverAPI.RoomserverInternalAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
-
-	creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{
-		Alias: alias,
-	}
-	var creatorQueryRes roomserverAPI.GetCreatorIDForAliasResponse
-	if err := aliasAPI.GetCreatorIDForAlias(req.Context(), &creatorQueryReq, &creatorQueryRes); err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.GetCreatorIDForAlias failed")
-		return jsonerror.InternalServerError()
-	}
-
-	if creatorQueryRes.UserID == "" {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("Alias does not exist"),
-		}
-	}
-
-	if creatorQueryRes.UserID != device.UserID {
-		// TODO: Still allow deletion if user is admin
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("You do not have permission to delete this alias"),
-		}
-	}
-
 	queryReq := roomserverAPI.RemoveRoomAliasRequest{
 		Alias:  alias,
 		UserID: device.UserID,
 	}
 	var queryRes roomserverAPI.RemoveRoomAliasResponse
-	if err := aliasAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
+	if err := rsAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
 		util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed")
 		return jsonerror.InternalServerError()
 	}
 
+	if !queryRes.Found {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("The alias does not exist."),
+		}
+	}
+
+	if !queryRes.Removed {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You do not have permission to remove this alias."),
+		}
+	}
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: struct{}{},
@@ -294,9 +282,9 @@ func SetVisibility(
 		return jsonerror.InternalServerError()
 	}
 
-	// NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event
+	// NOTSPEC: Check if the user's power is greater than power required to change m.room.canonical_alias event
 	power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event)
-	if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) {
+	if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomCanonicalAlias, true) {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"),

clientapi/routing/routing.go

@@ -275,6 +275,14 @@ func Setup(
 		return OnIncomingStateRequest(req.Context(), device, rsAPI, vars["roomID"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
+	r0mux.Handle("/rooms/{roomID}/aliases", httputil.MakeAuthAPI("aliases", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		return GetAliases(req, rsAPI, device, vars["roomID"])
+	})).Methods(http.MethodGet, http.MethodOptions)
+
 	r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
 		vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {

cmd/dendrite-demo-libp2p/main.go

@@ -44,6 +44,8 @@ import (
 	"github.com/matrix-org/dendrite/eduserver/cache"
 
 	"github.com/sirupsen/logrus"
+
+	_ "github.com/mattn/go-sqlite3"
 )
 
 func createKeyDB(

cmd/dendrite-demo-pinecone/main.go

@@ -54,6 +54,8 @@ import (
 	pineconeSessions "github.com/matrix-org/pinecone/sessions"
 
 	"github.com/sirupsen/logrus"
+
+	_ "github.com/mattn/go-sqlite3"
 )
 
 var (

cmd/dendrite-demo-yggdrasil/main.go

@@ -44,6 +44,8 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/sirupsen/logrus"
+
+	_ "github.com/mattn/go-sqlite3"
 )
 
 var (

cmd/dendrite-monolith-server/main.go

@@ -31,6 +31,8 @@ import (
 	"github.com/matrix-org/dendrite/signingkeyserver"
 	"github.com/matrix-org/dendrite/userapi"
 	"github.com/sirupsen/logrus"
+
+	_ "github.com/mattn/go-sqlite3"
 )
 
 var (

cmd/dendrite-polylith-multi/main.go

@@ -23,6 +23,8 @@ import (
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/sirupsen/logrus"
+
+	_ "github.com/mattn/go-sqlite3"
 )
 
 type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite)

cmd/dendritejs-pinecone/main.go

@@ -144,6 +144,13 @@ func generateKey() ed25519.PrivateKey {
 }
 
 func main() {
+	startup()
+
+	// We want to block forever to let the fetch and libp2p handler serve the APIs
+	select {}
+}
+
+func startup() {
 	sk := generateKey()
 	pk := sk.Public().(ed25519.PublicKey)
 
@@ -250,7 +257,4 @@ func main() {
 			}
 		}
 	}()
-
-	// We want to block forever to let the fetch and libp2p handler serve the APIs
-	select {}
 }

cmd/dendritejs-pinecone/main_test.go

@@ -0,0 +1,25 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build wasm
+
+package main
+
+import (
+	"testing"
+)
+
+func TestStartup(t *testing.T) {
+	startup()
+}

docs/p2p.md

@@ -2,14 +2,23 @@
 
 These are the instructions for setting up P2P Dendrite, current as of May 2020. There's both Go stuff and JS stuff to do to set this up.
 
-
 ### Dendrite
 
+#### Build
+
 - The `master` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`.
 - Build it and copy assets to riot-web.
 ```
-$ GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs
+$ ./build-dendritejs.sh
-$ cp main.wasm ../riot-web/src/vector/dendrite.wasm
+$ cp bin/main.wasm ../riot-web/src/vector/dendrite.wasm
+```
+
+#### Test
+
+To check that the Dendrite side is working well as Wasm, you can run the
+Wasm-specific tests:
+```
+$ ./test-dendritejs.sh
 ```
 
 ### Rendezvous

federationsender/storage/sqlite3/storage.go

@@ -18,8 +18,6 @@ package sqlite3
 import (
 	"database/sql"
 
-	_ "github.com/mattn/go-sqlite3"
-
 	"github.com/matrix-org/dendrite/federationsender/storage/shared"
 	"github.com/matrix-org/dendrite/federationsender/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/internal/caching"

go.mod

@@ -35,9 +35,9 @@ require (
 	github.com/lucas-clemente/quic-go v0.19.3
 	github.com/matrix-org/dugong v0.0.0-20180820122854-51a565b5666b
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
-	github.com/matrix-org/go-sqlite3-js v0.0.0-20210625141222-bd2b7124cee8
+	github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20210719140634-f44a103bb12e
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20210722110442-5061d6986876
 	github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb

go.sum

@@ -1024,13 +1024,13 @@ github.com/matrix-org/dugong v0.0.0-20180820122854-51a565b5666b h1:xpcmnpfUImRC4
 github.com/matrix-org/dugong v0.0.0-20180820122854-51a565b5666b/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 h1:eqE5OnGx9ZMWmrRbD3KF/3KtTunw0iQulI7YxOIdxo4=
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4/go.mod h1:3WluEZ9QXSwU30tWYqktnpC1x9mwZKx1r8uAv8Iq+a4=
-github.com/matrix-org/go-sqlite3-js v0.0.0-20210625141222-bd2b7124cee8 h1:/FKUeUlCATr1gXxYqlaJgH8FW/sw0Jz8t7s8BIlECfg=
+github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d h1:mGhPVaTht5NViFN/UpdrIlRApmH2FWcVaKUH5MdBKiY=
-github.com/matrix-org/go-sqlite3-js v0.0.0-20210625141222-bd2b7124cee8/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
+github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20210719140634-f44a103bb12e h1:dYPsAU1363AEbcohNzBJHV0CkbfVa8QiMNFib/i5pS8=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20210722110442-5061d6986876 h1:6ypwCtgRLK0v/hGWvnd847+KTo9BSkP9N0A4qSniP4E=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20210719140634-f44a103bb12e/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20210722110442-5061d6986876/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b h1:5X5vdWQ13xrNkJVqaJHPsrt7rKkMJH5iac0EtfOuxSg=
 github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b/go.mod h1:CVlrvs1R5iz7Omy2GqAjJJKbACn07GZgUq1Gli18FYE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=

internal/eventutil/eventcontent.go

@@ -53,7 +53,6 @@ func InitialPowerLevelsContent(roomCreator string) (c gomatrixserverlib.PowerLev
 		"m.room.history_visibility": 100,
 		"m.room.canonical_alias":    50,
 		"m.room.avatar":             50,
-		"m.room.aliases":            0, // anyone can publish aliases by default. Has to be 0 else state_default is used.
 	}
 	c.Users = map[string]int64{roomCreator: 100}
 	return c

internal/sqlutil/sql.go

@@ -19,7 +19,6 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
-	"runtime"
 	"strings"
 
 	"github.com/matrix-org/util"
@@ -113,13 +112,6 @@ func QueryVariadicOffset(count, offset int) string {
 	return str
 }
 
-func SQLiteDriverName() string {
-	if runtime.GOOS == "js" {
-		return "sqlite3_js"
-	}
-	return "sqlite3"
-}
-
 func minOfInts(a, b int) int {
 	if a <= b {
 		return a

internal/sqlutil/trace.go

@@ -104,7 +104,7 @@ func Open(dbProperties *config.DatabaseOptions) (*sql.DB, error) {
 	var driverName, dsn string
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		driverName = SQLiteDriverName()
+		driverName = "sqlite3"
 		dsn, err = ParseFileURI(dbProperties.ConnectionString)
 		if err != nil {
 			return nil, fmt.Errorf("ParseFileURI: %w", err)
@@ -123,11 +123,11 @@ func Open(dbProperties *config.DatabaseOptions) (*sql.DB, error) {
 	if err != nil {
 		return nil, err
 	}
-	if driverName != SQLiteDriverName() {
+	if driverName != "sqlite3" {
 		logrus.WithFields(logrus.Fields{
-			"MaxOpenConns":    dbProperties.MaxOpenConns,
+			"MaxOpenConns":    dbProperties.MaxOpenConns(),
-			"MaxIdleConns":    dbProperties.MaxIdleConns,
+			"MaxIdleConns":    dbProperties.MaxIdleConns(),
-			"ConnMaxLifetime": dbProperties.ConnMaxLifetime,
+			"ConnMaxLifetime": dbProperties.ConnMaxLifetime(),
 			"dataSourceName":  regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"),
 		}).Debug("Setting DB connection limits")
 		db.SetMaxOpenConns(dbProperties.MaxOpenConns())

mediaapi/storage/sqlite3/storage.go

@@ -24,7 +24,6 @@ import (
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
-	_ "github.com/mattn/go-sqlite3"
 )
 
 // Database is used to store metadata about a repository of media files.

roomserver/api/alias.go

@@ -78,4 +78,9 @@ type RemoveRoomAliasRequest struct {
 }
 
 // RemoveRoomAliasResponse is a response to RemoveRoomAlias
-type RemoveRoomAliasResponse struct{}
+type RemoveRoomAliasResponse struct {
+	// Did the alias exist before?
+	Found bool `json:"found"`
+	// Did we remove it?
+	Removed bool `json:"removed"`
+}

roomserver/api/query.go

@@ -181,11 +181,8 @@ type QueryServerJoinedToRoomRequest struct {
 type QueryServerJoinedToRoomResponse struct {
 	// True if the room exists on the server
 	RoomExists bool `json:"room_exists"`
-	// True if we still believe that we are participating in the room
+	// True if we still believe that the server is participating in the room
 	IsInRoom bool `json:"is_in_room"`
-	// List of servers that are also in the room. This will not be populated
-	// if the queried ServerName is the local server name.
-	ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
 }
 
 // QueryServerAllowedToSeeEventRequest is a request to QueryServerAllowedToSeeEvent

roomserver/internal/alias.go

@@ -16,10 +16,7 @@ package internal
 
 import (
 	"context"
-	"encoding/json"
-	"errors"
 	"fmt"
-	"time"
 
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -73,11 +70,7 @@ func (r *RoomserverInternalAPI) SetRoomAlias(
 		return err
 	}
 
-	// Send a m.room.aliases event with the updated list of aliases for this room
+	return nil
-	// At this point we've already committed the alias to the database so we
-	// shouldn't cancel this request.
-	// TODO: Ensure that we send unsent events when if server restarts.
-	return r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, request.RoomID)
 }
 
 // GetRoomIDForAlias implements alias.RoomserverInternalAPI
@@ -157,122 +150,44 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias(
 	request *api.RemoveRoomAliasRequest,
 	response *api.RemoveRoomAliasResponse,
 ) error {
-	// Look up the room ID in the database
 	roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias)
 	if err != nil {
-		return err
+		return fmt.Errorf("r.DB.GetRoomIDForAlias: %w", err)
+	}
+	if roomID == "" {
+		response.Found = false
+		response.Removed = false
+		return nil
 	}
 
-	// Remove the dalias from the database
+	response.Found = true
+	creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias)
+	if err != nil {
+		return fmt.Errorf("r.DB.GetCreatorIDForAlias: %w", err)
+	}
+
+	if creatorID != request.UserID {
+		plEvent, err := r.DB.GetStateEvent(ctx, roomID, gomatrixserverlib.MRoomPowerLevels, "")
+		if err != nil {
+			return fmt.Errorf("r.DB.GetStateEvent: %w", err)
+		}
+
+		pls, err := plEvent.PowerLevels()
+		if err != nil {
+			return fmt.Errorf("plEvent.PowerLevels: %w", err)
+		}
+
+		if pls.UserLevel(request.UserID) < pls.EventLevel(gomatrixserverlib.MRoomCanonicalAlias, true) {
+			response.Removed = false
+			return nil
+		}
+	}
+
+	// Remove the alias from the database
 	if err := r.DB.RemoveRoomAlias(ctx, request.Alias); err != nil {
 		return err
 	}
 
-	// Send an updated m.room.aliases event
+	response.Removed = true
-	// At this point we've already committed the alias to the database so we
+	return nil
-	// shouldn't cancel this request.
-	// TODO: Ensure that we send unsent events when if server restarts.
-	return r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, roomID)
-}
-
-type roomAliasesContent struct {
-	Aliases []string `json:"aliases"`
-}
-
-// Build the updated m.room.aliases event to send to the room after addition or
-// removal of an alias
-func (r *RoomserverInternalAPI) sendUpdatedAliasesEvent(
-	ctx context.Context, userID string, roomID string,
-) error {
-	serverName := string(r.Cfg.Matrix.ServerName)
-
-	builder := gomatrixserverlib.EventBuilder{
-		Sender:   userID,
-		RoomID:   roomID,
-		Type:     "m.room.aliases",
-		StateKey: &serverName,
-	}
-
-	// Retrieve the updated list of aliases, marhal it and set it as the
-	// event's content
-	aliases, err := r.DB.GetAliasesForRoomID(ctx, roomID)
-	if err != nil {
-		return err
-	}
-	content := roomAliasesContent{Aliases: aliases}
-	rawContent, err := json.Marshal(content)
-	if err != nil {
-		return err
-	}
-	err = builder.SetContent(json.RawMessage(rawContent))
-	if err != nil {
-		return err
-	}
-
-	// Get needed state events and depth
-	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(&builder)
-	if err != nil {
-		return err
-	}
-	if len(eventsNeeded.Tuples()) == 0 {
-		return errors.New("expecting state tuples for event builder, got none")
-	}
-	req := api.QueryLatestEventsAndStateRequest{
-		RoomID:       roomID,
-		StateToFetch: eventsNeeded.Tuples(),
-	}
-	var res api.QueryLatestEventsAndStateResponse
-	if err = r.QueryLatestEventsAndState(ctx, &req, &res); err != nil {
-		return err
-	}
-	builder.Depth = res.Depth
-	builder.PrevEvents = res.LatestEvents
-
-	// Add auth events
-	authEvents := gomatrixserverlib.NewAuthEvents(nil)
-	for i := range res.StateEvents {
-		err = authEvents.AddEvent(res.StateEvents[i].Event)
-		if err != nil {
-			return err
-		}
-	}
-	refs, err := eventsNeeded.AuthEventReferences(&authEvents)
-	if err != nil {
-		return err
-	}
-	builder.AuthEvents = refs
-
-	roomInfo, err := r.DB.RoomInfo(ctx, roomID)
-	if err != nil {
-		return err
-	}
-	if roomInfo == nil {
-		return fmt.Errorf("room %s does not exist", roomID)
-	}
-
-	// Build the event
-	now := time.Now()
-	event, err := builder.Build(
-		now, r.Cfg.Matrix.ServerName, r.Cfg.Matrix.KeyID,
-		r.Cfg.Matrix.PrivateKey, roomInfo.RoomVersion,
-	)
-	if err != nil {
-		return err
-	}
-
-	// Create the request
-	ire := api.InputRoomEvent{
-		Kind:         api.KindNew,
-		Event:        event.Headered(roomInfo.RoomVersion),
-		AuthEventIDs: event.AuthEventIDs(),
-		SendAsServer: serverName,
-	}
-	inputReq := api.InputRoomEventsRequest{
-		InputRoomEvents: []api.InputRoomEvent{ire},
-	}
-	var inputRes api.InputRoomEventsResponse
-
-	// Send the request
-	r.InputRoomEvents(ctx, &inputReq, &inputRes)
-	return inputRes.Err()
 }

roomserver/internal/input/input_membership.go

@@ -136,6 +136,8 @@ func (r *Inputer) updateMembership(
 		return updateToJoinMembership(mu, add, updates)
 	case gomatrixserverlib.Leave, gomatrixserverlib.Ban:
 		return updateToLeaveMembership(mu, add, newMembership, updates)
+	case gomatrixserverlib.Knock:
+		return updateToKnockMembership(mu, add, updates)
 	default:
 		panic(fmt.Errorf(
 			"input: membership %q is not one of the allowed values", newMembership,
@@ -220,6 +222,18 @@ func updateToLeaveMembership(
 	return updates, nil
 }
 
+func updateToKnockMembership(
+	mu *shared.MembershipUpdater, add *gomatrixserverlib.Event, updates []api.OutputEvent,
+) ([]api.OutputEvent, error) {
+	if mu.IsLeave() {
+		_, err := mu.SetToKnock(add)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return updates, nil
+}
+
 // membershipChanges pairs up the membership state changes.
 func membershipChanges(removed, added []types.StateEntry) []stateChange {
 	changes := pairUpChanges(removed, added)

roomserver/internal/perform/perform_invite.go

@@ -223,8 +223,8 @@ func buildInviteStrippedState(
 	// https://matrix.org/docs/spec/client_server/r0.6.0#m-room-member
 	for _, t := range []string{
 		gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias,
-		gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules,
+		gomatrixserverlib.MRoomJoinRules, gomatrixserverlib.MRoomAvatar,
-		"m.room.avatar", "m.room.encryption", gomatrixserverlib.MRoomCreate,
+		gomatrixserverlib.MRoomEncryption, gomatrixserverlib.MRoomCreate,
 	} {
 		stateWanted = append(stateWanted, gomatrixserverlib.StateKeyTuple{
 			EventType: t,

roomserver/internal/query/query.go

@@ -330,46 +330,17 @@ func (r *Queryer) QueryServerJoinedToRoom(
 	response.RoomExists = true
 
 	if request.ServerName == r.ServerName || request.ServerName == "" {
-		var joined bool
+		response.IsInRoom, err = r.DB.GetLocalServerInRoom(ctx, info.RoomNID)
-		joined, err = r.DB.GetLocalServerInRoom(ctx, info.RoomNID)
 		if err != nil {
 			return fmt.Errorf("r.DB.GetLocalServerInRoom: %w", err)
 		}
-		response.IsInRoom = joined
+	} else {
-		return nil
+		response.IsInRoom, err = r.DB.GetServerInRoom(ctx, info.RoomNID, request.ServerName)
-	}
+		if err != nil {
-
+			return fmt.Errorf("r.DB.GetServerInRoom: %w", err)
-	eventNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, true, false)
-	if err != nil {
-		return fmt.Errorf("r.DB.GetMembershipEventNIDsForRoom: %w", err)
-	}
-	if len(eventNIDs) == 0 {
-		return nil
-	}
-
-	events, err := r.DB.Events(ctx, eventNIDs)
-	if err != nil {
-		return fmt.Errorf("r.DB.Events: %w", err)
-	}
-
-	servers := map[gomatrixserverlib.ServerName]struct{}{}
-	for _, e := range events {
-		if e.Type() == gomatrixserverlib.MRoomMember && e.StateKey() != nil {
-			_, serverName, err := gomatrixserverlib.SplitID('@', *e.StateKey())
-			if err != nil {
-				continue
-			}
-			servers[serverName] = struct{}{}
-			if serverName == request.ServerName {
-				response.IsInRoom = true
-			}
 		}
 	}
 
-	for server := range servers {
-		response.ServerNames = append(response.ServerNames, server)
-	}
-
 	return nil
 }
 

roomserver/storage/interface.go

@@ -156,6 +156,8 @@ type Database interface {
 	JoinedUsersSetInRooms(ctx context.Context, roomIDs []string) (map[string]int, error)
 	// GetLocalServerInRoom returns true if we think we're in a given room or false otherwise.
 	GetLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error)
+	// GetServerInRoom returns true if we think a server is in a given room or false otherwise.
+	GetServerInRoom(ctx context.Context, roomNID types.RoomNID, serverName gomatrixserverlib.ServerName) (bool, error)
 	// GetKnownUsers searches all users that userID knows about.
 	GetKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error)
 	// GetKnownRooms returns a list of all rooms we know about.

roomserver/storage/postgres/membership_table.go

@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 const membershipSchema = `
@@ -132,6 +133,16 @@ var selectKnownUsersSQL = "" +
 const selectLocalServerInRoomSQL = "" +
 	"SELECT room_nid FROM roomserver_membership WHERE target_local = true AND membership_nid = $1 AND room_nid = $2 LIMIT 1"
 
+// selectServerMembersInRoomSQL is an optimised case for checking for server members in a room.
+// The JOIN is significantly leaner than the previous case of looking up event NIDs and reading the
+// membership events from the database, as the JOIN query amounts to little more than two index
+// scans which are very fast. The presence of a single row from this query suggests the server is
+// in the room, no rows returned suggests they aren't.
+const selectServerInRoomSQL = "" +
+	"SELECT room_nid FROM roomserver_membership" +
+	" JOIN roomserver_event_state_keys ON roomserver_membership.target_nid = roomserver_event_state_keys.event_state_key_nid" +
+	" WHERE membership_nid = $1 AND room_nid = $2 AND event_state_key LIKE '%:' || $3 LIMIT 1"
+
 type membershipStatements struct {
 	insertMembershipStmt                            *sql.Stmt
 	selectMembershipForUpdateStmt                   *sql.Stmt
@@ -146,6 +157,7 @@ type membershipStatements struct {
 	selectKnownUsersStmt                            *sql.Stmt
 	updateMembershipForgetRoomStmt                  *sql.Stmt
 	selectLocalServerInRoomStmt                     *sql.Stmt
+	selectServerInRoomStmt                          *sql.Stmt
 }
 
 func createMembershipTable(db *sql.DB) error {
@@ -170,6 +182,7 @@ func prepareMembershipTable(db *sql.DB) (tables.Membership, error) {
 		{&s.selectKnownUsersStmt, selectKnownUsersSQL},
 		{&s.updateMembershipForgetRoomStmt, updateMembershipForgetRoom},
 		{&s.selectLocalServerInRoomStmt, selectLocalServerInRoomSQL},
+		{&s.selectServerInRoomStmt, selectServerInRoomSQL},
 	}.Prepare(db)
 }
 
@@ -347,3 +360,15 @@ func (s *membershipStatements) SelectLocalServerInRoom(ctx context.Context, room
 	found := nid > 0
 	return found, nil
 }
+
+func (s *membershipStatements) SelectServerInRoom(ctx context.Context, roomNID types.RoomNID, serverName gomatrixserverlib.ServerName) (bool, error) {
+	var nid types.RoomNID
+	err := s.selectServerInRoomStmt.QueryRowContext(ctx, tables.MembershipStateJoin, roomNID, serverName).Scan(&nid)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return false, nil
+		}
+		return false, err
+	}
+	return roomNID == nid, nil
+}

roomserver/storage/shared/membership_updater.go

@@ -86,6 +86,11 @@ func (u *MembershipUpdater) IsLeave() bool {
 	return u.membership == tables.MembershipStateLeaveOrBan
 }
 
+// IsKnock implements types.MembershipUpdater
+func (u *MembershipUpdater) IsKnock() bool {
+	return u.membership == tables.MembershipStateKnock
+}
+
 // SetToInvite implements types.MembershipUpdater
 func (u *MembershipUpdater) SetToInvite(event gomatrixserverlib.Event) (bool, error) {
 	var inserted bool
@@ -180,3 +185,27 @@ func (u *MembershipUpdater) SetToLeave(senderUserID string, eventID string) ([]s
 	})
 	return inviteEventIDs, err
 }
+
+// SetToKnock implements types.MembershipUpdater
+func (u *MembershipUpdater) SetToKnock(event *gomatrixserverlib.Event) (bool, error) {
+	var inserted bool
+	err := u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error {
+		senderUserNID, err := u.d.assignStateKeyNID(u.ctx, u.txn, event.Sender())
+		if err != nil {
+			return fmt.Errorf("u.d.AssignStateKeyNID: %w", err)
+		}
+		if u.membership != tables.MembershipStateKnock {
+			// Look up the NID of the new knock event
+			nIDs, err := u.d.EventNIDs(u.ctx, []string{event.EventID()})
+			if err != nil {
+				return fmt.Errorf("u.d.EventNIDs: %w", err)
+			}
+
+			if err = u.d.MembershipTable.UpdateMembership(u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateKnock, nIDs[event.EventID()], false); err != nil {
+				return fmt.Errorf("u.d.MembershipTable.UpdateMembership: %w", err)
+			}
+		}
+		return nil
+	})
+	return inserted, err
+}

roomserver/storage/shared/storage.go

@@ -857,6 +857,9 @@ func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey s
 	if err != nil {
 		return nil, err
 	}
+	if roomInfo == nil || roomInfo.IsStub {
+		return nil, fmt.Errorf("room %s doesn't exist", roomID)
+	}
 	eventTypeNID, err := d.EventTypesTable.SelectEventTypeNID(ctx, nil, evType)
 	if err == sql.ErrNoRows {
 		// No rooms have an event of this type, otherwise we'd have an event type NID
@@ -1068,6 +1071,11 @@ func (d *Database) GetLocalServerInRoom(ctx context.Context, roomNID types.RoomN
 	return d.MembershipTable.SelectLocalServerInRoom(ctx, roomNID)
 }
 
+// GetServerInRoom returns true if we think a server is in a given room or false otherwise.
+func (d *Database) GetServerInRoom(ctx context.Context, roomNID types.RoomNID, serverName gomatrixserverlib.ServerName) (bool, error) {
+	return d.MembershipTable.SelectServerInRoom(ctx, roomNID, serverName)
+}
+
 // GetKnownUsers searches all users that userID knows about.
 func (d *Database) GetKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error) {
 	stateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, userID)

roomserver/storage/sqlite3/membership_table.go

@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
 	"github.com/matrix-org/dendrite/roomserver/storage/tables"
 	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/gomatrixserverlib"
 )
 
 const membershipSchema = `
@@ -108,6 +109,16 @@ var selectKnownUsersSQL = "" +
 const selectLocalServerInRoomSQL = "" +
 	"SELECT room_nid FROM roomserver_membership WHERE target_local = 1 AND membership_nid = $1 AND room_nid = $2 LIMIT 1"
 
+// selectServerMembersInRoomSQL is an optimised case for checking for server members in a room.
+// The JOIN is significantly leaner than the previous case of looking up event NIDs and reading the
+// membership events from the database, as the JOIN query amounts to little more than two index
+// scans which are very fast. The presence of a single row from this query suggests the server is
+// in the room, no rows returned suggests they aren't.
+const selectServerInRoomSQL = "" +
+	"SELECT room_nid FROM roomserver_membership" +
+	" JOIN roomserver_event_state_keys ON roomserver_membership.target_nid = roomserver_event_state_keys.event_state_key_nid" +
+	" WHERE membership_nid = $1 AND room_nid = $2 AND event_state_key LIKE '%:' || $3 LIMIT 1"
+
 type membershipStatements struct {
 	db                                              *sql.DB
 	insertMembershipStmt                            *sql.Stmt
@@ -122,6 +133,7 @@ type membershipStatements struct {
 	selectKnownUsersStmt                            *sql.Stmt
 	updateMembershipForgetRoomStmt                  *sql.Stmt
 	selectLocalServerInRoomStmt                     *sql.Stmt
+	selectServerInRoomStmt                          *sql.Stmt
 }
 
 func createMembershipTable(db *sql.DB) error {
@@ -147,6 +159,7 @@ func prepareMembershipTable(db *sql.DB) (tables.Membership, error) {
 		{&s.selectKnownUsersStmt, selectKnownUsersSQL},
 		{&s.updateMembershipForgetRoomStmt, updateMembershipForgetRoom},
 		{&s.selectLocalServerInRoomStmt, selectLocalServerInRoomSQL},
+		{&s.selectServerInRoomStmt, selectServerInRoomSQL},
 	}.Prepare(db)
 }
 
@@ -327,3 +340,15 @@ func (s *membershipStatements) SelectLocalServerInRoom(ctx context.Context, room
 	found := nid > 0
 	return found, nil
 }
+
+func (s *membershipStatements) SelectServerInRoom(ctx context.Context, roomNID types.RoomNID, serverName gomatrixserverlib.ServerName) (bool, error) {
+	var nid types.RoomNID
+	err := s.selectServerInRoomStmt.QueryRowContext(ctx, tables.MembershipStateJoin, roomNID, serverName).Scan(&nid)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return false, nil
+		}
+		return false, err
+	}
+	return roomNID == nid, nil
+}

roomserver/storage/sqlite3/storage.go

@@ -19,8 +19,6 @@ import (
 	"context"
 	"database/sql"
 
-	_ "github.com/mattn/go-sqlite3"
-
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"

roomserver/storage/tables/interface.go

@@ -120,6 +120,7 @@ const (
 	MembershipStateLeaveOrBan MembershipState = 1
 	MembershipStateInvite     MembershipState = 2
 	MembershipStateJoin       MembershipState = 3
+	MembershipStateKnock      MembershipState = 4
 )
 
 type Membership interface {
@@ -136,6 +137,7 @@ type Membership interface {
 	SelectKnownUsers(ctx context.Context, userID types.EventStateKeyNID, searchString string, limit int) ([]string, error)
 	UpdateForgetMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, forget bool) error
 	SelectLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error)
+	SelectServerInRoom(ctx context.Context, roomNID types.RoomNID, serverName gomatrixserverlib.ServerName) (bool, error)
 }
 
 type Published interface {

setup/jetstream/nats.go

@@ -19,6 +19,10 @@ var natsServer *natsserver.Server
 var natsServerMutex sync.Mutex
 
 func SetupConsumerProducer(cfg *config.JetStream) (sarama.Consumer, sarama.SyncProducer) {
+	// check if we need an in-process NATS Server
+	if len(cfg.Addresses) != 0 {
+		return setupNATS(cfg, nil)
+	}
 	natsServerMutex.Lock()
 	if natsServer == nil {
 		var err error
@@ -64,14 +68,12 @@ func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (sarama.Consumer, sar
 	}
 
 	for _, stream := range streams {
+		stream.Name = cfg.TopicFor(stream.Name)
 		info, err := s.StreamInfo(stream.Name)
 		if err != nil && err != natsclient.ErrStreamNotFound {
 			logrus.WithError(err).Fatal("Unable to get stream info")
 		}
 		if info == nil {
-			stream.Name = cfg.TopicFor(stream.Name)
-			stream.Subjects = []string{stream.Name}
-
 			// If we're trying to keep everything in memory (e.g. unit tests)
 			// then overwrite the storage policy.
 			if cfg.InMemory {

setup/jetstream/streams.go

@@ -18,32 +18,38 @@ var (
 var streams = []*nats.StreamConfig{
 	{
 		Name:      OutputRoomEvent,
+		Subjects:  []string{OutputRoomEvent},
 		Retention: nats.InterestPolicy,
 		Storage:   nats.FileStorage,
 	},
 	{
 		Name:      OutputSendToDeviceEvent,
+		Subjects:  []string{OutputSendToDeviceEvent},
 		Retention: nats.InterestPolicy,
 		Storage:   nats.FileStorage,
 	},
 	{
 		Name:      OutputKeyChangeEvent,
+		Subjects:  []string{OutputKeyChangeEvent},
 		Retention: nats.LimitsPolicy,
 		Storage:   nats.FileStorage,
 	},
 	{
 		Name:      OutputTypingEvent,
+		Subjects:  []string{OutputTypingEvent},
 		Retention: nats.InterestPolicy,
 		Storage:   nats.MemoryStorage,
 		MaxAge:    time.Second * 60,
 	},
 	{
 		Name:      OutputClientData,
+		Subjects:  []string{OutputClientData},
 		Retention: nats.InterestPolicy,
 		Storage:   nats.FileStorage,
 	},
 	{
 		Name:      OutputReceiptEvent,
+		Subjects:  []string{OutputReceiptEvent},
 		Retention: nats.InterestPolicy,
 		Storage:   nats.FileStorage,
 	},

signingkeyserver/storage/sqlite3/keydb.go

@@ -23,8 +23,6 @@ import (
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/gomatrixserverlib"
-
-	_ "github.com/mattn/go-sqlite3"
 )
 
 // A Database implements gomatrixserverlib.KeyDatabase and is used to store

syncapi/consumers/roomserver.go

@@ -379,7 +379,7 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *gomatrixserverlib.Head
 		return event, err
 	}
 
-	if prevEvent == nil {
+	if prevEvent == nil || prevEvent.EventID() == event.EventID() {
 		return event, nil
 	}
 

syncapi/storage/sqlite3/syncserver.go

@@ -18,9 +18,6 @@ package sqlite3
 import (
 	"database/sql"
 
-	// Import the sqlite3 package
-	_ "github.com/mattn/go-sqlite3"
-
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/syncapi/storage/shared"

sytest-whitelist

@@ -537,3 +537,6 @@ Remote servers should reject attempts by non-creators to set the power levels
 Federation handles empty auth_events in state_ids sanely
 Key notary server should return an expired key if it can't find any others
 Key notary server must not overwrite a valid key with a spurious result from the origin server
+GET /rooms/:room_id/aliases lists aliases
+Only room members can list aliases of a room
+Users with sufficient power-level can delete other's aliases

test-dendritejs.sh

@@ -0,0 +1,3 @@
+#!/bin/sh -eu
+
+GOOS=js GOARCH=wasm go test -v -exec "$(pwd)/test/wasm/index.js" ./cmd/dendritejs-pinecone

test/wasm/index.js

@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+const fs = require('fs');
+const path = require('path');
+const childProcess = require('child_process');
+
+(async function() {
+    // sql.js
+    const initSqlJs = require('sql.js');
+    await initSqlJs().then(SQL => {
+        global._go_sqlite = SQL;
+        console.log("Loaded sqlite")
+    });
+    // dendritejs expects to write to `/idb` so we create that here
+    // Since this is testing only, we use the default in-memory FS
+    global._go_sqlite.FS.mkdir("/idb");
+
+    // WebSocket
+    const WebSocket = require('isomorphic-ws');
+    global.WebSocket = WebSocket;
+
+    // Load the generic Go Wasm exec helper inline to trigger built-in run call
+    // This approach avoids copying `wasm_exec.js` into the repo, which is nice
+    // to aim for since it can differ between Go versions.
+    const goRoot = await new Promise((resolve, reject) => {
+        childProcess.execFile('go', ['env', 'GOROOT'], (err, out) => {
+            if (err) {
+                reject("Can't find go");
+            }
+            resolve(out.trim());
+        });
+    });
+    const execPath = path.join(goRoot, 'misc/wasm/wasm_exec.js');
+    const execCode = fs.readFileSync(execPath, 'utf8');
+    eval(execCode);
+})();

test/wasm/package-lock.json

@@ -0,0 +1,67 @@
+{
+  "name": "wasm",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "dependencies": {
+        "isomorphic-ws": "^4.0.1",
+        "sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae",
+        "ws": "^7.5.2"
+      }
+    },
+    "node_modules/isomorphic-ws": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz",
+      "integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==",
+      "peerDependencies": {
+        "ws": "*"
+      }
+    },
+    "node_modules/sql.js": {
+      "version": "1.5.0",
+      "resolved": "git+ssh://git@github.com/neilalexander/sql.js.git#252a72bf57b0538cbd49bbd6f70af71e516966ae",
+      "integrity": "sha512-EFYI/yMoQ1U08nZxQOZ7+4S0nOpKF45EVoWGef8L1kvSCMP3B3xSzwZeOmoF2tBVpbMssAgHEz43cf0ZulRDSQ==",
+      "license": "MIT"
+    },
+    "node_modules/ws": {
+      "version": "7.5.2",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz",
+      "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==",
+      "engines": {
+        "node": ">=8.3.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": "^5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    }
+  },
+  "dependencies": {
+    "isomorphic-ws": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz",
+      "integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==",
+      "requires": {}
+    },
+    "sql.js": {
+      "version": "git+ssh://git@github.com/neilalexander/sql.js.git#252a72bf57b0538cbd49bbd6f70af71e516966ae",
+      "integrity": "sha512-EFYI/yMoQ1U08nZxQOZ7+4S0nOpKF45EVoWGef8L1kvSCMP3B3xSzwZeOmoF2tBVpbMssAgHEz43cf0ZulRDSQ==",
+      "from": "sql.js@github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae"
+    },
+    "ws": {
+      "version": "7.5.2",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz",
+      "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==",
+      "requires": {}
+    }
+  }
+}

test/wasm/package.json

@@ -0,0 +1,7 @@
+{
+  "dependencies": {
+    "isomorphic-ws": "^4.0.1",
+    "sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae",
+    "ws": "^7.5.2"
+  }
+}

userapi/storage/devices/sqlite3/storage.go

@@ -25,8 +25,6 @@ import (
 	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3/deltas"
 	"github.com/matrix-org/gomatrixserverlib"
-
-	_ "github.com/mattn/go-sqlite3"
 )
 
 // The length of generated device IDs