diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml new file mode 100644 index 00000000..4889283a --- /dev/null +++ b/.github/workflows/wasm.yml @@ -0,0 +1,49 @@ +name: WebAssembly + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.16.5 + + - uses: actions/cache@v2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Install Node + uses: actions/setup-node@v2 + with: + node-version: 14 + + - uses: actions/cache@v2 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Reconfigure Git to use HTTPS auth for repo packages + run: > + git config --global url."https://github.com/".insteadOf + ssh://git@github.com/ + + - name: Install test dependencies + working-directory: ./test/wasm + run: npm ci + + - name: Test + run: ./test-dendritejs.sh diff --git a/.gitignore b/.gitignore index c5bf92cc..6a13ed37 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ # Hidden files .* +# Allow GitHub config +!.github + # Downloads /.downloads @@ -36,6 +39,7 @@ _testmain.go *.exe *.test *.prof +*.wasm # Generated keys *.pem @@ -53,3 +57,6 @@ dendrite.yaml # Generated code cmd/dendrite-demo-yggdrasil/embed/fs*.go + +# Test dependencies +test/wasm/node_modules diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 6ba5a6f6..51bfe710 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" - _ "github.com/mattn/go-sqlite3" ) // Database stores events intended to be later sent to application services diff --git a/build-dendritejs.sh b/build-dendritejs.sh index 83ec3699..1c44bdd9 100755 --- a/build-dendritejs.sh +++ b/build-dendritejs.sh @@ -1,4 +1,4 @@ #!/bin/sh -eu export GIT_COMMIT=$(git rev-list -1 HEAD) && \ -GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o main.wasm ./cmd/dendritejs +GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o bin/main.wasm ./cmd/dendritejs-pinecone diff --git a/build.sh b/build.sh index a4981408..8196fc65 100755 --- a/build.sh +++ b/build.sh @@ -21,4 +21,4 @@ mkdir -p bin CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/... -CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs +CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone diff --git a/clientapi/routing/aliases.go b/clientapi/routing/aliases.go new file mode 100644 index 00000000..8c483053 --- /dev/null +++ b/clientapi/routing/aliases.go @@ -0,0 +1,96 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "fmt" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/util" +) + +// GetAliases implements GET /_matrix/client/r0/rooms/{roomId}/aliases +func GetAliases( + req *http.Request, rsAPI api.RoomserverInternalAPI, device *userapi.Device, roomID string, +) util.JSONResponse { + stateTuple := gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomHistoryVisibility, + StateKey: "", + } + stateReq := &api.QueryCurrentStateRequest{ + RoomID: roomID, + StateTuples: []gomatrixserverlib.StateKeyTuple{stateTuple}, + } + stateRes := &api.QueryCurrentStateResponse{} + if err := rsAPI.QueryCurrentState(req.Context(), stateReq, stateRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryCurrentState failed") + return util.ErrorResponse(fmt.Errorf("rsAPI.QueryCurrentState: %w", err)) + } + + visibility := "invite" + if historyVisEvent, ok := stateRes.StateEvents[stateTuple]; ok { + var err error + visibility, err = historyVisEvent.HistoryVisibility() + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("historyVisEvent.HistoryVisibility failed") + return util.ErrorResponse(fmt.Errorf("historyVisEvent.HistoryVisibility: %w", err)) + } + } + if visibility != gomatrixserverlib.WorldReadable { + queryReq := api.QueryMembershipForUserRequest{ + RoomID: roomID, + UserID: device.UserID, + } + var queryRes api.QueryMembershipForUserResponse + if err := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed") + return jsonerror.InternalServerError() + } + if !queryRes.IsInRoom { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("You aren't a member of this room."), + } + } + } + + aliasesReq := api.GetAliasesForRoomIDRequest{ + RoomID: roomID, + } + aliasesRes := api.GetAliasesForRoomIDResponse{} + if err := rsAPI.GetAliasesForRoomID(req.Context(), &aliasesReq, &aliasesRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("rsAPI.GetAliasesForRoomID failed") + return util.ErrorResponse(fmt.Errorf("rsAPI.GetAliasesForRoomID: %w", err)) + } + + response := struct { + Aliases []string `json:"aliases"` + }{ + Aliases: aliasesRes.Aliases, + } + if response.Aliases == nil { + response.Aliases = []string{} // pleases sytest + } + + return util.JSONResponse{ + Code: 200, + JSON: response, + } +} diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 2d886746..b3b996ec 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -43,7 +43,7 @@ type createRoomRequest struct { Visibility string `json:"visibility"` Topic string `json:"topic"` Preset string `json:"preset"` - CreationContent map[string]interface{} `json:"creation_content"` + CreationContent json.RawMessage `json:"creation_content"` InitialState []fledglingEvent `json:"initial_state"` RoomAliasName string `json:"room_alias_name"` GuestCanJoin bool `json:"guest_can_join"` @@ -177,11 +177,6 @@ func createRoom( // Clobber keys: creator, room_version - if r.CreationContent == nil { - r.CreationContent = make(map[string]interface{}, 2) - } - - r.CreationContent["creator"] = userID roomVersion := roomserverVersion.DefaultRoomVersion() if r.RoomVersion != "" { candidateVersion := gomatrixserverlib.RoomVersion(r.RoomVersion) @@ -194,7 +189,6 @@ func createRoom( } roomVersion = candidateVersion } - r.CreationContent["room_version"] = roomVersion // TODO: visibility/presets/raw initial state // TODO: Create room alias association @@ -203,7 +197,7 @@ func createRoom( logger.WithFields(log.Fields{ "userID": userID, "roomID": roomID, - "roomVersion": r.CreationContent["room_version"], + "roomVersion": roomVersion, }).Info("Creating new room") profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) @@ -212,6 +206,109 @@ func createRoom( return jsonerror.InternalServerError() } + createContent := map[string]interface{}{} + if len(r.CreationContent) > 0 { + if err = json.Unmarshal(r.CreationContent, &createContent); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal for creation_content failed") + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("invalid create content"), + } + } + } + createContent["creator"] = userID + createContent["room_version"] = roomVersion + powerLevelContent := eventutil.InitialPowerLevelsContent(userID) + joinRuleContent := gomatrixserverlib.JoinRuleContent{ + JoinRule: gomatrixserverlib.Invite, + } + historyVisibilityContent := gomatrixserverlib.HistoryVisibilityContent{ + HistoryVisibility: historyVisibilityShared, + } + + if r.PowerLevelContentOverride != nil { + // Merge powerLevelContentOverride fields by unmarshalling it atop the defaults + err = json.Unmarshal(r.PowerLevelContentOverride, &powerLevelContent) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal for power_level_content_override failed") + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("malformed power_level_content_override"), + } + } + } + + switch r.Preset { + case presetPrivateChat: + joinRuleContent.JoinRule = gomatrixserverlib.Invite + historyVisibilityContent.HistoryVisibility = historyVisibilityShared + case presetTrustedPrivateChat: + joinRuleContent.JoinRule = gomatrixserverlib.Invite + historyVisibilityContent.HistoryVisibility = historyVisibilityShared + // TODO If trusted_private_chat, all invitees are given the same power level as the room creator. + case presetPublicChat: + joinRuleContent.JoinRule = gomatrixserverlib.Public + historyVisibilityContent.HistoryVisibility = historyVisibilityShared + } + + createEvent := fledglingEvent{ + Type: gomatrixserverlib.MRoomCreate, + Content: createContent, + } + powerLevelEvent := fledglingEvent{ + Type: gomatrixserverlib.MRoomPowerLevels, + Content: powerLevelContent, + } + joinRuleEvent := fledglingEvent{ + Type: gomatrixserverlib.MRoomJoinRules, + Content: joinRuleContent, + } + historyVisibilityEvent := fledglingEvent{ + Type: gomatrixserverlib.MRoomHistoryVisibility, + Content: historyVisibilityContent, + } + membershipEvent := fledglingEvent{ + Type: gomatrixserverlib.MRoomMember, + StateKey: userID, + Content: gomatrixserverlib.MemberContent{ + Membership: gomatrixserverlib.Join, + DisplayName: profile.DisplayName, + AvatarURL: profile.AvatarURL, + }, + } + + var nameEvent *fledglingEvent + var topicEvent *fledglingEvent + var guestAccessEvent *fledglingEvent + var aliasEvent *fledglingEvent + + if r.Name != "" { + nameEvent = &fledglingEvent{ + Type: gomatrixserverlib.MRoomName, + Content: eventutil.NameContent{ + Name: r.Name, + }, + } + } + + if r.Topic != "" { + topicEvent = &fledglingEvent{ + Type: gomatrixserverlib.MRoomTopic, + Content: eventutil.TopicContent{ + Topic: r.Topic, + }, + } + } + + if r.GuestCanJoin { + guestAccessEvent = &fledglingEvent{ + Type: gomatrixserverlib.MRoomGuestAccess, + Content: eventutil.GuestAccessContent{ + GuestAccess: "can_join", + }, + } + } + var roomAlias string if r.RoomAliasName != "" { roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, cfg.Matrix.ServerName) @@ -230,44 +327,46 @@ func createRoom( if aliasResp.RoomID != "" { return util.MessageResponse(400, "Alias already exists") } + + aliasEvent = &fledglingEvent{ + Type: gomatrixserverlib.MRoomCanonicalAlias, + Content: eventutil.CanonicalAlias{ + Alias: roomAlias, + }, + } } - membershipContent := gomatrixserverlib.MemberContent{ - Membership: gomatrixserverlib.Join, - DisplayName: profile.DisplayName, - AvatarURL: profile.AvatarURL, - } + var initialStateEvents []fledglingEvent + for i := range r.InitialState { + if r.InitialState[i].StateKey != "" { + initialStateEvents = append(initialStateEvents, r.InitialState[i]) + continue + } - var joinRules, historyVisibility string - switch r.Preset { - case presetPrivateChat: - joinRules = gomatrixserverlib.Invite - historyVisibility = historyVisibilityShared - case presetTrustedPrivateChat: - joinRules = gomatrixserverlib.Invite - historyVisibility = historyVisibilityShared - // TODO If trusted_private_chat, all invitees are given the same power level as the room creator. - case presetPublicChat: - joinRules = gomatrixserverlib.Public - historyVisibility = historyVisibilityShared - default: - // Default room rules, r.Preset was previously checked for valid values so - // only a request with no preset should end up here. - joinRules = gomatrixserverlib.Invite - historyVisibility = historyVisibilityShared - } + switch r.InitialState[i].Type { + case gomatrixserverlib.MRoomCreate: + continue - var builtEvents []*gomatrixserverlib.HeaderedEvent + case gomatrixserverlib.MRoomPowerLevels: + powerLevelEvent = r.InitialState[i] - powerLevelContent := eventutil.InitialPowerLevelsContent(userID) - if r.PowerLevelContentOverride != nil { - // Merge powerLevelContentOverride fields by unmarshalling it atop the defaults - err = json.Unmarshal(r.PowerLevelContentOverride, &powerLevelContent) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("malformed power_level_content_override"), - } + case gomatrixserverlib.MRoomJoinRules: + joinRuleEvent = r.InitialState[i] + + case gomatrixserverlib.MRoomHistoryVisibility: + historyVisibilityEvent = r.InitialState[i] + + case gomatrixserverlib.MRoomGuestAccess: + guestAccessEvent = &r.InitialState[i] + + case gomatrixserverlib.MRoomName: + nameEvent = &r.InitialState[i] + + case gomatrixserverlib.MRoomTopic: + topicEvent = &r.InitialState[i] + + default: + initialStateEvents = append(initialStateEvents, r.InitialState[i]) } } @@ -284,37 +383,33 @@ func createRoom( // 10- m.room.topic (opt) // 11- invite events (opt) - with is_direct flag if applicable TODO // 12- 3pid invite events (opt) TODO - // 13- m.room.aliases event for HS (if alias specified) TODO // This differs from Synapse slightly. Synapse would vary the ordering of 3-7 // depending on if those events were in "initial_state" or not. This made it // harder to reason about, hence sticking to a strict static ordering. // TODO: Synapse has txn/token ID on each event. Do we need to do this here? eventsToMake := []fledglingEvent{ - {"m.room.create", "", r.CreationContent}, - {"m.room.member", userID, membershipContent}, - {"m.room.power_levels", "", powerLevelContent}, - {"m.room.join_rules", "", gomatrixserverlib.JoinRuleContent{JoinRule: joinRules}}, - {"m.room.history_visibility", "", eventutil.HistoryVisibilityContent{HistoryVisibility: historyVisibility}}, + createEvent, membershipEvent, powerLevelEvent, joinRuleEvent, historyVisibilityEvent, } - if roomAlias != "" { + if guestAccessEvent != nil { + eventsToMake = append(eventsToMake, *guestAccessEvent) + } + eventsToMake = append(eventsToMake, initialStateEvents...) + if nameEvent != nil { + eventsToMake = append(eventsToMake, *nameEvent) + } + if topicEvent != nil { + eventsToMake = append(eventsToMake, *topicEvent) + } + if aliasEvent != nil { // TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room. // This means we might fail creating the alias but say the canonical alias is something that doesn't exist. - // m.room.aliases is handled when we call roomserver.SetRoomAlias - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", eventutil.CanonicalAlias{Alias: roomAlias}}) - } - if r.GuestCanJoin { - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", eventutil.GuestAccessContent{GuestAccess: "can_join"}}) - } - eventsToMake = append(eventsToMake, r.InitialState...) - if r.Name != "" { - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", eventutil.NameContent{Name: r.Name}}) - } - if r.Topic != "" { - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", eventutil.TopicContent{Topic: r.Topic}}) + eventsToMake = append(eventsToMake, *aliasEvent) } + // TODO: invite events // TODO: 3pid invite events + var builtEvents []*gomatrixserverlib.HeaderedEvent authEvents := gomatrixserverlib.NewAuthEvents(nil) for i, e := range eventsToMake { depth := i + 1 // depth starts at 1 @@ -403,7 +498,7 @@ func createRoom( fallthrough case gomatrixserverlib.MRoomCanonicalAlias: fallthrough - case "m.room.encryption": // TODO: move this to gmsl + case gomatrixserverlib.MRoomEncryption: fallthrough case gomatrixserverlib.MRoomMember: fallthrough diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 0e994b64..ae466065 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -113,13 +113,12 @@ func DirectoryRoom( } // SetLocalAlias implements PUT /directory/room/{roomAlias} -// TODO: Check if the user has the power level to set an alias func SetLocalAlias( req *http.Request, device *api.Device, alias string, cfg *config.ClientAPI, - aliasAPI roomserverAPI.RoomserverInternalAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', alias) if err != nil { @@ -172,7 +171,7 @@ func SetLocalAlias( Alias: alias, } var queryRes roomserverAPI.SetRoomAliasResponse - if err := aliasAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil { + if err := rsAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed") return jsonerror.InternalServerError() } @@ -195,43 +194,32 @@ func RemoveLocalAlias( req *http.Request, device *api.Device, alias string, - aliasAPI roomserverAPI.RoomserverInternalAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { - - creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{ - Alias: alias, - } - var creatorQueryRes roomserverAPI.GetCreatorIDForAliasResponse - if err := aliasAPI.GetCreatorIDForAlias(req.Context(), &creatorQueryReq, &creatorQueryRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.GetCreatorIDForAlias failed") - return jsonerror.InternalServerError() - } - - if creatorQueryRes.UserID == "" { - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("Alias does not exist"), - } - } - - if creatorQueryRes.UserID != device.UserID { - // TODO: Still allow deletion if user is admin - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("You do not have permission to delete this alias"), - } - } - queryReq := roomserverAPI.RemoveRoomAliasRequest{ Alias: alias, UserID: device.UserID, } var queryRes roomserverAPI.RemoveRoomAliasResponse - if err := aliasAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil { + if err := rsAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed") return jsonerror.InternalServerError() } + if !queryRes.Found { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The alias does not exist."), + } + } + + if !queryRes.Removed { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("You do not have permission to remove this alias."), + } + } + return util.JSONResponse{ Code: http.StatusOK, JSON: struct{}{}, @@ -294,9 +282,9 @@ func SetVisibility( return jsonerror.InternalServerError() } - // NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event + // NOTSPEC: Check if the user's power is greater than power required to change m.room.canonical_alias event power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event) - if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) { + if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomCanonicalAlias, true) { return util.JSONResponse{ Code: http.StatusForbidden, JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"), diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 37279e8e..d768247a 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -275,6 +275,14 @@ func Setup( return OnIncomingStateRequest(req.Context(), device, rsAPI, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/aliases", httputil.MakeAuthAPI("aliases", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetAliases(req, rsAPI, device, vars["roomID"]) + })).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 8ebe9541..6568be52 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -44,6 +44,8 @@ import ( "github.com/matrix-org/dendrite/eduserver/cache" "github.com/sirupsen/logrus" + + _ "github.com/mattn/go-sqlite3" ) func createKeyDB( diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index e6a88fcc..52acd16d 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -54,6 +54,8 @@ import ( pineconeSessions "github.com/matrix-org/pinecone/sessions" "github.com/sirupsen/logrus" + + _ "github.com/mattn/go-sqlite3" ) var ( diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 3b8ceca2..d3134c2c 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -44,6 +44,8 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" + + _ "github.com/mattn/go-sqlite3" ) var ( diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 5efbe856..3785371a 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -31,6 +31,8 @@ import ( "github.com/matrix-org/dendrite/signingkeyserver" "github.com/matrix-org/dendrite/userapi" "github.com/sirupsen/logrus" + + _ "github.com/mattn/go-sqlite3" ) var ( diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go index d3c52967..c2208ca2 100644 --- a/cmd/dendrite-polylith-multi/main.go +++ b/cmd/dendrite-polylith-multi/main.go @@ -23,6 +23,8 @@ import ( "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" "github.com/sirupsen/logrus" + + _ "github.com/mattn/go-sqlite3" ) type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite) diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index 1a97e778..ec6e2d8b 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -144,6 +144,13 @@ func generateKey() ed25519.PrivateKey { } func main() { + startup() + + // We want to block forever to let the fetch and libp2p handler serve the APIs + select {} +} + +func startup() { sk := generateKey() pk := sk.Public().(ed25519.PublicKey) @@ -250,7 +257,4 @@ func main() { } } }() - - // We want to block forever to let the fetch and libp2p handler serve the APIs - select {} } diff --git a/cmd/dendritejs-pinecone/main_test.go b/cmd/dendritejs-pinecone/main_test.go new file mode 100644 index 00000000..751700cb --- /dev/null +++ b/cmd/dendritejs-pinecone/main_test.go @@ -0,0 +1,25 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package main + +import ( + "testing" +) + +func TestStartup(t *testing.T) { + startup() +} diff --git a/docs/p2p.md b/docs/p2p.md index d69b47be..e858ba11 100644 --- a/docs/p2p.md +++ b/docs/p2p.md @@ -2,14 +2,23 @@ These are the instructions for setting up P2P Dendrite, current as of May 2020. There's both Go stuff and JS stuff to do to set this up. - ### Dendrite +#### Build + - The `master` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`. - Build it and copy assets to riot-web. ``` -$ GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs -$ cp main.wasm ../riot-web/src/vector/dendrite.wasm +$ ./build-dendritejs.sh +$ cp bin/main.wasm ../riot-web/src/vector/dendrite.wasm +``` + +#### Test + +To check that the Dendrite side is working well as Wasm, you can run the +Wasm-specific tests: +``` +$ ./test-dendritejs.sh ``` ### Rendezvous diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index c5d1d1ac..18fa418f 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -18,8 +18,6 @@ package sqlite3 import ( "database/sql" - _ "github.com/mattn/go-sqlite3" - "github.com/matrix-org/dendrite/federationsender/storage/shared" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3/deltas" "github.com/matrix-org/dendrite/internal/caching" diff --git a/go.mod b/go.mod index b55da547..b779c1f0 100644 --- a/go.mod +++ b/go.mod @@ -35,9 +35,9 @@ require ( github.com/lucas-clemente/quic-go v0.19.3 github.com/matrix-org/dugong v0.0.0-20180820122854-51a565b5666b github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 - github.com/matrix-org/go-sqlite3-js v0.0.0-20210625141222-bd2b7124cee8 + github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20210719140634-f44a103bb12e + github.com/matrix-org/gomatrixserverlib v0.0.0-20210722110442-5061d6986876 github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb diff --git a/go.sum b/go.sum index 65b03c5f..b5d9357c 100644 --- a/go.sum +++ b/go.sum @@ -1024,13 +1024,13 @@ github.com/matrix-org/dugong v0.0.0-20180820122854-51a565b5666b h1:xpcmnpfUImRC4 github.com/matrix-org/dugong v0.0.0-20180820122854-51a565b5666b/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 h1:eqE5OnGx9ZMWmrRbD3KF/3KtTunw0iQulI7YxOIdxo4= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4/go.mod h1:3WluEZ9QXSwU30tWYqktnpC1x9mwZKx1r8uAv8Iq+a4= -github.com/matrix-org/go-sqlite3-js v0.0.0-20210625141222-bd2b7124cee8 h1:/FKUeUlCATr1gXxYqlaJgH8FW/sw0Jz8t7s8BIlECfg= -github.com/matrix-org/go-sqlite3-js v0.0.0-20210625141222-bd2b7124cee8/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d h1:mGhPVaTht5NViFN/UpdrIlRApmH2FWcVaKUH5MdBKiY= +github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210719140634-f44a103bb12e h1:dYPsAU1363AEbcohNzBJHV0CkbfVa8QiMNFib/i5pS8= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210719140634-f44a103bb12e/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210722110442-5061d6986876 h1:6ypwCtgRLK0v/hGWvnd847+KTo9BSkP9N0A4qSniP4E= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210722110442-5061d6986876/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b h1:5X5vdWQ13xrNkJVqaJHPsrt7rKkMJH5iac0EtfOuxSg= github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b/go.mod h1:CVlrvs1R5iz7Omy2GqAjJJKbACn07GZgUq1Gli18FYE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= @@ -2065,4 +2065,4 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= \ No newline at end of file +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/internal/eventutil/eventcontent.go b/internal/eventutil/eventcontent.go index 873e20a8..4ecb5fb5 100644 --- a/internal/eventutil/eventcontent.go +++ b/internal/eventutil/eventcontent.go @@ -53,7 +53,6 @@ func InitialPowerLevelsContent(roomCreator string) (c gomatrixserverlib.PowerLev "m.room.history_visibility": 100, "m.room.canonical_alias": 50, "m.room.avatar": 50, - "m.room.aliases": 0, // anyone can publish aliases by default. Has to be 0 else state_default is used. } c.Users = map[string]int64{roomCreator: 100} return c diff --git a/internal/sqlutil/sql.go b/internal/sqlutil/sql.go index a3885d66..6cf17bea 100644 --- a/internal/sqlutil/sql.go +++ b/internal/sqlutil/sql.go @@ -19,7 +19,6 @@ import ( "database/sql" "errors" "fmt" - "runtime" "strings" "github.com/matrix-org/util" @@ -113,13 +112,6 @@ func QueryVariadicOffset(count, offset int) string { return str } -func SQLiteDriverName() string { - if runtime.GOOS == "js" { - return "sqlite3_js" - } - return "sqlite3" -} - func minOfInts(a, b int) int { if a <= b { return a diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go index ad004455..51eaa1b4 100644 --- a/internal/sqlutil/trace.go +++ b/internal/sqlutil/trace.go @@ -104,7 +104,7 @@ func Open(dbProperties *config.DatabaseOptions) (*sql.DB, error) { var driverName, dsn string switch { case dbProperties.ConnectionString.IsSQLite(): - driverName = SQLiteDriverName() + driverName = "sqlite3" dsn, err = ParseFileURI(dbProperties.ConnectionString) if err != nil { return nil, fmt.Errorf("ParseFileURI: %w", err) @@ -123,11 +123,11 @@ func Open(dbProperties *config.DatabaseOptions) (*sql.DB, error) { if err != nil { return nil, err } - if driverName != SQLiteDriverName() { + if driverName != "sqlite3" { logrus.WithFields(logrus.Fields{ - "MaxOpenConns": dbProperties.MaxOpenConns, - "MaxIdleConns": dbProperties.MaxIdleConns, - "ConnMaxLifetime": dbProperties.ConnMaxLifetime, + "MaxOpenConns": dbProperties.MaxOpenConns(), + "MaxIdleConns": dbProperties.MaxIdleConns(), + "ConnMaxLifetime": dbProperties.ConnMaxLifetime(), "dataSourceName": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"), }).Debug("Setting DB connection limits") db.SetMaxOpenConns(dbProperties.MaxOpenConns()) diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index 0edfc08e..fa442173 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" - _ "github.com/mattn/go-sqlite3" ) // Database is used to store metadata about a repository of media files. diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index 2eb91129..df69e5b4 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -78,4 +78,9 @@ type RemoveRoomAliasRequest struct { } // RemoveRoomAliasResponse is a response to RemoveRoomAlias -type RemoveRoomAliasResponse struct{} +type RemoveRoomAliasResponse struct { + // Did the alias exist before? + Found bool `json:"found"` + // Did we remove it? + Removed bool `json:"removed"` +} diff --git a/roomserver/api/query.go b/roomserver/api/query.go index c70db65c..599156bb 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -181,11 +181,8 @@ type QueryServerJoinedToRoomRequest struct { type QueryServerJoinedToRoomResponse struct { // True if the room exists on the server RoomExists bool `json:"room_exists"` - // True if we still believe that we are participating in the room + // True if we still believe that the server is participating in the room IsInRoom bool `json:"is_in_room"` - // List of servers that are also in the room. This will not be populated - // if the queried ServerName is the local server name. - ServerNames []gomatrixserverlib.ServerName `json:"server_names"` } // QueryServerAllowedToSeeEventRequest is a request to QueryServerAllowedToSeeEvent diff --git a/roomserver/internal/alias.go b/roomserver/internal/alias.go index f15881d7..7995279d 100644 --- a/roomserver/internal/alias.go +++ b/roomserver/internal/alias.go @@ -16,10 +16,7 @@ package internal import ( "context" - "encoding/json" - "errors" "fmt" - "time" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -73,11 +70,7 @@ func (r *RoomserverInternalAPI) SetRoomAlias( return err } - // Send a m.room.aliases event with the updated list of aliases for this room - // At this point we've already committed the alias to the database so we - // shouldn't cancel this request. - // TODO: Ensure that we send unsent events when if server restarts. - return r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, request.RoomID) + return nil } // GetRoomIDForAlias implements alias.RoomserverInternalAPI @@ -157,122 +150,44 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias( request *api.RemoveRoomAliasRequest, response *api.RemoveRoomAliasResponse, ) error { - // Look up the room ID in the database roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) if err != nil { - return err + return fmt.Errorf("r.DB.GetRoomIDForAlias: %w", err) + } + if roomID == "" { + response.Found = false + response.Removed = false + return nil } - // Remove the dalias from the database + response.Found = true + creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias) + if err != nil { + return fmt.Errorf("r.DB.GetCreatorIDForAlias: %w", err) + } + + if creatorID != request.UserID { + plEvent, err := r.DB.GetStateEvent(ctx, roomID, gomatrixserverlib.MRoomPowerLevels, "") + if err != nil { + return fmt.Errorf("r.DB.GetStateEvent: %w", err) + } + + pls, err := plEvent.PowerLevels() + if err != nil { + return fmt.Errorf("plEvent.PowerLevels: %w", err) + } + + if pls.UserLevel(request.UserID) < pls.EventLevel(gomatrixserverlib.MRoomCanonicalAlias, true) { + response.Removed = false + return nil + } + } + + // Remove the alias from the database if err := r.DB.RemoveRoomAlias(ctx, request.Alias); err != nil { return err } - // Send an updated m.room.aliases event - // At this point we've already committed the alias to the database so we - // shouldn't cancel this request. - // TODO: Ensure that we send unsent events when if server restarts. - return r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, roomID) -} - -type roomAliasesContent struct { - Aliases []string `json:"aliases"` -} - -// Build the updated m.room.aliases event to send to the room after addition or -// removal of an alias -func (r *RoomserverInternalAPI) sendUpdatedAliasesEvent( - ctx context.Context, userID string, roomID string, -) error { - serverName := string(r.Cfg.Matrix.ServerName) - - builder := gomatrixserverlib.EventBuilder{ - Sender: userID, - RoomID: roomID, - Type: "m.room.aliases", - StateKey: &serverName, - } - - // Retrieve the updated list of aliases, marhal it and set it as the - // event's content - aliases, err := r.DB.GetAliasesForRoomID(ctx, roomID) - if err != nil { - return err - } - content := roomAliasesContent{Aliases: aliases} - rawContent, err := json.Marshal(content) - if err != nil { - return err - } - err = builder.SetContent(json.RawMessage(rawContent)) - if err != nil { - return err - } - - // Get needed state events and depth - eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(&builder) - if err != nil { - return err - } - if len(eventsNeeded.Tuples()) == 0 { - return errors.New("expecting state tuples for event builder, got none") - } - req := api.QueryLatestEventsAndStateRequest{ - RoomID: roomID, - StateToFetch: eventsNeeded.Tuples(), - } - var res api.QueryLatestEventsAndStateResponse - if err = r.QueryLatestEventsAndState(ctx, &req, &res); err != nil { - return err - } - builder.Depth = res.Depth - builder.PrevEvents = res.LatestEvents - - // Add auth events - authEvents := gomatrixserverlib.NewAuthEvents(nil) - for i := range res.StateEvents { - err = authEvents.AddEvent(res.StateEvents[i].Event) - if err != nil { - return err - } - } - refs, err := eventsNeeded.AuthEventReferences(&authEvents) - if err != nil { - return err - } - builder.AuthEvents = refs - - roomInfo, err := r.DB.RoomInfo(ctx, roomID) - if err != nil { - return err - } - if roomInfo == nil { - return fmt.Errorf("room %s does not exist", roomID) - } - - // Build the event - now := time.Now() - event, err := builder.Build( - now, r.Cfg.Matrix.ServerName, r.Cfg.Matrix.KeyID, - r.Cfg.Matrix.PrivateKey, roomInfo.RoomVersion, - ) - if err != nil { - return err - } - - // Create the request - ire := api.InputRoomEvent{ - Kind: api.KindNew, - Event: event.Headered(roomInfo.RoomVersion), - AuthEventIDs: event.AuthEventIDs(), - SendAsServer: serverName, - } - inputReq := api.InputRoomEventsRequest{ - InputRoomEvents: []api.InputRoomEvent{ire}, - } - var inputRes api.InputRoomEventsResponse - - // Send the request - r.InputRoomEvents(ctx, &inputReq, &inputRes) - return inputRes.Err() + response.Removed = true + return nil } diff --git a/roomserver/internal/input/input_membership.go b/roomserver/internal/input/input_membership.go index 44435bfd..2511097d 100644 --- a/roomserver/internal/input/input_membership.go +++ b/roomserver/internal/input/input_membership.go @@ -136,6 +136,8 @@ func (r *Inputer) updateMembership( return updateToJoinMembership(mu, add, updates) case gomatrixserverlib.Leave, gomatrixserverlib.Ban: return updateToLeaveMembership(mu, add, newMembership, updates) + case gomatrixserverlib.Knock: + return updateToKnockMembership(mu, add, updates) default: panic(fmt.Errorf( "input: membership %q is not one of the allowed values", newMembership, @@ -220,6 +222,18 @@ func updateToLeaveMembership( return updates, nil } +func updateToKnockMembership( + mu *shared.MembershipUpdater, add *gomatrixserverlib.Event, updates []api.OutputEvent, +) ([]api.OutputEvent, error) { + if mu.IsLeave() { + _, err := mu.SetToKnock(add) + if err != nil { + return nil, err + } + } + return updates, nil +} + // membershipChanges pairs up the membership state changes. func membershipChanges(removed, added []types.StateEntry) []stateChange { changes := pairUpChanges(removed, added) diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go index fa65ce9b..c6ad79d9 100644 --- a/roomserver/internal/perform/perform_invite.go +++ b/roomserver/internal/perform/perform_invite.go @@ -223,8 +223,8 @@ func buildInviteStrippedState( // https://matrix.org/docs/spec/client_server/r0.6.0#m-room-member for _, t := range []string{ gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias, - gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules, - "m.room.avatar", "m.room.encryption", gomatrixserverlib.MRoomCreate, + gomatrixserverlib.MRoomJoinRules, gomatrixserverlib.MRoomAvatar, + gomatrixserverlib.MRoomEncryption, gomatrixserverlib.MRoomCreate, } { stateWanted = append(stateWanted, gomatrixserverlib.StateKeyTuple{ EventType: t, diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 4af0e639..b80f08ab 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -330,46 +330,17 @@ func (r *Queryer) QueryServerJoinedToRoom( response.RoomExists = true if request.ServerName == r.ServerName || request.ServerName == "" { - var joined bool - joined, err = r.DB.GetLocalServerInRoom(ctx, info.RoomNID) + response.IsInRoom, err = r.DB.GetLocalServerInRoom(ctx, info.RoomNID) if err != nil { return fmt.Errorf("r.DB.GetLocalServerInRoom: %w", err) } - response.IsInRoom = joined - return nil - } - - eventNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, true, false) - if err != nil { - return fmt.Errorf("r.DB.GetMembershipEventNIDsForRoom: %w", err) - } - if len(eventNIDs) == 0 { - return nil - } - - events, err := r.DB.Events(ctx, eventNIDs) - if err != nil { - return fmt.Errorf("r.DB.Events: %w", err) - } - - servers := map[gomatrixserverlib.ServerName]struct{}{} - for _, e := range events { - if e.Type() == gomatrixserverlib.MRoomMember && e.StateKey() != nil { - _, serverName, err := gomatrixserverlib.SplitID('@', *e.StateKey()) - if err != nil { - continue - } - servers[serverName] = struct{}{} - if serverName == request.ServerName { - response.IsInRoom = true - } + } else { + response.IsInRoom, err = r.DB.GetServerInRoom(ctx, info.RoomNID, request.ServerName) + if err != nil { + return fmt.Errorf("r.DB.GetServerInRoom: %w", err) } } - for server := range servers { - response.ServerNames = append(response.ServerNames, server) - } - return nil } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index c25820aa..62aa73ad 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -156,6 +156,8 @@ type Database interface { JoinedUsersSetInRooms(ctx context.Context, roomIDs []string) (map[string]int, error) // GetLocalServerInRoom returns true if we think we're in a given room or false otherwise. GetLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) + // GetServerInRoom returns true if we think a server is in a given room or false otherwise. + GetServerInRoom(ctx context.Context, roomNID types.RoomNID, serverName gomatrixserverlib.ServerName) (bool, error) // GetKnownUsers searches all users that userID knows about. GetKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error) // GetKnownRooms returns a list of all rooms we know about. diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 9102f26a..b4a27900 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) const membershipSchema = ` @@ -132,6 +133,16 @@ var selectKnownUsersSQL = "" + const selectLocalServerInRoomSQL = "" + "SELECT room_nid FROM roomserver_membership WHERE target_local = true AND membership_nid = $1 AND room_nid = $2 LIMIT 1" +// selectServerMembersInRoomSQL is an optimised case for checking for server members in a room. +// The JOIN is significantly leaner than the previous case of looking up event NIDs and reading the +// membership events from the database, as the JOIN query amounts to little more than two index +// scans which are very fast. The presence of a single row from this query suggests the server is +// in the room, no rows returned suggests they aren't. +const selectServerInRoomSQL = "" + + "SELECT room_nid FROM roomserver_membership" + + " JOIN roomserver_event_state_keys ON roomserver_membership.target_nid = roomserver_event_state_keys.event_state_key_nid" + + " WHERE membership_nid = $1 AND room_nid = $2 AND event_state_key LIKE '%:' || $3 LIMIT 1" + type membershipStatements struct { insertMembershipStmt *sql.Stmt selectMembershipForUpdateStmt *sql.Stmt @@ -146,6 +157,7 @@ type membershipStatements struct { selectKnownUsersStmt *sql.Stmt updateMembershipForgetRoomStmt *sql.Stmt selectLocalServerInRoomStmt *sql.Stmt + selectServerInRoomStmt *sql.Stmt } func createMembershipTable(db *sql.DB) error { @@ -170,6 +182,7 @@ func prepareMembershipTable(db *sql.DB) (tables.Membership, error) { {&s.selectKnownUsersStmt, selectKnownUsersSQL}, {&s.updateMembershipForgetRoomStmt, updateMembershipForgetRoom}, {&s.selectLocalServerInRoomStmt, selectLocalServerInRoomSQL}, + {&s.selectServerInRoomStmt, selectServerInRoomSQL}, }.Prepare(db) } @@ -347,3 +360,15 @@ func (s *membershipStatements) SelectLocalServerInRoom(ctx context.Context, room found := nid > 0 return found, nil } + +func (s *membershipStatements) SelectServerInRoom(ctx context.Context, roomNID types.RoomNID, serverName gomatrixserverlib.ServerName) (bool, error) { + var nid types.RoomNID + err := s.selectServerInRoomStmt.QueryRowContext(ctx, tables.MembershipStateJoin, roomNID, serverName).Scan(&nid) + if err != nil { + if err == sql.ErrNoRows { + return false, nil + } + return false, err + } + return roomNID == nid, nil +} diff --git a/roomserver/storage/shared/membership_updater.go b/roomserver/storage/shared/membership_updater.go index 57f3a520..f1f589a3 100644 --- a/roomserver/storage/shared/membership_updater.go +++ b/roomserver/storage/shared/membership_updater.go @@ -86,6 +86,11 @@ func (u *MembershipUpdater) IsLeave() bool { return u.membership == tables.MembershipStateLeaveOrBan } +// IsKnock implements types.MembershipUpdater +func (u *MembershipUpdater) IsKnock() bool { + return u.membership == tables.MembershipStateKnock +} + // SetToInvite implements types.MembershipUpdater func (u *MembershipUpdater) SetToInvite(event gomatrixserverlib.Event) (bool, error) { var inserted bool @@ -180,3 +185,27 @@ func (u *MembershipUpdater) SetToLeave(senderUserID string, eventID string) ([]s }) return inviteEventIDs, err } + +// SetToKnock implements types.MembershipUpdater +func (u *MembershipUpdater) SetToKnock(event *gomatrixserverlib.Event) (bool, error) { + var inserted bool + err := u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error { + senderUserNID, err := u.d.assignStateKeyNID(u.ctx, u.txn, event.Sender()) + if err != nil { + return fmt.Errorf("u.d.AssignStateKeyNID: %w", err) + } + if u.membership != tables.MembershipStateKnock { + // Look up the NID of the new knock event + nIDs, err := u.d.EventNIDs(u.ctx, []string{event.EventID()}) + if err != nil { + return fmt.Errorf("u.d.EventNIDs: %w", err) + } + + if err = u.d.MembershipTable.UpdateMembership(u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateKnock, nIDs[event.EventID()], false); err != nil { + return fmt.Errorf("u.d.MembershipTable.UpdateMembership: %w", err) + } + } + return nil + }) + return inserted, err +} diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 8e787851..e1851679 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -857,6 +857,9 @@ func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey s if err != nil { return nil, err } + if roomInfo == nil || roomInfo.IsStub { + return nil, fmt.Errorf("room %s doesn't exist", roomID) + } eventTypeNID, err := d.EventTypesTable.SelectEventTypeNID(ctx, nil, evType) if err == sql.ErrNoRows { // No rooms have an event of this type, otherwise we'd have an event type NID @@ -1068,6 +1071,11 @@ func (d *Database) GetLocalServerInRoom(ctx context.Context, roomNID types.RoomN return d.MembershipTable.SelectLocalServerInRoom(ctx, roomNID) } +// GetServerInRoom returns true if we think a server is in a given room or false otherwise. +func (d *Database) GetServerInRoom(ctx context.Context, roomNID types.RoomNID, serverName gomatrixserverlib.ServerName) (bool, error) { + return d.MembershipTable.SelectServerInRoom(ctx, roomNID, serverName) +} + // GetKnownUsers searches all users that userID knows about. func (d *Database) GetKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error) { stateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, userID) diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 82babe0d..911a2516 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) const membershipSchema = ` @@ -108,6 +109,16 @@ var selectKnownUsersSQL = "" + const selectLocalServerInRoomSQL = "" + "SELECT room_nid FROM roomserver_membership WHERE target_local = 1 AND membership_nid = $1 AND room_nid = $2 LIMIT 1" +// selectServerMembersInRoomSQL is an optimised case for checking for server members in a room. +// The JOIN is significantly leaner than the previous case of looking up event NIDs and reading the +// membership events from the database, as the JOIN query amounts to little more than two index +// scans which are very fast. The presence of a single row from this query suggests the server is +// in the room, no rows returned suggests they aren't. +const selectServerInRoomSQL = "" + + "SELECT room_nid FROM roomserver_membership" + + " JOIN roomserver_event_state_keys ON roomserver_membership.target_nid = roomserver_event_state_keys.event_state_key_nid" + + " WHERE membership_nid = $1 AND room_nid = $2 AND event_state_key LIKE '%:' || $3 LIMIT 1" + type membershipStatements struct { db *sql.DB insertMembershipStmt *sql.Stmt @@ -122,6 +133,7 @@ type membershipStatements struct { selectKnownUsersStmt *sql.Stmt updateMembershipForgetRoomStmt *sql.Stmt selectLocalServerInRoomStmt *sql.Stmt + selectServerInRoomStmt *sql.Stmt } func createMembershipTable(db *sql.DB) error { @@ -147,6 +159,7 @@ func prepareMembershipTable(db *sql.DB) (tables.Membership, error) { {&s.selectKnownUsersStmt, selectKnownUsersSQL}, {&s.updateMembershipForgetRoomStmt, updateMembershipForgetRoom}, {&s.selectLocalServerInRoomStmt, selectLocalServerInRoomSQL}, + {&s.selectServerInRoomStmt, selectServerInRoomSQL}, }.Prepare(db) } @@ -327,3 +340,15 @@ func (s *membershipStatements) SelectLocalServerInRoom(ctx context.Context, room found := nid > 0 return found, nil } + +func (s *membershipStatements) SelectServerInRoom(ctx context.Context, roomNID types.RoomNID, serverName gomatrixserverlib.ServerName) (bool, error) { + var nid types.RoomNID + err := s.selectServerInRoomStmt.QueryRowContext(ctx, tables.MembershipStateJoin, roomNID, serverName).Scan(&nid) + if err != nil { + if err == sql.ErrNoRows { + return false, nil + } + return false, err + } + return roomNID == nid, nil +} diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index c07ab507..e081acdb 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -19,8 +19,6 @@ import ( "context" "database/sql" - _ "github.com/mattn/go-sqlite3" - "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 4a893663..8720d400 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -120,6 +120,7 @@ const ( MembershipStateLeaveOrBan MembershipState = 1 MembershipStateInvite MembershipState = 2 MembershipStateJoin MembershipState = 3 + MembershipStateKnock MembershipState = 4 ) type Membership interface { @@ -136,6 +137,7 @@ type Membership interface { SelectKnownUsers(ctx context.Context, userID types.EventStateKeyNID, searchString string, limit int) ([]string, error) UpdateForgetMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, forget bool) error SelectLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) + SelectServerInRoom(ctx context.Context, roomNID types.RoomNID, serverName gomatrixserverlib.ServerName) (bool, error) } type Published interface { diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 1f5a8bf9..002ed168 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -19,6 +19,10 @@ var natsServer *natsserver.Server var natsServerMutex sync.Mutex func SetupConsumerProducer(cfg *config.JetStream) (sarama.Consumer, sarama.SyncProducer) { + // check if we need an in-process NATS Server + if len(cfg.Addresses) != 0 { + return setupNATS(cfg, nil) + } natsServerMutex.Lock() if natsServer == nil { var err error @@ -64,14 +68,12 @@ func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (sarama.Consumer, sar } for _, stream := range streams { + stream.Name = cfg.TopicFor(stream.Name) info, err := s.StreamInfo(stream.Name) if err != nil && err != natsclient.ErrStreamNotFound { logrus.WithError(err).Fatal("Unable to get stream info") } if info == nil { - stream.Name = cfg.TopicFor(stream.Name) - stream.Subjects = []string{stream.Name} - // If we're trying to keep everything in memory (e.g. unit tests) // then overwrite the storage policy. if cfg.InMemory { diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index 326e62a9..b533d04e 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -18,32 +18,38 @@ var ( var streams = []*nats.StreamConfig{ { Name: OutputRoomEvent, + Subjects: []string{OutputRoomEvent}, Retention: nats.InterestPolicy, Storage: nats.FileStorage, }, { Name: OutputSendToDeviceEvent, + Subjects: []string{OutputSendToDeviceEvent}, Retention: nats.InterestPolicy, Storage: nats.FileStorage, }, { Name: OutputKeyChangeEvent, + Subjects: []string{OutputKeyChangeEvent}, Retention: nats.LimitsPolicy, Storage: nats.FileStorage, }, { Name: OutputTypingEvent, + Subjects: []string{OutputTypingEvent}, Retention: nats.InterestPolicy, Storage: nats.MemoryStorage, MaxAge: time.Second * 60, }, { Name: OutputClientData, + Subjects: []string{OutputClientData}, Retention: nats.InterestPolicy, Storage: nats.FileStorage, }, { Name: OutputReceiptEvent, + Subjects: []string{OutputReceiptEvent}, Retention: nats.InterestPolicy, Storage: nats.FileStorage, }, diff --git a/signingkeyserver/storage/sqlite3/keydb.go b/signingkeyserver/storage/sqlite3/keydb.go index 8825d697..1f85a09c 100644 --- a/signingkeyserver/storage/sqlite3/keydb.go +++ b/signingkeyserver/storage/sqlite3/keydb.go @@ -23,8 +23,6 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" - - _ "github.com/mattn/go-sqlite3" ) // A Database implements gomatrixserverlib.KeyDatabase and is used to store diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 68a80952..7ddf2c09 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -379,7 +379,7 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *gomatrixserverlib.Head return event, err } - if prevEvent == nil { + if prevEvent == nil || prevEvent.EventID() == event.EventID() { return event, nil } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index ae0647fc..706d43f8 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -18,9 +18,6 @@ package sqlite3 import ( "database/sql" - // Import the sqlite3 package - _ "github.com/mattn/go-sqlite3" - "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/storage/shared" diff --git a/sytest-whitelist b/sytest-whitelist index 4d0b9fcf..d90ba4fb 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -537,3 +537,6 @@ Remote servers should reject attempts by non-creators to set the power levels Federation handles empty auth_events in state_ids sanely Key notary server should return an expired key if it can't find any others Key notary server must not overwrite a valid key with a spurious result from the origin server +GET /rooms/:room_id/aliases lists aliases +Only room members can list aliases of a room +Users with sufficient power-level can delete other's aliases diff --git a/test-dendritejs.sh b/test-dendritejs.sh new file mode 100755 index 00000000..73e3d735 --- /dev/null +++ b/test-dendritejs.sh @@ -0,0 +1,3 @@ +#!/bin/sh -eu + +GOOS=js GOARCH=wasm go test -v -exec "$(pwd)/test/wasm/index.js" ./cmd/dendritejs-pinecone diff --git a/test/wasm/index.js b/test/wasm/index.js new file mode 100755 index 00000000..fce5753c --- /dev/null +++ b/test/wasm/index.js @@ -0,0 +1,52 @@ +#!/usr/bin/env node + +/* +Copyright 2021 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +const fs = require('fs'); +const path = require('path'); +const childProcess = require('child_process'); + +(async function() { + // sql.js + const initSqlJs = require('sql.js'); + await initSqlJs().then(SQL => { + global._go_sqlite = SQL; + console.log("Loaded sqlite") + }); + // dendritejs expects to write to `/idb` so we create that here + // Since this is testing only, we use the default in-memory FS + global._go_sqlite.FS.mkdir("/idb"); + + // WebSocket + const WebSocket = require('isomorphic-ws'); + global.WebSocket = WebSocket; + + // Load the generic Go Wasm exec helper inline to trigger built-in run call + // This approach avoids copying `wasm_exec.js` into the repo, which is nice + // to aim for since it can differ between Go versions. + const goRoot = await new Promise((resolve, reject) => { + childProcess.execFile('go', ['env', 'GOROOT'], (err, out) => { + if (err) { + reject("Can't find go"); + } + resolve(out.trim()); + }); + }); + const execPath = path.join(goRoot, 'misc/wasm/wasm_exec.js'); + const execCode = fs.readFileSync(execPath, 'utf8'); + eval(execCode); +})(); diff --git a/test/wasm/package-lock.json b/test/wasm/package-lock.json new file mode 100644 index 00000000..f26d55ab --- /dev/null +++ b/test/wasm/package-lock.json @@ -0,0 +1,67 @@ +{ + "name": "wasm", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "dependencies": { + "isomorphic-ws": "^4.0.1", + "sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae", + "ws": "^7.5.2" + } + }, + "node_modules/isomorphic-ws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz", + "integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==", + "peerDependencies": { + "ws": "*" + } + }, + "node_modules/sql.js": { + "version": "1.5.0", + "resolved": "git+ssh://git@github.com/neilalexander/sql.js.git#252a72bf57b0538cbd49bbd6f70af71e516966ae", + "integrity": "sha512-EFYI/yMoQ1U08nZxQOZ7+4S0nOpKF45EVoWGef8L1kvSCMP3B3xSzwZeOmoF2tBVpbMssAgHEz43cf0ZulRDSQ==", + "license": "MIT" + }, + "node_modules/ws": { + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz", + "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + }, + "dependencies": { + "isomorphic-ws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz", + "integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==", + "requires": {} + }, + "sql.js": { + "version": "git+ssh://git@github.com/neilalexander/sql.js.git#252a72bf57b0538cbd49bbd6f70af71e516966ae", + "integrity": "sha512-EFYI/yMoQ1U08nZxQOZ7+4S0nOpKF45EVoWGef8L1kvSCMP3B3xSzwZeOmoF2tBVpbMssAgHEz43cf0ZulRDSQ==", + "from": "sql.js@github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae" + }, + "ws": { + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz", + "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==", + "requires": {} + } + } +} diff --git a/test/wasm/package.json b/test/wasm/package.json new file mode 100644 index 00000000..b28c30b1 --- /dev/null +++ b/test/wasm/package.json @@ -0,0 +1,7 @@ +{ + "dependencies": { + "isomorphic-ws": "^4.0.1", + "sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae", + "ws": "^7.5.2" + } +} diff --git a/userapi/storage/devices/sqlite3/storage.go b/userapi/storage/devices/sqlite3/storage.go index 8afa9fb4..53864483 100644 --- a/userapi/storage/devices/sqlite3/storage.go +++ b/userapi/storage/devices/sqlite3/storage.go @@ -25,8 +25,6 @@ import ( "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3/deltas" "github.com/matrix-org/gomatrixserverlib" - - _ "github.com/mattn/go-sqlite3" ) // The length of generated device IDs