diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml new file mode 100644 index 00000000..0322866d --- /dev/null +++ b/.github/workflows/docker-hub.yml @@ -0,0 +1,71 @@ +# Based on https://github.com/docker/build-push-action + +name: "Docker Hub" + +on: + release: + types: [published] + +env: + DOCKER_NAMESPACE: matrixdotorg + DOCKER_HUB_USER: dendritegithub + PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 + +jobs: + Monolith: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Get release tag + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ env.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build monolith image + id: docker_build_monolith + uses: docker/build-push-action@v2 + with: + context: . + file: ./build/docker/Dockerfile.monolith + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }} + + Polylith: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Get release tag + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ env.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build polylith image + id: docker_build_polylith + uses: docker/build-push-action@v2 + with: + context: . + file: ./build/docker/Dockerfile.polylith + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }} diff --git a/CHANGES.md b/CHANGES.md index ea754a00..a91dea64 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,12 +1,31 @@ # Changelog +## Dendrite 0.3.6 (2021-01-18) + +### Features + +* Experimental support for MSC2946 (Spaces Summary) has been merged +* Send-to-device messages have been refactored and now take advantage of having their own stream position, making delivery more reliable +* Unstable features and MSCs are now listed in `/versions` (contributed by [sumitks866](https://github.com/sumitks866)) +* Well-known and DNS SRV record results for federated servers are now cached properly, improving outbound federation performance and reducing traffic + +### Fixes + +* Updating forward extremities will no longer result in so many unnecessary state snapshots, reducing on-going disk usage in the roomserver database +* Pagination tokens for `/messages` have been fixed, which should improve the reliability of scrollback/pagination +* Dendrite now avoids returning `null`s in fields of the `/sync` response, and omitting some fields altogether when not needed, which should fix sync issues with Element Android +* Requests for user device lists now time out quicker, which prevents federated `/send` requests from also timing out in many cases +* Empty push rules are no longer sent over and over again in `/sync` +* An integer overflow in the device list updater which could result in panics on 32-bit platforms has been fixed (contributed by [Lesterpig](https://github.com/Lesterpig)) +* Event IDs are now logged properly in federation sender and sync API consumer errors + ## Dendrite 0.3.5 (2021-01-11) ### Features * All `/sync` streams are now logically separate after a refactoring exercise -## Fixes +### Fixes * Event references are now deeply checked properly when calculating forward extremities, reducing the amount of forward extremities in most cases, which improves RAM utilisation and reduces the work done by state resolution * Sync no longer sends incorrect `next_batch` tokens with old stream positions, reducing flashbacks of old messages in clients diff --git a/build.sh b/build.sh index e5e7fe8f..a4981408 100755 --- a/build.sh +++ b/build.sh @@ -17,6 +17,8 @@ else export FLAGS="" fi -go install -trimpath -ldflags "$FLAGS" -v $PWD/`dirname $0`/cmd/... +mkdir -p bin -GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs +CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/... + +CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile deleted file mode 100644 index 5cab0530..00000000 --- a/build/docker/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM docker.io/golang:1.15-alpine AS builder - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN sh ./build.sh \ No newline at end of file diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith index 3e9d0cba..eb099c4c 100644 --- a/build/docker/Dockerfile.monolith +++ b/build/docker/Dockerfile.monolith @@ -1,11 +1,20 @@ -FROM matrixdotorg/dendrite:latest AS base +FROM docker.io/golang:1.15-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server +RUN go build -trimpath -o bin/ ./cmd/goose +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys FROM alpine:latest -COPY --from=base /build/bin/dendrite-monolith-server /usr/bin -COPY --from=base /build/bin/goose /usr/bin -COPY --from=base /build/bin/create-account /usr/bin -COPY --from=base /build/bin/generate-keys /usr/bin +COPY --from=base /build/bin/* /usr/bin VOLUME /etc/dendrite WORKDIR /etc/dendrite diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith index dd4cbd38..1a7ba193 100644 --- a/build/docker/Dockerfile.polylith +++ b/build/docker/Dockerfile.polylith @@ -1,11 +1,20 @@ -FROM matrixdotorg/dendrite:latest AS base +FROM docker.io/golang:1.15-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi +RUN go build -trimpath -o bin/ ./cmd/goose +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys FROM alpine:latest -COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin -COPY --from=base /build/bin/goose /usr/bin -COPY --from=base /build/bin/create-account /usr/bin -COPY --from=base /build/bin/generate-keys /usr/bin +COPY --from=base /build/bin/* /usr/bin VOLUME /etc/dendrite WORKDIR /etc/dendrite diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index f80f6bed..eaed5f6d 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -6,7 +6,5 @@ TAG=${1:-latest} echo "Building tag '${TAG}'" -docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} . - docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith . docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith . \ No newline at end of file diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 8dbfc551..7c320253 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -62,13 +62,19 @@ func Setup( rateLimits := newRateLimits(&cfg.RateLimiting) userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg) + unstableFeatures := make(map[string]bool) + for _, msc := range cfg.MSCs.MSCs { + unstableFeatures["org.matrix."+msc] = true + } + publicAPIMux.Handle("/versions", httputil.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse { return util.JSONResponse{ Code: http.StatusOK, JSON: struct { - Versions []string `json:"versions"` - }{[]string{ + Versions []string `json:"versions"` + UnstableFeatures map[string]bool `json:"unstable_features"` + }{Versions: []string{ "r0.0.1", "r0.1.0", "r0.2.0", @@ -76,7 +82,7 @@ func Setup( "r0.4.0", "r0.5.0", "r0.6.1", - }}, + }, UnstableFeatures: unstableFeatures}, } }), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index c7ad1b37..ff0b311a 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -63,7 +63,7 @@ func main() { if *defaultsForCI { cfg.ClientAPI.RateLimiting.Enabled = false cfg.FederationSender.DisableTLSValidation = true - cfg.MSCs.MSCs = []string{"msc2836"} + cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} cfg.Logging[0].Level = "trace" // don't hit matrix.org when running tests!!! cfg.SigningKeyServer.KeyPerspectives = config.KeyPerspectives{} diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 585d466b..978b1800 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -257,7 +257,8 @@ media_api: mscs: # A list of enabled MSC's # Currently valid values are: - # - msc2836 (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) + # - msc2836 (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) + # - msc2946 (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) mscs: [] database: connection_string: file:mscs.db diff --git a/docs/systemd/monolith-example.service b/docs/systemd/monolith-example.service index 7dd7755d..731c6159 100644 --- a/docs/systemd/monolith-example.service +++ b/docs/systemd/monolith-example.service @@ -5,6 +5,7 @@ After=network.target After=postgresql.service [Service] +Environment=GODEBUG=madvdontneed=1 RestartSec=2s Type=simple User=dendrite diff --git a/federationsender/api/api.go b/federationsender/api/api.go index e4d176b1..dfc2dd8a 100644 --- a/federationsender/api/api.go +++ b/federationsender/api/api.go @@ -22,6 +22,7 @@ type FederationClient interface { GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) GetServerKeys(ctx context.Context, matrixServer gomatrixserverlib.ServerName) (gomatrixserverlib.ServerKeys, error) MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) + MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, r gomatrixserverlib.MSC2946SpacesRequest) (res gomatrixserverlib.MSC2946SpacesResponse, err error) LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error) } diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index b53cb251..513919c6 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -102,6 +102,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { default: // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ + "event_id": ev.EventID(), "event": string(ev.JSON()), "add": output.NewRoomEvent.AddsStateEventIDs, "del": output.NewRoomEvent.RemovesStateEventIDs, diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index 407e7ffe..1de774ef 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -244,3 +244,17 @@ func (a *FederationSenderInternalAPI) MSC2836EventRelationships( } return ires.(gomatrixserverlib.MSC2836EventRelationshipsResponse), nil } + +func (a *FederationSenderInternalAPI) MSC2946Spaces( + ctx context.Context, s gomatrixserverlib.ServerName, roomID string, r gomatrixserverlib.MSC2946SpacesRequest, +) (res gomatrixserverlib.MSC2946SpacesResponse, err error) { + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + ires, err := a.doRequest(s, func() (interface{}, error) { + return a.federation.MSC2946Spaces(ctx, s, roomID, r) + }) + if err != nil { + return res, err + } + return ires.(gomatrixserverlib.MSC2946SpacesResponse), nil +} diff --git a/federationsender/inthttp/client.go b/federationsender/inthttp/client.go index fe98ff33..81d3fc51 100644 --- a/federationsender/inthttp/client.go +++ b/federationsender/inthttp/client.go @@ -33,6 +33,7 @@ const ( FederationSenderGetServerKeysPath = "/federationsender/client/getServerKeys" FederationSenderLookupServerKeysPath = "/federationsender/client/lookupServerKeys" FederationSenderEventRelationshipsPath = "/federationsender/client/msc2836eventRelationships" + FederationSenderSpacesSummaryPath = "/federationsender/client/msc2946spacesSummary" ) // NewFederationSenderClient creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API. @@ -449,3 +450,34 @@ func (h *httpFederationSenderInternalAPI) MSC2836EventRelationships( } return response.Res, nil } + +type spacesReq struct { + S gomatrixserverlib.ServerName + Req gomatrixserverlib.MSC2946SpacesRequest + RoomID string + Res gomatrixserverlib.MSC2946SpacesResponse + Err *api.FederationClientError +} + +func (h *httpFederationSenderInternalAPI) MSC2946Spaces( + ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, r gomatrixserverlib.MSC2946SpacesRequest, +) (res gomatrixserverlib.MSC2946SpacesResponse, err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "MSC2946Spaces") + defer span.Finish() + + request := spacesReq{ + S: dst, + Req: r, + RoomID: roomID, + } + var response spacesReq + apiURL := h.federationSenderURL + FederationSenderSpacesSummaryPath + err = httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response) + if err != nil { + return res, err + } + if response.Err != nil { + return res, response.Err + } + return response.Res, nil +} diff --git a/federationsender/inthttp/server.go b/federationsender/inthttp/server.go index 293fb420..be995111 100644 --- a/federationsender/inthttp/server.go +++ b/federationsender/inthttp/server.go @@ -329,4 +329,26 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route return util.JSONResponse{Code: http.StatusOK, JSON: request} }), ) + internalAPIMux.Handle( + FederationSenderSpacesSummaryPath, + httputil.MakeInternalAPI("MSC2946SpacesSummary", func(req *http.Request) util.JSONResponse { + var request spacesReq + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + res, err := intAPI.MSC2946Spaces(req.Context(), request.S, request.RoomID, request.Req) + if err != nil { + ferr, ok := err.(*api.FederationClientError) + if ok { + request.Err = ferr + } else { + request.Err = &api.FederationClientError{ + Err: err.Error(), + } + } + } + request.Res = res + return util.JSONResponse{Code: http.StatusOK, JSON: request} + }), + ) } diff --git a/go.mod b/go.mod index c9438841..58871dfb 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20210113173004-b1c67ac867cc + github.com/matrix-org/gomatrixserverlib v0.0.0-20210119115951-bd57c7cff614 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 @@ -33,16 +33,15 @@ require ( github.com/pressly/goose v2.7.0-rc5+incompatible github.com/prometheus/client_golang v1.7.1 github.com/sirupsen/logrus v1.7.0 - github.com/tidwall/gjson v1.6.3 - github.com/tidwall/match v1.0.2 // indirect - github.com/tidwall/sjson v1.1.2 + github.com/tidwall/gjson v1.6.7 + github.com/tidwall/sjson v1.1.4 github.com/uber/jaeger-client-go v2.25.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20201006093556-760d9a7fd5ee go.uber.org/atomic v1.6.0 - golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9 + golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad golang.org/x/net v0.0.0-20200528225125-3c3fba18258b - golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 // indirect + golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 // indirect gopkg.in/h2non/bimg.v1 v1.1.4 gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index 7accb06e..b79337fa 100644 --- a/go.sum +++ b/go.sum @@ -567,8 +567,12 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210113173004-b1c67ac867cc h1:n2Hnbg8RZ4102Qmxie1riLkIyrqeqShJUILg1miSmDI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210113173004-b1c67ac867cc/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210115150839-9ba5f3e11086 h1:nfGXVXx+cg1iBAWatukPsBe5OKsW+TdmF/qydnt04eg= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210115150839-9ba5f3e11086/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210115152401-7c4619994337 h1:HJ9iH00PwMDaXsH7vWpO7nRucz+d92QLoH0PNW7hs58= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210115152401-7c4619994337/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210119115951-bd57c7cff614 h1:X5FP1YOiGmPfpK4IAc8KyX8lOW4nC81/YZPTbOWAyKs= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210119115951-bd57c7cff614/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= @@ -810,13 +814,12 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= -github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= -github.com/tidwall/gjson v1.6.3 h1:aHoiiem0dr7GHkW001T1SMTJ7X5PvyekH5WX0whWGnI= -github.com/tidwall/gjson v1.6.3/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= +github.com/tidwall/gjson v1.6.7 h1:Mb1M9HZCRWEcXQ8ieJo7auYyyiSux6w9XN3AdTpxJrE= +github.com/tidwall/gjson v1.6.7/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= -github.com/tidwall/match v1.0.2 h1:uuqvHuBGSedK7awZ2YoAtpnimfwBGFjHuWLuLqQj+bU= -github.com/tidwall/match v1.0.2/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= +github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -824,8 +827,8 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8= github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= -github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ= -github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= +github.com/tidwall/sjson v1.1.4 h1:bTSsPLdAYF5QNLSwYsKfBKKTnlGbIuhqL3CpRsjzGhg= +github.com/tidwall/sjson v1.1.4/go.mod h1:wXpKXu8CtDjKAZ+3DrKY5ROCorDFahq8l0tey/Lx1fg= github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= @@ -906,8 +909,8 @@ golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzht golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9 h1:phUcVbl53swtrUN8kQEXFhUxPlIlWyBfKmidCu7P95o= -golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -996,8 +999,8 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 h1:nVuTkr9L6Bq62qpUqKo/RnZCFfzDBL0bYo6w9OJUqZY= +golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/version.go b/internal/version.go index 2a65bfbf..bb6d7038 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 3 - VersionPatch = 5 + VersionPatch = 6 VersionTag = "" // example: "rc1" ) diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index ff57be00..c4950a11 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -245,7 +245,7 @@ func (u *DeviceListUpdater) notifyWorkers(userID string) { } hash := fnv.New32a() _, _ = hash.Write([]byte(remoteServer)) - index := int(hash.Sum32()) % len(u.workerChans) + index := int(int64(hash.Sum32()) % int64(len(u.workerChans))) ch := u.assignChannel(userID) u.workerChans[index] <- remoteServer diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index e5ec8c4d..c9264a27 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -100,7 +100,8 @@ type latestEventsUpdater struct { // The eventID of the event that was processed before this one. lastEventIDSent string // The latest events in the room after processing this event. - latest []types.StateAtEventAndReference + oldLatest []types.StateAtEventAndReference + latest []types.StateAtEventAndReference // The state entries removed from and added to the current state of the // room as a result of processing this event. They are sorted lists. removed []types.StateEntry @@ -123,10 +124,10 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { // state snapshot from somewhere else, e.g. a federated room join, // then start with an empty set - none of the forward extremities // that we knew about before matter anymore. - oldLatest := []types.StateAtEventAndReference{} + u.oldLatest = []types.StateAtEventAndReference{} if !u.rewritesState { u.oldStateNID = u.updater.CurrentStateSnapshotNID() - oldLatest = u.updater.LatestEvents() + u.oldLatest = u.updater.LatestEvents() } // If the event has already been written to the output log then we @@ -140,7 +141,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { // Work out what the latest events are. This will include the new // event if it is not already referenced. extremitiesChanged, err := u.calculateLatest( - oldLatest, u.event, + u.oldLatest, u.event, types.StateAtEventAndReference{ EventReference: u.event.EventReference(), StateAtEvent: u.stateAtEvent, @@ -200,6 +201,37 @@ func (u *latestEventsUpdater) latestState() error { var err error roomState := state.NewStateResolution(u.api.DB, *u.roomInfo) + // Work out if the state at the extremities has actually changed + // or not. If they haven't then we won't bother doing all of the + // hard work. + if u.event.StateKey() == nil { + stateChanged := false + oldStateNIDs := make([]types.StateSnapshotNID, 0, len(u.oldLatest)) + newStateNIDs := make([]types.StateSnapshotNID, 0, len(u.latest)) + for _, old := range u.oldLatest { + oldStateNIDs = append(oldStateNIDs, old.BeforeStateSnapshotNID) + } + for _, new := range u.latest { + newStateNIDs = append(newStateNIDs, new.BeforeStateSnapshotNID) + } + oldStateNIDs = state.UniqueStateSnapshotNIDs(oldStateNIDs) + newStateNIDs = state.UniqueStateSnapshotNIDs(newStateNIDs) + if len(oldStateNIDs) != len(newStateNIDs) { + stateChanged = true + } else { + for i := range oldStateNIDs { + if oldStateNIDs[i] != newStateNIDs[i] { + stateChanged = true + break + } + } + } + if !stateChanged { + u.newStateNID = u.oldStateNID + return nil + } + } + // Get a list of the current latest events. This may or may not // include the new event from the input path, depending on whether // it is a forward extremity or not. diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 87715af4..953276b2 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -116,7 +116,7 @@ func (v StateResolution) LoadCombinedStateAfterEvents( // Deduplicate the IDs before passing them to the database. // There could be duplicates because the events could be state events where // the snapshot of the room state before them was the same. - stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs)) + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, UniqueStateSnapshotNIDs(stateNIDs)) if err != nil { return nil, fmt.Errorf("v.db.StateBlockNIDs: %w", err) } @@ -1103,7 +1103,7 @@ func (s stateNIDSorter) Len() int { return len(s) } func (s stateNIDSorter) Less(i, j int) bool { return s[i] < s[j] } func (s stateNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func uniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID { +func UniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID { return nids[:util.SortAndUnique(stateNIDSorter(nids))] } diff --git a/setup/config/config.go b/setup/config/config.go index b8b12d0c..b9114407 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -344,6 +344,7 @@ func (c *Dendrite) Wiring() { c.ClientAPI.Derived = &c.Derived c.AppServiceAPI.Derived = &c.Derived + c.ClientAPI.MSCs = &c.MSCs } // Error returns a string detailing how many errors were contained within a diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index 52115491..c7cb9c33 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -37,6 +37,8 @@ type ClientAPI struct { // Rate-limiting options RateLimiting RateLimiting `yaml:"rate_limiting"` + + MSCs *MSCs `yaml:"mscs"` } func (c *ClientAPI) Defaults() { diff --git a/setup/config/config_mscs.go b/setup/config/config_mscs.go index 776d0b64..4b53495f 100644 --- a/setup/config/config_mscs.go +++ b/setup/config/config_mscs.go @@ -3,7 +3,7 @@ package config type MSCs struct { Matrix *Global `yaml:"-"` - // The MSCs to enable, currently only `msc2836` is supported. + // The MSCs to enable MSCs []string `yaml:"mscs"` Database DatabaseOptions `yaml:"database"` diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go new file mode 100644 index 00000000..c3a68632 --- /dev/null +++ b/setup/mscs/msc2946/msc2946.go @@ -0,0 +1,607 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package msc2946 'Spaces Summary' implements https://github.com/matrix-org/matrix-doc/pull/2946 +package msc2946 + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/gorilla/mux" + chttputil "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + fs "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal/hooks" + "github.com/matrix-org/dendrite/internal/httputil" + roomserver "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/tidwall/gjson" +) + +const ( + ConstCreateEventContentKey = "org.matrix.msc1772.type" + ConstSpaceChildEventType = "org.matrix.msc1772.space.child" + ConstSpaceParentEventType = "org.matrix.msc1772.space.parent" +) + +// Defaults sets the request defaults +func Defaults(r *gomatrixserverlib.MSC2946SpacesRequest) { + r.Limit = 100 + r.MaxRoomsPerSpace = -1 +} + +// Enable this MSC +func Enable( + base *setup.BaseDendrite, rsAPI roomserver.RoomserverInternalAPI, userAPI userapi.UserInternalAPI, + fsAPI fs.FederationSenderInternalAPI, keyRing gomatrixserverlib.JSONVerifier, +) error { + db, err := NewDatabase(&base.Cfg.MSCs.Database) + if err != nil { + return fmt.Errorf("Cannot enable MSC2946: %w", err) + } + hooks.Enable() + hooks.Attach(hooks.KindNewEventPersisted, func(headeredEvent interface{}) { + he := headeredEvent.(*gomatrixserverlib.HeaderedEvent) + hookErr := db.StoreReference(context.Background(), he) + if hookErr != nil { + util.GetLogger(context.Background()).WithError(hookErr).WithField("event_id", he.EventID()).Error( + "failed to StoreReference", + ) + } + }) + + base.PublicClientAPIMux.Handle("/unstable/rooms/{roomID}/spaces", + httputil.MakeAuthAPI("spaces", userAPI, spacesHandler(db, rsAPI, fsAPI, base.Cfg.Global.ServerName)), + ).Methods(http.MethodPost, http.MethodOptions) + + base.PublicFederationAPIMux.Handle("/unstable/spaces/{roomID}", httputil.MakeExternalAPI( + "msc2946_fed_spaces", func(req *http.Request) util.JSONResponse { + fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest( + req, time.Now(), base.Cfg.Global.ServerName, keyRing, + ) + if fedReq == nil { + return errResp + } + // Extract the room ID from the request. Sanity check request data. + params, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + roomID := params["roomID"] + return federatedSpacesHandler(req.Context(), fedReq, roomID, db, rsAPI, fsAPI, base.Cfg.Global.ServerName) + }, + )).Methods(http.MethodPost, http.MethodOptions) + return nil +} + +func federatedSpacesHandler( + ctx context.Context, fedReq *gomatrixserverlib.FederationRequest, roomID string, db Database, + rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationSenderInternalAPI, + thisServer gomatrixserverlib.ServerName, +) util.JSONResponse { + inMemoryBatchCache := make(map[string]set) + var r gomatrixserverlib.MSC2946SpacesRequest + Defaults(&r) + if err := json.Unmarshal(fedReq.Content(), &r); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()), + } + } + if r.Limit > 100 { + r.Limit = 100 + } + w := walker{ + req: &r, + rootRoomID: roomID, + serverName: fedReq.Origin(), + thisServer: thisServer, + ctx: ctx, + + db: db, + rsAPI: rsAPI, + fsAPI: fsAPI, + inMemoryBatchCache: inMemoryBatchCache, + } + res := w.walk() + return util.JSONResponse{ + Code: 200, + JSON: res, + } +} + +func spacesHandler( + db Database, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationSenderInternalAPI, + thisServer gomatrixserverlib.ServerName, +) func(*http.Request, *userapi.Device) util.JSONResponse { + return func(req *http.Request, device *userapi.Device) util.JSONResponse { + inMemoryBatchCache := make(map[string]set) + // Extract the room ID from the request. Sanity check request data. + params, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + roomID := params["roomID"] + var r gomatrixserverlib.MSC2946SpacesRequest + Defaults(&r) + if resErr := chttputil.UnmarshalJSONRequest(req, &r); resErr != nil { + return *resErr + } + if r.Limit > 100 { + r.Limit = 100 + } + w := walker{ + req: &r, + rootRoomID: roomID, + caller: device, + thisServer: thisServer, + ctx: req.Context(), + + db: db, + rsAPI: rsAPI, + fsAPI: fsAPI, + inMemoryBatchCache: inMemoryBatchCache, + } + res := w.walk() + return util.JSONResponse{ + Code: 200, + JSON: res, + } + } +} + +type walker struct { + req *gomatrixserverlib.MSC2946SpacesRequest + rootRoomID string + caller *userapi.Device + serverName gomatrixserverlib.ServerName + thisServer gomatrixserverlib.ServerName + db Database + rsAPI roomserver.RoomserverInternalAPI + fsAPI fs.FederationSenderInternalAPI + ctx context.Context + + // user ID|device ID|batch_num => event/room IDs sent to client + inMemoryBatchCache map[string]set + mu sync.Mutex +} + +func (w *walker) roomIsExcluded(roomID string) bool { + for _, exclRoom := range w.req.ExcludeRooms { + if exclRoom == roomID { + return true + } + } + return false +} + +func (w *walker) callerID() string { + if w.caller != nil { + return w.caller.UserID + "|" + w.caller.ID + } + return string(w.serverName) +} + +func (w *walker) alreadySent(id string) bool { + w.mu.Lock() + defer w.mu.Unlock() + m, ok := w.inMemoryBatchCache[w.callerID()] + if !ok { + return false + } + return m[id] +} + +func (w *walker) markSent(id string) { + w.mu.Lock() + defer w.mu.Unlock() + m := w.inMemoryBatchCache[w.callerID()] + if m == nil { + m = make(set) + } + m[id] = true + w.inMemoryBatchCache[w.callerID()] = m +} + +// nolint:gocyclo +func (w *walker) walk() *gomatrixserverlib.MSC2946SpacesResponse { + var res gomatrixserverlib.MSC2946SpacesResponse + // Begin walking the graph starting with the room ID in the request in a queue of unvisited rooms + unvisited := []string{w.rootRoomID} + processed := make(set) + for len(unvisited) > 0 { + roomID := unvisited[0] + unvisited = unvisited[1:] + // If this room has already been processed, skip. NB: do not remember this between calls + if processed[roomID] || roomID == "" { + continue + } + // Mark this room as processed. + processed[roomID] = true + + // Is the caller currently joined to the room or is the room `world_readable` + // If no, skip this room. If yes, continue. + if !w.roomExists(roomID) || !w.authorised(roomID) { + // attempt to query this room over federation, as either we've never heard of it before + // or we've left it and hence are not authorised (but info may be exposed regardless) + fedRes, err := w.federatedRoomInfo(roomID) + if err != nil { + util.GetLogger(w.ctx).WithError(err).WithField("room_id", roomID).Errorf("failed to query federated spaces") + continue + } + if fedRes != nil { + res = combineResponses(res, *fedRes) + } + continue + } + // Get all `m.space.child` and `m.space.parent` state events for the room. *In addition*, get + // all `m.space.child` and `m.space.parent` state events which *point to* (via `state_key` or `content.room_id`) + // this room. This requires servers to store reverse lookups. + refs, err := w.references(roomID) + if err != nil { + util.GetLogger(w.ctx).WithError(err).WithField("room_id", roomID).Error("failed to extract references for room") + continue + } + + // If this room has not ever been in `rooms` (across multiple requests), extract the + // `PublicRoomsChunk` for this room. + if !w.alreadySent(roomID) && !w.roomIsExcluded(roomID) { + pubRoom := w.publicRoomsChunk(roomID) + roomType := "" + create := w.stateEvent(roomID, gomatrixserverlib.MRoomCreate, "") + if create != nil { + // escape the `.`s so gjson doesn't think it's nested + roomType = gjson.GetBytes(create.Content(), strings.ReplaceAll(ConstCreateEventContentKey, ".", `\.`)).Str + } + + // Add the total number of events to `PublicRoomsChunk` under `num_refs`. Add `PublicRoomsChunk` to `rooms`. + res.Rooms = append(res.Rooms, gomatrixserverlib.MSC2946Room{ + PublicRoom: *pubRoom, + NumRefs: refs.len(), + RoomType: roomType, + }) + w.markSent(roomID) + } + + uniqueRooms := make(set) + + // If this is the root room from the original request, insert all these events into `events` if + // they haven't been added before (across multiple requests). + if w.rootRoomID == roomID { + for _, ev := range refs.events() { + if !w.alreadySent(ev.EventID()) { + strip := stripped(ev.Event) + if strip == nil { + continue + } + res.Events = append(res.Events, *strip) + uniqueRooms[ev.RoomID()] = true + uniqueRooms[SpaceTarget(ev)] = true + w.markSent(ev.EventID()) + } + } + } else { + // Else add them to `events` honouring the `limit` and `max_rooms_per_space` values. If either + // are exceeded, stop adding events. If the event has already been added, do not add it again. + numAdded := 0 + for _, ev := range refs.events() { + if w.req.Limit > 0 && len(res.Events) >= w.req.Limit { + break + } + if w.req.MaxRoomsPerSpace > 0 && numAdded >= w.req.MaxRoomsPerSpace { + break + } + if w.alreadySent(ev.EventID()) { + continue + } + // Skip the room if it's part of exclude_rooms but ONLY IF the source matches, as we still + // want to catch arrows which point to excluded rooms. + if w.roomIsExcluded(ev.RoomID()) { + continue + } + strip := stripped(ev.Event) + if strip == nil { + continue + } + res.Events = append(res.Events, *strip) + uniqueRooms[ev.RoomID()] = true + uniqueRooms[SpaceTarget(ev)] = true + w.markSent(ev.EventID()) + // we don't distinguish between child state events and parent state events for the purposes of + // max_rooms_per_space, maybe we should? + numAdded++ + } + } + + // For each referenced room ID in the events being returned to the caller (both parent and child) + // add the room ID to the queue of unvisited rooms. Loop from the beginning. + for roomID := range uniqueRooms { + unvisited = append(unvisited, roomID) + } + } + return &res +} + +func (w *walker) stateEvent(roomID, evType, stateKey string) *gomatrixserverlib.HeaderedEvent { + var queryRes roomserver.QueryCurrentStateResponse + tuple := gomatrixserverlib.StateKeyTuple{ + EventType: evType, + StateKey: stateKey, + } + err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: roomID, + StateTuples: []gomatrixserverlib.StateKeyTuple{tuple}, + }, &queryRes) + if err != nil { + return nil + } + return queryRes.StateEvents[tuple] +} + +func (w *walker) publicRoomsChunk(roomID string) *gomatrixserverlib.PublicRoom { + pubRooms, err := roomserver.PopulatePublicRooms(w.ctx, []string{roomID}, w.rsAPI) + if err != nil { + util.GetLogger(w.ctx).WithError(err).Error("failed to PopulatePublicRooms") + return nil + } + if len(pubRooms) == 0 { + return nil + } + return &pubRooms[0] +} + +// federatedRoomInfo returns more of the spaces graph from another server. Returns nil if this was +// unsuccessful. +func (w *walker) federatedRoomInfo(roomID string) (*gomatrixserverlib.MSC2946SpacesResponse, error) { + // only do federated requests for client requests + if w.caller == nil { + return nil, nil + } + // extract events which point to this room ID and extract their vias + events, err := w.db.References(w.ctx, roomID) + if err != nil { + return nil, fmt.Errorf("failed to get References events: %w", err) + } + vias := make(set) + for _, ev := range events { + if ev.StateKeyEquals(roomID) { + // event points at this room, extract vias + content := struct { + Vias []string `json:"via"` + }{} + if err = json.Unmarshal(ev.Content(), &content); err != nil { + continue // silently ignore corrupted state events + } + for _, v := range content.Vias { + vias[v] = true + } + } + } + util.GetLogger(w.ctx).Infof("Querying federatedRoomInfo via %+v", vias) + ctx := context.Background() + // query more of the spaces graph using these servers + for serverName := range vias { + if serverName == string(w.thisServer) { + continue + } + res, err := w.fsAPI.MSC2946Spaces(ctx, gomatrixserverlib.ServerName(serverName), roomID, gomatrixserverlib.MSC2946SpacesRequest{ + Limit: w.req.Limit, + MaxRoomsPerSpace: w.req.MaxRoomsPerSpace, + }) + if err != nil { + util.GetLogger(w.ctx).WithError(err).Warnf("failed to call MSC2946Spaces on server %s", serverName) + continue + } + return &res, nil + } + return nil, nil +} + +func (w *walker) roomExists(roomID string) bool { + var queryRes roomserver.QueryServerJoinedToRoomResponse + err := w.rsAPI.QueryServerJoinedToRoom(w.ctx, &roomserver.QueryServerJoinedToRoomRequest{ + RoomID: roomID, + ServerName: w.thisServer, + }, &queryRes) + if err != nil { + util.GetLogger(w.ctx).WithError(err).Error("failed to QueryServerJoinedToRoom") + return false + } + // if the room exists but we aren't in the room then we might have stale data so we want to fetch + // it fresh via federation + return queryRes.RoomExists && queryRes.IsInRoom +} + +// authorised returns true iff the user is joined this room or the room is world_readable +func (w *walker) authorised(roomID string) bool { + if w.caller != nil { + return w.authorisedUser(roomID) + } + return w.authorisedServer(roomID) +} + +// authorisedServer returns true iff the server is joined this room or the room is world_readable +func (w *walker) authorisedServer(roomID string) bool { + // Check history visibility first + hisVisTuple := gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomHistoryVisibility, + StateKey: "", + } + var queryRoomRes roomserver.QueryCurrentStateResponse + err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: roomID, + StateTuples: []gomatrixserverlib.StateKeyTuple{ + hisVisTuple, + }, + }, &queryRoomRes) + if err != nil { + util.GetLogger(w.ctx).WithError(err).Error("failed to QueryCurrentState") + return false + } + hisVisEv := queryRoomRes.StateEvents[hisVisTuple] + if hisVisEv != nil { + hisVis, _ := hisVisEv.HistoryVisibility() + if hisVis == "world_readable" { + return true + } + } + // check if server is joined to the room + var queryRes fs.QueryJoinedHostServerNamesInRoomResponse + err = w.fsAPI.QueryJoinedHostServerNamesInRoom(w.ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ + RoomID: roomID, + }, &queryRes) + if err != nil { + util.GetLogger(w.ctx).WithError(err).Error("failed to QueryJoinedHostServerNamesInRoom") + return false + } + for _, srv := range queryRes.ServerNames { + if srv == w.serverName { + return true + } + } + return false +} + +// authorisedUser returns true iff the user is joined this room or the room is world_readable +func (w *walker) authorisedUser(roomID string) bool { + hisVisTuple := gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomHistoryVisibility, + StateKey: "", + } + roomMemberTuple := gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomMember, + StateKey: w.caller.UserID, + } + var queryRes roomserver.QueryCurrentStateResponse + err := w.rsAPI.QueryCurrentState(w.ctx, &roomserver.QueryCurrentStateRequest{ + RoomID: roomID, + StateTuples: []gomatrixserverlib.StateKeyTuple{ + hisVisTuple, roomMemberTuple, + }, + }, &queryRes) + if err != nil { + util.GetLogger(w.ctx).WithError(err).Error("failed to QueryCurrentState") + return false + } + memberEv := queryRes.StateEvents[roomMemberTuple] + hisVisEv := queryRes.StateEvents[hisVisTuple] + if memberEv != nil { + membership, _ := memberEv.Membership() + if membership == gomatrixserverlib.Join { + return true + } + } + if hisVisEv != nil { + hisVis, _ := hisVisEv.HistoryVisibility() + if hisVis == "world_readable" { + return true + } + } + return false +} + +// references returns all references pointing to or from this room. +func (w *walker) references(roomID string) (eventLookup, error) { + events, err := w.db.References(w.ctx, roomID) + if err != nil { + return nil, err + } + el := make(eventLookup) + for _, ev := range events { + // only return events that have a `via` key as per MSC1772 + // else we'll incorrectly walk redacted events (as the link + // is in the state_key) + if gjson.GetBytes(ev.Content(), "via").Exists() { + el.set(ev) + } + } + return el, nil +} + +// state event lookup across multiple rooms keyed on event type +// NOT THREAD SAFE +type eventLookup map[string][]*gomatrixserverlib.HeaderedEvent + +func (el eventLookup) set(ev *gomatrixserverlib.HeaderedEvent) { + evs := el[ev.Type()] + if evs == nil { + evs = make([]*gomatrixserverlib.HeaderedEvent, 0) + } + evs = append(evs, ev) + el[ev.Type()] = evs +} + +func (el eventLookup) len() int { + sum := 0 + for _, evs := range el { + sum += len(evs) + } + return sum +} + +func (el eventLookup) events() (events []*gomatrixserverlib.HeaderedEvent) { + for _, evs := range el { + events = append(events, evs...) + } + return +} + +type set map[string]bool + +func stripped(ev *gomatrixserverlib.Event) *gomatrixserverlib.MSC2946StrippedEvent { + if ev.StateKey() == nil { + return nil + } + return &gomatrixserverlib.MSC2946StrippedEvent{ + Type: ev.Type(), + StateKey: *ev.StateKey(), + Content: ev.Content(), + Sender: ev.Sender(), + RoomID: ev.RoomID(), + } +} + +func combineResponses(local, remote gomatrixserverlib.MSC2946SpacesResponse) gomatrixserverlib.MSC2946SpacesResponse { + knownRooms := make(set) + for _, room := range local.Rooms { + knownRooms[room.RoomID] = true + } + knownEvents := make(set) + for _, event := range local.Events { + knownEvents[event.RoomID+event.Type+event.StateKey] = true + } + // mux in remote entries if and only if they aren't present already + for _, room := range remote.Rooms { + if knownRooms[room.RoomID] { + continue + } + local.Rooms = append(local.Rooms, room) + } + for _, event := range remote.Events { + if knownEvents[event.RoomID+event.Type+event.StateKey] { + continue + } + local.Events = append(local.Events, event) + } + return local +} diff --git a/setup/mscs/msc2946/msc2946_test.go b/setup/mscs/msc2946/msc2946_test.go new file mode 100644 index 00000000..4f180a98 --- /dev/null +++ b/setup/mscs/msc2946/msc2946_test.go @@ -0,0 +1,503 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package msc2946_test + +import ( + "bytes" + "context" + "crypto/ed25519" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/internal/hooks" + "github.com/matrix-org/dendrite/internal/httputil" + roomserver "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/mscs/msc2946" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" +) + +var ( + client = &http.Client{ + Timeout: 10 * time.Second, + } + roomVer = gomatrixserverlib.RoomVersionV6 +) + +// Basic sanity check of MSC2946 logic. Tests a single room with a few state events +// and a bit of recursion to subspaces. Makes a graph like: +// Root +// ____|_____ +// | | | +// R1 R2 S1 +// |_________ +// | | | +// R3 R4 S2 +// | <-- this link is just a parent, not a child +// R5 +// +// Alice is not joined to R4, but R4 is "world_readable". +func TestMSC2946(t *testing.T) { + alice := "@alice:localhost" + // give access token to alice + nopUserAPI := &testUserAPI{ + accessTokens: make(map[string]userapi.Device), + } + nopUserAPI.accessTokens["alice"] = userapi.Device{ + AccessToken: "alice", + DisplayName: "Alice", + UserID: alice, + } + rootSpace := "!rootspace:localhost" + subSpaceS1 := "!subspaceS1:localhost" + subSpaceS2 := "!subspaceS2:localhost" + room1 := "!room1:localhost" + room2 := "!room2:localhost" + room3 := "!room3:localhost" + room4 := "!room4:localhost" + empty := "" + room5 := "!room5:localhost" + allRooms := []string{ + rootSpace, subSpaceS1, subSpaceS2, + room1, room2, room3, room4, room5, + } + rootToR1 := mustCreateEvent(t, fledglingEvent{ + RoomID: rootSpace, + Sender: alice, + Type: msc2946.ConstSpaceChildEventType, + StateKey: &room1, + Content: map[string]interface{}{ + "via": []string{"localhost"}, + }, + }) + rootToR2 := mustCreateEvent(t, fledglingEvent{ + RoomID: rootSpace, + Sender: alice, + Type: msc2946.ConstSpaceChildEventType, + StateKey: &room2, + Content: map[string]interface{}{ + "via": []string{"localhost"}, + }, + }) + rootToS1 := mustCreateEvent(t, fledglingEvent{ + RoomID: rootSpace, + Sender: alice, + Type: msc2946.ConstSpaceChildEventType, + StateKey: &subSpaceS1, + Content: map[string]interface{}{ + "via": []string{"localhost"}, + }, + }) + s1ToR3 := mustCreateEvent(t, fledglingEvent{ + RoomID: subSpaceS1, + Sender: alice, + Type: msc2946.ConstSpaceChildEventType, + StateKey: &room3, + Content: map[string]interface{}{ + "via": []string{"localhost"}, + }, + }) + s1ToR4 := mustCreateEvent(t, fledglingEvent{ + RoomID: subSpaceS1, + Sender: alice, + Type: msc2946.ConstSpaceChildEventType, + StateKey: &room4, + Content: map[string]interface{}{ + "via": []string{"localhost"}, + }, + }) + s1ToS2 := mustCreateEvent(t, fledglingEvent{ + RoomID: subSpaceS1, + Sender: alice, + Type: msc2946.ConstSpaceChildEventType, + StateKey: &subSpaceS2, + Content: map[string]interface{}{ + "via": []string{"localhost"}, + }, + }) + // This is a parent link only + s2ToR5 := mustCreateEvent(t, fledglingEvent{ + RoomID: room5, + Sender: alice, + Type: msc2946.ConstSpaceParentEventType, + StateKey: &subSpaceS2, + Content: map[string]interface{}{ + "via": []string{"localhost"}, + }, + }) + // history visibility for R4 + r4HisVis := mustCreateEvent(t, fledglingEvent{ + RoomID: room4, + Sender: "@someone:localhost", + Type: gomatrixserverlib.MRoomHistoryVisibility, + StateKey: &empty, + Content: map[string]interface{}{ + "history_visibility": "world_readable", + }, + }) + var joinEvents []*gomatrixserverlib.HeaderedEvent + for _, roomID := range allRooms { + if roomID == room4 { + continue // not joined to that room + } + joinEvents = append(joinEvents, mustCreateEvent(t, fledglingEvent{ + RoomID: roomID, + Sender: alice, + StateKey: &alice, + Type: gomatrixserverlib.MRoomMember, + Content: map[string]interface{}{ + "membership": "join", + }, + })) + } + roomNameTuple := gomatrixserverlib.StateKeyTuple{ + EventType: "m.room.name", + StateKey: "", + } + hisVisTuple := gomatrixserverlib.StateKeyTuple{ + EventType: "m.room.history_visibility", + StateKey: "", + } + nopRsAPI := &testRoomserverAPI{ + joinEvents: joinEvents, + events: map[string]*gomatrixserverlib.HeaderedEvent{ + rootToR1.EventID(): rootToR1, + rootToR2.EventID(): rootToR2, + rootToS1.EventID(): rootToS1, + s1ToR3.EventID(): s1ToR3, + s1ToR4.EventID(): s1ToR4, + s1ToS2.EventID(): s1ToS2, + s2ToR5.EventID(): s2ToR5, + r4HisVis.EventID(): r4HisVis, + }, + pubRoomState: map[string]map[gomatrixserverlib.StateKeyTuple]string{ + rootSpace: { + roomNameTuple: "Root", + hisVisTuple: "shared", + }, + subSpaceS1: { + roomNameTuple: "Sub-Space 1", + hisVisTuple: "joined", + }, + subSpaceS2: { + roomNameTuple: "Sub-Space 2", + hisVisTuple: "shared", + }, + room1: { + hisVisTuple: "joined", + }, + room2: { + hisVisTuple: "joined", + }, + room3: { + hisVisTuple: "joined", + }, + room4: { + hisVisTuple: "world_readable", + }, + room5: { + hisVisTuple: "joined", + }, + }, + } + allEvents := []*gomatrixserverlib.HeaderedEvent{ + rootToR1, rootToR2, rootToS1, + s1ToR3, s1ToR4, s1ToS2, + s2ToR5, r4HisVis, + } + allEvents = append(allEvents, joinEvents...) + router := injectEvents(t, nopUserAPI, nopRsAPI, allEvents) + cancel := runServer(t, router) + defer cancel() + + t.Run("returns no events for unknown rooms", func(t *testing.T) { + res := postSpaces(t, 200, "alice", "!unknown:localhost", newReq(t, map[string]interface{}{})) + if len(res.Events) > 0 { + t.Errorf("got %d events, want 0", len(res.Events)) + } + if len(res.Rooms) > 0 { + t.Errorf("got %d rooms, want 0", len(res.Rooms)) + } + }) + t.Run("returns the entire graph", func(t *testing.T) { + res := postSpaces(t, 200, "alice", rootSpace, newReq(t, map[string]interface{}{})) + if len(res.Events) != 7 { + t.Errorf("got %d events, want 7", len(res.Events)) + } + if len(res.Rooms) != len(allRooms) { + t.Errorf("got %d rooms, want %d", len(res.Rooms), len(allRooms)) + } + }) + t.Run("can update the graph", func(t *testing.T) { + // remove R3 from the graph + rmS1ToR3 := mustCreateEvent(t, fledglingEvent{ + RoomID: subSpaceS1, + Sender: alice, + Type: msc2946.ConstSpaceChildEventType, + StateKey: &room3, + Content: map[string]interface{}{}, // redacted + }) + nopRsAPI.events[rmS1ToR3.EventID()] = rmS1ToR3 + hooks.Run(hooks.KindNewEventPersisted, rmS1ToR3) + + res := postSpaces(t, 200, "alice", rootSpace, newReq(t, map[string]interface{}{})) + if len(res.Events) != 6 { // one less since we don't return redacted events + t.Errorf("got %d events, want 6", len(res.Events)) + } + if len(res.Rooms) != (len(allRooms) - 1) { // one less due to lack of R3 + t.Errorf("got %d rooms, want %d", len(res.Rooms), len(allRooms)-1) + } + }) +} + +func newReq(t *testing.T, jsonBody map[string]interface{}) *gomatrixserverlib.MSC2946SpacesRequest { + t.Helper() + b, err := json.Marshal(jsonBody) + if err != nil { + t.Fatalf("Failed to marshal request: %s", err) + } + var r gomatrixserverlib.MSC2946SpacesRequest + if err := json.Unmarshal(b, &r); err != nil { + t.Fatalf("Failed to unmarshal request: %s", err) + } + return &r +} + +func runServer(t *testing.T, router *mux.Router) func() { + t.Helper() + externalServ := &http.Server{ + Addr: string(":8010"), + WriteTimeout: 60 * time.Second, + Handler: router, + } + go func() { + externalServ.ListenAndServe() + }() + // wait to listen on the port + time.Sleep(500 * time.Millisecond) + return func() { + externalServ.Shutdown(context.TODO()) + } +} + +func postSpaces(t *testing.T, expectCode int, accessToken, roomID string, req *gomatrixserverlib.MSC2946SpacesRequest) *gomatrixserverlib.MSC2946SpacesResponse { + t.Helper() + var r gomatrixserverlib.MSC2946SpacesRequest + msc2946.Defaults(&r) + data, err := json.Marshal(req) + if err != nil { + t.Fatalf("failed to marshal request: %s", err) + } + httpReq, err := http.NewRequest( + "POST", "http://localhost:8010/_matrix/client/unstable/rooms/"+url.PathEscape(roomID)+"/spaces", + bytes.NewBuffer(data), + ) + httpReq.Header.Set("Authorization", "Bearer "+accessToken) + if err != nil { + t.Fatalf("failed to prepare request: %s", err) + } + res, err := client.Do(httpReq) + if err != nil { + t.Fatalf("failed to do request: %s", err) + } + if res.StatusCode != expectCode { + body, _ := ioutil.ReadAll(res.Body) + t.Fatalf("wrong response code, got %d want %d - body: %s", res.StatusCode, expectCode, string(body)) + } + if res.StatusCode == 200 { + var result gomatrixserverlib.MSC2946SpacesResponse + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("response 200 OK but failed to read response body: %s", err) + } + t.Logf("Body: %s", string(body)) + if err := json.Unmarshal(body, &result); err != nil { + t.Fatalf("response 200 OK but failed to deserialise JSON : %s\nbody: %s", err, string(body)) + } + return &result + } + return nil +} + +type testUserAPI struct { + accessTokens map[string]userapi.Device +} + +func (u *testUserAPI) InputAccountData(ctx context.Context, req *userapi.InputAccountDataRequest, res *userapi.InputAccountDataResponse) error { + return nil +} +func (u *testUserAPI) PerformAccountCreation(ctx context.Context, req *userapi.PerformAccountCreationRequest, res *userapi.PerformAccountCreationResponse) error { + return nil +} +func (u *testUserAPI) PerformPasswordUpdate(ctx context.Context, req *userapi.PerformPasswordUpdateRequest, res *userapi.PerformPasswordUpdateResponse) error { + return nil +} +func (u *testUserAPI) PerformDeviceCreation(ctx context.Context, req *userapi.PerformDeviceCreationRequest, res *userapi.PerformDeviceCreationResponse) error { + return nil +} +func (u *testUserAPI) PerformDeviceDeletion(ctx context.Context, req *userapi.PerformDeviceDeletionRequest, res *userapi.PerformDeviceDeletionResponse) error { + return nil +} +func (u *testUserAPI) PerformDeviceUpdate(ctx context.Context, req *userapi.PerformDeviceUpdateRequest, res *userapi.PerformDeviceUpdateResponse) error { + return nil +} +func (u *testUserAPI) PerformLastSeenUpdate(ctx context.Context, req *userapi.PerformLastSeenUpdateRequest, res *userapi.PerformLastSeenUpdateResponse) error { + return nil +} +func (u *testUserAPI) PerformAccountDeactivation(ctx context.Context, req *userapi.PerformAccountDeactivationRequest, res *userapi.PerformAccountDeactivationResponse) error { + return nil +} +func (u *testUserAPI) QueryProfile(ctx context.Context, req *userapi.QueryProfileRequest, res *userapi.QueryProfileResponse) error { + return nil +} +func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error { + dev, ok := u.accessTokens[req.AccessToken] + if !ok { + res.Err = fmt.Errorf("unknown token") + return nil + } + res.Device = &dev + return nil +} +func (u *testUserAPI) QueryDevices(ctx context.Context, req *userapi.QueryDevicesRequest, res *userapi.QueryDevicesResponse) error { + return nil +} +func (u *testUserAPI) QueryAccountData(ctx context.Context, req *userapi.QueryAccountDataRequest, res *userapi.QueryAccountDataResponse) error { + return nil +} +func (u *testUserAPI) QueryDeviceInfos(ctx context.Context, req *userapi.QueryDeviceInfosRequest, res *userapi.QueryDeviceInfosResponse) error { + return nil +} +func (u *testUserAPI) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error { + return nil +} + +type testRoomserverAPI struct { + // use a trace API as it implements method stubs so we don't need to have them here. + // We'll override the functions we care about. + roomserver.RoomserverInternalAPITrace + joinEvents []*gomatrixserverlib.HeaderedEvent + events map[string]*gomatrixserverlib.HeaderedEvent + pubRoomState map[string]map[gomatrixserverlib.StateKeyTuple]string +} + +func (r *testRoomserverAPI) QueryServerJoinedToRoom(ctx context.Context, req *roomserver.QueryServerJoinedToRoomRequest, res *roomserver.QueryServerJoinedToRoomResponse) error { + res.IsInRoom = true + res.RoomExists = true + return nil +} + +func (r *testRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *roomserver.QueryBulkStateContentRequest, res *roomserver.QueryBulkStateContentResponse) error { + res.Rooms = make(map[string]map[gomatrixserverlib.StateKeyTuple]string) + for _, roomID := range req.RoomIDs { + pubRoomData, ok := r.pubRoomState[roomID] + if ok { + res.Rooms[roomID] = pubRoomData + } + } + return nil +} + +func (r *testRoomserverAPI) QueryCurrentState(ctx context.Context, req *roomserver.QueryCurrentStateRequest, res *roomserver.QueryCurrentStateResponse) error { + res.StateEvents = make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent) + checkEvent := func(he *gomatrixserverlib.HeaderedEvent) { + if he.RoomID() != req.RoomID { + return + } + if he.StateKey() == nil { + return + } + tuple := gomatrixserverlib.StateKeyTuple{ + EventType: he.Type(), + StateKey: *he.StateKey(), + } + for _, t := range req.StateTuples { + if t == tuple { + res.StateEvents[t] = he + } + } + } + for _, he := range r.joinEvents { + checkEvent(he) + } + for _, he := range r.events { + checkEvent(he) + } + return nil +} + +func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserver.RoomserverInternalAPI, events []*gomatrixserverlib.HeaderedEvent) *mux.Router { + t.Helper() + cfg := &config.Dendrite{} + cfg.Defaults() + cfg.Global.ServerName = "localhost" + cfg.MSCs.Database.ConnectionString = "file:msc2946_test.db" + cfg.MSCs.MSCs = []string{"msc2946"} + base := &setup.BaseDendrite{ + Cfg: cfg, + PublicClientAPIMux: mux.NewRouter().PathPrefix(httputil.PublicClientPathPrefix).Subrouter(), + PublicFederationAPIMux: mux.NewRouter().PathPrefix(httputil.PublicFederationPathPrefix).Subrouter(), + } + + err := msc2946.Enable(base, rsAPI, userAPI, nil, nil) + if err != nil { + t.Fatalf("failed to enable MSC2946: %s", err) + } + for _, ev := range events { + hooks.Run(hooks.KindNewEventPersisted, ev) + } + return base.PublicClientAPIMux +} + +type fledglingEvent struct { + Type string + StateKey *string + Content interface{} + Sender string + RoomID string +} + +func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *gomatrixserverlib.HeaderedEvent) { + t.Helper() + seed := make([]byte, ed25519.SeedSize) // zero seed + key := ed25519.NewKeyFromSeed(seed) + eb := gomatrixserverlib.EventBuilder{ + Sender: ev.Sender, + Depth: 999, + Type: ev.Type, + StateKey: ev.StateKey, + RoomID: ev.RoomID, + } + err := eb.SetContent(ev.Content) + if err != nil { + t.Fatalf("mustCreateEvent: failed to marshal event content %+v", ev.Content) + } + // make sure the origin_server_ts changes so we can test recency + time.Sleep(1 * time.Millisecond) + signedEvent, err := eb.Build(time.Now(), gomatrixserverlib.ServerName("localhost"), "ed25519:test", key, roomVer) + if err != nil { + t.Fatalf("mustCreateEvent: failed to sign event: %s", err) + } + h := signedEvent.Headered(roomVer) + return h +} diff --git a/setup/mscs/msc2946/storage.go b/setup/mscs/msc2946/storage.go new file mode 100644 index 00000000..20db1859 --- /dev/null +++ b/setup/mscs/msc2946/storage.go @@ -0,0 +1,182 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package msc2946 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" +) + +var ( + relTypes = map[string]int{ + ConstSpaceChildEventType: 1, + ConstSpaceParentEventType: 2, + } +) + +type Database interface { + // StoreReference persists a child or parent space mapping. + StoreReference(ctx context.Context, he *gomatrixserverlib.HeaderedEvent) error + // References returns all events which have the given roomID as a parent or child space. + References(ctx context.Context, roomID string) ([]*gomatrixserverlib.HeaderedEvent, error) +} + +type DB struct { + db *sql.DB + writer sqlutil.Writer + insertEdgeStmt *sql.Stmt + selectEdgesStmt *sql.Stmt +} + +// NewDatabase loads the database for msc2836 +func NewDatabase(dbOpts *config.DatabaseOptions) (Database, error) { + if dbOpts.ConnectionString.IsPostgres() { + return newPostgresDatabase(dbOpts) + } + return newSQLiteDatabase(dbOpts) +} + +func newPostgresDatabase(dbOpts *config.DatabaseOptions) (Database, error) { + d := DB{ + writer: sqlutil.NewDummyWriter(), + } + var err error + if d.db, err = sqlutil.Open(dbOpts); err != nil { + return nil, err + } + _, err = d.db.Exec(` + CREATE TABLE IF NOT EXISTS msc2946_edges ( + room_version TEXT NOT NULL, + -- the room ID of the event, the source of the arrow + source_room_id TEXT NOT NULL, + -- the target room ID, the arrow destination + dest_room_id TEXT NOT NULL, + -- the kind of relation, either child or parent (1,2) + rel_type SMALLINT NOT NULL, + event_json TEXT NOT NULL, + CONSTRAINT msc2946_edges_uniq UNIQUE (source_room_id, dest_room_id, rel_type) + ); + `) + if err != nil { + return nil, err + } + if d.insertEdgeStmt, err = d.db.Prepare(` + INSERT INTO msc2946_edges(room_version, source_room_id, dest_room_id, rel_type, event_json) + VALUES($1, $2, $3, $4, $5) + ON CONFLICT ON CONSTRAINT msc2946_edges_uniq DO UPDATE SET event_json = $5 + `); err != nil { + return nil, err + } + if d.selectEdgesStmt, err = d.db.Prepare(` + SELECT room_version, event_json FROM msc2946_edges + WHERE source_room_id = $1 OR dest_room_id = $2 + `); err != nil { + return nil, err + } + return &d, err +} + +func newSQLiteDatabase(dbOpts *config.DatabaseOptions) (Database, error) { + d := DB{ + writer: sqlutil.NewExclusiveWriter(), + } + var err error + if d.db, err = sqlutil.Open(dbOpts); err != nil { + return nil, err + } + _, err = d.db.Exec(` + CREATE TABLE IF NOT EXISTS msc2946_edges ( + room_version TEXT NOT NULL, + -- the room ID of the event, the source of the arrow + source_room_id TEXT NOT NULL, + -- the target room ID, the arrow destination + dest_room_id TEXT NOT NULL, + -- the kind of relation, either child or parent (1,2) + rel_type SMALLINT NOT NULL, + event_json TEXT NOT NULL, + UNIQUE (source_room_id, dest_room_id, rel_type) + ); + `) + if err != nil { + return nil, err + } + if d.insertEdgeStmt, err = d.db.Prepare(` + INSERT INTO msc2946_edges(room_version, source_room_id, dest_room_id, rel_type, event_json) + VALUES($1, $2, $3, $4, $5) + ON CONFLICT (source_room_id, dest_room_id, rel_type) DO UPDATE SET event_json = $5 + `); err != nil { + return nil, err + } + if d.selectEdgesStmt, err = d.db.Prepare(` + SELECT room_version, event_json FROM msc2946_edges + WHERE source_room_id = $1 OR dest_room_id = $2 + `); err != nil { + return nil, err + } + return &d, err +} + +func (d *DB) StoreReference(ctx context.Context, he *gomatrixserverlib.HeaderedEvent) error { + target := SpaceTarget(he) + if target == "" { + return nil // malformed event + } + relType := relTypes[he.Type()] + _, err := d.insertEdgeStmt.ExecContext(ctx, he.RoomVersion, he.RoomID(), target, relType, he.JSON()) + return err +} + +func (d *DB) References(ctx context.Context, roomID string) ([]*gomatrixserverlib.HeaderedEvent, error) { + rows, err := d.selectEdgesStmt.QueryContext(ctx, roomID, roomID) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "failed to close References") + refs := make([]*gomatrixserverlib.HeaderedEvent, 0) + for rows.Next() { + var roomVer string + var jsonBytes []byte + if err := rows.Scan(&roomVer, &jsonBytes); err != nil { + return nil, err + } + ev, err := gomatrixserverlib.NewEventFromTrustedJSON(jsonBytes, false, gomatrixserverlib.RoomVersion(roomVer)) + if err != nil { + return nil, err + } + he := ev.Headered(gomatrixserverlib.RoomVersion(roomVer)) + refs = append(refs, he) + } + return refs, nil +} + +// SpaceTarget returns the destination room ID for the space event. This is either a child or a parent +// depending on the event type. +func SpaceTarget(he *gomatrixserverlib.HeaderedEvent) string { + if he.StateKey() == nil { + return "" // no-op + } + switch he.Type() { + case ConstSpaceParentEventType: + return *he.StateKey() + case ConstSpaceChildEventType: + return *he.StateKey() + } + return "" +} diff --git a/setup/mscs/mscs.go b/setup/mscs/mscs.go index a8e5668e..027885c8 100644 --- a/setup/mscs/mscs.go +++ b/setup/mscs/mscs.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/mscs/msc2836" + "github.com/matrix-org/dendrite/setup/mscs/msc2946" "github.com/matrix-org/util" ) @@ -39,6 +40,8 @@ func EnableMSC(base *setup.BaseDendrite, monolith *setup.Monolith, msc string) e switch msc { case "msc2836": return msc2836.Enable(base, monolith.RoomserverAPI, monolith.FederationSenderAPI, monolith.UserAPI, monolith.KeyRing) + case "msc2946": + return msc2946.Enable(base, monolith.RoomserverAPI, monolith.UserAPI, monolith.FederationSenderAPI, monolith.KeyRing) default: return fmt.Errorf("EnableMSC: unknown msc '%s'", msc) } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 1d47b73a..a8cc5f71 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -173,6 +173,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ + "event_id": ev.EventID(), "event": string(ev.JSON()), log.ErrorKey: err, "add": msg.AddsStateEventIDs, @@ -215,6 +216,7 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent( if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ + "event_id": ev.EventID(), "event": string(ev.JSON()), log.ErrorKey: err, }).Panicf("roomserver output log: write old event failure") @@ -276,6 +278,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ + "event_id": msg.Event.EventID(), "event": string(msg.Event.JSON()), "pdupos": pduPos, log.ErrorKey: err, diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index e294c880..ba739148 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -235,12 +235,15 @@ func (r *messagesReq) retrieveEvents() ( clientEvents []gomatrixserverlib.ClientEvent, start, end types.TopologyToken, err error, ) { + eventFilter := gomatrixserverlib.DefaultRoomEventFilter() + eventFilter.Limit = r.limit + // Retrieve the events from the local database. var streamEvents []types.StreamEvent if r.fromStream != nil { toStream := r.to.StreamToken() streamEvents, err = r.db.GetEventsInStreamingRange( - r.ctx, r.fromStream, &toStream, r.roomID, r.limit, r.backwardOrdering, + r.ctx, r.fromStream, &toStream, r.roomID, &eventFilter, r.backwardOrdering, ) } else { streamEvents, err = r.db.GetEventsInTopologicalRange( diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index a51ab4e0..22d80161 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -40,7 +40,7 @@ type Database interface { GetStateDeltas(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error) RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) - RecentEvents(ctx context.Context, roomID string, r types.Range, limit int, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) + RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) GetBackwardTopologyPos(ctx context.Context, events []types.StreamEvent) (types.TopologyToken, error) PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error) @@ -105,7 +105,7 @@ type Database interface { // Returns an error if there was a problem communicating with the database. DeletePeeks(ctx context.Context, RoomID, UserID string) (types.StreamPosition, error) // GetEventsInStreamingRange retrieves all of the events on a given ordering using the given extremities and limit. - GetEventsInStreamingRange(ctx context.Context, from, to *types.StreamingToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) + GetEventsInStreamingRange(ctx context.Context, from, to *types.StreamingToken, roomID string, eventFilter *gomatrixserverlib.RoomEventFilter, backwardOrdering bool) (events []types.StreamEvent, err error) // GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) // EventPositionInTopology returns the depth and stream position of the given event. diff --git a/syncapi/storage/postgres/filter_table.go b/syncapi/storage/postgres/filter_table.go index beeb864b..dfd3d696 100644 --- a/syncapi/storage/postgres/filter_table.go +++ b/syncapi/storage/postgres/filter_table.go @@ -83,7 +83,7 @@ func (s *filterStatements) SelectFilter( } // Unmarshal JSON into Filter struct - var filter gomatrixserverlib.Filter + filter := gomatrixserverlib.DefaultFilter() if err = json.Unmarshal(filterData, &filter); err != nil { return nil, err } diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index f4bbebd2..28668de0 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -84,17 +84,29 @@ const selectEventsSQL = "" + const selectRecentEventsSQL = "" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + - " ORDER BY id DESC LIMIT $4" + " AND ( $4::text[] IS NULL OR sender = ANY($4) )" + + " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + + " AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" + + " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + + " ORDER BY id DESC LIMIT $8" const selectRecentEventsForSyncSQL = "" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + - " ORDER BY id DESC LIMIT $4" + " AND ( $4::text[] IS NULL OR sender = ANY($4) )" + + " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + + " AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" + + " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + + " ORDER BY id DESC LIMIT $8" const selectEarlyEventsSQL = "" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + - " ORDER BY id ASC LIMIT $4" + " AND ( $4::text[] IS NULL OR sender = ANY($4) )" + + " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + + " AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" + + " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + + " ORDER BY id ASC LIMIT $8" const selectMaxEventIDSQL = "" + "SELECT MAX(id) FROM syncapi_output_room_events" @@ -322,7 +334,7 @@ func (s *outputRoomEventsStatements) InsertEvent( // from sync. func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, - roomID string, r types.Range, limit int, + roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool, ) ([]types.StreamEvent, bool, error) { var stmt *sql.Stmt @@ -331,7 +343,14 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( } else { stmt = sqlutil.TxStmt(txn, s.selectRecentEventsStmt) } - rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit+1) + rows, err := stmt.QueryContext( + ctx, roomID, r.Low(), r.High(), + pq.StringArray(eventFilter.Senders), + pq.StringArray(eventFilter.NotSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(eventFilter.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(eventFilter.NotTypes)), + eventFilter.Limit+1, + ) if err != nil { return nil, false, err } @@ -350,7 +369,7 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( } // we queried for 1 more than the limit, so if we returned one more mark limited=true limited := false - if len(events) > limit { + if len(events) > eventFilter.Limit { limited = true // re-slice the extra (oldest) event out: in chronological order this is the first entry, else the last. if chronologicalOrder { @@ -367,10 +386,17 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( // from a given position, up to a maximum of 'limit'. func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, - roomID string, r types.Range, limit int, + roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, ) ([]types.StreamEvent, error) { stmt := sqlutil.TxStmt(txn, s.selectEarlyEventsStmt) - rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) + rows, err := stmt.QueryContext( + ctx, roomID, r.Low(), r.High(), + pq.StringArray(eventFilter.Senders), + pq.StringArray(eventFilter.NotSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(eventFilter.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(eventFilter.NotTypes)), + eventFilter.Limit, + ) if err != nil { return nil, err } diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 5b06aabc..9df07693 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -110,8 +110,8 @@ func (d *Database) RoomIDsWithMembership(ctx context.Context, userID string, mem return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, membership) } -func (d *Database) RecentEvents(ctx context.Context, roomID string, r types.Range, limit int, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) { - return d.OutputEvents.SelectRecentEvents(ctx, nil, roomID, r, limit, chronologicalOrder, onlySyncEvents) +func (d *Database) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) { + return d.OutputEvents.SelectRecentEvents(ctx, nil, roomID, r, eventFilter, chronologicalOrder, onlySyncEvents) } func (d *Database) PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error) { @@ -151,7 +151,7 @@ func (d *Database) Events(ctx context.Context, eventIDs []string) ([]*gomatrixse func (d *Database) GetEventsInStreamingRange( ctx context.Context, from, to *types.StreamingToken, - roomID string, limit int, + roomID string, eventFilter *gomatrixserverlib.RoomEventFilter, backwardOrdering bool, ) (events []types.StreamEvent, err error) { r := types.Range{ @@ -162,14 +162,14 @@ func (d *Database) GetEventsInStreamingRange( if backwardOrdering { // When using backward ordering, we want the most recent events first. if events, _, err = d.OutputEvents.SelectRecentEvents( - ctx, nil, roomID, r, limit, false, false, + ctx, nil, roomID, r, eventFilter, false, false, ); err != nil { return } } else { // When using forward ordering, we want the least recent events first. if events, err = d.OutputEvents.SelectEarlyEvents( - ctx, nil, roomID, r, limit, + ctx, nil, roomID, r, eventFilter, ); err != nil { return } diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index 4bcc06ed..1c65cb6a 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -82,7 +82,7 @@ func (s *accountDataStatements) InsertAccountData( ctx context.Context, txn *sql.Tx, userID, roomID, dataType string, ) (pos types.StreamPosition, err error) { - pos, err = s.streamIDStatements.nextStreamID(ctx, txn) + pos, err = s.streamIDStatements.nextAccountDataID(ctx, txn) if err != nil { return } diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index ac659057..55ed27a4 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -19,6 +19,7 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "strings" "github.com/matrix-org/dendrite/internal" @@ -66,13 +67,8 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectCurrentStateSQL = "" + - "SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + - " AND ( $2 IS NULL OR sender IN ($2) )" + - " AND ( $3 IS NULL OR NOT(sender IN ($3)) )" + - " AND ( $4 IS NULL OR type IN ($4) )" + - " AND ( $5 IS NULL OR NOT(type IN ($5)) )" + - " AND ( $6 IS NULL OR contains_url = $6 )" + - " LIMIT $7" + "SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + // WHEN, ORDER BY and LIMIT will be added by prepareWithFilter const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" @@ -95,7 +91,6 @@ type currentRoomStateStatements struct { deleteRoomStateByEventIDStmt *sql.Stmt DeleteRoomStateForRoomStmt *sql.Stmt selectRoomIDsWithMembershipStmt *sql.Stmt - selectCurrentStateStmt *sql.Stmt selectJoinedUsersStmt *sql.Stmt selectStateEventStmt *sql.Stmt } @@ -121,9 +116,6 @@ func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *streamIDStatements) (t if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil { return nil, err } - if s.selectCurrentStateStmt, err = db.Prepare(selectCurrentStateSQL); err != nil { - return nil, err - } if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil { return nil, err } @@ -185,17 +177,22 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( // CurrentState returns all the current state events for the given room. func (s *currentRoomStateStatements) SelectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, - stateFilterPart *gomatrixserverlib.StateFilter, + stateFilter *gomatrixserverlib.StateFilter, ) ([]*gomatrixserverlib.HeaderedEvent, error) { - stmt := sqlutil.TxStmt(txn, s.selectCurrentStateStmt) - rows, err := stmt.QueryContext(ctx, roomID, - nil, // FIXME: pq.StringArray(stateFilterPart.Senders), - nil, // FIXME: pq.StringArray(stateFilterPart.NotSenders), - nil, // FIXME: pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), - nil, // FIXME: pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), - stateFilterPart.ContainsURL, - stateFilterPart.Limit, + stmt, params, err := prepareWithFilters( + s.db, txn, selectCurrentStateSQL, + []interface{}{ + roomID, + }, + stateFilter.Senders, stateFilter.NotSenders, + stateFilter.Types, stateFilter.NotTypes, + stateFilter.Limit, FilterOrderNone, ) + if err != nil { + return nil, fmt.Errorf("s.prepareWithFilters: %w", err) + } + + rows, err := stmt.QueryContext(ctx, params...) if err != nil { return nil, err } diff --git a/syncapi/storage/sqlite3/filter_table.go b/syncapi/storage/sqlite3/filter_table.go index 3092bcd7..0cfebef2 100644 --- a/syncapi/storage/sqlite3/filter_table.go +++ b/syncapi/storage/sqlite3/filter_table.go @@ -87,7 +87,7 @@ func (s *filterStatements) SelectFilter( } // Unmarshal JSON into Filter struct - var filter gomatrixserverlib.Filter + filter := gomatrixserverlib.DefaultFilter() if err = json.Unmarshal(filterData, &filter); err != nil { return nil, err } diff --git a/syncapi/storage/sqlite3/filtering.go b/syncapi/storage/sqlite3/filtering.go new file mode 100644 index 00000000..0faf5297 --- /dev/null +++ b/syncapi/storage/sqlite3/filtering.go @@ -0,0 +1,76 @@ +package sqlite3 + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" +) + +type FilterOrder int + +const ( + FilterOrderNone = iota + FilterOrderAsc + FilterOrderDesc +) + +// prepareWithFilters returns a prepared statement with the +// relevant filters included. It also includes an []interface{} +// list of all the relevant parameters to pass straight to +// QueryContext, QueryRowContext etc. +// We don't take the filter object directly here because the +// fields might come from either a StateFilter or an EventFilter, +// and it's easier just to have the caller extract the relevant +// parts. +func prepareWithFilters( + db *sql.DB, txn *sql.Tx, query string, params []interface{}, + senders, notsenders, types, nottypes []string, + limit int, order FilterOrder, +) (*sql.Stmt, []interface{}, error) { + offset := len(params) + if count := len(senders); count > 0 { + query += " AND sender IN " + sqlutil.QueryVariadicOffset(count, offset) + for _, v := range senders { + params, offset = append(params, v), offset+1 + } + } + if count := len(notsenders); count > 0 { + query += " AND sender NOT IN " + sqlutil.QueryVariadicOffset(count, offset) + for _, v := range notsenders { + params, offset = append(params, v), offset+1 + } + } + if count := len(types); count > 0 { + query += " AND type IN " + sqlutil.QueryVariadicOffset(count, offset) + for _, v := range types { + params, offset = append(params, v), offset+1 + } + } + if count := len(nottypes); count > 0 { + query += " AND type NOT IN " + sqlutil.QueryVariadicOffset(count, offset) + for _, v := range nottypes { + params, offset = append(params, v), offset+1 + } + } + switch order { + case FilterOrderAsc: + query += " ORDER BY id ASC" + case FilterOrderDesc: + query += " ORDER BY id DESC" + } + query += fmt.Sprintf(" LIMIT $%d", offset+1) + params = append(params, limit) + + var stmt *sql.Stmt + var err error + if txn != nil { + stmt, err = txn.Prepare(query) + } else { + stmt, err = db.Prepare(query) + } + if err != nil { + return nil, nil, fmt.Errorf("s.db.Prepare: %w", err) + } + return stmt, params, nil +} diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index f9dcfdbc..7498fd68 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -93,7 +93,7 @@ func NewSqliteInvitesTable(db *sql.DB, streamID *streamIDStatements) (tables.Inv func (s *inviteEventsStatements) InsertInviteEvent( ctx context.Context, txn *sql.Tx, inviteEvent *gomatrixserverlib.HeaderedEvent, ) (streamPos types.StreamPosition, err error) { - streamPos, err = s.streamIDStatements.nextStreamID(ctx, txn) + streamPos, err = s.streamIDStatements.nextInviteID(ctx, txn) if err != nil { return } @@ -119,7 +119,7 @@ func (s *inviteEventsStatements) InsertInviteEvent( func (s *inviteEventsStatements) DeleteInviteEvent( ctx context.Context, txn *sql.Tx, inviteEventID string, ) (types.StreamPosition, error) { - streamPos, err := s.streamIDStatements.nextStreamID(ctx, txn) + streamPos, err := s.streamIDStatements.nextInviteID(ctx, txn) if err != nil { return streamPos, err } diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index edbd36fb..019aba8b 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -19,6 +19,7 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "sort" "github.com/matrix-org/dendrite/internal" @@ -60,18 +61,18 @@ const selectEventsSQL = "" + const selectRecentEventsSQL = "" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + - " WHERE room_id = $1 AND id > $2 AND id <= $3" + - " ORDER BY id DESC LIMIT $4" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters const selectRecentEventsForSyncSQL = "" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + - " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + - " ORDER BY id DESC LIMIT $4" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters const selectEarlyEventsSQL = "" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + - " WHERE room_id = $1 AND id > $2 AND id <= $3" + - " ORDER BY id ASC LIMIT $4" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters const selectMaxEventIDSQL = "" + "SELECT MAX(id) FROM syncapi_output_room_events" @@ -79,45 +80,24 @@ const selectMaxEventIDSQL = "" + const updateEventJSONSQL = "" + "UPDATE syncapi_output_room_events SET headered_event_json=$1 WHERE event_id=$2" -// In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). -/* - $1 = oldPos, - $2 = newPos, - $3 = pq.StringArray(stateFilterPart.Senders), - $4 = pq.StringArray(stateFilterPart.NotSenders), - $5 = pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), - $6 = pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), - $7 = stateFilterPart.ContainsURL, - $8 = stateFilterPart.Limit, -*/ const selectStateInRangeSQL = "" + "SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + - " WHERE (id > $1 AND id <= $2)" + // old/new pos - " AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + - /* " AND ( $3 IS NULL OR sender IN ($3) )" + // sender - " AND ( $4 IS NULL OR NOT(sender IN ($4)) )" + // not sender - " AND ( $5 IS NULL OR type IN ($5) )" + // type - " AND ( $6 IS NULL OR NOT(type IN ($6)) )" + // not type - " AND ( $7 IS NULL OR contains_url = $7)" + // contains URL? */ - " ORDER BY id ASC" + - " LIMIT $8" // limit + " WHERE (id > $1 AND id <= $2)" + + " AND ((add_state_ids IS NOT NULL AND add_state_ids != '') OR (remove_state_ids IS NOT NULL AND remove_state_ids != ''))" + // WHEN, ORDER BY and LIMIT are appended by prepareWithFilters const deleteEventsForRoomSQL = "" + "DELETE FROM syncapi_output_room_events WHERE room_id = $1" type outputRoomEventsStatements struct { - db *sql.DB - streamIDStatements *streamIDStatements - insertEventStmt *sql.Stmt - selectEventsStmt *sql.Stmt - selectMaxEventIDStmt *sql.Stmt - selectRecentEventsStmt *sql.Stmt - selectRecentEventsForSyncStmt *sql.Stmt - selectEarlyEventsStmt *sql.Stmt - selectStateInRangeStmt *sql.Stmt - updateEventJSONStmt *sql.Stmt - deleteEventsForRoomStmt *sql.Stmt + db *sql.DB + streamIDStatements *streamIDStatements + insertEventStmt *sql.Stmt + selectEventsStmt *sql.Stmt + selectMaxEventIDStmt *sql.Stmt + updateEventJSONStmt *sql.Stmt + deleteEventsForRoomStmt *sql.Stmt } func NewSqliteEventsTable(db *sql.DB, streamID *streamIDStatements) (tables.Events, error) { @@ -138,18 +118,6 @@ func NewSqliteEventsTable(db *sql.DB, streamID *streamIDStatements) (tables.Even if s.selectMaxEventIDStmt, err = db.Prepare(selectMaxEventIDSQL); err != nil { return nil, err } - if s.selectRecentEventsStmt, err = db.Prepare(selectRecentEventsSQL); err != nil { - return nil, err - } - if s.selectRecentEventsForSyncStmt, err = db.Prepare(selectRecentEventsForSyncSQL); err != nil { - return nil, err - } - if s.selectEarlyEventsStmt, err = db.Prepare(selectEarlyEventsSQL); err != nil { - return nil, err - } - if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { - return nil, err - } if s.updateEventJSONStmt, err = db.Prepare(updateEventJSONSQL); err != nil { return nil, err } @@ -173,19 +141,22 @@ func (s *outputRoomEventsStatements) UpdateEventJSON(ctx context.Context, event // two positions, only the most recent state is returned. func (s *outputRoomEventsStatements) SelectStateInRange( ctx context.Context, txn *sql.Tx, r types.Range, - stateFilterPart *gomatrixserverlib.StateFilter, + stateFilter *gomatrixserverlib.StateFilter, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { - stmt := sqlutil.TxStmt(txn, s.selectStateInRangeStmt) - - rows, err := stmt.QueryContext( - ctx, r.Low(), r.High(), - /*pq.StringArray(stateFilterPart.Senders), - pq.StringArray(stateFilterPart.NotSenders), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), - stateFilterPart.ContainsURL,*/ - stateFilterPart.Limit, + stmt, params, err := prepareWithFilters( + s.db, txn, selectStateInRangeSQL, + []interface{}{ + r.Low(), r.High(), + }, + stateFilter.Senders, stateFilter.NotSenders, + stateFilter.Types, stateFilter.NotTypes, + stateFilter.Limit, FilterOrderAsc, ) + if err != nil { + return nil, nil, fmt.Errorf("s.prepareWithFilters: %w", err) + } + + rows, err := stmt.QueryContext(ctx, params...) if err != nil { return nil, nil, err } @@ -298,16 +269,21 @@ func (s *outputRoomEventsStatements) InsertEvent( return 0, err } - addStateJSON, err := json.Marshal(addState) - if err != nil { - return 0, err + var addStateJSON, removeStateJSON []byte + if len(addState) > 0 { + addStateJSON, err = json.Marshal(addState) } - removeStateJSON, err := json.Marshal(removeState) if err != nil { - return 0, err + return 0, fmt.Errorf("json.Marshal(addState): %w", err) + } + if len(removeState) > 0 { + removeStateJSON, err = json.Marshal(removeState) + } + if err != nil { + return 0, fmt.Errorf("json.Marshal(removeState): %w", err) } - streamPos, err := s.streamIDStatements.nextStreamID(ctx, txn) + streamPos, err := s.streamIDStatements.nextPDUID(ctx, txn) if err != nil { return 0, err } @@ -333,17 +309,30 @@ func (s *outputRoomEventsStatements) InsertEvent( func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, - roomID string, r types.Range, limit int, + roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool, ) ([]types.StreamEvent, bool, error) { - var stmt *sql.Stmt + var query string if onlySyncEvents { - stmt = sqlutil.TxStmt(txn, s.selectRecentEventsForSyncStmt) + query = selectRecentEventsForSyncSQL } else { - stmt = sqlutil.TxStmt(txn, s.selectRecentEventsStmt) + query = selectRecentEventsSQL } - rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit+1) + stmt, params, err := prepareWithFilters( + s.db, txn, query, + []interface{}{ + roomID, r.Low(), r.High(), + }, + eventFilter.Senders, eventFilter.NotSenders, + eventFilter.Types, eventFilter.NotTypes, + eventFilter.Limit+1, FilterOrderDesc, + ) + if err != nil { + return nil, false, fmt.Errorf("s.prepareWithFilters: %w", err) + } + + rows, err := stmt.QueryContext(ctx, params...) if err != nil { return nil, false, err } @@ -362,7 +351,7 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( } // we queried for 1 more than the limit, so if we returned one more mark limited=true limited := false - if len(events) > limit { + if len(events) > eventFilter.Limit { limited = true // re-slice the extra (oldest) event out: in chronological order this is the first entry, else the last. if chronologicalOrder { @@ -376,10 +365,21 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, - roomID string, r types.Range, limit int, + roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, ) ([]types.StreamEvent, error) { - stmt := sqlutil.TxStmt(txn, s.selectEarlyEventsStmt) - rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) + stmt, params, err := prepareWithFilters( + s.db, txn, selectEarlyEventsSQL, + []interface{}{ + roomID, r.Low(), r.High(), + }, + eventFilter.Senders, eventFilter.NotSenders, + eventFilter.Types, eventFilter.NotTypes, + eventFilter.Limit, FilterOrderAsc, + ) + if err != nil { + return nil, fmt.Errorf("s.prepareWithFilters: %w", err) + } + rows, err := stmt.QueryContext(ctx, params...) if err != nil { return nil, err } diff --git a/syncapi/storage/sqlite3/peeks_table.go b/syncapi/storage/sqlite3/peeks_table.go index d755e28c..c93c8205 100644 --- a/syncapi/storage/sqlite3/peeks_table.go +++ b/syncapi/storage/sqlite3/peeks_table.go @@ -108,7 +108,7 @@ func NewSqlitePeeksTable(db *sql.DB, streamID *streamIDStatements) (tables.Peeks func (s *peekStatements) InsertPeek( ctx context.Context, txn *sql.Tx, roomID, userID, deviceID string, ) (streamPos types.StreamPosition, err error) { - streamPos, err = s.streamIDStatements.nextStreamID(ctx, txn) + streamPos, err = s.streamIDStatements.nextPDUID(ctx, txn) if err != nil { return } @@ -120,7 +120,7 @@ func (s *peekStatements) InsertPeek( func (s *peekStatements) DeletePeek( ctx context.Context, txn *sql.Tx, roomID, userID, deviceID string, ) (streamPos types.StreamPosition, err error) { - streamPos, err = s.streamIDStatements.nextStreamID(ctx, txn) + streamPos, err = s.streamIDStatements.nextPDUID(ctx, txn) if err != nil { return } @@ -131,7 +131,7 @@ func (s *peekStatements) DeletePeek( func (s *peekStatements) DeletePeeks( ctx context.Context, txn *sql.Tx, roomID, userID string, ) (types.StreamPosition, error) { - streamPos, err := s.streamIDStatements.nextStreamID(ctx, txn) + streamPos, err := s.streamIDStatements.nextPDUID(ctx, txn) if err != nil { return 0, err } diff --git a/syncapi/storage/sqlite3/stream_id_table.go b/syncapi/storage/sqlite3/stream_id_table.go index f73be422..b614271d 100644 --- a/syncapi/storage/sqlite3/stream_id_table.go +++ b/syncapi/storage/sqlite3/stream_id_table.go @@ -20,6 +20,10 @@ INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("global", 0) ON CONFLICT DO NOTHING; INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("receipt", 0) ON CONFLICT DO NOTHING; +INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("accountdata", 0) + ON CONFLICT DO NOTHING; +INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("invite", 0) + ON CONFLICT DO NOTHING; ` const increaseStreamIDStmt = "" + @@ -49,7 +53,7 @@ func (s *streamIDStatements) prepare(db *sql.DB) (err error) { return } -func (s *streamIDStatements) nextStreamID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { +func (s *streamIDStatements) nextPDUID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt) selectStmt := sqlutil.TxStmt(txn, s.selectStreamIDStmt) if _, err = increaseStmt.ExecContext(ctx, "global"); err != nil { @@ -68,3 +72,23 @@ func (s *streamIDStatements) nextReceiptID(ctx context.Context, txn *sql.Tx) (po err = selectStmt.QueryRowContext(ctx, "receipt").Scan(&pos) return } + +func (s *streamIDStatements) nextInviteID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { + increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt) + selectStmt := sqlutil.TxStmt(txn, s.selectStreamIDStmt) + if _, err = increaseStmt.ExecContext(ctx, "invite"); err != nil { + return + } + err = selectStmt.QueryRowContext(ctx, "invite").Scan(&pos) + return +} + +func (s *streamIDStatements) nextAccountDataID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { + increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt) + selectStmt := sqlutil.TxStmt(txn, s.selectStreamIDStmt) + if _, err = increaseStmt.ExecContext(ctx, "accountdata"); err != nil { + return + } + err = selectStmt.QueryRowContext(ctx, "accountdata").Scan(&pos) + return +} diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index fca88824..73967677 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -56,9 +56,9 @@ type Events interface { // SelectRecentEvents returns events between the two stream positions: exclusive of low and inclusive of high. // If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude from sync. // Returns up to `limit` events. Returns `limited=true` if there are more events in this range but we hit the `limit`. - SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) + SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) // SelectEarlyEvents returns the earliest events in the given room. - SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int) ([]types.StreamEvent, error) + SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter) ([]types.StreamEvent, error) SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]types.StreamEvent, error) UpdateEventJSON(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error // DeleteEventsForRoom removes all event information for a room. This should only be done when removing the room entirely. diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index d403d164..3f531318 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -129,6 +129,7 @@ func (p *PDUStreamProvider) IncrementalSync( var joinedRooms []string stateFilter := req.Filter.Room.State + eventFilter := req.Filter.Room.Timeline if req.WantFullState { if stateDeltas, joinedRooms, err = p.DB.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { @@ -155,7 +156,7 @@ func (p *PDUStreamProvider) IncrementalSync( } for _, delta := range stateDeltas { - if err = p.addRoomDeltaToResponse(ctx, req.Device, r, delta, req.Filter.Room.Timeline.Limit, req.Response); err != nil { + if err = p.addRoomDeltaToResponse(ctx, req.Device, r, delta, &eventFilter, req.Response); err != nil { req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed") return newPos } @@ -169,7 +170,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( device *userapi.Device, r types.Range, delta types.StateDelta, - numRecentEventsPerRoom int, + eventFilter *gomatrixserverlib.RoomEventFilter, res *types.Response, ) error { if delta.MembershipPos > 0 && delta.Membership == gomatrixserverlib.Leave { @@ -183,7 +184,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } recentStreamEvents, limited, err := p.DB.RecentEvents( ctx, delta.RoomID, r, - numRecentEventsPerRoom, true, true, + eventFilter, true, true, ) if err != nil { return err @@ -246,21 +247,27 @@ func (p *PDUStreamProvider) getResponseForCompleteSync( ctx context.Context, roomID string, r types.Range, - filter *gomatrixserverlib.Filter, + stateFilter *gomatrixserverlib.StateFilter, + eventFilter *gomatrixserverlib.RoomEventFilter, device *userapi.Device, ) ( recentEvents, stateEvents []*gomatrixserverlib.HeaderedEvent, prevBatch *types.TopologyToken, limited bool, err error, ) { + stateEvents, err = p.DB.CurrentState(ctx, roomID, stateFilter) + if err != nil { + return + } // TODO: When filters are added, we may need to call this multiple times to get enough events. // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 var recentStreamEvents []types.StreamEvent recentStreamEvents, limited, err = p.DB.RecentEvents( - ctx, roomID, r, filter.Room.Timeline.Limit, true, true, + ctx, roomID, r, eventFilter, true, true, ) if err != nil { return } + recentStreamEvents, limited = p.filterStreamEventsAccordingToHistoryVisibility(recentStreamEvents, stateEvents, device, limited) // TODO: How do we apply filter.Room.State to this as well? @@ -310,7 +317,9 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( filter *gomatrixserverlib.Filter, device *userapi.Device, ) (jr *types.JoinResponse, err error) { - recentEvents, stateEvents, prevBatch, limited, err := p.getResponseForCompleteSync(ctx, roomID, r, filter, device) + recentEvents, stateEvents, prevBatch, limited, err := p.getResponseForCompleteSync( + ctx, roomID, r, &filter.Room.State, &filter.Room.Timeline, device, + ) if err != nil { return nil, fmt.Errorf("p.getResponseForCompleteSync: %w", err) } @@ -330,7 +339,9 @@ func (p *PDUStreamProvider) getLeaveResponseForCompleteSync( filter *gomatrixserverlib.Filter, device *userapi.Device, ) (lr *types.LeaveResponse, err error) { - recentEvents, stateEvents, prevBatch, limited, err := p.getResponseForCompleteSync(ctx, roomID, r, filter, device) + recentEvents, stateEvents, prevBatch, limited, err := p.getResponseForCompleteSync( + ctx, roomID, r, &filter.Room.State, &filter.Room.Timeline, device, + ) if err != nil { return nil, fmt.Errorf("p.getResponseForCompleteSync: %w", err) } diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index 88e376e3..30adb86d 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -17,7 +17,6 @@ package sync import ( "encoding/json" "fmt" - "math" "net/http" "strconv" "time" @@ -31,7 +30,7 @@ import ( ) const defaultSyncTimeout = time.Duration(0) -const defaultTimelineLimit = 20 +const DefaultTimelineLimit = 20 func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Database) (*types.SyncRequest, error) { timeout := getTimeout(req.URL.Query().Get("timeout")) @@ -45,51 +44,37 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat return nil, fmt.Errorf("types.NewStreamTokenFromString: %w", err) } } - - var f *gomatrixserverlib.Filter // TODO: read from stored filters too + filter := gomatrixserverlib.DefaultFilter() filterQuery := req.URL.Query().Get("filter") if filterQuery != "" { if filterQuery[0] == '{' { - // attempt to parse the timeline limit at least - if err := json.Unmarshal([]byte(filterQuery), &f); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed") + // Parse the filter from the query string + if err := json.Unmarshal([]byte(filterQuery), &filter); err != nil { return nil, fmt.Errorf("json.Unmarshal: %w", err) } } else { - // attempt to load the filter ID + // Try to load the filter from the database localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") return nil, fmt.Errorf("gomatrixserverlib.SplitID: %w", err) } - f, err = syncDB.GetFilter(req.Context(), localpart, filterQuery) - if err != nil { + if f, err := syncDB.GetFilter(req.Context(), localpart, filterQuery); err != nil { util.GetLogger(req.Context()).WithError(err).Error("syncDB.GetFilter failed") return nil, fmt.Errorf("syncDB.GetFilter: %w", err) + } else { + filter = *f } } } - filter := gomatrixserverlib.DefaultFilter() - if f != nil { - filter = *f - } - // TODO: Get a better to default these - // Ideally, we could merge the default base filter with the parsed one - if filter.Room.Timeline.Limit == 0 { - filter.Room.Timeline.Limit = defaultTimelineLimit - } - if filter.Room.State.Limit == 0 { - filter.Room.State.Limit = math.MaxInt32 - } - // TODO: Additional query params: set_presence - logger := util.GetLogger(req.Context()).WithFields(logrus.Fields{ "user_id": device.UserID, "device_id": device.ID, "since": since, "timeout": timeout, + "limit": filter.Room.Timeline.Limit, }) return &types.SyncRequest{ diff --git a/sytest-whitelist b/sytest-whitelist index d8efcf24..4bff57fa 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -504,3 +504,8 @@ A next_batch token can be used in the v1 messages API Users receive device_list updates for their own devices m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users +State is included in the timeline in the initial sync +State from remote users is included in the state in the initial sync +Changes to state are included in an gapped incremental sync +A full_state incremental update returns all state +Can pass a JSON filter as a query parameter