From 3bd66ff196a84b0c96763850228c379b2a124fb6 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Fri, 2 Oct 2020 23:35:54 +0200 Subject: [PATCH 1/6] Route several paths to sync_api (#1473) --- docs/nginx/polylith-sample.conf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf index 658e0e4a..ab346184 100644 --- a/docs/nginx/polylith-sample.conf +++ b/docs/nginx/polylith-sample.conf @@ -18,6 +18,17 @@ server { return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; } + # route requests to: + # /_matrix/client/.*/sync + # /_matrix/client/.*/user/{userId}/filter + # /_matrix/client/.*/user/{userId}/filter/{filterID} + # /_matrix/client/.*/keys/changes + # /_matrix/client/.*/rooms/{roomId}/messages + # to sync_api + location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$ { + proxy_pass http://sync_api:8073; + } + location /_matrix/client { proxy_pass http://client_api:8071; } From c4756eee0e59ba730fa7855171e18eda9dba8b5f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Oct 2020 11:06:31 +0100 Subject: [PATCH 2/6] Don't store backfilled events using request context (#1478) --- syncapi/routing/messages.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 9c6c6a80..e5299f20 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -503,7 +503,7 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][] // up in responses to sync requests. for i := range res.Events { _, err = r.db.WriteEvent( - r.ctx, + context.Background(), &res.Events[i], []gomatrixserverlib.HeaderedEvent{}, []string{}, From 2bfab5f58b77002b1838b52f6edc369ea280ced7 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Mon, 5 Oct 2020 12:28:10 +0200 Subject: [PATCH 3/6] Add example goose sqlite3 migration (#1474) Signed-off-by: Till Faelligen --- cmd/goose/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/goose/README.md b/cmd/goose/README.md index c7f085d8..725c6a58 100644 --- a/cmd/goose/README.md +++ b/cmd/goose/README.md @@ -104,4 +104,6 @@ You __must__ import the package in `/cmd/goose/main.go` so `func init()` gets ca #### Database limitations - SQLite3 does NOT support `ALTER TABLE table_name DROP COLUMN` - you would have to rename the column or drop the table - entirely and recreate it. + entirely and recreate it. ([example](https://github.com/matrix-org/dendrite/blob/master/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql)) + + More information: [sqlite.org](https://www.sqlite.org/lang_altertable.html) From 4e6b7f726d6741e0faa77ad0c8ba373ea58ef3f8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Oct 2020 12:42:56 +0100 Subject: [PATCH 4/6] Update to matrix-org/gomatrixserverlib#230 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b1119f60..5808c55a 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290 + github.com/matrix-org/gomatrixserverlib v0.0.0-20201005114154-18abed1923fd github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index c657cc45..7dae1ef9 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290 h1:ilT9QNIh2KXfvzIALtAe31IvLVZH7mVjVtOOTxdd0tY= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201005114154-18abed1923fd h1:30Mlf0wdEj4dtVQdpSRT1aVkFu2qXh5fK9qrExPbPIM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201005114154-18abed1923fd/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= From 2e71d2708f7c922dcb6361c8a49022470b8b6e33 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Oct 2020 17:47:08 +0100 Subject: [PATCH 5/6] Resolve state after event against current room state when determining latest state changes (#1479) * Resolve state after event against current room state when determining latest state changes * Update sytest-whitelist * Update sytest-whitelist, blacklist --- .../internal/input/input_latest_events.go | 23 ++++++++++++++++--- roomserver/state/state.go | 8 ++++--- sytest-blacklist | 7 +++++- sytest-whitelist | 4 ---- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 5c2a1de6..2e9f3b4e 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -215,10 +215,27 @@ func (u *latestEventsUpdater) latestState() error { var err error roomState := state.NewStateResolution(u.api.DB, *u.roomInfo) - // Get a list of the current latest events. - latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)) + // Get a list of the current room state events if available. + var currentState []types.StateEntry + if u.roomInfo.StateSnapshotNID != 0 { + currentState, _ = roomState.LoadStateAtSnapshot(u.ctx, u.roomInfo.StateSnapshotNID) + } + + // Get a list of the current latest events. This will include both + // the current room state and the latest events after the input event. + // The idea is that we will perform state resolution on this set and + // any conflicting events will be resolved properly. + latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)+len(currentState)) + offset := 0 + for i := range currentState { + latestStateAtEvents[i] = types.StateAtEvent{ + BeforeStateSnapshotNID: u.roomInfo.StateSnapshotNID, + StateEntry: currentState[i], + } + offset++ + } for i := range u.latest { - latestStateAtEvents[i] = u.latest[i].StateAtEvent + latestStateAtEvents[offset+i] = u.latest[i].StateAtEvent } // Takes the NIDs of the latest events and creates a state snapshot diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 0663499e..2944f71c 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -118,7 +118,7 @@ func (v StateResolution) LoadCombinedStateAfterEvents( // the snapshot of the room state before them was the same. stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs)) if err != nil { - return nil, err + return nil, fmt.Errorf("v.db.StateBlockNIDs: %w", err) } var stateBlockNIDs []types.StateBlockNID @@ -131,7 +131,7 @@ func (v StateResolution) LoadCombinedStateAfterEvents( // multiple snapshots. stateEntryLists, err := v.db.StateEntries(ctx, uniqueStateBlockNIDs(stateBlockNIDs)) if err != nil { - return nil, err + return nil, fmt.Errorf("v.db.StateEntries: %w", err) } stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists) stateEntriesMap := stateEntryListMap(stateEntryLists) @@ -623,7 +623,7 @@ func (v StateResolution) calculateAndStoreStateAfterManyEvents( v.calculateStateAfterManyEvents(ctx, v.roomInfo.RoomVersion, prevStates) metrics.algorithm = algorithm if err != nil { - return metrics.stop(0, err) + return metrics.stop(0, fmt.Errorf("v.calculateStateAfterManyEvents: %w", err)) } // TODO: Check if we can encode the new state as a delta against the @@ -642,6 +642,7 @@ func (v StateResolution) calculateStateAfterManyEvents( // First stage: load the state after each of the prev events. combined, err = v.LoadCombinedStateAfterEvents(ctx, prevStates) if err != nil { + err = fmt.Errorf("v.LoadCombinedStateAfterEvents: %w", err) algorithm = "_load_combined_state" return } @@ -672,6 +673,7 @@ func (v StateResolution) calculateStateAfterManyEvents( var resolved []types.StateEntry resolved, err = v.resolveConflicts(ctx, roomVersion, notConflicted, conflicts) if err != nil { + err = fmt.Errorf("v.resolveConflits: %w", err) algorithm = "_resolve_conflicts" return } diff --git a/sytest-blacklist b/sytest-blacklist index 2f80fc78..ff7fdf7e 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -52,4 +52,9 @@ Inbound federation accepts a second soft-failed event Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state # We don't implement lazy membership loading yet. -The only membership state included in a gapped incremental sync is for senders in the timeline \ No newline at end of file +The only membership state included in a gapped incremental sync is for senders in the timeline + +# Blacklisted out of flakiness after #1479 +Invited user can reject local invite after originator leaves +Invited user can reject invite for empty room +If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes \ No newline at end of file diff --git a/sytest-whitelist b/sytest-whitelist index a811259f..9a013cbf 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -400,8 +400,6 @@ Uninvited users cannot join the room Users cannot invite themselves to a room Users cannot invite a user that is already in the room Invited user can reject invite -Invited user can reject invite for empty room -Invited user can reject local invite after originator leaves PUT /rooms/:room_id/typing/:user_id sets typing notification Typing notification sent to local room members Typing notifications also sent to remote room members @@ -431,7 +429,6 @@ A prev_batch token can be used in the v1 messages API We don't send redundant membership state across incremental syncs by default Typing notifications don't leak Users cannot kick users from a room they are not in -Users cannot kick users who have already left a room User appears in user directory User directory correctly update on display name change User in shared private room does appear in user directory @@ -451,7 +448,6 @@ Banned servers cannot backfill Inbound /v1/send_leave rejects leaves from other servers Guest users can accept invites to private rooms over federation AS user (not ghost) can join room without registering -If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes Can search public room list Can get remote public room list Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list From 52ddded72d05924434f7284bd686629e1cca06c6 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Oct 2020 17:56:18 +0100 Subject: [PATCH 6/6] Update to matrix-org/gomatrixserverlib#232 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5808c55a..5793897a 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201005114154-18abed1923fd + github.com/matrix-org/gomatrixserverlib v0.0.0-20201005165532-90f2e192f625 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 7dae1ef9..20479015 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201005114154-18abed1923fd h1:30Mlf0wdEj4dtVQdpSRT1aVkFu2qXh5fK9qrExPbPIM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201005114154-18abed1923fd/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201005165532-90f2e192f625 h1:9ShgY0ZkfLzqe3gv18V5WxDAZ4dgUvJwnGORycox680= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201005165532-90f2e192f625/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=