Merge branch 'master' into v0.1.0

cmd/goose/README.md

@@ -104,4 +104,6 @@ You __must__ import the package in `/cmd/goose/main.go` so `func init()` gets ca
 #### Database limitations
 
 - SQLite3 does NOT support `ALTER TABLE table_name DROP COLUMN` - you would have to rename the column or drop the table
-  entirely and recreate it.
+  entirely and recreate it. ([example](https://github.com/matrix-org/dendrite/blob/master/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql))
+  
+  More information: [sqlite.org](https://www.sqlite.org/lang_altertable.html)

docs/nginx/polylith-sample.conf

@@ -18,6 +18,17 @@ server {
         return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }';
     }
 
+    # route requests to:
+    # /_matrix/client/.*/sync
+    # /_matrix/client/.*/user/{userId}/filter
+    # /_matrix/client/.*/user/{userId}/filter/{filterID}
+    # /_matrix/client/.*/keys/changes
+    # /_matrix/client/.*/rooms/{roomId}/messages
+    # to sync_api
+    location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$  {
+        proxy_pass http://sync_api:8073;
+    }
+
     location /_matrix/client {
         proxy_pass http://client_api:8071;
     }

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20201005165532-90f2e192f625
 	github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.2

go.sum

@@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290 h1:ilT9QNIh2KXfvzIALtAe31IvLVZH7mVjVtOOTxdd0tY=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20201005165532-90f2e192f625 h1:9ShgY0ZkfLzqe3gv18V5WxDAZ4dgUvJwnGORycox680=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20201005165532-90f2e192f625/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=

roomserver/internal/input/input_latest_events.go

@@ -215,10 +215,27 @@ func (u *latestEventsUpdater) latestState() error {
 	var err error
 	roomState := state.NewStateResolution(u.api.DB, *u.roomInfo)
 
-	// Get a list of the current latest events.
-	latestStateAtEvents := make([]types.StateAtEvent, len(u.latest))
+	// Get a list of the current room state events if available.
+	var currentState []types.StateEntry
+	if u.roomInfo.StateSnapshotNID != 0 {
+		currentState, _ = roomState.LoadStateAtSnapshot(u.ctx, u.roomInfo.StateSnapshotNID)
+	}
+
+	// Get a list of the current latest events. This will include both
+	// the current room state and the latest events after the input event.
+	// The idea is that we will perform state resolution on this set and
+	// any conflicting events will be resolved properly.
+	latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)+len(currentState))
+	offset := 0
+	for i := range currentState {
+		latestStateAtEvents[i] = types.StateAtEvent{
+			BeforeStateSnapshotNID: u.roomInfo.StateSnapshotNID,
+			StateEntry:             currentState[i],
+		}
+		offset++
+	}
 	for i := range u.latest {
-		latestStateAtEvents[i] = u.latest[i].StateAtEvent
+		latestStateAtEvents[offset+i] = u.latest[i].StateAtEvent
 	}
 
 	// Takes the NIDs of the latest events and creates a state snapshot

roomserver/state/state.go

@@ -118,7 +118,7 @@ func (v StateResolution) LoadCombinedStateAfterEvents(
 	// the snapshot of the room state before them was the same.
 	stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs))
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("v.db.StateBlockNIDs: %w", err)
 	}
 
 	var stateBlockNIDs []types.StateBlockNID
@@ -131,7 +131,7 @@ func (v StateResolution) LoadCombinedStateAfterEvents(
 	// multiple snapshots.
 	stateEntryLists, err := v.db.StateEntries(ctx, uniqueStateBlockNIDs(stateBlockNIDs))
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("v.db.StateEntries: %w", err)
 	}
 	stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists)
 	stateEntriesMap := stateEntryListMap(stateEntryLists)
@@ -623,7 +623,7 @@ func (v StateResolution) calculateAndStoreStateAfterManyEvents(
 		v.calculateStateAfterManyEvents(ctx, v.roomInfo.RoomVersion, prevStates)
 	metrics.algorithm = algorithm
 	if err != nil {
-		return metrics.stop(0, err)
+		return metrics.stop(0, fmt.Errorf("v.calculateStateAfterManyEvents: %w", err))
 	}
 
 	// TODO: Check if we can encode the new state as a delta against the
@@ -642,6 +642,7 @@ func (v StateResolution) calculateStateAfterManyEvents(
 	// First stage: load the state after each of the prev events.
 	combined, err = v.LoadCombinedStateAfterEvents(ctx, prevStates)
 	if err != nil {
+		err = fmt.Errorf("v.LoadCombinedStateAfterEvents: %w", err)
 		algorithm = "_load_combined_state"
 		return
 	}
@@ -672,6 +673,7 @@ func (v StateResolution) calculateStateAfterManyEvents(
 		var resolved []types.StateEntry
 		resolved, err = v.resolveConflicts(ctx, roomVersion, notConflicted, conflicts)
 		if err != nil {
+			err = fmt.Errorf("v.resolveConflits: %w", err)
 			algorithm = "_resolve_conflicts"
 			return
 		}

syncapi/routing/messages.go

@@ -503,7 +503,7 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]
 	// up in responses to sync requests.
 	for i := range res.Events {
 		_, err = r.db.WriteEvent(
-			r.ctx,
+			context.Background(),
 			&res.Events[i],
 			[]gomatrixserverlib.HeaderedEvent{},
 			[]string{},

sytest-blacklist

@@ -52,4 +52,9 @@ Inbound federation accepts a second soft-failed event
 Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state
 
 # We don't implement lazy membership loading yet.
-The only membership state included in a gapped incremental sync is for senders in the timeline
+The only membership state included in a gapped incremental sync is for senders in the timeline
+
+# Blacklisted out of flakiness after #1479
+Invited user can reject local invite after originator leaves
+Invited user can reject invite for empty room
+If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes

sytest-whitelist

@@ -400,8 +400,6 @@ Uninvited users cannot join the room
 Users cannot invite themselves to a room
 Users cannot invite a user that is already in the room
 Invited user can reject invite
-Invited user can reject invite for empty room
-Invited user can reject local invite after originator leaves
 PUT /rooms/:room_id/typing/:user_id sets typing notification
 Typing notification sent to local room members
 Typing notifications also sent to remote room members
@@ -431,7 +429,6 @@ A prev_batch token can be used in the v1 messages API
 We don't send redundant membership state across incremental syncs by default
 Typing notifications don't leak
 Users cannot kick users from a room they are not in
-Users cannot kick users who have already left a room
 User appears in user directory
 User directory correctly update on display name change
 User in shared private room does appear in user directory
@@ -451,7 +448,6 @@ Banned servers cannot backfill
 Inbound /v1/send_leave rejects leaves from other servers
 Guest users can accept invites to private rooms over federation
 AS user (not ghost) can join room without registering
-If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes
 Can search public room list
 Can get remote public room list
 Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list