Handle state with input event as new events (#1415)

* SendEventWithState events as new

* Use cumulative state IDs for final event

* Error wrapping in calculateAndSetState

* Handle overwriting same event type and state key

* Hacky way to spot historical events

* Don't exclude from sync

* Don't generate output events when rewriting forward extremities

* Update output event check

* Historical output events

* Define output room event type

* Notify key changes on state

* Don't send our membership event twice

* Deduplicate state entries

* Tweaks

* Remove unnecessary nolint

* Fix current state upsert in sync API

* Send auth events as outliers, state events as rewrite

* Sync API don't consume state events

* Process events actually

* Improve outlier check

* Fix local room check

* Remove extra room check, it seems to break the whole damn world

* Fix federated join check

* Fix nil pointer exception

* Better comments on DeduplicateStateEntries

* Reflow forced federated joins

* Don't force federated join for possibly even local invites

* Comment SendEventWithState better

* Rewrite room state in sync API storage

* Add TODO

* Clean up all room data when receiving create event

* Don't generate output events for rewrites, but instead notify that state is rewritten on the final new event

* Rename to PurgeRoom

* Exclude backfilled messages from /sync

* Split out rewriting state from updating state from state res

Co-authored-by: Kegan Dougal <kegan@matrix.org>

roomserver/types/types.go

@@ -16,6 +16,8 @@
 package types
 
 import (
+	"sort"
+
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -72,6 +74,25 @@ func (a StateEntry) LessThan(b StateEntry) bool {
 	return a.EventNID < b.EventNID
 }
 
+// Deduplicate takes a set of state entries and ensures that there are no
+// duplicate (event type, state key) tuples. If there are then we dedupe
+// them, making sure that the latest/highest NIDs are always chosen.
+func DeduplicateStateEntries(a []StateEntry) []StateEntry {
+	if len(a) < 2 {
+		return a
+	}
+	sort.SliceStable(a, func(i, j int) bool {
+		return a[i].LessThan(a[j])
+	})
+	for i := 0; i < len(a)-1; i++ {
+		if a[i].StateKeyTuple == a[i+1].StateKeyTuple {
+			a = append(a[:i], a[i+1:]...)
+			i--
+		}
+	}
+	return a
+}
+
 // StateAtEvent is the state before and after a matrix event.
 type StateAtEvent struct {
 	// Should this state overwrite the latest events and memberships of the room?

roomserver/types/types_test.go

@@ -0,0 +1,26 @@
+package types
+
+import (
+	"testing"
+)
+
+func TestDeduplicateStateEntries(t *testing.T) {
+	entries := []StateEntry{
+		{StateKeyTuple{1, 1}, 1},
+		{StateKeyTuple{1, 1}, 2},
+		{StateKeyTuple{1, 1}, 3},
+		{StateKeyTuple{2, 2}, 4},
+		{StateKeyTuple{2, 3}, 5},
+		{StateKeyTuple{3, 3}, 6},
+	}
+	expected := []EventNID{3, 4, 5, 6}
+	entries = DeduplicateStateEntries(entries)
+	if len(entries) != 4 {
+		t.Fatalf("Expected 4 entries, got %d entries", len(entries))
+	}
+	for i, v := range entries {
+		if v.EventNID != expected[i] {
+			t.Fatalf("Expected position %d to be %d but got %d", i, expected[i], v.EventNID)
+		}
+	}
+}