From 0096fc850a127fa4ea39fb222b746a79ce42885b Mon Sep 17 00:00:00 2001
From: Neil Alexander <neilalexander@users.noreply.github.com>
Date: Wed, 20 Jan 2021 10:29:32 +0000
Subject: [PATCH] Refactor removeDuplicates (might use slightly more memory but
 less CPU)

---
 syncapi/routing/filter.go     |  1 -
 syncapi/streams/stream_pdu.go | 46 ++++++++++-------------------------
 2 files changed, 13 insertions(+), 34 deletions(-)

diff --git a/syncapi/routing/filter.go b/syncapi/routing/filter.go
index 4cefd753..d1e3c7f8 100644
--- a/syncapi/routing/filter.go
+++ b/syncapi/routing/filter.go
@@ -64,7 +64,6 @@ type filterResponse struct {
 	FilterID string `json:"filter_id"`
 }
 
-// TODO: asdf
 //PutFilter implements POST /_matrix/client/r0/user/{userId}/filter
 func PutFilter(
 	req *http.Request, device *api.Device, syncDB storage.Database, userID string,
diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go
index 06612e0b..a183fd23 100644
--- a/syncapi/streams/stream_pdu.go
+++ b/syncapi/streams/stream_pdu.go
@@ -8,7 +8,6 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/sirupsen/logrus"
 )
 
 type PDUStreamProvider struct {
@@ -103,7 +102,6 @@ func (p *PDUStreamProvider) CompleteSync(
 				req.Log.WithError(err).Error("p.getLeaveResponseForCompleteSync failed")
 				return from
 			}
-
 			req.Response.Rooms.Leave[roomID] = *lr
 		}
 	}
@@ -347,9 +345,6 @@ func (p *PDUStreamProvider) filterStreamEventsAccordingToHistoryVisibility(
 			}
 			switch content.HistoryVisibility {
 			case "world_readable", "shared":
-				logrus.WithFields(logrus.Fields{
-					"content.HistoryVisibility": content.HistoryVisibility,
-				}).Info("filterStreamEventsAccordingToHistoryVisibility: No filtering needed for these events")
 				return recentStreamEvents, limited
 			default:
 				break
@@ -399,38 +394,23 @@ func (p *PDUStreamProvider) filterStreamEventsAccordingToHistoryVisibility(
 		sliceEnd = leaveEventIndex + 1
 	}
 
-	events := make([]string, len(recentStreamEvents))
-	for i, v := range recentStreamEvents {
-		events[i] = string(v.HeaderedEvent.Event.JSON())
-	}
-
-	logrus.WithFields(logrus.Fields{
-		"sliceStart":                sliceStart,
-		"sliceEnd":                  sliceEnd,
-		"before recentStreamEvents": fmt.Sprintf("%+v", events),
-	}).Info("filterStreamEventsAccordingToHistoryVisibility: cutting down the events")
-
 	return recentStreamEvents[sliceStart:sliceEnd], limited
 }
 
 func removeDuplicates(stateEvents, recentEvents []*gomatrixserverlib.HeaderedEvent) []*gomatrixserverlib.HeaderedEvent {
-	for _, recentEv := range recentEvents {
-		if recentEv.StateKey() == nil {
-			continue // not a state event
-		}
-		// TODO: This is a linear scan over all the current state events in this room. This will
-		//       be slow for big rooms. We should instead sort the state events by event ID  (ORDER BY)
-		//       then do a binary search to find matching events, similar to what roomserver does.
-		for j := 0; j < len(stateEvents); j++ {
-			if stateEvents[j].EventID() == recentEv.EventID() {
-				// overwrite the element to remove with the last element then pop the last element.
-				// This is orders of magnitude faster than re-slicing, but doesn't preserve ordering
-				// (we don't care about the order of stateEvents)
-				stateEvents[j] = stateEvents[len(stateEvents)-1]
-				stateEvents = stateEvents[:len(stateEvents)-1]
-				break // there shouldn't be multiple events with the same event ID
-			}
+	timeline := map[string]struct{}{}
+	for _, event := range recentEvents {
+		if event.StateKey() == nil {
+			continue
 		}
+		timeline[event.EventID()] = struct{}{}
 	}
-	return stateEvents
+	state := []*gomatrixserverlib.HeaderedEvent{}
+	for _, event := range stateEvents {
+		if _, ok := timeline[event.EventID()]; ok {
+			continue
+		}
+		state = append(state, event)
+	}
+	return state
 }