Add presence module V2 (#2312)

* Syncapi presence

* Clientapi http presence handler

* Why is this here?

* Missing files

* FederationAPI presence implementation

* Add new presence stream

* Pinecone update

* Pinecone update

* Add passing tests

* Make linter happy

* Add presence producer

* Add presence config option

* Set user to unavailable after x minutes

* Only set currently_active if online
Avoid unneeded presence updates when syncing

* Tweaks

* Query devices for last_active_ts
Fixes & tweaks

* Export SharedUsers/SharedUsers

* Presence stream in MemoryStorage

* Remove status_msg_nil

* Fix sytest crashes

* Make presence types const and use stringer for it

* Change options to allow inbound/outbound presence

* Fix option & typo

* Update configs

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>

syncapi/streams/stream_presence.go

@@ -0,0 +1,179 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package streams
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"sync"
+
+	"github.com/matrix-org/dendrite/syncapi/notifier"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type PresenceStreamProvider struct {
+	StreamProvider
+	// cache contains previously sent presence updates to avoid unneeded updates
+	cache    sync.Map
+	notifier *notifier.Notifier
+}
+
+func (p *PresenceStreamProvider) Setup() {
+	p.StreamProvider.Setup()
+
+	id, err := p.DB.MaxStreamPositionForPresence(context.Background())
+	if err != nil {
+		panic(err)
+	}
+	p.latest = id
+}
+
+func (p *PresenceStreamProvider) CompleteSync(
+	ctx context.Context,
+	req *types.SyncRequest,
+) types.StreamPosition {
+	return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx))
+}
+
+func (p *PresenceStreamProvider) IncrementalSync(
+	ctx context.Context,
+	req *types.SyncRequest,
+	from, to types.StreamPosition,
+) types.StreamPosition {
+	presences, err := p.DB.PresenceAfter(ctx, from)
+	if err != nil {
+		req.Log.WithError(err).Error("p.DB.PresenceAfter failed")
+		return from
+	}
+
+	if len(presences) == 0 {
+		return to
+	}
+
+	// get all joined users
+	// TODO: SharedUsers might get out of sync
+	sharedUsers := p.notifier.SharedUsers(req.Device.UserID)
+
+	sharedUsersMap := map[string]bool{
+		req.Device.UserID: true,
+	}
+	// convert array to a map for easier checking if a user exists
+	for i := range sharedUsers {
+		sharedUsersMap[sharedUsers[i]] = true
+	}
+
+	// add newly joined rooms user presences
+	newlyJoined := joinedRooms(req.Response, req.Device.UserID)
+	if len(newlyJoined) > 0 {
+		// TODO: This refreshes all lists and is quite expensive
+		// The notifier should update the lists itself
+		if err = p.notifier.Load(ctx, p.DB); err != nil {
+			req.Log.WithError(err).Error("unable to refresh notifier lists")
+			return from
+		}
+		for _, roomID := range newlyJoined {
+			roomUsers := p.notifier.JoinedUsers(roomID)
+			for i := range roomUsers {
+				sharedUsersMap[roomUsers[i]] = true
+				// we already got a presence from this user
+				if _, ok := presences[roomUsers[i]]; ok {
+					continue
+				}
+				presences[roomUsers[i]], err = p.DB.GetPresence(ctx, roomUsers[i])
+				if err != nil {
+					if err == sql.ErrNoRows {
+						continue
+					}
+					req.Log.WithError(err).Error("unable to query presence for user")
+					return from
+				}
+			}
+		}
+	}
+
+	lastPos := to
+	for i := range presences {
+		presence := presences[i]
+		// Ignore users we don't share a room with
+		if !sharedUsersMap[presence.UserID] {
+			continue
+		}
+		cacheKey := req.Device.UserID + req.Device.ID + presence.UserID
+		pres, ok := p.cache.Load(cacheKey)
+		if ok {
+			// skip already sent presence
+			prevPresence := pres.(*types.PresenceInternal)
+			currentlyActive := prevPresence.CurrentlyActive()
+			skip := prevPresence.Equals(presence) && currentlyActive && req.Device.UserID != presence.UserID
+			if skip {
+				req.Log.Debugf("Skipping presence, no change (%s)", presence.UserID)
+				continue
+			}
+		}
+		presence.ClientFields.LastActiveAgo = presence.LastActiveAgo()
+		if presence.ClientFields.Presence == "online" {
+			currentlyActive := presence.CurrentlyActive()
+			presence.ClientFields.CurrentlyActive = &currentlyActive
+		}
+
+		content, err := json.Marshal(presence.ClientFields)
+		if err != nil {
+			return from
+		}
+
+		req.Response.Presence.Events = append(req.Response.Presence.Events, gomatrixserverlib.ClientEvent{
+			Content: content,
+			Sender:  presence.UserID,
+			Type:    gomatrixserverlib.MPresence,
+		})
+		if presence.StreamPos > lastPos {
+			lastPos = presence.StreamPos
+		}
+		p.cache.Store(cacheKey, presence)
+	}
+
+	return lastPos
+}
+
+func joinedRooms(res *types.Response, userID string) []string {
+	var roomIDs []string
+	for roomID, join := range res.Rooms.Join {
+		// we would expect to see our join event somewhere if we newly joined the room.
+		// Normal events get put in the join section so it's not enough to know the room ID is present in 'join'.
+		newlyJoined := membershipEventPresent(join.State.Events, userID)
+		if newlyJoined {
+			roomIDs = append(roomIDs, roomID)
+			continue
+		}
+		newlyJoined = membershipEventPresent(join.Timeline.Events, userID)
+		if newlyJoined {
+			roomIDs = append(roomIDs, roomID)
+		}
+	}
+	return roomIDs
+}
+
+func membershipEventPresent(events []gomatrixserverlib.ClientEvent, userID string) bool {
+	for _, ev := range events {
+		// it's enough to know that we have our member event here, don't need to check membership content
+		// as it's implied by being in the respective section of the sync response.
+		if ev.Type == gomatrixserverlib.MRoomMember && ev.StateKey != nil && *ev.StateKey == userID {
+			return true
+		}
+	}
+	return false
+}

syncapi/streams/streams.go

@@ -6,6 +6,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/caching"
 	keyapi "github.com/matrix-org/dendrite/keyserver/api"
 	rsapi "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/notifier"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
@@ -20,12 +21,13 @@ type Streams struct {
 	AccountDataStreamProvider      types.StreamProvider
 	DeviceListStreamProvider       types.StreamProvider
 	NotificationDataStreamProvider types.StreamProvider
+	PresenceStreamProvider         types.StreamProvider
 }
 
 func NewSyncStreamProviders(
 	d storage.Database, userAPI userapi.UserInternalAPI,
 	rsAPI rsapi.RoomserverInternalAPI, keyAPI keyapi.KeyInternalAPI,
-	eduCache *caching.EDUCache,
+	eduCache *caching.EDUCache, notifier *notifier.Notifier,
 ) *Streams {
 	streams := &Streams{
 		PDUStreamProvider: &PDUStreamProvider{
@@ -56,6 +58,10 @@ func NewSyncStreamProviders(
 			rsAPI:          rsAPI,
 			keyAPI:         keyAPI,
 		},
+		PresenceStreamProvider: &PresenceStreamProvider{
+			StreamProvider: StreamProvider{DB: d},
+			notifier:       notifier,
+		},
 	}
 
 	streams.PDUStreamProvider.Setup()
@@ -66,6 +72,7 @@ func NewSyncStreamProviders(
 	streams.AccountDataStreamProvider.Setup()
 	streams.NotificationDataStreamProvider.Setup()
 	streams.DeviceListStreamProvider.Setup()
+	streams.PresenceStreamProvider.Setup()
 
 	return streams
 }
@@ -80,5 +87,6 @@ func (s *Streams) Latest(ctx context.Context) types.StreamingToken {
 		AccountDataPosition:      s.AccountDataStreamProvider.LatestPosition(ctx),
 		NotificationDataPosition: s.NotificationDataStreamProvider.LatestPosition(ctx),
 		DeviceListPosition:       s.DeviceListStreamProvider.LatestPosition(ctx),
+		PresencePosition:         s.PresenceStreamProvider.LatestPosition(ctx),
 	}
 }