mirror of
https://github.com/hoernschen/dendrite.git
synced 2025-08-01 05:42:46 +00:00
syncapi: Rename and split out tokens (#1025)
* syncapi: Rename and split out tokens Previously we used the badly named `PaginationToken` which was used for both `/sync` and `/messages` requests. This quickly became confusing because named fields like `PDUPosition` meant different things depending on the token type. Instead, we now have two token types: `TopologyToken` and `StreamingToken`, both of which have fields which make more sense for their specific situations. Updated the codebase to use one or the other. `PaginationToken` still lives on as `syncToken`, an unexported type which both tokens rely on. This allows us to guarantee that the specific mappings of positions to a string remain solely under the control of the `types` package. This enables us to move high-level conceptual things like "decrement this topological token" to function calls e.g `TopologicalToken.Decrement()`. Currently broken because `/messages` seemingly used both stream and topological tokens, though I need to confirm this. * final tweaks/hacks * spurious logging * Review comments and linting
This commit is contained in:
Kegsay
GitHub
parent
31e6a7f193
commit
5e9dce1c0c
15 changed files with 457 additions and 439 deletions
|
|
@ -50,13 +50,13 @@ type Database interface {
|
|||
// Returns an error if there was an issue with the retrieval.
|
||||
GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.HeaderedEvent, err error)
|
||||
// SyncPosition returns the latest positions for syncing.
|
||||
SyncPosition(ctx context.Context) (types.PaginationToken, error)
|
||||
SyncPosition(ctx context.Context) (types.StreamingToken, error)
|
||||
// IncrementalSync returns all the data needed in order to create an incremental
|
||||
// sync response for the given user. Events returned will include any client
|
||||
// transaction IDs associated with the given device. These transaction IDs come
|
||||
// from when the device sent the event via an API that included a transaction
|
||||
// ID.
|
||||
IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error)
|
||||
IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.StreamingToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error)
|
||||
// CompleteSync returns a complete /sync API response for the given user.
|
||||
CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error)
|
||||
// GetAccountDataInRange returns all account data for a given user inserted or
|
||||
|
|
@ -88,9 +88,10 @@ type Database interface {
|
|||
// RemoveTypingUser removes a typing user from the typing cache.
|
||||
// Returns the newly calculated sync position for typing notifications.
|
||||
RemoveTypingUser(userID, roomID string) types.StreamPosition
|
||||
// GetEventsInRange retrieves all of the events on a given ordering using the
|
||||
// given extremities and limit.
|
||||
GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error)
|
||||
// GetEventsInStreamingRange retrieves all of the events on a given ordering using the given extremities and limit.
|
||||
GetEventsInStreamingRange(ctx context.Context, from, to *types.StreamingToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error)
|
||||
// GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit.
|
||||
GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error)
|
||||
// EventPositionInTopology returns the depth and stream position of the given event.
|
||||
EventPositionInTopology(ctx context.Context, eventID string) (depth types.StreamPosition, stream types.StreamPosition, err error)
|
||||
// EventsAtTopologicalPosition returns all of the events matching a given
|
||||
|
|
|
|||
|
|
@ -228,69 +228,68 @@ func (d *SyncServerDatasource) GetStateEventsForRoom(
|
|||
return
|
||||
}
|
||||
|
||||
func (d *SyncServerDatasource) GetEventsInRange(
|
||||
func (d *SyncServerDatasource) GetEventsInTopologicalRange(
|
||||
ctx context.Context,
|
||||
from, to *types.PaginationToken,
|
||||
from, to *types.TopologyToken,
|
||||
roomID string, limit int,
|
||||
backwardOrdering bool,
|
||||
) (events []types.StreamEvent, err error) {
|
||||
// If the pagination token's type is types.PaginationTokenTypeTopology, the
|
||||
// events must be retrieved from the rooms' topology table rather than the
|
||||
// table contaning the syncapi server's whole stream of events.
|
||||
if from.Type == types.PaginationTokenTypeTopology {
|
||||
// Determine the backward and forward limit, i.e. the upper and lower
|
||||
// limits to the selection in the room's topology, from the direction.
|
||||
var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition
|
||||
if backwardOrdering {
|
||||
// Backward ordering is antichronological (latest event to oldest
|
||||
// one).
|
||||
backwardLimit = to.PDUPosition
|
||||
forwardLimit = from.PDUPosition
|
||||
forwardMicroLimit = from.EDUTypingPosition
|
||||
} else {
|
||||
// Forward ordering is chronological (oldest event to latest one).
|
||||
backwardLimit = from.PDUPosition
|
||||
forwardLimit = to.PDUPosition
|
||||
}
|
||||
// Determine the backward and forward limit, i.e. the upper and lower
|
||||
// limits to the selection in the room's topology, from the direction.
|
||||
var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition
|
||||
if backwardOrdering {
|
||||
// Backward ordering is antichronological (latest event to oldest
|
||||
// one).
|
||||
backwardLimit = to.Depth()
|
||||
forwardLimit = from.Depth()
|
||||
forwardMicroLimit = from.PDUPosition()
|
||||
} else {
|
||||
// Forward ordering is chronological (oldest event to latest one).
|
||||
backwardLimit = from.Depth()
|
||||
forwardLimit = to.Depth()
|
||||
}
|
||||
|
||||
// Select the event IDs from the defined range.
|
||||
var eIDs []string
|
||||
eIDs, err = d.topology.selectEventIDsInRange(
|
||||
ctx, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering,
|
||||
)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Retrieve the events' contents using their IDs.
|
||||
events, err = d.events.selectEvents(ctx, nil, eIDs)
|
||||
// Select the event IDs from the defined range.
|
||||
var eIDs []string
|
||||
eIDs, err = d.topology.selectEventIDsInRange(
|
||||
ctx, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering,
|
||||
)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// If the pagination token's type is types.PaginationTokenTypeStream, the
|
||||
// events must be retrieved from the table contaning the syncapi server's
|
||||
// whole stream of events.
|
||||
// Retrieve the events' contents using their IDs.
|
||||
events, err = d.events.selectEvents(ctx, nil, eIDs)
|
||||
return
|
||||
}
|
||||
|
||||
// GetEventsInStreamingRange retrieves all of the events on a given ordering using the
|
||||
// given extremities and limit.
|
||||
func (d *SyncServerDatasource) GetEventsInStreamingRange(
|
||||
ctx context.Context,
|
||||
from, to *types.StreamingToken,
|
||||
roomID string, limit int,
|
||||
backwardOrdering bool,
|
||||
) (events []types.StreamEvent, err error) {
|
||||
if backwardOrdering {
|
||||
// When using backward ordering, we want the most recent events first.
|
||||
if events, err = d.events.selectRecentEvents(
|
||||
ctx, nil, roomID, to.PDUPosition, from.PDUPosition, limit, false, false,
|
||||
ctx, nil, roomID, to.PDUPosition(), from.PDUPosition(), limit, false, false,
|
||||
); err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// When using forward ordering, we want the least recent events first.
|
||||
if events, err = d.events.selectEarlyEvents(
|
||||
ctx, nil, roomID, from.PDUPosition, to.PDUPosition, limit,
|
||||
ctx, nil, roomID, from.PDUPosition(), to.PDUPosition(), limit,
|
||||
); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
return events, err
|
||||
}
|
||||
|
||||
func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.PaginationToken, error) {
|
||||
func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.StreamingToken, error) {
|
||||
return d.syncPositionTx(ctx, nil)
|
||||
}
|
||||
|
||||
|
|
@ -353,7 +352,7 @@ func (d *SyncServerDatasource) syncStreamPositionTx(
|
|||
|
||||
func (d *SyncServerDatasource) syncPositionTx(
|
||||
ctx context.Context, txn *sql.Tx,
|
||||
) (sp types.PaginationToken, err error) {
|
||||
) (sp types.StreamingToken, err error) {
|
||||
|
||||
maxEventID, err := d.events.selectMaxEventID(ctx, txn)
|
||||
if err != nil {
|
||||
|
|
@ -373,8 +372,7 @@ func (d *SyncServerDatasource) syncPositionTx(
|
|||
if maxInviteID > maxEventID {
|
||||
maxEventID = maxInviteID
|
||||
}
|
||||
sp.PDUPosition = types.StreamPosition(maxEventID)
|
||||
sp.EDUTypingPosition = types.StreamPosition(d.eduCache.GetLatestSyncPosition())
|
||||
sp = types.NewStreamToken(types.StreamPosition(maxEventID), types.StreamPosition(d.eduCache.GetLatestSyncPosition()))
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -439,7 +437,7 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse(
|
|||
// addTypingDeltaToResponse adds all typing notifications to a sync response
|
||||
// since the specified position.
|
||||
func (d *SyncServerDatasource) addTypingDeltaToResponse(
|
||||
since types.PaginationToken,
|
||||
since types.StreamingToken,
|
||||
joinedRoomIDs []string,
|
||||
res *types.Response,
|
||||
) error {
|
||||
|
|
@ -448,7 +446,7 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse(
|
|||
var err error
|
||||
for _, roomID := range joinedRoomIDs {
|
||||
if typingUsers, updated := d.eduCache.GetTypingUsersIfUpdatedAfter(
|
||||
roomID, int64(since.EDUTypingPosition),
|
||||
roomID, int64(since.EDUPosition()),
|
||||
); updated {
|
||||
ev := gomatrixserverlib.ClientEvent{
|
||||
Type: gomatrixserverlib.MTyping,
|
||||
|
|
@ -473,12 +471,12 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse(
|
|||
// addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if
|
||||
// the positions of that type are not equal in fromPos and toPos.
|
||||
func (d *SyncServerDatasource) addEDUDeltaToResponse(
|
||||
fromPos, toPos types.PaginationToken,
|
||||
fromPos, toPos types.StreamingToken,
|
||||
joinedRoomIDs []string,
|
||||
res *types.Response,
|
||||
) (err error) {
|
||||
|
||||
if fromPos.EDUTypingPosition != toPos.EDUTypingPosition {
|
||||
if fromPos.EDUPosition() != toPos.EDUPosition() {
|
||||
err = d.addTypingDeltaToResponse(
|
||||
fromPos, joinedRoomIDs, res,
|
||||
)
|
||||
|
|
@ -490,7 +488,7 @@ func (d *SyncServerDatasource) addEDUDeltaToResponse(
|
|||
func (d *SyncServerDatasource) IncrementalSync(
|
||||
ctx context.Context,
|
||||
device authtypes.Device,
|
||||
fromPos, toPos types.PaginationToken,
|
||||
fromPos, toPos types.StreamingToken,
|
||||
numRecentEventsPerRoom int,
|
||||
wantFullState bool,
|
||||
) (*types.Response, error) {
|
||||
|
|
@ -499,9 +497,9 @@ func (d *SyncServerDatasource) IncrementalSync(
|
|||
|
||||
var joinedRoomIDs []string
|
||||
var err error
|
||||
if fromPos.PDUPosition != toPos.PDUPosition || wantFullState {
|
||||
if fromPos.PDUPosition() != toPos.PDUPosition() || wantFullState {
|
||||
joinedRoomIDs, err = d.addPDUDeltaToResponse(
|
||||
ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, wantFullState, res,
|
||||
ctx, device, fromPos.PDUPosition(), toPos.PDUPosition(), numRecentEventsPerRoom, wantFullState, res,
|
||||
)
|
||||
} else {
|
||||
joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(
|
||||
|
|
@ -530,7 +528,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
|
|||
numRecentEventsPerRoom int,
|
||||
) (
|
||||
res *types.Response,
|
||||
toPos types.PaginationToken,
|
||||
toPos types.StreamingToken,
|
||||
joinedRoomIDs []string,
|
||||
err error,
|
||||
) {
|
||||
|
|
@ -577,7 +575,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
|
|||
// See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316
|
||||
var recentStreamEvents []types.StreamEvent
|
||||
recentStreamEvents, err = d.events.selectRecentEvents(
|
||||
ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition,
|
||||
ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition(),
|
||||
numRecentEventsPerRoom, true, true,
|
||||
)
|
||||
if err != nil {
|
||||
|
|
@ -588,27 +586,25 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
|
|||
// oldest event in the room's topology.
|
||||
var backwardTopologyPos, backwardStreamPos types.StreamPosition
|
||||
backwardTopologyPos, backwardStreamPos, err = d.topology.selectPositionInTopology(ctx, recentStreamEvents[0].EventID())
|
||||
if backwardTopologyPos-1 <= 0 {
|
||||
backwardTopologyPos = types.StreamPosition(1)
|
||||
} else {
|
||||
backwardTopologyPos--
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
prevBatch := types.NewTopologyToken(backwardTopologyPos, backwardStreamPos)
|
||||
prevBatch.Decrement()
|
||||
|
||||
// We don't include a device here as we don't need to send down
|
||||
// transaction IDs for complete syncs
|
||||
recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents)
|
||||
stateEvents = removeDuplicates(stateEvents, recentEvents)
|
||||
jr := types.NewJoinResponse()
|
||||
jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition(
|
||||
types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos,
|
||||
).String()
|
||||
jr.Timeline.PrevBatch = prevBatch.String()
|
||||
jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
|
||||
jr.Timeline.Limited = true
|
||||
jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync)
|
||||
res.Rooms.Join[roomID] = *jr
|
||||
}
|
||||
|
||||
if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition, res); err != nil {
|
||||
if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition(), res); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -628,7 +624,7 @@ func (d *SyncServerDatasource) CompleteSync(
|
|||
|
||||
// Use a zero value SyncPosition for fromPos so all EDU states are added.
|
||||
err = d.addEDUDeltaToResponse(
|
||||
types.PaginationToken{}, toPos, joinedRoomIDs, res,
|
||||
types.NewStreamToken(0, 0), toPos, joinedRoomIDs, res,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -757,14 +753,15 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse(
|
|||
recentEvents := d.StreamEventsToEvents(device, recentStreamEvents)
|
||||
delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back
|
||||
backwardTopologyPos, backwardStreamPos := d.getBackwardTopologyPos(ctx, recentStreamEvents)
|
||||
prevBatch := types.NewTopologyToken(
|
||||
backwardTopologyPos, backwardStreamPos,
|
||||
)
|
||||
|
||||
switch delta.membership {
|
||||
case gomatrixserverlib.Join:
|
||||
jr := types.NewJoinResponse()
|
||||
|
||||
jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition(
|
||||
types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos,
|
||||
).String()
|
||||
jr.Timeline.PrevBatch = prevBatch.String()
|
||||
jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
|
||||
jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true
|
||||
jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync)
|
||||
|
|
@ -775,9 +772,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse(
|
|||
// TODO: recentEvents may contain events that this user is not allowed to see because they are
|
||||
// no longer in the room.
|
||||
lr := types.NewLeaveResponse()
|
||||
lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition(
|
||||
types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos,
|
||||
).String()
|
||||
lr.Timeline.PrevBatch = prevBatch.String()
|
||||
lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
|
||||
lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true
|
||||
lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync)
|
||||
|
|
|
|||
|
|
@ -269,63 +269,63 @@ func (d *SyncServerDatasource) GetStateEventsForRoom(
|
|||
return
|
||||
}
|
||||
|
||||
// GetEventsInRange retrieves all of the events on a given ordering using the
|
||||
// GetEventsInTopologicalRange retrieves all of the events on a given ordering using the
|
||||
// given extremities and limit.
|
||||
func (d *SyncServerDatasource) GetEventsInRange(
|
||||
func (d *SyncServerDatasource) GetEventsInTopologicalRange(
|
||||
ctx context.Context,
|
||||
from, to *types.PaginationToken,
|
||||
from, to *types.TopologyToken,
|
||||
roomID string, limit int,
|
||||
backwardOrdering bool,
|
||||
) (events []types.StreamEvent, err error) {
|
||||
// If the pagination token's type is types.PaginationTokenTypeTopology, the
|
||||
// events must be retrieved from the rooms' topology table rather than the
|
||||
// table contaning the syncapi server's whole stream of events.
|
||||
if from.Type == types.PaginationTokenTypeTopology {
|
||||
// TODO: ARGH CONFUSING
|
||||
// Determine the backward and forward limit, i.e. the upper and lower
|
||||
// limits to the selection in the room's topology, from the direction.
|
||||
var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition
|
||||
if backwardOrdering {
|
||||
// Backward ordering is antichronological (latest event to oldest
|
||||
// one).
|
||||
backwardLimit = to.PDUPosition
|
||||
forwardLimit = from.PDUPosition
|
||||
forwardMicroLimit = from.EDUTypingPosition
|
||||
} else {
|
||||
// Forward ordering is chronological (oldest event to latest one).
|
||||
backwardLimit = from.PDUPosition
|
||||
forwardLimit = to.PDUPosition
|
||||
}
|
||||
// TODO: ARGH CONFUSING
|
||||
// Determine the backward and forward limit, i.e. the upper and lower
|
||||
// limits to the selection in the room's topology, from the direction.
|
||||
var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition
|
||||
if backwardOrdering {
|
||||
// Backward ordering is antichronological (latest event to oldest
|
||||
// one).
|
||||
backwardLimit = to.Depth()
|
||||
forwardLimit = from.Depth()
|
||||
forwardMicroLimit = from.PDUPosition()
|
||||
} else {
|
||||
// Forward ordering is chronological (oldest event to latest one).
|
||||
backwardLimit = from.Depth()
|
||||
forwardLimit = to.Depth()
|
||||
}
|
||||
|
||||
// Select the event IDs from the defined range.
|
||||
var eIDs []string
|
||||
eIDs, err = d.topology.selectEventIDsInRange(
|
||||
ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering,
|
||||
)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Retrieve the events' contents using their IDs.
|
||||
events, err = d.events.selectEvents(ctx, nil, eIDs)
|
||||
// Select the event IDs from the defined range.
|
||||
var eIDs []string
|
||||
eIDs, err = d.topology.selectEventIDsInRange(
|
||||
ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering,
|
||||
)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// If the pagination token's type is types.PaginationTokenTypeStream, the
|
||||
// events must be retrieved from the table contaning the syncapi server's
|
||||
// whole stream of events.
|
||||
// Retrieve the events' contents using their IDs.
|
||||
events, err = d.events.selectEvents(ctx, nil, eIDs)
|
||||
return
|
||||
}
|
||||
|
||||
// GetEventsInStreamingRange retrieves all of the events on a given ordering using the
|
||||
// given extremities and limit.
|
||||
func (d *SyncServerDatasource) GetEventsInStreamingRange(
|
||||
ctx context.Context,
|
||||
from, to *types.StreamingToken,
|
||||
roomID string, limit int,
|
||||
backwardOrdering bool,
|
||||
) (events []types.StreamEvent, err error) {
|
||||
if backwardOrdering {
|
||||
// When using backward ordering, we want the most recent events first.
|
||||
if events, err = d.events.selectRecentEvents(
|
||||
ctx, nil, roomID, to.PDUPosition, from.PDUPosition, limit, false, false,
|
||||
ctx, nil, roomID, to.PDUPosition(), from.PDUPosition(), limit, false, false,
|
||||
); err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// When using forward ordering, we want the least recent events first.
|
||||
if events, err = d.events.selectEarlyEvents(
|
||||
ctx, nil, roomID, from.PDUPosition, to.PDUPosition, limit,
|
||||
ctx, nil, roomID, from.PDUPosition(), to.PDUPosition(), limit,
|
||||
); err != nil {
|
||||
return
|
||||
}
|
||||
|
|
@ -334,10 +334,14 @@ func (d *SyncServerDatasource) GetEventsInRange(
|
|||
}
|
||||
|
||||
// SyncPosition returns the latest positions for syncing.
|
||||
func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.PaginationToken, err error) {
|
||||
func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.StreamingToken, err error) {
|
||||
err = common.WithTransaction(d.db, func(txn *sql.Tx) error {
|
||||
tok, err = d.syncPositionTx(ctx, txn)
|
||||
return err
|
||||
pos, err := d.syncPositionTx(ctx, txn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tok = *pos
|
||||
return nil
|
||||
})
|
||||
return
|
||||
}
|
||||
|
|
@ -412,30 +416,31 @@ func (d *SyncServerDatasource) syncStreamPositionTx(
|
|||
|
||||
func (d *SyncServerDatasource) syncPositionTx(
|
||||
ctx context.Context, txn *sql.Tx,
|
||||
) (sp types.PaginationToken, err error) {
|
||||
) (*types.StreamingToken, error) {
|
||||
|
||||
maxEventID, err := d.events.selectMaxEventID(ctx, txn)
|
||||
if err != nil {
|
||||
return sp, err
|
||||
return nil, err
|
||||
}
|
||||
maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn)
|
||||
if err != nil {
|
||||
return sp, err
|
||||
return nil, err
|
||||
}
|
||||
if maxAccountDataID > maxEventID {
|
||||
maxEventID = maxAccountDataID
|
||||
}
|
||||
maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn)
|
||||
if err != nil {
|
||||
return sp, err
|
||||
return nil, err
|
||||
}
|
||||
if maxInviteID > maxEventID {
|
||||
maxEventID = maxInviteID
|
||||
}
|
||||
sp.PDUPosition = types.StreamPosition(maxEventID)
|
||||
sp.EDUTypingPosition = types.StreamPosition(d.eduCache.GetLatestSyncPosition())
|
||||
sp.Type = types.PaginationTokenTypeStream
|
||||
return
|
||||
sp := types.NewStreamToken(
|
||||
types.StreamPosition(maxEventID),
|
||||
types.StreamPosition(d.eduCache.GetLatestSyncPosition()),
|
||||
)
|
||||
return &sp, nil
|
||||
}
|
||||
|
||||
// addPDUDeltaToResponse adds all PDU deltas to a sync response.
|
||||
|
|
@ -499,7 +504,7 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse(
|
|||
// addTypingDeltaToResponse adds all typing notifications to a sync response
|
||||
// since the specified position.
|
||||
func (d *SyncServerDatasource) addTypingDeltaToResponse(
|
||||
since types.PaginationToken,
|
||||
since types.StreamingToken,
|
||||
joinedRoomIDs []string,
|
||||
res *types.Response,
|
||||
) error {
|
||||
|
|
@ -508,7 +513,7 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse(
|
|||
var err error
|
||||
for _, roomID := range joinedRoomIDs {
|
||||
if typingUsers, updated := d.eduCache.GetTypingUsersIfUpdatedAfter(
|
||||
roomID, int64(since.EDUTypingPosition),
|
||||
roomID, int64(since.EDUPosition()),
|
||||
); updated {
|
||||
ev := gomatrixserverlib.ClientEvent{
|
||||
Type: gomatrixserverlib.MTyping,
|
||||
|
|
@ -533,12 +538,12 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse(
|
|||
// addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if
|
||||
// the positions of that type are not equal in fromPos and toPos.
|
||||
func (d *SyncServerDatasource) addEDUDeltaToResponse(
|
||||
fromPos, toPos types.PaginationToken,
|
||||
fromPos, toPos types.StreamingToken,
|
||||
joinedRoomIDs []string,
|
||||
res *types.Response,
|
||||
) (err error) {
|
||||
|
||||
if fromPos.EDUTypingPosition != toPos.EDUTypingPosition {
|
||||
if fromPos.EDUPosition() != toPos.EDUPosition() {
|
||||
err = d.addTypingDeltaToResponse(
|
||||
fromPos, joinedRoomIDs, res,
|
||||
)
|
||||
|
|
@ -555,18 +560,21 @@ func (d *SyncServerDatasource) addEDUDeltaToResponse(
|
|||
func (d *SyncServerDatasource) IncrementalSync(
|
||||
ctx context.Context,
|
||||
device authtypes.Device,
|
||||
fromPos, toPos types.PaginationToken,
|
||||
fromPos, toPos types.StreamingToken,
|
||||
numRecentEventsPerRoom int,
|
||||
wantFullState bool,
|
||||
) (*types.Response, error) {
|
||||
fmt.Println("from ", fromPos, "to", toPos)
|
||||
nextBatchPos := fromPos.WithUpdates(toPos)
|
||||
res := types.NewResponse(nextBatchPos)
|
||||
fmt.Println("from ", fromPos, "to", toPos, "next", nextBatchPos)
|
||||
|
||||
var joinedRoomIDs []string
|
||||
var err error
|
||||
if fromPos.PDUPosition != toPos.PDUPosition || wantFullState {
|
||||
fmt.Println("from", fromPos.PDUPosition(), "to", toPos.PDUPosition())
|
||||
if fromPos.PDUPosition() != toPos.PDUPosition() || wantFullState {
|
||||
joinedRoomIDs, err = d.addPDUDeltaToResponse(
|
||||
ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, wantFullState, res,
|
||||
ctx, device, fromPos.PDUPosition(), toPos.PDUPosition(), numRecentEventsPerRoom, wantFullState, res,
|
||||
)
|
||||
} else {
|
||||
joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(
|
||||
|
|
@ -595,7 +603,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
|
|||
numRecentEventsPerRoom int,
|
||||
) (
|
||||
res *types.Response,
|
||||
toPos types.PaginationToken,
|
||||
toPos *types.StreamingToken,
|
||||
joinedRoomIDs []string,
|
||||
err error,
|
||||
) {
|
||||
|
|
@ -621,7 +629,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
|
|||
return
|
||||
}
|
||||
|
||||
res = types.NewResponse(toPos)
|
||||
res = types.NewResponse(*toPos)
|
||||
|
||||
// Extract room state and recent events for all rooms the user is joined to.
|
||||
joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join)
|
||||
|
|
@ -643,7 +651,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
|
|||
// See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316
|
||||
var recentStreamEvents []types.StreamEvent
|
||||
recentStreamEvents, err = d.events.selectRecentEvents(
|
||||
ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition,
|
||||
ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition(),
|
||||
numRecentEventsPerRoom, true, true,
|
||||
)
|
||||
if err != nil {
|
||||
|
|
@ -655,28 +663,22 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
|
|||
// oldest event in the room's topology.
|
||||
var backwardTopologyPos, backwardTopologyStreamPos types.StreamPosition
|
||||
backwardTopologyPos, backwardTopologyStreamPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID())
|
||||
if backwardTopologyPos-1 <= 0 {
|
||||
backwardTopologyPos = types.StreamPosition(1)
|
||||
} else {
|
||||
backwardTopologyPos--
|
||||
backwardTopologyStreamPos += 1000 // this has to be bigger than the number of events we backfill per request
|
||||
}
|
||||
prevBatch := types.NewTopologyToken(backwardTopologyPos, backwardTopologyStreamPos)
|
||||
prevBatch.Decrement()
|
||||
|
||||
// We don't include a device here as we don't need to send down
|
||||
// transaction IDs for complete syncs
|
||||
recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents)
|
||||
stateEvents = removeDuplicates(stateEvents, recentEvents)
|
||||
jr := types.NewJoinResponse()
|
||||
jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition(
|
||||
types.PaginationTokenTypeTopology, backwardTopologyPos, backwardTopologyStreamPos,
|
||||
).String()
|
||||
jr.Timeline.PrevBatch = prevBatch.String()
|
||||
jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
|
||||
jr.Timeline.Limited = true
|
||||
jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync)
|
||||
res.Rooms.Join[roomID] = *jr
|
||||
}
|
||||
|
||||
if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition, res); err != nil {
|
||||
if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition(), res); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -697,7 +699,7 @@ func (d *SyncServerDatasource) CompleteSync(
|
|||
|
||||
// Use a zero value SyncPosition for fromPos so all EDU states are added.
|
||||
err = d.addEDUDeltaToResponse(
|
||||
types.PaginationToken{}, toPos, joinedRoomIDs, res,
|
||||
types.NewStreamToken(0, 0), *toPos, joinedRoomIDs, res,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -860,14 +862,14 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse(
|
|||
recentEvents := d.StreamEventsToEvents(device, recentStreamEvents)
|
||||
delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents)
|
||||
backwardTopologyPos, backwardStreamPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents)
|
||||
prevBatch := types.NewTopologyToken(
|
||||
backwardTopologyPos, backwardStreamPos,
|
||||
)
|
||||
|
||||
switch delta.membership {
|
||||
case gomatrixserverlib.Join:
|
||||
jr := types.NewJoinResponse()
|
||||
|
||||
jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition(
|
||||
types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos,
|
||||
).String()
|
||||
jr.Timeline.PrevBatch = prevBatch.String()
|
||||
jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
|
||||
jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true
|
||||
jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync)
|
||||
|
|
@ -878,9 +880,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse(
|
|||
// TODO: recentEvents may contain events that this user is not allowed to see because they are
|
||||
// no longer in the room.
|
||||
lr := types.NewLeaveResponse()
|
||||
lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition(
|
||||
types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos,
|
||||
).String()
|
||||
lr.Timeline.PrevBatch = prevBatch.String()
|
||||
lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
|
||||
lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true
|
||||
lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync)
|
||||
|
|
|
|||
|
|
@ -154,10 +154,10 @@ func TestSyncResponse(t *testing.T) {
|
|||
{
|
||||
Name: "IncrementalSync penultimate",
|
||||
DoSync: func() (*types.Response, error) {
|
||||
from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are at the penultimate event
|
||||
types.PaginationTokenTypeStream, positions[len(positions)-2], types.StreamPosition(0),
|
||||
from := types.NewStreamToken( // pretend we are at the penultimate event
|
||||
positions[len(positions)-2], types.StreamPosition(0),
|
||||
)
|
||||
return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false)
|
||||
return db.IncrementalSync(ctx, testUserDeviceA, from, latest, 5, false)
|
||||
},
|
||||
WantTimeline: events[len(events)-1:],
|
||||
},
|
||||
|
|
@ -166,11 +166,11 @@ func TestSyncResponse(t *testing.T) {
|
|||
{
|
||||
Name: "IncrementalSync limited",
|
||||
DoSync: func() (*types.Response, error) {
|
||||
from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are 10 events behind
|
||||
types.PaginationTokenTypeStream, positions[len(positions)-11], types.StreamPosition(0),
|
||||
from := types.NewStreamToken( // pretend we are 10 events behind
|
||||
positions[len(positions)-11], types.StreamPosition(0),
|
||||
)
|
||||
// limit is set to 5
|
||||
return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false)
|
||||
return db.IncrementalSync(ctx, testUserDeviceA, from, latest, 5, false)
|
||||
},
|
||||
// want the last 5 events, NOT the last 10.
|
||||
WantTimeline: events[len(events)-5:],
|
||||
|
|
@ -207,7 +207,7 @@ func TestSyncResponse(t *testing.T) {
|
|||
if err != nil {
|
||||
st.Fatalf("failed to do sync: %s", err)
|
||||
}
|
||||
next := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeStream, latest.PDUPosition, latest.EDUTypingPosition)
|
||||
next := types.NewStreamToken(latest.PDUPosition(), latest.EDUPosition())
|
||||
if res.NextBatch != next.String() {
|
||||
st.Errorf("NextBatch got %s want %s", res.NextBatch, next.String())
|
||||
}
|
||||
|
|
@ -230,11 +230,11 @@ func TestGetEventsInRangeWithPrevBatch(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("failed to get SyncPosition: %s", err)
|
||||
}
|
||||
from := types.NewPaginationTokenFromTypeAndPosition(
|
||||
types.PaginationTokenTypeStream, positions[len(positions)-2], types.StreamPosition(0),
|
||||
from := types.NewStreamToken(
|
||||
positions[len(positions)-2], types.StreamPosition(0),
|
||||
)
|
||||
|
||||
res, err := db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false)
|
||||
res, err := db.IncrementalSync(ctx, testUserDeviceA, from, latest, 5, false)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to IncrementalSync with latest token")
|
||||
}
|
||||
|
|
@ -249,14 +249,14 @@ func TestGetEventsInRangeWithPrevBatch(t *testing.T) {
|
|||
if prev == "" {
|
||||
t.Fatalf("IncrementalSync expected prev_batch token")
|
||||
}
|
||||
prevBatchToken, err := types.NewPaginationTokenFromString(prev)
|
||||
prevBatchToken, err := types.NewTopologyTokenFromString(prev)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to NewPaginationTokenFromString : %s", err)
|
||||
t.Fatalf("failed to NewTopologyTokenFromString : %s", err)
|
||||
}
|
||||
// backpaginate 5 messages starting at the latest position.
|
||||
// head towards the beginning of time
|
||||
to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0)
|
||||
paginatedEvents, err := db.GetEventsInRange(ctx, prevBatchToken, to, testRoomID, 5, true)
|
||||
to := types.NewTopologyToken(0, 0)
|
||||
paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &prevBatchToken, &to, testRoomID, 5, true)
|
||||
if err != nil {
|
||||
t.Fatalf("GetEventsInRange returned an error: %s", err)
|
||||
}
|
||||
|
|
@ -275,10 +275,10 @@ func TestGetEventsInRangeWithStreamToken(t *testing.T) {
|
|||
t.Fatalf("failed to get SyncPosition: %s", err)
|
||||
}
|
||||
// head towards the beginning of time
|
||||
to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0)
|
||||
to := types.NewStreamToken(0, 0)
|
||||
|
||||
// backpaginate 5 messages starting at the latest position.
|
||||
paginatedEvents, err := db.GetEventsInRange(ctx, &latest, to, testRoomID, 5, true)
|
||||
paginatedEvents, err := db.GetEventsInStreamingRange(ctx, &latest, &to, testRoomID, 5, true)
|
||||
if err != nil {
|
||||
t.Fatalf("GetEventsInRange returned an error: %s", err)
|
||||
}
|
||||
|
|
@ -296,12 +296,12 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("failed to get MaxTopologicalPosition: %s", err)
|
||||
}
|
||||
from := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latest, latestStream)
|
||||
from := types.NewTopologyToken(latest, latestStream)
|
||||
// head towards the beginning of time
|
||||
to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0)
|
||||
to := types.NewTopologyToken(0, 0)
|
||||
|
||||
// backpaginate 5 messages starting at the latest position.
|
||||
paginatedEvents, err := db.GetEventsInRange(ctx, from, to, testRoomID, 5, true)
|
||||
paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &from, &to, testRoomID, 5, true)
|
||||
if err != nil {
|
||||
t.Fatalf("GetEventsInRange returned an error: %s", err)
|
||||
}
|
||||
|
|
@ -366,14 +366,14 @@ func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("failed to get EventPositionInTopology for event: %s", err)
|
||||
}
|
||||
fromLatest := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latestPos, latestStreamPos)
|
||||
fromFork := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, topoPos, streamPos)
|
||||
fromLatest := types.NewTopologyToken(latestPos, latestStreamPos)
|
||||
fromFork := types.NewTopologyToken(topoPos, streamPos)
|
||||
// head towards the beginning of time
|
||||
to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0)
|
||||
to := types.NewTopologyToken(0, 0)
|
||||
|
||||
testCases := []struct {
|
||||
Name string
|
||||
From *types.PaginationToken
|
||||
From types.TopologyToken
|
||||
Limit int
|
||||
Wants []gomatrixserverlib.HeaderedEvent
|
||||
}{
|
||||
|
|
@ -399,7 +399,7 @@ func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) {
|
|||
|
||||
for _, tc := range testCases {
|
||||
// backpaginate messages starting at the latest position.
|
||||
paginatedEvents, err := db.GetEventsInRange(ctx, tc.From, to, testRoomID, tc.Limit, true)
|
||||
paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &tc.From, &to, testRoomID, tc.Limit, true)
|
||||
if err != nil {
|
||||
t.Fatalf("%s GetEventsInRange returned an error: %s", tc.Name, err)
|
||||
}
|
||||
|
|
@ -446,13 +446,13 @@ func TestGetEventsInRangeWithEventsInsertedLikeBackfill(t *testing.T) {
|
|||
}
|
||||
|
||||
// head towards the beginning of time
|
||||
to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0)
|
||||
to := types.NewTopologyToken(0, 0)
|
||||
|
||||
// starting at `from`, backpaginate to the beginning of time, asserting as we go.
|
||||
chunkSize = 3
|
||||
events = reversed(events)
|
||||
for i := 0; i < len(events); i += chunkSize {
|
||||
paginatedEvents, err := db.GetEventsInRange(ctx, from, to, testRoomID, chunkSize, true)
|
||||
paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, from, &to, testRoomID, chunkSize, true)
|
||||
if err != nil {
|
||||
t.Fatalf("GetEventsInRange returned an error: %s", err)
|
||||
}
|
||||
|
|
@ -506,19 +506,15 @@ func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatr
|
|||
}
|
||||
}
|
||||
|
||||
func topologyTokenBefore(t *testing.T, db storage.Database, eventID string) *types.PaginationToken {
|
||||
func topologyTokenBefore(t *testing.T, db storage.Database, eventID string) *types.TopologyToken {
|
||||
pos, spos, err := db.EventPositionInTopology(ctx, eventID)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get EventPositionInTopology: %s", err)
|
||||
}
|
||||
|
||||
if pos-1 <= 0 {
|
||||
pos = types.StreamPosition(1)
|
||||
} else {
|
||||
pos = pos - 1
|
||||
spos += 1000 // this has to be bigger than the chunk limit
|
||||
}
|
||||
return types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, spos)
|
||||
tok := types.NewTopologyToken(pos, spos)
|
||||
tok.Decrement()
|
||||
return &tok
|
||||
}
|
||||
|
||||
func reversed(in []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue