mirror of
https://github.com/hoernschen/dendrite.git
synced 2025-08-02 14:12:47 +00:00
syncapi: Rename and split out tokens (#1025)
* syncapi: Rename and split out tokens Previously we used the badly named `PaginationToken` which was used for both `/sync` and `/messages` requests. This quickly became confusing because named fields like `PDUPosition` meant different things depending on the token type. Instead, we now have two token types: `TopologyToken` and `StreamingToken`, both of which have fields which make more sense for their specific situations. Updated the codebase to use one or the other. `PaginationToken` still lives on as `syncToken`, an unexported type which both tokens rely on. This allows us to guarantee that the specific mappings of positions to a string remain solely under the control of the `types` package. This enables us to move high-level conceptual things like "decrement this topological token" to function calls e.g `TopologicalToken.Decrement()`. Currently broken because `/messages` seemingly used both stream and topological tokens, though I need to confirm this. * final tweaks/hacks * spurious logging * Review comments and linting
This commit is contained in:
Kegsay
GitHub
parent
31e6a7f193
commit
5e9dce1c0c
15 changed files with 457 additions and 439 deletions
|
|
@ -269,63 +269,63 @@ func (d *SyncServerDatasource) GetStateEventsForRoom(
|
|||
return
|
||||
}
|
||||
|
||||
// GetEventsInRange retrieves all of the events on a given ordering using the
|
||||
// GetEventsInTopologicalRange retrieves all of the events on a given ordering using the
|
||||
// given extremities and limit.
|
||||
func (d *SyncServerDatasource) GetEventsInRange(
|
||||
func (d *SyncServerDatasource) GetEventsInTopologicalRange(
|
||||
ctx context.Context,
|
||||
from, to *types.PaginationToken,
|
||||
from, to *types.TopologyToken,
|
||||
roomID string, limit int,
|
||||
backwardOrdering bool,
|
||||
) (events []types.StreamEvent, err error) {
|
||||
// If the pagination token's type is types.PaginationTokenTypeTopology, the
|
||||
// events must be retrieved from the rooms' topology table rather than the
|
||||
// table contaning the syncapi server's whole stream of events.
|
||||
if from.Type == types.PaginationTokenTypeTopology {
|
||||
// TODO: ARGH CONFUSING
|
||||
// Determine the backward and forward limit, i.e. the upper and lower
|
||||
// limits to the selection in the room's topology, from the direction.
|
||||
var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition
|
||||
if backwardOrdering {
|
||||
// Backward ordering is antichronological (latest event to oldest
|
||||
// one).
|
||||
backwardLimit = to.PDUPosition
|
||||
forwardLimit = from.PDUPosition
|
||||
forwardMicroLimit = from.EDUTypingPosition
|
||||
} else {
|
||||
// Forward ordering is chronological (oldest event to latest one).
|
||||
backwardLimit = from.PDUPosition
|
||||
forwardLimit = to.PDUPosition
|
||||
}
|
||||
// TODO: ARGH CONFUSING
|
||||
// Determine the backward and forward limit, i.e. the upper and lower
|
||||
// limits to the selection in the room's topology, from the direction.
|
||||
var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition
|
||||
if backwardOrdering {
|
||||
// Backward ordering is antichronological (latest event to oldest
|
||||
// one).
|
||||
backwardLimit = to.Depth()
|
||||
forwardLimit = from.Depth()
|
||||
forwardMicroLimit = from.PDUPosition()
|
||||
} else {
|
||||
// Forward ordering is chronological (oldest event to latest one).
|
||||
backwardLimit = from.Depth()
|
||||
forwardLimit = to.Depth()
|
||||
}
|
||||
|
||||
// Select the event IDs from the defined range.
|
||||
var eIDs []string
|
||||
eIDs, err = d.topology.selectEventIDsInRange(
|
||||
ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering,
|
||||
)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Retrieve the events' contents using their IDs.
|
||||
events, err = d.events.selectEvents(ctx, nil, eIDs)
|
||||
// Select the event IDs from the defined range.
|
||||
var eIDs []string
|
||||
eIDs, err = d.topology.selectEventIDsInRange(
|
||||
ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering,
|
||||
)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// If the pagination token's type is types.PaginationTokenTypeStream, the
|
||||
// events must be retrieved from the table contaning the syncapi server's
|
||||
// whole stream of events.
|
||||
// Retrieve the events' contents using their IDs.
|
||||
events, err = d.events.selectEvents(ctx, nil, eIDs)
|
||||
return
|
||||
}
|
||||
|
||||
// GetEventsInStreamingRange retrieves all of the events on a given ordering using the
|
||||
// given extremities and limit.
|
||||
func (d *SyncServerDatasource) GetEventsInStreamingRange(
|
||||
ctx context.Context,
|
||||
from, to *types.StreamingToken,
|
||||
roomID string, limit int,
|
||||
backwardOrdering bool,
|
||||
) (events []types.StreamEvent, err error) {
|
||||
if backwardOrdering {
|
||||
// When using backward ordering, we want the most recent events first.
|
||||
if events, err = d.events.selectRecentEvents(
|
||||
ctx, nil, roomID, to.PDUPosition, from.PDUPosition, limit, false, false,
|
||||
ctx, nil, roomID, to.PDUPosition(), from.PDUPosition(), limit, false, false,
|
||||
); err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// When using forward ordering, we want the least recent events first.
|
||||
if events, err = d.events.selectEarlyEvents(
|
||||
ctx, nil, roomID, from.PDUPosition, to.PDUPosition, limit,
|
||||
ctx, nil, roomID, from.PDUPosition(), to.PDUPosition(), limit,
|
||||
); err != nil {
|
||||
return
|
||||
}
|
||||
|
|
@ -334,10 +334,14 @@ func (d *SyncServerDatasource) GetEventsInRange(
|
|||
}
|
||||
|
||||
// SyncPosition returns the latest positions for syncing.
|
||||
func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.PaginationToken, err error) {
|
||||
func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.StreamingToken, err error) {
|
||||
err = common.WithTransaction(d.db, func(txn *sql.Tx) error {
|
||||
tok, err = d.syncPositionTx(ctx, txn)
|
||||
return err
|
||||
pos, err := d.syncPositionTx(ctx, txn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tok = *pos
|
||||
return nil
|
||||
})
|
||||
return
|
||||
}
|
||||
|
|
@ -412,30 +416,31 @@ func (d *SyncServerDatasource) syncStreamPositionTx(
|
|||
|
||||
func (d *SyncServerDatasource) syncPositionTx(
|
||||
ctx context.Context, txn *sql.Tx,
|
||||
) (sp types.PaginationToken, err error) {
|
||||
) (*types.StreamingToken, error) {
|
||||
|
||||
maxEventID, err := d.events.selectMaxEventID(ctx, txn)
|
||||
if err != nil {
|
||||
return sp, err
|
||||
return nil, err
|
||||
}
|
||||
maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn)
|
||||
if err != nil {
|
||||
return sp, err
|
||||
return nil, err
|
||||
}
|
||||
if maxAccountDataID > maxEventID {
|
||||
maxEventID = maxAccountDataID
|
||||
}
|
||||
maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn)
|
||||
if err != nil {
|
||||
return sp, err
|
||||
return nil, err
|
||||
}
|
||||
if maxInviteID > maxEventID {
|
||||
maxEventID = maxInviteID
|
||||
}
|
||||
sp.PDUPosition = types.StreamPosition(maxEventID)
|
||||
sp.EDUTypingPosition = types.StreamPosition(d.eduCache.GetLatestSyncPosition())
|
||||
sp.Type = types.PaginationTokenTypeStream
|
||||
return
|
||||
sp := types.NewStreamToken(
|
||||
types.StreamPosition(maxEventID),
|
||||
types.StreamPosition(d.eduCache.GetLatestSyncPosition()),
|
||||
)
|
||||
return &sp, nil
|
||||
}
|
||||
|
||||
// addPDUDeltaToResponse adds all PDU deltas to a sync response.
|
||||
|
|
@ -499,7 +504,7 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse(
|
|||
// addTypingDeltaToResponse adds all typing notifications to a sync response
|
||||
// since the specified position.
|
||||
func (d *SyncServerDatasource) addTypingDeltaToResponse(
|
||||
since types.PaginationToken,
|
||||
since types.StreamingToken,
|
||||
joinedRoomIDs []string,
|
||||
res *types.Response,
|
||||
) error {
|
||||
|
|
@ -508,7 +513,7 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse(
|
|||
var err error
|
||||
for _, roomID := range joinedRoomIDs {
|
||||
if typingUsers, updated := d.eduCache.GetTypingUsersIfUpdatedAfter(
|
||||
roomID, int64(since.EDUTypingPosition),
|
||||
roomID, int64(since.EDUPosition()),
|
||||
); updated {
|
||||
ev := gomatrixserverlib.ClientEvent{
|
||||
Type: gomatrixserverlib.MTyping,
|
||||
|
|
@ -533,12 +538,12 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse(
|
|||
// addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if
|
||||
// the positions of that type are not equal in fromPos and toPos.
|
||||
func (d *SyncServerDatasource) addEDUDeltaToResponse(
|
||||
fromPos, toPos types.PaginationToken,
|
||||
fromPos, toPos types.StreamingToken,
|
||||
joinedRoomIDs []string,
|
||||
res *types.Response,
|
||||
) (err error) {
|
||||
|
||||
if fromPos.EDUTypingPosition != toPos.EDUTypingPosition {
|
||||
if fromPos.EDUPosition() != toPos.EDUPosition() {
|
||||
err = d.addTypingDeltaToResponse(
|
||||
fromPos, joinedRoomIDs, res,
|
||||
)
|
||||
|
|
@ -555,18 +560,21 @@ func (d *SyncServerDatasource) addEDUDeltaToResponse(
|
|||
func (d *SyncServerDatasource) IncrementalSync(
|
||||
ctx context.Context,
|
||||
device authtypes.Device,
|
||||
fromPos, toPos types.PaginationToken,
|
||||
fromPos, toPos types.StreamingToken,
|
||||
numRecentEventsPerRoom int,
|
||||
wantFullState bool,
|
||||
) (*types.Response, error) {
|
||||
fmt.Println("from ", fromPos, "to", toPos)
|
||||
nextBatchPos := fromPos.WithUpdates(toPos)
|
||||
res := types.NewResponse(nextBatchPos)
|
||||
fmt.Println("from ", fromPos, "to", toPos, "next", nextBatchPos)
|
||||
|
||||
var joinedRoomIDs []string
|
||||
var err error
|
||||
if fromPos.PDUPosition != toPos.PDUPosition || wantFullState {
|
||||
fmt.Println("from", fromPos.PDUPosition(), "to", toPos.PDUPosition())
|
||||
if fromPos.PDUPosition() != toPos.PDUPosition() || wantFullState {
|
||||
joinedRoomIDs, err = d.addPDUDeltaToResponse(
|
||||
ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, wantFullState, res,
|
||||
ctx, device, fromPos.PDUPosition(), toPos.PDUPosition(), numRecentEventsPerRoom, wantFullState, res,
|
||||
)
|
||||
} else {
|
||||
joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(
|
||||
|
|
@ -595,7 +603,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
|
|||
numRecentEventsPerRoom int,
|
||||
) (
|
||||
res *types.Response,
|
||||
toPos types.PaginationToken,
|
||||
toPos *types.StreamingToken,
|
||||
joinedRoomIDs []string,
|
||||
err error,
|
||||
) {
|
||||
|
|
@ -621,7 +629,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
|
|||
return
|
||||
}
|
||||
|
||||
res = types.NewResponse(toPos)
|
||||
res = types.NewResponse(*toPos)
|
||||
|
||||
// Extract room state and recent events for all rooms the user is joined to.
|
||||
joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join)
|
||||
|
|
@ -643,7 +651,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
|
|||
// See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316
|
||||
var recentStreamEvents []types.StreamEvent
|
||||
recentStreamEvents, err = d.events.selectRecentEvents(
|
||||
ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition,
|
||||
ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition(),
|
||||
numRecentEventsPerRoom, true, true,
|
||||
)
|
||||
if err != nil {
|
||||
|
|
@ -655,28 +663,22 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
|
|||
// oldest event in the room's topology.
|
||||
var backwardTopologyPos, backwardTopologyStreamPos types.StreamPosition
|
||||
backwardTopologyPos, backwardTopologyStreamPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID())
|
||||
if backwardTopologyPos-1 <= 0 {
|
||||
backwardTopologyPos = types.StreamPosition(1)
|
||||
} else {
|
||||
backwardTopologyPos--
|
||||
backwardTopologyStreamPos += 1000 // this has to be bigger than the number of events we backfill per request
|
||||
}
|
||||
prevBatch := types.NewTopologyToken(backwardTopologyPos, backwardTopologyStreamPos)
|
||||
prevBatch.Decrement()
|
||||
|
||||
// We don't include a device here as we don't need to send down
|
||||
// transaction IDs for complete syncs
|
||||
recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents)
|
||||
stateEvents = removeDuplicates(stateEvents, recentEvents)
|
||||
jr := types.NewJoinResponse()
|
||||
jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition(
|
||||
types.PaginationTokenTypeTopology, backwardTopologyPos, backwardTopologyStreamPos,
|
||||
).String()
|
||||
jr.Timeline.PrevBatch = prevBatch.String()
|
||||
jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
|
||||
jr.Timeline.Limited = true
|
||||
jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync)
|
||||
res.Rooms.Join[roomID] = *jr
|
||||
}
|
||||
|
||||
if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition, res); err != nil {
|
||||
if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition(), res); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -697,7 +699,7 @@ func (d *SyncServerDatasource) CompleteSync(
|
|||
|
||||
// Use a zero value SyncPosition for fromPos so all EDU states are added.
|
||||
err = d.addEDUDeltaToResponse(
|
||||
types.PaginationToken{}, toPos, joinedRoomIDs, res,
|
||||
types.NewStreamToken(0, 0), *toPos, joinedRoomIDs, res,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -860,14 +862,14 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse(
|
|||
recentEvents := d.StreamEventsToEvents(device, recentStreamEvents)
|
||||
delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents)
|
||||
backwardTopologyPos, backwardStreamPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents)
|
||||
prevBatch := types.NewTopologyToken(
|
||||
backwardTopologyPos, backwardStreamPos,
|
||||
)
|
||||
|
||||
switch delta.membership {
|
||||
case gomatrixserverlib.Join:
|
||||
jr := types.NewJoinResponse()
|
||||
|
||||
jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition(
|
||||
types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos,
|
||||
).String()
|
||||
jr.Timeline.PrevBatch = prevBatch.String()
|
||||
jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
|
||||
jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true
|
||||
jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync)
|
||||
|
|
@ -878,9 +880,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse(
|
|||
// TODO: recentEvents may contain events that this user is not allowed to see because they are
|
||||
// no longer in the room.
|
||||
lr := types.NewLeaveResponse()
|
||||
lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition(
|
||||
types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos,
|
||||
).String()
|
||||
lr.Timeline.PrevBatch = prevBatch.String()
|
||||
lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
|
||||
lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true
|
||||
lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue