De-map device list positions in streaming tokens (#1642)

* De-map device list positions in streaming tokens

* Fix lint error

* Tweak toOffset

syncapi/consumers/keychange.go

@@ -23,7 +23,6 @@ import (
 	"github.com/matrix-org/dendrite/internal"
 	"github.com/matrix-org/dendrite/keyserver/api"
 	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
-	syncinternal "github.com/matrix-org/dendrite/syncapi/internal"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	syncapi "github.com/matrix-org/dendrite/syncapi/sync"
 	"github.com/matrix-org/dendrite/syncapi/types"
@@ -115,11 +114,9 @@ func (s *OutputKeyChangeEventConsumer) onMessage(msg *sarama.ConsumerMessage) er
 	}
 	// TODO: f.e queryRes.UserIDsToCount : notify users by waking up streams
 	posUpdate := types.StreamingToken{
-		Logs: map[string]*types.LogPosition{
+		DeviceListPosition: types.LogPosition{
-			syncinternal.DeviceListLogName: {
+			Offset:    msg.Offset,
-				Offset:    msg.Offset,
+			Partition: msg.Partition,
-				Partition: msg.Partition,
-			},
 		},
 	}
 	for userID := range queryRes.UserIDsToCount {

syncapi/internal/keychange.go

@@ -73,15 +73,13 @@ func DeviceListCatchup(
 	offset = sarama.OffsetOldest
 	// Extract partition/offset from sync token
 	// TODO: In a world where keyserver is sharded there will be multiple partitions and hence multiple QueryKeyChanges to make.
-	logOffset := from.Log(DeviceListLogName)
+	if !from.DeviceListPosition.IsEmpty() {
-	if logOffset != nil {
+		partition = from.DeviceListPosition.Partition
-		partition = logOffset.Partition
+		offset = from.DeviceListPosition.Offset
-		offset = logOffset.Offset
 	}
 	var toOffset int64
 	toOffset = sarama.OffsetNewest
-	toLog := to.Log(DeviceListLogName)
+	if toLog := to.DeviceListPosition; toLog.Partition == partition && toLog.Offset > 0 {
-	if toLog != nil && toLog.Offset > 0 {
 		toOffset = toLog.Offset
 	}
 	var queryRes api.QueryKeyChangesResponse
@@ -130,10 +128,10 @@ func DeviceListCatchup(
 		}
 	}
 	// set the new token
-	to.SetLog(DeviceListLogName, &types.LogPosition{
+	to.DeviceListPosition = types.LogPosition{
 		Partition: queryRes.Partition,
 		Offset:    queryRes.Offset,
-	})
+	}
 	res.NextBatch = to.String()
 
 	return hasNew, nil

syncapi/internal/keychange_test.go

@@ -18,11 +18,9 @@ var (
 	syncingUser = "@alice:localhost"
 	emptyToken  = types.StreamingToken{}
 	newestToken = types.StreamingToken{
-		Logs: map[string]*types.LogPosition{
+		DeviceListPosition: types.LogPosition{
-			DeviceListLogName: {
+			Offset:    sarama.OffsetNewest,
-				Offset:    sarama.OffsetNewest,
+			Partition: 0,
-				Partition: 0,
-			},
 		},
 	}
 )

syncapi/types/types.go

@@ -17,7 +17,6 @@ package types
 import (
 	"encoding/json"
 	"fmt"
-	"sort"
 	"strconv"
 	"strings"
 
@@ -45,6 +44,10 @@ type LogPosition struct {
 	Offset    int64
 }
 
+func (p *LogPosition) IsEmpty() bool {
+	return p.Offset == 0
+}
+
 // IsAfter returns true if this position is after `lp`.
 func (p *LogPosition) IsAfter(lp *LogPosition) bool {
 	if lp == nil {
@@ -110,22 +113,7 @@ type StreamingToken struct {
 	TypingPosition       StreamPosition
 	ReceiptPosition      StreamPosition
 	SendToDevicePosition StreamPosition
-	Logs                 map[string]*LogPosition
+	DeviceListPosition   LogPosition
-}
-
-func (t *StreamingToken) SetLog(name string, lp *LogPosition) {
-	if t.Logs == nil {
-		t.Logs = make(map[string]*LogPosition)
-	}
-	t.Logs[name] = lp
-}
-
-func (t *StreamingToken) Log(name string) *LogPosition {
-	l, ok := t.Logs[name]
-	if !ok {
-		return nil
-	}
-	return l
 }
 
 func (t StreamingToken) String() string {
@@ -134,14 +122,10 @@ func (t StreamingToken) String() string {
 		t.PDUPosition, t.TypingPosition,
 		t.ReceiptPosition, t.SendToDevicePosition,
 	)
-	var logStrings []string
+	if dl := t.DeviceListPosition; !dl.IsEmpty() {
-	for name, lp := range t.Logs {
+		posStr += fmt.Sprintf(".dl-%d-%d", dl.Partition, dl.Offset)
-		logStr := fmt.Sprintf("%s-%d-%d", name, lp.Partition, lp.Offset)
-		logStrings = append(logStrings, logStr)
 	}
-	sort.Strings(logStrings)
+	return posStr
-	// E.g s11_22_33_44.dl0-134.ab1-441
-	return strings.Join(append([]string{posStr}, logStrings...), ".")
 }
 
 // IsAfter returns true if ANY position in this token is greater than `other`.
@@ -155,21 +139,14 @@ func (t *StreamingToken) IsAfter(other StreamingToken) bool {
 		return true
 	case t.SendToDevicePosition > other.SendToDevicePosition:
 		return true
-	}
+	case t.DeviceListPosition.IsAfter(&other.DeviceListPosition):
-	for name := range t.Logs {
+		return true
-		otherLog := other.Log(name)
-		if otherLog == nil {
-			continue
-		}
-		if t.Logs[name].IsAfter(otherLog) {
-			return true
-		}
 	}
 	return false
 }
 
 func (t *StreamingToken) IsEmpty() bool {
-	return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition == 0
+	return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition == 0 && t.DeviceListPosition.IsEmpty()
 }
 
 // WithUpdates returns a copy of the StreamingToken with updates applied from another StreamingToken.
@@ -187,15 +164,8 @@ func (t *StreamingToken) WithUpdates(other StreamingToken) (ret StreamingToken)
 		ret.ReceiptPosition = other.ReceiptPosition
 	case other.SendToDevicePosition > 0:
 		ret.SendToDevicePosition = other.SendToDevicePosition
-	}
+	case other.DeviceListPosition.Offset > 0:
-	ret.Logs = make(map[string]*LogPosition)
+		ret.DeviceListPosition = other.DeviceListPosition
-	for name := range t.Logs {
-		otherLog := other.Log(name)
-		if otherLog == nil {
-			continue
-		}
-		copy := *otherLog
-		ret.Logs[name] = &copy
 	}
 	return ret
 }
@@ -294,30 +264,31 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
 		TypingPosition:       positions[1],
 		ReceiptPosition:      positions[2],
 		SendToDevicePosition: positions[3],
-		Logs:                 make(map[string]*LogPosition),
 	}
 	// dl-0-1234
 	// $log_name-$partition-$offset
 	for _, logStr := range categories[1:] {
 		segments := strings.Split(logStr, "-")
 		if len(segments) != 3 {
-			err = fmt.Errorf("token %s - invalid log: %s", tok, logStr)
+			err = fmt.Errorf("invalid log position %q", logStr)
 			return
 		}
-		var partition int64
+		switch segments[0] {
-		partition, err = strconv.ParseInt(segments[1], 10, 32)
+		case "dl":
-		if err != nil {
+			// Device list syncing
+			var partition, offset int
+			if partition, err = strconv.Atoi(segments[1]); err != nil {
+				return
+			}
+			if offset, err = strconv.Atoi(segments[2]); err != nil {
+				return
+			}
+			token.DeviceListPosition.Partition = int32(partition)
+			token.DeviceListPosition.Offset = int64(offset)
+		default:
+			err = fmt.Errorf("unrecognised token type %q", segments[0])
 			return
 		}
-		var offset int64
-		offset, err = strconv.ParseInt(segments[2], 10, 64)
-		if err != nil {
-			return
-		}
-		token.Logs[segments[0]] = &LogPosition{
-			Partition: int32(partition),
-			Offset:    offset,
-		}
 	}
 	return token, nil
 }

syncapi/types/types_test.go

@@ -12,28 +12,12 @@ func TestNewSyncTokenWithLogs(t *testing.T) {
 	tests := map[string]*StreamingToken{
 		"s4_0_0_0": {
 			PDUPosition: 4,
-			Logs:        make(map[string]*LogPosition),
 		},
 		"s4_0_0_0.dl-0-123": {
 			PDUPosition: 4,
-			Logs: map[string]*LogPosition{
+			DeviceListPosition: LogPosition{
-				"dl": {
+				Partition: 0,
-					Partition: 0,
+				Offset:    123,
-					Offset:    123,
-				},
-			},
-		},
-		"s4_0_0_0.ab-1-14419482332.dl-0-123": {
-			PDUPosition: 4,
-			Logs: map[string]*LogPosition{
-				"ab": {
-					Partition: 1,
-					Offset:    14419482332,
-				},
-				"dl": {
-					Partition: 0,
-					Offset:    123,
-				},
 			},
 		},
 	}
@@ -58,10 +42,10 @@ func TestNewSyncTokenWithLogs(t *testing.T) {
 
 func TestSyncTokens(t *testing.T) {
 	shouldPass := map[string]string{
-		"s4_0_0_0": StreamingToken{4, 0, 0, 0, nil}.String(),
+		"s4_0_0_0":        StreamingToken{4, 0, 0, 0, LogPosition{}}.String(),
-		"s3_1_0_0": StreamingToken{3, 1, 0, 0, nil}.String(),
+		"s3_1_0_0.dl-1-2": StreamingToken{3, 1, 0, 0, LogPosition{1, 2}}.String(),
-		"s3_1_2_3": StreamingToken{3, 1, 2, 3, nil}.String(),
+		"s3_1_2_3":        StreamingToken{3, 1, 2, 3, LogPosition{}}.String(),
-		"t3_1":     TopologyToken{3, 1}.String(),
+		"t3_1":            TopologyToken{3, 1}.String(),
 	}
 
 	for a, b := range shouldPass {