Merge branch 'master' into 1323-archived-rooms-sync-left-rooms

syncapi/types/provider.go

@@ -0,0 +1,53 @@
+package types
+
+import (
+	"context"
+	"time"
+
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+)
+
+type SyncRequest struct {
+	Context       context.Context
+	Log           *logrus.Entry
+	Device        *userapi.Device
+	Response      *Response
+	Filter        gomatrixserverlib.Filter
+	Since         StreamingToken
+	Limit         int
+	Timeout       time.Duration
+	WantFullState bool
+
+	// Updated by the PDU stream.
+	Rooms map[string]string
+}
+
+type StreamProvider interface {
+	Setup()
+
+	// Advance will update the latest position of the stream based on
+	// an update and will wake callers waiting on StreamNotifyAfter.
+	Advance(latest StreamPosition)
+
+	// CompleteSync will update the response to include all updates as needed
+	// for a complete sync. It will always return immediately.
+	CompleteSync(ctx context.Context, req *SyncRequest) StreamPosition
+
+	// IncrementalSync will update the response to include all updates between
+	// the from and to sync positions. It will always return immediately,
+	// making no changes if the range contains no updates.
+	IncrementalSync(ctx context.Context, req *SyncRequest, from, to StreamPosition) StreamPosition
+
+	// LatestPosition returns the latest stream position for this stream.
+	LatestPosition(ctx context.Context) StreamPosition
+}
+
+type PartitionedStreamProvider interface {
+	Setup()
+	Advance(latest LogPosition)
+	CompleteSync(ctx context.Context, req *SyncRequest) LogPosition
+	IncrementalSync(ctx context.Context, req *SyncRequest, from, to LogPosition) LogPosition
+	LatestPosition(ctx context.Context) LogPosition
+}

syncapi/types/types.go

@@ -35,6 +35,15 @@ var (
 	ErrInvalidSyncTokenLen = fmt.Errorf("Sync token has an invalid length")
 )
 
+type StateDelta struct {
+	RoomID      string
+	StateEvents []*gomatrixserverlib.HeaderedEvent
+	Membership  string
+	// The PDU stream position of the latest membership event for this user, if applicable.
+	// Can be 0 if there is no membership event in this delta.
+	MembershipPos StreamPosition
+}
+
 // StreamPosition represents the offset in the sync stream a client is at.
 type StreamPosition int64
 
@@ -114,6 +123,7 @@ type StreamingToken struct {
 	ReceiptPosition      StreamPosition
 	SendToDevicePosition StreamPosition
 	InvitePosition       StreamPosition
+	AccountDataPosition  StreamPosition
 	DeviceListPosition   LogPosition
 }
 
@@ -130,10 +140,10 @@ func (s *StreamingToken) UnmarshalText(text []byte) (err error) {
 
 func (t StreamingToken) String() string {
 	posStr := fmt.Sprintf(
-		"s%d_%d_%d_%d_%d",
+		"s%d_%d_%d_%d_%d_%d",
 		t.PDUPosition, t.TypingPosition,
 		t.ReceiptPosition, t.SendToDevicePosition,
-		t.InvitePosition,
+		t.InvitePosition, t.AccountDataPosition,
 	)
 	if dl := t.DeviceListPosition; !dl.IsEmpty() {
 		posStr += fmt.Sprintf(".dl-%d-%d", dl.Partition, dl.Offset)
@@ -154,6 +164,8 @@ func (t *StreamingToken) IsAfter(other StreamingToken) bool {
 		return true
 	case t.InvitePosition > other.InvitePosition:
 		return true
+	case t.AccountDataPosition > other.AccountDataPosition:
+		return true
 	case t.DeviceListPosition.IsAfter(&other.DeviceListPosition):
 		return true
 	}
@@ -161,7 +173,7 @@ func (t *StreamingToken) IsAfter(other StreamingToken) bool {
 }
 
 func (t *StreamingToken) IsEmpty() bool {
-	return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition+t.InvitePosition == 0 && t.DeviceListPosition.IsEmpty()
+	return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition+t.InvitePosition+t.AccountDataPosition == 0 && t.DeviceListPosition.IsEmpty()
 }
 
 // WithUpdates returns a copy of the StreamingToken with updates applied from another StreamingToken.
@@ -178,22 +190,25 @@ func (t *StreamingToken) WithUpdates(other StreamingToken) StreamingToken {
 // streaming token contains any positions that are not 0, they are considered updates
 // and will overwrite the value in the token.
 func (t *StreamingToken) ApplyUpdates(other StreamingToken) {
-	if other.PDUPosition > 0 {
+	if other.PDUPosition > t.PDUPosition {
 		t.PDUPosition = other.PDUPosition
 	}
-	if other.TypingPosition > 0 {
+	if other.TypingPosition > t.TypingPosition {
 		t.TypingPosition = other.TypingPosition
 	}
-	if other.ReceiptPosition > 0 {
+	if other.ReceiptPosition > t.ReceiptPosition {
 		t.ReceiptPosition = other.ReceiptPosition
 	}
-	if other.SendToDevicePosition > 0 {
+	if other.SendToDevicePosition > t.SendToDevicePosition {
 		t.SendToDevicePosition = other.SendToDevicePosition
 	}
-	if other.InvitePosition > 0 {
+	if other.InvitePosition > t.InvitePosition {
 		t.InvitePosition = other.InvitePosition
 	}
-	if other.DeviceListPosition.Offset > 0 {
+	if other.AccountDataPosition > t.AccountDataPosition {
+		t.AccountDataPosition = other.AccountDataPosition
+	}
+	if other.DeviceListPosition.IsAfter(&t.DeviceListPosition) {
 		t.DeviceListPosition = other.DeviceListPosition
 	}
 }
@@ -286,7 +301,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
 	}
 	categories := strings.Split(tok[1:], ".")
 	parts := strings.Split(categories[0], "_")
-	var positions [5]StreamPosition
+	var positions [6]StreamPosition
 	for i, p := range parts {
 		if i > len(positions) {
 			break
@@ -304,6 +319,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
 		ReceiptPosition:      positions[2],
 		SendToDevicePosition: positions[3],
 		InvitePosition:       positions[4],
+		AccountDataPosition:  positions[5],
 	}
 	// dl-0-1234
 	// $log_name-$partition-$offset

syncapi/types/types_test.go

@@ -10,10 +10,10 @@ import (
 
 func TestNewSyncTokenWithLogs(t *testing.T) {
 	tests := map[string]*StreamingToken{
-		"s4_0_0_0_0": {
+		"s4_0_0_0_0_0": {
 			PDUPosition: 4,
 		},
-		"s4_0_0_0_0.dl-0-123": {
+		"s4_0_0_0_0_0.dl-0-123": {
 			PDUPosition: 4,
 			DeviceListPosition: LogPosition{
 				Partition: 0,
@@ -42,10 +42,10 @@ func TestNewSyncTokenWithLogs(t *testing.T) {
 
 func TestSyncTokens(t *testing.T) {
 	shouldPass := map[string]string{
-		"s4_0_0_0_0":        StreamingToken{4, 0, 0, 0, 0, LogPosition{}}.String(),
-		"s3_1_0_0_0.dl-1-2": StreamingToken{3, 1, 0, 0, 0, LogPosition{1, 2}}.String(),
-		"s3_1_2_3_5":        StreamingToken{3, 1, 2, 3, 5, LogPosition{}}.String(),
-		"t3_1":              TopologyToken{3, 1}.String(),
+		"s4_0_0_0_0_0":        StreamingToken{4, 0, 0, 0, 0, 0, LogPosition{}}.String(),
+		"s3_1_0_0_0_0.dl-1-2": StreamingToken{3, 1, 0, 0, 0, 0, LogPosition{1, 2}}.String(),
+		"s3_1_2_3_5_0":        StreamingToken{3, 1, 2, 3, 5, 0, LogPosition{}}.String(),
+		"t3_1":                TopologyToken{3, 1}.String(),
 	}
 
 	for a, b := range shouldPass {