use go module for dependencies (#594)

roomserver/input/authevents.go

@@ -0,0 +1,243 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package input
+
+import (
+	"context"
+	"sort"
+
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// checkAuthEvents checks that the event passes authentication checks
+// Returns the numeric IDs for the auth events.
+func checkAuthEvents(
+	ctx context.Context,
+	db RoomEventDatabase,
+	event gomatrixserverlib.Event,
+	authEventIDs []string,
+) ([]types.EventNID, error) {
+	// Grab the numeric IDs for the supplied auth state events from the database.
+	authStateEntries, err := db.StateEntriesForEventIDs(ctx, authEventIDs)
+	if err != nil {
+		return nil, err
+	}
+	// TODO: check for duplicate state keys here.
+
+	// Work out which of the state events we actually need.
+	stateNeeded := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{event})
+
+	// Load the actual auth events from the database.
+	authEvents, err := loadAuthEvents(ctx, db, stateNeeded, authStateEntries)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if the event is allowed.
+	if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil {
+		return nil, err
+	}
+
+	// Return the numeric IDs for the auth events.
+	result := make([]types.EventNID, len(authStateEntries))
+	for i := range authStateEntries {
+		result[i] = authStateEntries[i].EventNID
+	}
+	return result, nil
+}
+
+type authEvents struct {
+	stateKeyNIDMap map[string]types.EventStateKeyNID
+	state          stateEntryMap
+	events         eventMap
+}
+
+// Create implements gomatrixserverlib.AuthEventProvider
+func (ae *authEvents) Create() (*gomatrixserverlib.Event, error) {
+	return ae.lookupEventWithEmptyStateKey(types.MRoomCreateNID), nil
+}
+
+// PowerLevels implements gomatrixserverlib.AuthEventProvider
+func (ae *authEvents) PowerLevels() (*gomatrixserverlib.Event, error) {
+	return ae.lookupEventWithEmptyStateKey(types.MRoomPowerLevelsNID), nil
+}
+
+// JoinRules implements gomatrixserverlib.AuthEventProvider
+func (ae *authEvents) JoinRules() (*gomatrixserverlib.Event, error) {
+	return ae.lookupEventWithEmptyStateKey(types.MRoomJoinRulesNID), nil
+}
+
+// Memmber implements gomatrixserverlib.AuthEventProvider
+func (ae *authEvents) Member(stateKey string) (*gomatrixserverlib.Event, error) {
+	return ae.lookupEvent(types.MRoomMemberNID, stateKey), nil
+}
+
+// ThirdPartyInvite implements gomatrixserverlib.AuthEventProvider
+func (ae *authEvents) ThirdPartyInvite(stateKey string) (*gomatrixserverlib.Event, error) {
+	return ae.lookupEvent(types.MRoomThirdPartyInviteNID, stateKey), nil
+}
+
+func (ae *authEvents) lookupEventWithEmptyStateKey(typeNID types.EventTypeNID) *gomatrixserverlib.Event {
+	eventNID, ok := ae.state.lookup(types.StateKeyTuple{
+		EventTypeNID:     typeNID,
+		EventStateKeyNID: types.EmptyStateKeyNID,
+	})
+	if !ok {
+		return nil
+	}
+	event, ok := ae.events.lookup(eventNID)
+	if !ok {
+		return nil
+	}
+	return &event.Event
+}
+
+func (ae *authEvents) lookupEvent(typeNID types.EventTypeNID, stateKey string) *gomatrixserverlib.Event {
+	stateKeyNID, ok := ae.stateKeyNIDMap[stateKey]
+	if !ok {
+		return nil
+	}
+	eventNID, ok := ae.state.lookup(types.StateKeyTuple{
+		EventTypeNID:     typeNID,
+		EventStateKeyNID: stateKeyNID,
+	})
+	if !ok {
+		return nil
+	}
+	event, ok := ae.events.lookup(eventNID)
+	if !ok {
+		return nil
+	}
+	return &event.Event
+}
+
+// loadAuthEvents loads the events needed for authentication from the supplied room state.
+func loadAuthEvents(
+	ctx context.Context,
+	db RoomEventDatabase,
+	needed gomatrixserverlib.StateNeeded,
+	state []types.StateEntry,
+) (result authEvents, err error) {
+	// Look up the numeric IDs for the state keys needed for auth.
+	var neededStateKeys []string
+	neededStateKeys = append(neededStateKeys, needed.Member...)
+	neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...)
+	if result.stateKeyNIDMap, err = db.EventStateKeyNIDs(ctx, neededStateKeys); err != nil {
+		return
+	}
+
+	// Load the events we need.
+	result.state = state
+	var eventNIDs []types.EventNID
+	keyTuplesNeeded := stateKeyTuplesNeeded(result.stateKeyNIDMap, needed)
+	for _, keyTuple := range keyTuplesNeeded {
+		eventNID, ok := result.state.lookup(keyTuple)
+		if ok {
+			eventNIDs = append(eventNIDs, eventNID)
+		}
+	}
+	if result.events, err = db.Events(ctx, eventNIDs); err != nil {
+		return
+	}
+	return
+}
+
+// stateKeyTuplesNeeded works out which numeric state key tuples we need to authenticate some events.
+func stateKeyTuplesNeeded(
+	stateKeyNIDMap map[string]types.EventStateKeyNID,
+	stateNeeded gomatrixserverlib.StateNeeded,
+) []types.StateKeyTuple {
+	var keyTuples []types.StateKeyTuple
+	if stateNeeded.Create {
+		keyTuples = append(keyTuples, types.StateKeyTuple{
+			EventTypeNID:     types.MRoomCreateNID,
+			EventStateKeyNID: types.EmptyStateKeyNID,
+		})
+	}
+	if stateNeeded.PowerLevels {
+		keyTuples = append(keyTuples, types.StateKeyTuple{
+			EventTypeNID:     types.MRoomPowerLevelsNID,
+			EventStateKeyNID: types.EmptyStateKeyNID,
+		})
+	}
+	if stateNeeded.JoinRules {
+		keyTuples = append(keyTuples, types.StateKeyTuple{
+			EventTypeNID:     types.MRoomJoinRulesNID,
+			EventStateKeyNID: types.EmptyStateKeyNID,
+		})
+	}
+	for _, member := range stateNeeded.Member {
+		stateKeyNID, ok := stateKeyNIDMap[member]
+		if ok {
+			keyTuples = append(keyTuples, types.StateKeyTuple{
+				EventTypeNID:     types.MRoomMemberNID,
+				EventStateKeyNID: stateKeyNID,
+			})
+		}
+	}
+	for _, token := range stateNeeded.ThirdPartyInvite {
+		stateKeyNID, ok := stateKeyNIDMap[token]
+		if ok {
+			keyTuples = append(keyTuples, types.StateKeyTuple{
+				EventTypeNID:     types.MRoomThirdPartyInviteNID,
+				EventStateKeyNID: stateKeyNID,
+			})
+		}
+	}
+	return keyTuples
+}
+
+// Map from event type, state key tuple to numeric event ID.
+// Implemented using binary search on a sorted array.
+type stateEntryMap []types.StateEntry
+
+// lookup an entry in the event map.
+func (m stateEntryMap) lookup(stateKey types.StateKeyTuple) (eventNID types.EventNID, ok bool) {
+	// Since the list is sorted we can implement this using binary search.
+	// This is faster than using a hash map.
+	// We don't have to worry about pathological cases because the keys are fixed
+	// size and are controlled by us.
+	list := []types.StateEntry(m)
+	i := sort.Search(len(list), func(i int) bool {
+		return !list[i].StateKeyTuple.LessThan(stateKey)
+	})
+	if i < len(list) && list[i].StateKeyTuple == stateKey {
+		ok = true
+		eventNID = list[i].EventNID
+	}
+	return
+}
+
+// Map from numeric event ID to event.
+// Implemented using binary search on a sorted array.
+type eventMap []types.Event
+
+// lookup an entry in the event map.
+func (m eventMap) lookup(eventNID types.EventNID) (event *types.Event, ok bool) {
+	// Since the list is sorted we can implement this using binary search.
+	// This is faster than using a hash map.
+	// We don't have to worry about pathological cases because the keys are fixed
+	// size are controlled by us.
+	list := []types.Event(m)
+	i := sort.Search(len(list), func(i int) bool {
+		return list[i].EventNID >= eventNID
+	})
+	if i < len(list) && list[i].EventNID == eventNID {
+		ok = true
+		event = &list[i]
+	}
+	return
+}

roomserver/input/authevents_test.go

@@ -0,0 +1,136 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package input
+
+import (
+	"testing"
+
+	"github.com/matrix-org/dendrite/roomserver/types"
+)
+
+func benchmarkStateEntryMapLookup(entries, lookups int64, b *testing.B) {
+	var list []types.StateEntry
+	for i := int64(0); i < entries; i++ {
+		list = append(list, types.StateEntry{
+			StateKeyTuple: types.StateKeyTuple{
+				EventTypeNID:     types.EventTypeNID(i),
+				EventStateKeyNID: types.EventStateKeyNID(i),
+			},
+			EventNID: types.EventNID(i),
+		})
+	}
+
+	for i := 0; i < b.N; i++ {
+		entryMap := stateEntryMap(list)
+		for j := int64(0); j < lookups; j++ {
+			entryMap.lookup(types.StateKeyTuple{
+				EventTypeNID:     types.EventTypeNID(j),
+				EventStateKeyNID: types.EventStateKeyNID(j),
+			})
+		}
+	}
+}
+
+func BenchmarkStateEntryMap100Lookup10(b *testing.B) {
+	benchmarkStateEntryMapLookup(100, 10, b)
+}
+
+func BenchmarkStateEntryMap1000Lookup100(b *testing.B) {
+	benchmarkStateEntryMapLookup(1000, 100, b)
+}
+
+func BenchmarkStateEntryMap100Lookup100(b *testing.B) {
+	benchmarkStateEntryMapLookup(100, 100, b)
+}
+
+func BenchmarkStateEntryMap1000Lookup10000(b *testing.B) {
+	benchmarkStateEntryMapLookup(1000, 10000, b)
+}
+
+func TestStateEntryMap(t *testing.T) {
+	entryMap := stateEntryMap([]types.StateEntry{
+		{StateKeyTuple: types.StateKeyTuple{EventTypeNID: 1, EventStateKeyNID: 1}, EventNID: 1},
+		{StateKeyTuple: types.StateKeyTuple{EventTypeNID: 1, EventStateKeyNID: 3}, EventNID: 2},
+		{StateKeyTuple: types.StateKeyTuple{EventTypeNID: 2, EventStateKeyNID: 1}, EventNID: 3},
+	})
+
+	testCases := []struct {
+		inputTypeNID  types.EventTypeNID
+		inputStateKey types.EventStateKeyNID
+		wantOK        bool
+		wantEventNID  types.EventNID
+	}{
+		// Check that tuples that in the array are in the map.
+		{1, 1, true, 1},
+		{1, 3, true, 2},
+		{2, 1, true, 3},
+		// Check that tuples that aren't in the array aren't in the map.
+		{0, 0, false, 0},
+		{1, 2, false, 0},
+		{3, 1, false, 0},
+	}
+
+	for _, testCase := range testCases {
+		keyTuple := types.StateKeyTuple{EventTypeNID: testCase.inputTypeNID, EventStateKeyNID: testCase.inputStateKey}
+		gotEventNID, gotOK := entryMap.lookup(keyTuple)
+		if testCase.wantOK != gotOK {
+			t.Fatalf("stateEntryMap lookup(%v): want ok to be %v, got %v", keyTuple, testCase.wantOK, gotOK)
+		}
+		if testCase.wantEventNID != gotEventNID {
+			t.Fatalf("stateEntryMap lookup(%v): want eventNID to be %v, got %v", keyTuple, testCase.wantEventNID, gotEventNID)
+		}
+	}
+}
+
+func TestEventMap(t *testing.T) {
+	events := eventMap([]types.Event{
+		{EventNID: 1},
+		{EventNID: 2},
+		{EventNID: 3},
+		{EventNID: 5},
+		{EventNID: 8},
+	})
+
+	testCases := []struct {
+		inputEventNID types.EventNID
+		wantOK        bool
+		wantEvent     *types.Event
+	}{
+		// Check that the IDs that are in the array are in the map.
+		{1, true, &events[0]},
+		{2, true, &events[1]},
+		{3, true, &events[2]},
+		{5, true, &events[3]},
+		{8, true, &events[4]},
+		// Check that tuples that aren't in the array aren't in the map.
+		{0, false, nil},
+		{4, false, nil},
+		{6, false, nil},
+		{7, false, nil},
+		{9, false, nil},
+	}
+
+	for _, testCase := range testCases {
+		gotEvent, gotOK := events.lookup(testCase.inputEventNID)
+		if testCase.wantOK != gotOK {
+			t.Fatalf("eventMap lookup(%v): want ok to be %v, got %v", testCase.inputEventNID, testCase.wantOK, gotOK)
+		}
+
+		if testCase.wantEvent != gotEvent {
+			t.Fatalf("eventMap lookup(%v): want event to be %v, got %v", testCase.inputEventNID, testCase.wantEvent, gotEvent)
+		}
+	}
+
+}

roomserver/input/events.go

@@ -0,0 +1,235 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package input
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/state"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// A RoomEventDatabase has the storage APIs needed to store a room event.
+type RoomEventDatabase interface {
+	state.RoomStateDatabase
+	// Stores a matrix room event in the database
+	StoreEvent(
+		ctx context.Context,
+		event gomatrixserverlib.Event,
+		txnAndDeviceID *api.TransactionID,
+		authEventNIDs []types.EventNID,
+	) (types.RoomNID, types.StateAtEvent, error)
+	// Look up the state entries for a list of string event IDs
+	// Returns an error if the there is an error talking to the database
+	// Returns a types.MissingEventError if the event IDs aren't in the database.
+	StateEntriesForEventIDs(
+		ctx context.Context, eventIDs []string,
+	) ([]types.StateEntry, error)
+	// Set the state at an event.
+	SetState(
+		ctx context.Context,
+		eventNID types.EventNID,
+		stateNID types.StateSnapshotNID,
+	) error
+	// Look up the latest events in a room in preparation for an update.
+	// The RoomRecentEventsUpdater must have Commit or Rollback called on it if this doesn't return an error.
+	// Returns the latest events in the room and the last eventID sent to the log along with an updater.
+	// If this returns an error then no further action is required.
+	GetLatestEventsForUpdate(
+		ctx context.Context, roomNID types.RoomNID,
+	) (updater types.RoomRecentEventsUpdater, err error)
+	// Look up the string event IDs for a list of numeric event IDs
+	EventIDs(
+		ctx context.Context, eventNIDs []types.EventNID,
+	) (map[types.EventNID]string, error)
+	// Build a membership updater for the target user in a room.
+	MembershipUpdater(
+		ctx context.Context, roomID, targerUserID string,
+	) (types.MembershipUpdater, error)
+	// Look up event ID by transaction's info.
+	// This is used to determine if the room event is processed/processing already.
+	// Returns an empty string if no such event exists.
+	GetTransactionEventID(
+		ctx context.Context, transactionID string,
+		deviceID string, userID string,
+	) (string, error)
+}
+
+// OutputRoomEventWriter has the APIs needed to write an event to the output logs.
+type OutputRoomEventWriter interface {
+	// Write a list of events for a room
+	WriteOutputEvents(roomID string, updates []api.OutputEvent) error
+}
+
+// processRoomEvent can only be called once at a time
+//
+// TODO(#375): This should be rewritten to allow concurrent calls. The
+// difficulty is in ensuring that we correctly annotate events with the correct
+// state deltas when sending to kafka streams
+func processRoomEvent(
+	ctx context.Context,
+	db RoomEventDatabase,
+	ow OutputRoomEventWriter,
+	input api.InputRoomEvent,
+) (eventID string, err error) {
+	// Parse and validate the event JSON
+	event := input.Event
+
+	// Check that the event passes authentication checks and work out the numeric IDs for the auth events.
+	authEventNIDs, err := checkAuthEvents(ctx, db, event, input.AuthEventIDs)
+	if err != nil {
+		return
+	}
+
+	if input.TransactionID != nil {
+		tdID := input.TransactionID
+		eventID, err = db.GetTransactionEventID(
+			ctx, tdID.TransactionID, tdID.DeviceID, input.Event.Sender(),
+		)
+		// On error OR event with the transaction already processed/processesing
+		if err != nil || eventID != "" {
+			return
+		}
+	}
+
+	// Store the event
+	roomNID, stateAtEvent, err := db.StoreEvent(ctx, event, input.TransactionID, authEventNIDs)
+	if err != nil {
+		return
+	}
+
+	if input.Kind == api.KindOutlier {
+		// For outliers we can stop after we've stored the event itself as it
+		// doesn't have any associated state to store and we don't need to
+		// notify anyone about it.
+		return event.EventID(), nil
+	}
+
+	if stateAtEvent.BeforeStateSnapshotNID == 0 {
+		// We haven't calculated a state for this event yet.
+		// Lets calculate one.
+		err = calculateAndSetState(ctx, db, input, roomNID, &stateAtEvent, event)
+		if err != nil {
+			return
+		}
+	}
+
+	if input.Kind == api.KindBackfill {
+		// Backfill is not implemented.
+		panic("Not implemented")
+	}
+
+	// Update the extremities of the event graph for the room
+	return event.EventID(), updateLatestEvents(
+		ctx, db, ow, roomNID, stateAtEvent, event, input.SendAsServer, input.TransactionID,
+	)
+}
+
+func calculateAndSetState(
+	ctx context.Context,
+	db RoomEventDatabase,
+	input api.InputRoomEvent,
+	roomNID types.RoomNID,
+	stateAtEvent *types.StateAtEvent,
+	event gomatrixserverlib.Event,
+) error {
+	var err error
+	if input.HasState {
+		// We've been told what the state at the event is so we don't need to calculate it.
+		// Check that those state events are in the database and store the state.
+		var entries []types.StateEntry
+		if entries, err = db.StateEntriesForEventIDs(ctx, input.StateEventIDs); err != nil {
+			return err
+		}
+
+		if stateAtEvent.BeforeStateSnapshotNID, err = db.AddState(ctx, roomNID, nil, entries); err != nil {
+			return err
+		}
+	} else {
+		// We haven't been told what the state at the event is so we need to calculate it from the prev_events
+		if stateAtEvent.BeforeStateSnapshotNID, err = state.CalculateAndStoreStateBeforeEvent(ctx, db, event, roomNID); err != nil {
+			return err
+		}
+	}
+	return db.SetState(ctx, stateAtEvent.EventNID, stateAtEvent.BeforeStateSnapshotNID)
+}
+
+func processInviteEvent(
+	ctx context.Context,
+	db RoomEventDatabase,
+	ow OutputRoomEventWriter,
+	input api.InputInviteEvent,
+) (err error) {
+	if input.Event.StateKey() == nil {
+		return fmt.Errorf("invite must be a state event")
+	}
+
+	roomID := input.Event.RoomID()
+	targetUserID := *input.Event.StateKey()
+
+	updater, err := db.MembershipUpdater(ctx, roomID, targetUserID)
+	if err != nil {
+		return err
+	}
+	succeeded := false
+	defer common.EndTransaction(updater, &succeeded)
+
+	if updater.IsJoin() {
+		// If the user is joined to the room then that takes precedence over this
+		// invite event. It makes little sense to move a user that is already
+		// joined to the room into the invite state.
+		// This could plausibly happen if an invite request raced with a join
+		// request for a user. For example if a user was invited to a public
+		// room and they joined the room at the same time as the invite was sent.
+		// The other way this could plausibly happen is if an invite raced with
+		// a kick. For example if a user was kicked from a room in error and in
+		// response someone else in the room re-invited them then it is possible
+		// for the invite request to race with the leave event so that the
+		// target receives invite before it learns that it has been kicked.
+		// There are a few ways this could be plausibly handled in the roomserver.
+		// 1) Store the invite, but mark it as retired. That will result in the
+		//    permanent rejection of that invite event. So even if the target
+		//    user leaves the room and the invite is retransmitted it will be
+		//    ignored. However a new invite with a new event ID would still be
+		//    accepted.
+		// 2) Silently discard the invite event. This means that if the event
+		//    was retransmitted at a later date after the target user had left
+		//    the room we would accept the invite. However since we hadn't told
+		//    the sending server that the invite had been discarded it would
+		//    have no reason to attempt to retry.
+		// 3) Signal the sending server that the user is already joined to the
+		//    room.
+		// For now we will implement option 2. Since in the abesence of a retry
+		// mechanism it will be equivalent to option 1, and we don't have a
+		// signalling mechanism to implement option 3.
+		return nil
+	}
+
+	outputUpdates, err := updateToInviteMembership(updater, &input.Event, nil)
+	if err != nil {
+		return err
+	}
+
+	if err = ow.WriteOutputEvents(roomID, outputUpdates); err != nil {
+		return err
+	}
+
+	succeeded = true
+	return nil
+}

roomserver/input/input.go

@@ -0,0 +1,95 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package input contains the code processes new room events
+package input
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"sync"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/util"
+	sarama "gopkg.in/Shopify/sarama.v1"
+)
+
+// RoomserverInputAPI implements api.RoomserverInputAPI
+type RoomserverInputAPI struct {
+	DB       RoomEventDatabase
+	Producer sarama.SyncProducer
+	// The kafkaesque topic to output new room events to.
+	// This is the name used in kafka to identify the stream to write events to.
+	OutputRoomEventTopic string
+	// Protects calls to processRoomEvent
+	mutex sync.Mutex
+}
+
+// WriteOutputEvents implements OutputRoomEventWriter
+func (r *RoomserverInputAPI) WriteOutputEvents(roomID string, updates []api.OutputEvent) error {
+	messages := make([]*sarama.ProducerMessage, len(updates))
+	for i := range updates {
+		value, err := json.Marshal(updates[i])
+		if err != nil {
+			return err
+		}
+		messages[i] = &sarama.ProducerMessage{
+			Topic: r.OutputRoomEventTopic,
+			Key:   sarama.StringEncoder(roomID),
+			Value: sarama.ByteEncoder(value),
+		}
+	}
+	return r.Producer.SendMessages(messages)
+}
+
+// InputRoomEvents implements api.RoomserverInputAPI
+func (r *RoomserverInputAPI) InputRoomEvents(
+	ctx context.Context,
+	request *api.InputRoomEventsRequest,
+	response *api.InputRoomEventsResponse,
+) (err error) {
+	// We lock as processRoomEvent can only be called once at a time
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	for i := range request.InputRoomEvents {
+		if response.EventID, err = processRoomEvent(ctx, r.DB, r, request.InputRoomEvents[i]); err != nil {
+			return err
+		}
+	}
+	for i := range request.InputInviteEvents {
+		if err = processInviteEvent(ctx, r.DB, r, request.InputInviteEvents[i]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// SetupHTTP adds the RoomserverInputAPI handlers to the http.ServeMux.
+func (r *RoomserverInputAPI) SetupHTTP(servMux *http.ServeMux) {
+	servMux.Handle(api.RoomserverInputRoomEventsPath,
+		common.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse {
+			var request api.InputRoomEventsRequest
+			var response api.InputRoomEventsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+}

roomserver/input/latest_events.go

@@ -0,0 +1,293 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package input
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/state"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+// updateLatestEvents updates the list of latest events for this room in the database and writes the
+// event to the output log.
+// The latest events are the events that aren't referenced by another event in the database:
+//
+//     Time goes down the page. 1 is the m.room.create event (root).
+//
+//        1                 After storing 1 the latest events are {1}
+//        |                 After storing 2 the latest events are {2}
+//        2                 After storing 3 the latest events are {3}
+//       / \                After storing 4 the latest events are {3,4}
+//      3   4               After storing 5 the latest events are {5,4}
+//      |   |               After storing 6 the latest events are {5,6}
+//      5   6 <--- latest   After storing 7 the latest events are {6,7}
+//      |
+//      7 <----- latest
+//
+// Can only be called once at a time
+func updateLatestEvents(
+	ctx context.Context,
+	db RoomEventDatabase,
+	ow OutputRoomEventWriter,
+	roomNID types.RoomNID,
+	stateAtEvent types.StateAtEvent,
+	event gomatrixserverlib.Event,
+	sendAsServer string,
+	transactionID *api.TransactionID,
+) (err error) {
+	updater, err := db.GetLatestEventsForUpdate(ctx, roomNID)
+	if err != nil {
+		return
+	}
+	succeeded := false
+	defer common.EndTransaction(updater, &succeeded)
+
+	u := latestEventsUpdater{
+		ctx: ctx, db: db, updater: updater, ow: ow, roomNID: roomNID,
+		stateAtEvent: stateAtEvent, event: event, sendAsServer: sendAsServer,
+		transactionID: transactionID,
+	}
+	if err = u.doUpdateLatestEvents(); err != nil {
+		return err
+	}
+
+	succeeded = true
+	return
+}
+
+// latestEventsUpdater tracks the state used to update the latest events in the
+// room. It mostly just ferries state between the various function calls.
+// The state could be passed using function arguments, but it becomes impractical
+// when there are so many variables to pass around.
+type latestEventsUpdater struct {
+	ctx           context.Context
+	db            RoomEventDatabase
+	updater       types.RoomRecentEventsUpdater
+	ow            OutputRoomEventWriter
+	roomNID       types.RoomNID
+	stateAtEvent  types.StateAtEvent
+	event         gomatrixserverlib.Event
+	transactionID *api.TransactionID
+	// Which server to send this event as.
+	sendAsServer string
+	// The eventID of the event that was processed before this one.
+	lastEventIDSent string
+	// The latest events in the room after processing this event.
+	latest []types.StateAtEventAndReference
+	// The state entries removed from and added to the current state of the
+	// room as a result of processing this event. They are sorted lists.
+	removed []types.StateEntry
+	added   []types.StateEntry
+	// The state entries that are removed and added to recover the state before
+	// the event being processed. They are sorted lists.
+	stateBeforeEventRemoves []types.StateEntry
+	stateBeforeEventAdds    []types.StateEntry
+	// The snapshots of current state before and after processing this event
+	oldStateNID types.StateSnapshotNID
+	newStateNID types.StateSnapshotNID
+}
+
+func (u *latestEventsUpdater) doUpdateLatestEvents() error {
+	prevEvents := u.event.PrevEvents()
+	oldLatest := u.updater.LatestEvents()
+	u.lastEventIDSent = u.updater.LastEventIDSent()
+	u.oldStateNID = u.updater.CurrentStateSnapshotNID()
+
+	hasBeenSent, err := u.updater.HasEventBeenSent(u.stateAtEvent.EventNID)
+	if err != nil {
+		return err
+	} else if hasBeenSent {
+		// Already sent this event so we can stop processing
+		return nil
+	}
+
+	if err = u.updater.StorePreviousEvents(u.stateAtEvent.EventNID, prevEvents); err != nil {
+		return err
+	}
+
+	eventReference := u.event.EventReference()
+	// Check if this event is already referenced by another event in the room.
+	alreadyReferenced, err := u.updater.IsReferenced(eventReference)
+	if err != nil {
+		return err
+	}
+
+	u.latest = calculateLatest(oldLatest, alreadyReferenced, prevEvents, types.StateAtEventAndReference{
+		EventReference: eventReference,
+		StateAtEvent:   u.stateAtEvent,
+	})
+
+	if err = u.latestState(); err != nil {
+		return err
+	}
+
+	updates, err := updateMemberships(u.ctx, u.db, u.updater, u.removed, u.added)
+	if err != nil {
+		return err
+	}
+
+	update, err := u.makeOutputNewRoomEvent()
+	if err != nil {
+		return err
+	}
+	updates = append(updates, *update)
+
+	// Send the event to the output logs.
+	// We do this inside the database transaction to ensure that we only mark an event as sent if we sent it.
+	// (n.b. this means that it's possible that the same event will be sent twice if the transaction fails but
+	//  the write to the output log succeeds)
+	// TODO: This assumes that writing the event to the output log is synchronous. It should be possible to
+	// send the event asynchronously but we would need to ensure that 1) the events are written to the log in
+	// the correct order, 2) that pending writes are resent across restarts. In order to avoid writing all the
+	// necessary bookkeeping we'll keep the event sending synchronous for now.
+	if err = u.ow.WriteOutputEvents(u.event.RoomID(), updates); err != nil {
+		return err
+	}
+
+	if err = u.updater.SetLatestEvents(u.roomNID, u.latest, u.stateAtEvent.EventNID, u.newStateNID); err != nil {
+		return err
+	}
+
+	return u.updater.MarkEventAsSent(u.stateAtEvent.EventNID)
+}
+
+func (u *latestEventsUpdater) latestState() error {
+	var err error
+
+	latestStateAtEvents := make([]types.StateAtEvent, len(u.latest))
+	for i := range u.latest {
+		latestStateAtEvents[i] = u.latest[i].StateAtEvent
+	}
+	u.newStateNID, err = state.CalculateAndStoreStateAfterEvents(
+		u.ctx, u.db, u.roomNID, latestStateAtEvents,
+	)
+	if err != nil {
+		return err
+	}
+
+	u.removed, u.added, err = state.DifferenceBetweeenStateSnapshots(
+		u.ctx, u.db, u.oldStateNID, u.newStateNID,
+	)
+	if err != nil {
+		return err
+	}
+
+	u.stateBeforeEventRemoves, u.stateBeforeEventAdds, err = state.DifferenceBetweeenStateSnapshots(
+		u.ctx, u.db, u.newStateNID, u.stateAtEvent.BeforeStateSnapshotNID,
+	)
+	return err
+}
+
+func calculateLatest(
+	oldLatest []types.StateAtEventAndReference,
+	alreadyReferenced bool,
+	prevEvents []gomatrixserverlib.EventReference,
+	newEvent types.StateAtEventAndReference,
+) []types.StateAtEventAndReference {
+	var alreadyInLatest bool
+	var newLatest []types.StateAtEventAndReference
+	for _, l := range oldLatest {
+		keep := true
+		for _, prevEvent := range prevEvents {
+			if l.EventID == prevEvent.EventID && bytes.Equal(l.EventSHA256, prevEvent.EventSHA256) {
+				// This event can be removed from the latest events cause we've found an event that references it.
+				// (If an event is referenced by another event then it can't be one of the latest events in the room
+				//  because we have an event that comes after it)
+				keep = false
+				break
+			}
+		}
+		if l.EventNID == newEvent.EventNID {
+			alreadyInLatest = true
+		}
+		if keep {
+			// Keep the event in the latest events.
+			newLatest = append(newLatest, l)
+		}
+	}
+
+	if !alreadyReferenced && !alreadyInLatest {
+		// This event is not referenced by any of the events in the room
+		// and the event is not already in the latest events.
+		// Add it to the latest events
+		newLatest = append(newLatest, newEvent)
+	}
+
+	return newLatest
+}
+
+func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) {
+
+	latestEventIDs := make([]string, len(u.latest))
+	for i := range u.latest {
+		latestEventIDs[i] = u.latest[i].EventID
+	}
+
+	ore := api.OutputNewRoomEvent{
+		Event:           u.event,
+		LastSentEventID: u.lastEventIDSent,
+		LatestEventIDs:  latestEventIDs,
+		TransactionID:   u.transactionID,
+	}
+
+	var stateEventNIDs []types.EventNID
+	for _, entry := range u.added {
+		stateEventNIDs = append(stateEventNIDs, entry.EventNID)
+	}
+	for _, entry := range u.removed {
+		stateEventNIDs = append(stateEventNIDs, entry.EventNID)
+	}
+	for _, entry := range u.stateBeforeEventRemoves {
+		stateEventNIDs = append(stateEventNIDs, entry.EventNID)
+	}
+	for _, entry := range u.stateBeforeEventAdds {
+		stateEventNIDs = append(stateEventNIDs, entry.EventNID)
+	}
+	stateEventNIDs = stateEventNIDs[:util.SortAndUnique(eventNIDSorter(stateEventNIDs))]
+	eventIDMap, err := u.db.EventIDs(u.ctx, stateEventNIDs)
+	if err != nil {
+		return nil, err
+	}
+	for _, entry := range u.added {
+		ore.AddsStateEventIDs = append(ore.AddsStateEventIDs, eventIDMap[entry.EventNID])
+	}
+	for _, entry := range u.removed {
+		ore.RemovesStateEventIDs = append(ore.RemovesStateEventIDs, eventIDMap[entry.EventNID])
+	}
+	for _, entry := range u.stateBeforeEventRemoves {
+		ore.StateBeforeRemovesEventIDs = append(ore.StateBeforeRemovesEventIDs, eventIDMap[entry.EventNID])
+	}
+	for _, entry := range u.stateBeforeEventAdds {
+		ore.StateBeforeAddsEventIDs = append(ore.StateBeforeAddsEventIDs, eventIDMap[entry.EventNID])
+	}
+	ore.SendAsServer = u.sendAsServer
+
+	return &api.OutputEvent{
+		Type:         api.OutputTypeNewRoomEvent,
+		NewRoomEvent: &ore,
+	}, nil
+}
+
+type eventNIDSorter []types.EventNID
+
+func (s eventNIDSorter) Len() int           { return len(s) }
+func (s eventNIDSorter) Less(i, j int) bool { return s[i] < s[j] }
+func (s eventNIDSorter) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

roomserver/input/membership.go

@@ -0,0 +1,310 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package input
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// Membership values
+// TODO: Factor these out somewhere sensible?
+const join = "join"
+const leave = "leave"
+const invite = "invite"
+const ban = "ban"
+
+// updateMembership updates the current membership and the invites for each
+// user affected by a change in the current state of the room.
+// Returns a list of output events to write to the kafka log to inform the
+// consumers about the invites added or retired by the change in current state.
+func updateMemberships(
+	ctx context.Context,
+	db RoomEventDatabase,
+	updater types.RoomRecentEventsUpdater,
+	removed, added []types.StateEntry,
+) ([]api.OutputEvent, error) {
+	changes := membershipChanges(removed, added)
+	var eventNIDs []types.EventNID
+	for _, change := range changes {
+		if change.addedEventNID != 0 {
+			eventNIDs = append(eventNIDs, change.addedEventNID)
+		}
+		if change.removedEventNID != 0 {
+			eventNIDs = append(eventNIDs, change.removedEventNID)
+		}
+	}
+
+	// Load the event JSON so we can look up the "membership" key.
+	// TODO: Maybe add a membership key to the events table so we can load that
+	// key without having to load the entire event JSON?
+	events, err := db.Events(ctx, eventNIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	var updates []api.OutputEvent
+
+	for _, change := range changes {
+		var ae *gomatrixserverlib.Event
+		var re *gomatrixserverlib.Event
+		targetUserNID := change.EventStateKeyNID
+		if change.removedEventNID != 0 {
+			ev, _ := eventMap(events).lookup(change.removedEventNID)
+			if ev != nil {
+				re = &ev.Event
+			}
+		}
+		if change.addedEventNID != 0 {
+			ev, _ := eventMap(events).lookup(change.addedEventNID)
+			if ev != nil {
+				ae = &ev.Event
+			}
+		}
+		if updates, err = updateMembership(updater, targetUserNID, re, ae, updates); err != nil {
+			return nil, err
+		}
+	}
+	return updates, nil
+}
+
+func updateMembership(
+	updater types.RoomRecentEventsUpdater, targetUserNID types.EventStateKeyNID,
+	remove, add *gomatrixserverlib.Event,
+	updates []api.OutputEvent,
+) ([]api.OutputEvent, error) {
+	var err error
+	// Default the membership to Leave if no event was added or removed.
+	oldMembership := leave
+	newMembership := leave
+
+	if remove != nil {
+		oldMembership, err = remove.Membership()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if add != nil {
+		newMembership, err = add.Membership()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if oldMembership == newMembership && newMembership != join {
+		// If the membership is the same then nothing changed and we can return
+		// immediately, unless it's a Join update (e.g. profile update).
+		return updates, nil
+	}
+
+	mu, err := updater.MembershipUpdater(targetUserNID)
+	if err != nil {
+		return nil, err
+	}
+
+	switch newMembership {
+	case invite:
+		return updateToInviteMembership(mu, add, updates)
+	case join:
+		return updateToJoinMembership(mu, add, updates)
+	case leave, ban:
+		return updateToLeaveMembership(mu, add, newMembership, updates)
+	default:
+		panic(fmt.Errorf(
+			"input: membership %q is not one of the allowed values", newMembership,
+		))
+	}
+}
+
+func updateToInviteMembership(
+	mu types.MembershipUpdater, add *gomatrixserverlib.Event, updates []api.OutputEvent,
+) ([]api.OutputEvent, error) {
+	// We may have already sent the invite to the user, either because we are
+	// reprocessing this event, or because the we received this invite from a
+	// remote server via the federation invite API. In those cases we don't need
+	// to send the event.
+	needsSending, err := mu.SetToInvite(*add)
+	if err != nil {
+		return nil, err
+	}
+	if needsSending {
+		// We notify the consumers using a special event even though we will
+		// notify them about the change in current state as part of the normal
+		// room event stream. This ensures that the consumers only have to
+		// consider a single stream of events when determining whether a user
+		// is invited, rather than having to combine multiple streams themselves.
+		onie := api.OutputNewInviteEvent{
+			Event: *add,
+		}
+		updates = append(updates, api.OutputEvent{
+			Type:           api.OutputTypeNewInviteEvent,
+			NewInviteEvent: &onie,
+		})
+	}
+	return updates, nil
+}
+
+func updateToJoinMembership(
+	mu types.MembershipUpdater, add *gomatrixserverlib.Event, updates []api.OutputEvent,
+) ([]api.OutputEvent, error) {
+	// If the user is already marked as being joined, we call SetToJoin to update
+	// the event ID then we can return immediately. Retired is ignored as there
+	// is no invite event to retire.
+	if mu.IsJoin() {
+		_, err := mu.SetToJoin(add.Sender(), add.EventID(), true)
+		if err != nil {
+			return nil, err
+		}
+		return updates, nil
+	}
+	// When we mark a user as being joined we will invalidate any invites that
+	// are active for that user. We notify the consumers that the invites have
+	// been retired using a special event, even though they could infer this
+	// by studying the state changes in the room event stream.
+	retired, err := mu.SetToJoin(add.Sender(), add.EventID(), false)
+	if err != nil {
+		return nil, err
+	}
+	for _, eventID := range retired {
+		orie := api.OutputRetireInviteEvent{
+			EventID:          eventID,
+			Membership:       join,
+			RetiredByEventID: add.EventID(),
+			TargetUserID:     *add.StateKey(),
+		}
+		updates = append(updates, api.OutputEvent{
+			Type:              api.OutputTypeRetireInviteEvent,
+			RetireInviteEvent: &orie,
+		})
+	}
+	return updates, nil
+}
+
+func updateToLeaveMembership(
+	mu types.MembershipUpdater, add *gomatrixserverlib.Event,
+	newMembership string, updates []api.OutputEvent,
+) ([]api.OutputEvent, error) {
+	// If the user is already neither joined, nor invited to the room then we
+	// can return immediately.
+	if mu.IsLeave() {
+		return updates, nil
+	}
+	// When we mark a user as having left we will invalidate any invites that
+	// are active for that user. We notify the consumers that the invites have
+	// been retired using a special event, even though they could infer this
+	// by studying the state changes in the room event stream.
+	retired, err := mu.SetToLeave(add.Sender(), add.EventID())
+	if err != nil {
+		return nil, err
+	}
+	for _, eventID := range retired {
+		orie := api.OutputRetireInviteEvent{
+			EventID:          eventID,
+			Membership:       newMembership,
+			RetiredByEventID: add.EventID(),
+			TargetUserID:     *add.StateKey(),
+		}
+		updates = append(updates, api.OutputEvent{
+			Type:              api.OutputTypeRetireInviteEvent,
+			RetireInviteEvent: &orie,
+		})
+	}
+	return updates, nil
+}
+
+// membershipChanges pairs up the membership state changes from a sorted list
+// of state removed and a sorted list of state added.
+func membershipChanges(removed, added []types.StateEntry) []stateChange {
+	changes := pairUpChanges(removed, added)
+	var result []stateChange
+	for _, c := range changes {
+		if c.EventTypeNID == types.MRoomMemberNID {
+			result = append(result, c)
+		}
+	}
+	return result
+}
+
+type stateChange struct {
+	types.StateKeyTuple
+	removedEventNID types.EventNID
+	addedEventNID   types.EventNID
+}
+
+// pairUpChanges pairs up the state events added and removed for each type,
+// state key tuple. Assumes that removed and added are sorted.
+func pairUpChanges(removed, added []types.StateEntry) []stateChange {
+	var ai int
+	var ri int
+	var result []stateChange
+	for {
+		switch {
+		case ai == len(added):
+			// We've reached the end of the added entries.
+			// The rest of the removed list are events that were removed without
+			// an event with the same state key being added.
+			for _, s := range removed[ri:] {
+				result = append(result, stateChange{
+					StateKeyTuple:   s.StateKeyTuple,
+					removedEventNID: s.EventNID,
+				})
+			}
+			return result
+		case ri == len(removed):
+			// We've reached the end of the removed entries.
+			// The rest of the added list are events that were added without
+			// an event with the same state key being removed.
+			for _, s := range added[ai:] {
+				result = append(result, stateChange{
+					StateKeyTuple: s.StateKeyTuple,
+					addedEventNID: s.EventNID,
+				})
+			}
+			return result
+		case added[ai].StateKeyTuple == removed[ri].StateKeyTuple:
+			// The tuple is in both lists so an event with that key is being
+			// removed and another event with the same key is being added.
+			result = append(result, stateChange{
+				StateKeyTuple:   added[ai].StateKeyTuple,
+				removedEventNID: removed[ri].EventNID,
+				addedEventNID:   added[ai].EventNID,
+			})
+			ai++
+			ri++
+		case added[ai].StateKeyTuple.LessThan(removed[ri].StateKeyTuple):
+			// The lists are sorted so the added entry being less than the
+			// removed entry means that the added event was added without an
+			// event with the same key being removed.
+			result = append(result, stateChange{
+				StateKeyTuple: added[ai].StateKeyTuple,
+				addedEventNID: added[ai].EventNID,
+			})
+			ai++
+		default:
+			// Reaching the default case implies that the removed entry is less
+			// than the added entry. Since the lists are sorted this means that
+			// the removed event was removed without an event with the same
+			// key being added.
+			result = append(result, stateChange{
+				StateKeyTuple:   removed[ai].StateKeyTuple,
+				removedEventNID: removed[ri].EventNID,
+			})
+			ri++
+		}
+	}
+}