Support sqlite in addition to postgres (#869)

* Move current work into single branch * Initial massaging of clientapi etc (not working yet) * Interfaces for accounts/devices databases * Duplicate postgres package for sqlite3 (no changes made to it yet) * Some keydb, accountdb, devicedb, common partition fixes, some more syncapi tweaking * Fix accounts DB, device DB * Update naffka dependency for SQLite * Naffka SQLite * Update naffka to latest master * SQLite support for federationsender * Mostly not-bad support for SQLite in syncapi (although there are problems where lots of events get classed incorrectly as backward extremities, probably because of IN/ANY clauses that are badly supported) * Update Dockerfile -> Go 1.13.7, add build-base (as gcc and friends are needed for SQLite) * Implement GET endpoints for account_data in clientapi * Nuke filtering for now... * Revert "Implement GET endpoints for account_data in clientapi" This reverts commit 4d80dff4583d278620d9b3ed437e9fcd8d4674ee. * Implement GET endpoints for account_data in clientapi (#861) * Implement GET endpoints for account_data in clientapi * Fix accountDB parameter * Remove fmt.Println * Fix insertAccountData SQLite query * Fix accountDB storage interfaces * Add empty push rules into account data on account creation (#862) * Put SaveAccountData into the right function this time * Not sure if roomserver is better or worse now * sqlite work * Allow empty last sent ID for the first event * sqlite: room creation works * Support sending messages * Nuke fmt.println * Move QueryVariadic etc into common, other device fixes * Fix some linter issues * Fix bugs * Fix some linting errors * Fix errcheck lint errors * Make naffka use postgres as fallback, fix couple of compile errors * What on earth happened to the /rooms/{roomID}/send/{eventType} routing Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
2025-07-31 05:12:46 +00:00 · 2020-02-13 17:27:33 +00:00 · 2020-02-13 17:27:33 +00:00 · b6ea1bc67a
commit b6ea1bc67a
parent 6942ee1de0
103 changed files with 9467 additions and 710 deletions
--- a/syncapi/storage/postgres/syncserver.go
+++ b/syncapi/storage/postgres/syncserver.go
@ -413,13 +413,18 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse(
 	numRecentEventsPerRoom int,
 	wantFullState bool,
 	res *types.Response,
-) ([]string, error) {
+) (joinedRoomIDs []string, err error) {
 	txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot)
 	if err != nil {
 		return nil, err
 	}
 	var succeeded bool
-	defer common.EndTransaction(txn, &succeeded)
+	defer func() {
+		txerr := common.EndTransaction(txn, &succeeded)
+		if err == nil && txerr != nil {
+			err = txerr
+		}
+	}()

 	stateFilter := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request

@ -428,7 +433,6 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse(
 	// This works out what the 'state' key should be for each room as well as which membership block
 	// to put the room into.
 	var deltas []stateDelta
-	var joinedRoomIDs []string
 	if !wantFullState {
 		deltas, joinedRoomIDs, err = d.getStateDeltas(
 			ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilter,
@ -570,7 +574,12 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
 		return
 	}
 	var succeeded bool
-	defer common.EndTransaction(txn, &succeeded)
+	defer func() {
+		txerr := common.EndTransaction(txn, &succeeded)
+		if err == nil && txerr != nil {
+			err = txerr
+		}
+	}()

 	// Get the current sync position which we will base the sync response on.
 	toPos, err = d.syncPositionTx(ctx, txn)
--- a/syncapi/storage/sqlite3/account_data_table.go
+++ b/syncapi/storage/sqlite3/account_data_table.go
@ -0,0 +1,143 @@
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const accountDataSchema = `
+CREATE TABLE IF NOT EXISTS syncapi_account_data_type (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    UNIQUE (user_id, room_id, type)
+);
+`
+
+const insertAccountDataSQL = "" +
+	"INSERT INTO syncapi_account_data_type (id, user_id, room_id, type) VALUES ($1, $2, $3, $4)" +
+	" ON CONFLICT (user_id, room_id, type) DO UPDATE" +
+	" SET id = EXCLUDED.id"
+
+const selectAccountDataInRangeSQL = "" +
+	"SELECT room_id, type FROM syncapi_account_data_type" +
+	" WHERE user_id = $1 AND id > $2 AND id <= $3" +
+	" AND ( $4 IS NULL OR     type IN ($4)  )" +
+	" AND ( $5 IS NULL OR NOT(type IN ($5)) )" +
+	" ORDER BY id ASC LIMIT $6"
+
+const selectMaxAccountDataIDSQL = "" +
+	"SELECT MAX(id) FROM syncapi_account_data_type"
+
+type accountDataStatements struct {
+	streamIDStatements           *streamIDStatements
+	insertAccountDataStmt        *sql.Stmt
+	selectAccountDataInRangeStmt *sql.Stmt
+	selectMaxAccountDataIDStmt   *sql.Stmt
+}
+
+func (s *accountDataStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) {
+	s.streamIDStatements = streamID
+	_, err = db.Exec(accountDataSchema)
+	if err != nil {
+		return
+	}
+	if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil {
+		return
+	}
+	if s.selectAccountDataInRangeStmt, err = db.Prepare(selectAccountDataInRangeSQL); err != nil {
+		return
+	}
+	if s.selectMaxAccountDataIDStmt, err = db.Prepare(selectMaxAccountDataIDSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *accountDataStatements) insertAccountData(
+	ctx context.Context, txn *sql.Tx,
+	userID, roomID, dataType string,
+) (pos types.StreamPosition, err error) {
+	pos, err = s.streamIDStatements.nextStreamID(ctx, txn)
+	if err != nil {
+		return
+	}
+	insertStmt := common.TxStmt(txn, s.insertAccountDataStmt)
+	_, err = insertStmt.ExecContext(ctx, pos, userID, roomID, dataType)
+	return
+}
+
+func (s *accountDataStatements) selectAccountDataInRange(
+	ctx context.Context,
+	userID string,
+	oldPos, newPos types.StreamPosition,
+	accountDataFilterPart *gomatrixserverlib.EventFilter,
+) (data map[string][]string, err error) {
+	data = make(map[string][]string)
+
+	// If both positions are the same, it means that the data was saved after the
+	// latest room event. In that case, we need to decrement the old position as
+	// it would prevent the SQL request from returning anything.
+	if oldPos == newPos {
+		oldPos--
+	}
+
+	rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, oldPos, newPos,
+		pq.StringArray(filterConvertTypeWildcardToSQL(accountDataFilterPart.Types)),
+		pq.StringArray(filterConvertTypeWildcardToSQL(accountDataFilterPart.NotTypes)),
+		accountDataFilterPart.Limit,
+	)
+	if err != nil {
+		return
+	}
+
+	for rows.Next() {
+		var dataType string
+		var roomID string
+
+		if err = rows.Scan(&roomID, &dataType); err != nil {
+			return
+		}
+
+		if len(data[roomID]) > 0 {
+			data[roomID] = append(data[roomID], dataType)
+		} else {
+			data[roomID] = []string{dataType}
+		}
+	}
+
+	return
+}
+
+func (s *accountDataStatements) selectMaxAccountDataID(
+	ctx context.Context, txn *sql.Tx,
+) (id int64, err error) {
+	var nullableID sql.NullInt64
+	stmt := common.TxStmt(txn, s.selectMaxAccountDataIDStmt)
+	err = stmt.QueryRowContext(ctx).Scan(&nullableID)
+	if nullableID.Valid {
+		id = nullableID.Int64
+	}
+	return
+}
--- a/syncapi/storage/sqlite3/backward_extremities_table.go
+++ b/syncapi/storage/sqlite3/backward_extremities_table.go
@ -0,0 +1,124 @@
+// Copyright 2018 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/common"
+)
+
+const backwardExtremitiesSchema = `
+-- Stores output room events received from the roomserver.
+CREATE TABLE IF NOT EXISTS syncapi_backward_extremities (
+	room_id TEXT NOT NULL,
+	event_id TEXT NOT NULL,
+
+	PRIMARY KEY(room_id, event_id)
+);
+`
+
+const insertBackwardExtremitySQL = "" +
+	"INSERT INTO syncapi_backward_extremities (room_id, event_id)" +
+	" VALUES ($1, $2)" +
+	" ON CONFLICT (room_id, event_id) DO NOTHING"
+
+const selectBackwardExtremitiesForRoomSQL = "" +
+	"SELECT event_id FROM syncapi_backward_extremities WHERE room_id = $1"
+
+const isBackwardExtremitySQL = "" +
+	"SELECT EXISTS (" +
+	" SELECT TRUE FROM syncapi_backward_extremities" +
+	" WHERE room_id = $1 AND event_id = $2" +
+	")"
+
+const deleteBackwardExtremitySQL = "" +
+	"DELETE FROM syncapi_backward_extremities" +
+	" WHERE room_id = $1 AND event_id = $2"
+
+type backwardExtremitiesStatements struct {
+	insertBackwardExtremityStmt          *sql.Stmt
+	selectBackwardExtremitiesForRoomStmt *sql.Stmt
+	isBackwardExtremityStmt              *sql.Stmt
+	deleteBackwardExtremityStmt          *sql.Stmt
+}
+
+func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(backwardExtremitiesSchema)
+	if err != nil {
+		return
+	}
+	if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil {
+		return
+	}
+	if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil {
+		return
+	}
+	if s.isBackwardExtremityStmt, err = db.Prepare(isBackwardExtremitySQL); err != nil {
+		return
+	}
+	if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *backwardExtremitiesStatements) insertsBackwardExtremity(
+	ctx context.Context, txn *sql.Tx, roomID, eventID string,
+) (err error) {
+	stmt := common.TxStmt(txn, s.insertBackwardExtremityStmt)
+	_, err = stmt.ExecContext(ctx, roomID, eventID)
+	return
+}
+
+func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom(
+	ctx context.Context, txn *sql.Tx, roomID string,
+) (eventIDs []string, err error) {
+	eventIDs = make([]string, 0)
+
+	stmt := common.TxStmt(txn, s.selectBackwardExtremitiesForRoomStmt)
+	rows, err := stmt.QueryContext(ctx, roomID)
+	if err != nil {
+		return
+	}
+
+	for rows.Next() {
+		var eID string
+		if err = rows.Scan(&eID); err != nil {
+			return
+		}
+
+		eventIDs = append(eventIDs, eID)
+	}
+
+	return
+}
+
+func (s *backwardExtremitiesStatements) isBackwardExtremity(
+	ctx context.Context, txn *sql.Tx, roomID, eventID string,
+) (isBE bool, err error) {
+	stmt := common.TxStmt(txn, s.isBackwardExtremityStmt)
+	err = stmt.QueryRowContext(ctx, roomID, eventID).Scan(&isBE)
+	return
+}
+
+func (s *backwardExtremitiesStatements) deleteBackwardExtremity(
+	ctx context.Context, txn *sql.Tx, roomID, eventID string,
+) (err error) {
+	stmt := common.TxStmt(txn, s.deleteBackwardExtremityStmt)
+	_, err = stmt.ExecContext(ctx, roomID, eventID)
+	return
+}
--- a/syncapi/storage/sqlite3/current_room_state_table.go
+++ b/syncapi/storage/sqlite3/current_room_state_table.go
@ -0,0 +1,276 @@
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const currentRoomStateSchema = `
+-- Stores the current room state for every room.
+CREATE TABLE IF NOT EXISTS syncapi_current_room_state (
+    room_id TEXT NOT NULL,
+    event_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    sender TEXT NOT NULL,
+    contains_url BOOL NOT NULL DEFAULT false,
+    state_key TEXT NOT NULL,
+    event_json TEXT NOT NULL,
+    membership TEXT,
+    added_at BIGINT,
+    UNIQUE (room_id, type, state_key)
+);
+-- for event deletion
+CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_state(event_id, room_id, type, sender, contains_url);
+-- for querying membership states of users
+-- CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave';
+`
+
+const upsertRoomStateSQL = "" +
+	"INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" +
+	" VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" +
+	" ON CONFLICT (event_id, room_id, type, sender, contains_url)" +
+	" DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9"
+
+const deleteRoomStateByEventIDSQL = "" +
+	"DELETE FROM syncapi_current_room_state WHERE event_id = $1"
+
+const selectRoomIDsWithMembershipSQL = "" +
+	"SELECT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2"
+
+const selectCurrentStateSQL = "" +
+	"SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" +
+	" AND ( $2 IS NULL OR     sender IN ($2)  )" +
+	" AND ( $3 IS NULL OR NOT(sender IN ($3)) )" +
+	" AND ( $4 IS NULL OR     type   IN ($4)  )" +
+	" AND ( $5 IS NULL OR NOT(type   IN ($5)) )" +
+	" AND ( $6 IS NULL OR     contains_url = $6  )" +
+	" LIMIT $7"
+
+const selectJoinedUsersSQL = "" +
+	"SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'"
+
+const selectStateEventSQL = "" +
+	"SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
+
+const selectEventsWithEventIDsSQL = "" +
+	// TODO: The session_id and transaction_id blanks are here because otherwise
+	// the rowsToStreamEvents expects there to be exactly five columns. We need to
+	// figure out if these really need to be in the DB, and if so, we need a
+	// better permanent fix for this. - neilalexander, 2 Jan 2020
+	"SELECT added_at, event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" +
+	" FROM syncapi_current_room_state WHERE event_id IN ($1)"
+
+type currentRoomStateStatements struct {
+	streamIDStatements              *streamIDStatements
+	upsertRoomStateStmt             *sql.Stmt
+	deleteRoomStateByEventIDStmt    *sql.Stmt
+	selectRoomIDsWithMembershipStmt *sql.Stmt
+	selectCurrentStateStmt          *sql.Stmt
+	selectJoinedUsersStmt           *sql.Stmt
+	selectEventsWithEventIDsStmt    *sql.Stmt
+	selectStateEventStmt            *sql.Stmt
+}
+
+func (s *currentRoomStateStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) {
+	s.streamIDStatements = streamID
+	_, err = db.Exec(currentRoomStateSchema)
+	if err != nil {
+		return
+	}
+	if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil {
+		return
+	}
+	if s.deleteRoomStateByEventIDStmt, err = db.Prepare(deleteRoomStateByEventIDSQL); err != nil {
+		return
+	}
+	if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil {
+		return
+	}
+	if s.selectCurrentStateStmt, err = db.Prepare(selectCurrentStateSQL); err != nil {
+		return
+	}
+	if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil {
+		return
+	}
+	if s.selectEventsWithEventIDsStmt, err = db.Prepare(selectEventsWithEventIDsSQL); err != nil {
+		return
+	}
+	if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil {
+		return
+	}
+	return
+}
+
+// JoinedMemberLists returns a map of room ID to a list of joined user IDs.
+func (s *currentRoomStateStatements) selectJoinedUsers(
+	ctx context.Context,
+) (map[string][]string, error) {
+	rows, err := s.selectJoinedUsersStmt.QueryContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close() // nolint: errcheck
+
+	result := make(map[string][]string)
+	for rows.Next() {
+		var roomID string
+		var userID string
+		if err := rows.Scan(&roomID, &userID); err != nil {
+			return nil, err
+		}
+		users := result[roomID]
+		users = append(users, userID)
+		result[roomID] = users
+	}
+	return result, nil
+}
+
+// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
+func (s *currentRoomStateStatements) selectRoomIDsWithMembership(
+	ctx context.Context,
+	txn *sql.Tx,
+	userID string,
+	membership string, // nolint: unparam
+) ([]string, error) {
+	stmt := common.TxStmt(txn, s.selectRoomIDsWithMembershipStmt)
+	rows, err := stmt.QueryContext(ctx, userID, membership)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close() // nolint: errcheck
+
+	var result []string
+	for rows.Next() {
+		var roomID string
+		if err := rows.Scan(&roomID); err != nil {
+			return nil, err
+		}
+		result = append(result, roomID)
+	}
+	return result, nil
+}
+
+// CurrentState returns all the current state events for the given room.
+func (s *currentRoomStateStatements) selectCurrentState(
+	ctx context.Context, txn *sql.Tx, roomID string,
+	stateFilterPart *gomatrixserverlib.StateFilter,
+) ([]gomatrixserverlib.Event, error) {
+	stmt := common.TxStmt(txn, s.selectCurrentStateStmt)
+	rows, err := stmt.QueryContext(ctx, roomID,
+		pq.StringArray(stateFilterPart.Senders),
+		pq.StringArray(stateFilterPart.NotSenders),
+		pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)),
+		pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)),
+		stateFilterPart.ContainsURL,
+		stateFilterPart.Limit,
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close() // nolint: errcheck
+
+	return rowsToEvents(rows)
+}
+
+func (s *currentRoomStateStatements) deleteRoomStateByEventID(
+	ctx context.Context, txn *sql.Tx, eventID string,
+) error {
+	stmt := common.TxStmt(txn, s.deleteRoomStateByEventIDStmt)
+	_, err := stmt.ExecContext(ctx, eventID)
+	return err
+}
+
+func (s *currentRoomStateStatements) upsertRoomState(
+	ctx context.Context, txn *sql.Tx,
+	event gomatrixserverlib.Event, membership *string, addedAt types.StreamPosition,
+) error {
+	// Parse content as JSON and search for an "url" key
+	containsURL := false
+	var content map[string]interface{}
+	if json.Unmarshal(event.Content(), &content) != nil {
+		// Set containsURL to true if url is present
+		_, containsURL = content["url"]
+	}
+
+	// upsert state event
+	stmt := common.TxStmt(txn, s.upsertRoomStateStmt)
+	_, err := stmt.ExecContext(
+		ctx,
+		event.RoomID(),
+		event.EventID(),
+		event.Type(),
+		event.Sender(),
+		containsURL,
+		*event.StateKey(),
+		event.JSON(),
+		membership,
+		addedAt,
+	)
+	return err
+}
+
+func (s *currentRoomStateStatements) selectEventsWithEventIDs(
+	ctx context.Context, txn *sql.Tx, eventIDs []string,
+) ([]types.StreamEvent, error) {
+	stmt := common.TxStmt(txn, s.selectEventsWithEventIDsStmt)
+	rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs))
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close() // nolint: errcheck
+	return rowsToStreamEvents(rows)
+}
+
+func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) {
+	result := []gomatrixserverlib.Event{}
+	for rows.Next() {
+		var eventBytes []byte
+		if err := rows.Scan(&eventBytes); err != nil {
+			return nil, err
+		}
+		// TODO: Handle redacted events
+		ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false)
+		if err != nil {
+			return nil, err
+		}
+		result = append(result, ev)
+	}
+	return result, nil
+}
+
+func (s *currentRoomStateStatements) selectStateEvent(
+	ctx context.Context, roomID, evType, stateKey string,
+) (*gomatrixserverlib.Event, error) {
+	stmt := s.selectStateEventStmt
+	var res []byte
+	err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	ev, err := gomatrixserverlib.NewEventFromTrustedJSON(res, false)
+	return &ev, err
+}
--- a/syncapi/storage/sqlite3/filtering.go
+++ b/syncapi/storage/sqlite3/filtering.go
@ -0,0 +1,36 @@
+// Copyright 2017 Thibaut CHARLES
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"strings"
+)
+
+// filterConvertWildcardToSQL converts wildcards as defined in
+// https://matrix.org/docs/spec/client_server/r0.3.0.html#post-matrix-client-r0-user-userid-filter
+// to SQL wildcards that can be used with LIKE()
+func filterConvertTypeWildcardToSQL(values []string) []string {
+	if values == nil {
+		// Return nil instead of []string{} so IS NULL can work correctly when
+		// the return value is passed into SQL queries
+		return nil
+	}
+
+	ret := make([]string, len(values))
+	for i := range values {
+		ret[i] = strings.Replace(values[i], "*", "%", -1)
+	}
+	return ret
+}
--- a/syncapi/storage/sqlite3/invites_table.go
+++ b/syncapi/storage/sqlite3/invites_table.go
@ -0,0 +1,157 @@
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const inviteEventsSchema = `
+CREATE TABLE IF NOT EXISTS syncapi_invite_events (
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	event_id TEXT NOT NULL,
+	room_id TEXT NOT NULL,
+	target_user_id TEXT NOT NULL,
+	event_json TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS syncapi_invites_target_user_id_idx ON syncapi_invite_events (target_user_id, id);
+CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx ON syncapi_invite_events (event_id);
+`
+
+const insertInviteEventSQL = "" +
+	"INSERT INTO syncapi_invite_events" +
+	" (room_id, event_id, target_user_id, event_json)" +
+	" VALUES ($1, $2, $3, $4)"
+
+const selectLastInsertedInviteEventSQL = "" +
+	"SELECT id FROM syncapi_invite_events WHERE rowid = last_insert_rowid()"
+
+const deleteInviteEventSQL = "" +
+	"DELETE FROM syncapi_invite_events WHERE event_id = $1"
+
+const selectInviteEventsInRangeSQL = "" +
+	"SELECT room_id, event_json FROM syncapi_invite_events" +
+	" WHERE target_user_id = $1 AND id > $2 AND id <= $3" +
+	" ORDER BY id DESC"
+
+const selectMaxInviteIDSQL = "" +
+	"SELECT MAX(id) FROM syncapi_invite_events"
+
+type inviteEventsStatements struct {
+	streamIDStatements                *streamIDStatements
+	insertInviteEventStmt             *sql.Stmt
+	selectLastInsertedInviteEventStmt *sql.Stmt
+	selectInviteEventsInRangeStmt     *sql.Stmt
+	deleteInviteEventStmt             *sql.Stmt
+	selectMaxInviteIDStmt             *sql.Stmt
+}
+
+func (s *inviteEventsStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) {
+	s.streamIDStatements = streamID
+	_, err = db.Exec(inviteEventsSchema)
+	if err != nil {
+		return
+	}
+	if s.insertInviteEventStmt, err = db.Prepare(insertInviteEventSQL); err != nil {
+		return
+	}
+	if s.selectLastInsertedInviteEventStmt, err = db.Prepare(selectLastInsertedInviteEventSQL); err != nil {
+		return
+	}
+	if s.selectInviteEventsInRangeStmt, err = db.Prepare(selectInviteEventsInRangeSQL); err != nil {
+		return
+	}
+	if s.deleteInviteEventStmt, err = db.Prepare(deleteInviteEventSQL); err != nil {
+		return
+	}
+	if s.selectMaxInviteIDStmt, err = db.Prepare(selectMaxInviteIDSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *inviteEventsStatements) insertInviteEvent(
+	ctx context.Context, inviteEvent gomatrixserverlib.Event,
+) (streamPos types.StreamPosition, err error) {
+	_, err = s.insertInviteEventStmt.ExecContext(
+		ctx,
+		inviteEvent.RoomID(),
+		inviteEvent.EventID(),
+		*inviteEvent.StateKey(),
+		inviteEvent.JSON(),
+	)
+	if err != nil {
+		return
+	}
+	err = s.selectLastInsertedInviteEventStmt.QueryRowContext(ctx).Scan(&streamPos)
+	return
+}
+
+func (s *inviteEventsStatements) deleteInviteEvent(
+	ctx context.Context, inviteEventID string,
+) error {
+	_, err := s.deleteInviteEventStmt.ExecContext(ctx, inviteEventID)
+	return err
+}
+
+// selectInviteEventsInRange returns a map of room ID to invite event for the
+// active invites for the target user ID in the supplied range.
+func (s *inviteEventsStatements) selectInviteEventsInRange(
+	ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition,
+) (map[string]gomatrixserverlib.Event, error) {
+	stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt)
+	rows, err := stmt.QueryContext(ctx, targetUserID, startPos, endPos)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close() // nolint: errcheck
+	result := map[string]gomatrixserverlib.Event{}
+	for rows.Next() {
+		var (
+			roomID    string
+			eventJSON []byte
+		)
+		if err = rows.Scan(&roomID, &eventJSON); err != nil {
+			return nil, err
+		}
+
+		event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false)
+		if err != nil {
+			return nil, err
+		}
+
+		result[roomID] = event
+	}
+	return result, nil
+}
+
+func (s *inviteEventsStatements) selectMaxInviteID(
+	ctx context.Context, txn *sql.Tx,
+) (id int64, err error) {
+	var nullableID sql.NullInt64
+	stmt := common.TxStmt(txn, s.selectMaxInviteIDStmt)
+	err = stmt.QueryRowContext(ctx).Scan(&nullableID)
+	if nullableID.Valid {
+		id = nullableID.Int64
+	}
+	return
+}
--- a/syncapi/storage/sqlite3/output_room_events_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_table.go
@ -0,0 +1,411 @@
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"sort"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/syncapi/types"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/gomatrixserverlib"
+	log "github.com/sirupsen/logrus"
+)
+
+const outputRoomEventsSchema = `
+-- Stores output room events received from the roomserver.
+CREATE TABLE IF NOT EXISTS syncapi_output_room_events (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  event_id TEXT NOT NULL UNIQUE,
+  room_id TEXT NOT NULL,
+  event_json TEXT NOT NULL,
+  type TEXT NOT NULL,
+  sender TEXT NOT NULL,
+  contains_url BOOL NOT NULL,
+  add_state_ids TEXT[],
+  remove_state_ids TEXT[],
+  session_id BIGINT,
+  transaction_id TEXT,
+  exclude_from_sync BOOL DEFAULT FALSE
+);
+`
+
+const insertEventSQL = "" +
+	"INSERT INTO syncapi_output_room_events (" +
+	"id, room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" +
+	") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " +
+	"ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $11"
+
+const selectLastInsertedEventSQL = "" +
+	"SELECT id FROM syncapi_output_room_events WHERE rowid = last_insert_rowid()"
+
+const selectEventsSQL = "" +
+	"SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1"
+
+const selectRecentEventsSQL = "" +
+	"SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
+	" WHERE room_id = $1 AND id > $2 AND id <= $3" +
+	" ORDER BY id DESC LIMIT $4"
+
+const selectRecentEventsForSyncSQL = "" +
+	"SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
+	" WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" +
+	" ORDER BY id DESC LIMIT $4"
+
+const selectEarlyEventsSQL = "" +
+	"SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" +
+	" WHERE room_id = $1 AND id > $2 AND id <= $3" +
+	" ORDER BY id ASC LIMIT $4"
+
+const selectMaxEventIDSQL = "" +
+	"SELECT MAX(id) FROM syncapi_output_room_events"
+
+// In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id).
+/*
+	$1 = oldPos,
+	$2 = newPos,
+	$3 = pq.StringArray(stateFilterPart.Senders),
+	$4 = pq.StringArray(stateFilterPart.NotSenders),
+	$5 = pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)),
+	$6 = pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)),
+	$7 = stateFilterPart.ContainsURL,
+	$8 = stateFilterPart.Limit,
+*/
+const selectStateInRangeSQL = "" +
+	"SELECT id, event_json, exclude_from_sync, add_state_ids, remove_state_ids" +
+	" FROM syncapi_output_room_events" +
+	" WHERE (id > $1 AND id <= $2)" + // old/new pos
+	" AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" +
+	/*	" AND ( $3 IS NULL OR     sender  IN ($3)  )" + // sender
+		" AND ( $4 IS NULL OR NOT(sender  IN ($4)) )" + // not sender
+		" AND ( $5 IS NULL OR     type    IN ($5)  )" + // type
+		" AND ( $6 IS NULL OR NOT(type    IN ($6)) )" + // not type
+		" AND ( $7 IS NULL OR     contains_url = $7)" + // contains URL? */
+	" ORDER BY id ASC" +
+	" LIMIT $8" // limit
+
+type outputRoomEventsStatements struct {
+	streamIDStatements            *streamIDStatements
+	insertEventStmt               *sql.Stmt
+	selectLastInsertedEventStmt   *sql.Stmt
+	selectEventsStmt              *sql.Stmt
+	selectMaxEventIDStmt          *sql.Stmt
+	selectRecentEventsStmt        *sql.Stmt
+	selectRecentEventsForSyncStmt *sql.Stmt
+	selectEarlyEventsStmt         *sql.Stmt
+	selectStateInRangeStmt        *sql.Stmt
+}
+
+func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) {
+	s.streamIDStatements = streamID
+	_, err = db.Exec(outputRoomEventsSchema)
+	if err != nil {
+		return
+	}
+	if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil {
+		return
+	}
+	if s.selectLastInsertedEventStmt, err = db.Prepare(selectLastInsertedEventSQL); err != nil {
+		return
+	}
+	if s.selectEventsStmt, err = db.Prepare(selectEventsSQL); err != nil {
+		return
+	}
+	if s.selectMaxEventIDStmt, err = db.Prepare(selectMaxEventIDSQL); err != nil {
+		return
+	}
+	if s.selectRecentEventsStmt, err = db.Prepare(selectRecentEventsSQL); err != nil {
+		return
+	}
+	if s.selectRecentEventsForSyncStmt, err = db.Prepare(selectRecentEventsForSyncSQL); err != nil {
+		return
+	}
+	if s.selectEarlyEventsStmt, err = db.Prepare(selectEarlyEventsSQL); err != nil {
+		return
+	}
+	if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil {
+		return
+	}
+	return
+}
+
+// selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos.
+// Results are bucketed based on the room ID. If the same state is overwritten multiple times between the
+// two positions, only the most recent state is returned.
+func (s *outputRoomEventsStatements) selectStateInRange(
+	ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition,
+	stateFilterPart *gomatrixserverlib.StateFilter,
+) (map[string]map[string]bool, map[string]types.StreamEvent, error) {
+	stmt := common.TxStmt(txn, s.selectStateInRangeStmt)
+
+	rows, err := stmt.QueryContext(
+		ctx, oldPos, newPos,
+		/*pq.StringArray(stateFilterPart.Senders),
+		pq.StringArray(stateFilterPart.NotSenders),
+		pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)),
+		pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)),
+		stateFilterPart.ContainsURL,*/
+		stateFilterPart.Limit,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+	// Fetch all the state change events for all rooms between the two positions then loop each event and:
+	//  - Keep a cache of the event by ID (99% of state change events are for the event itself)
+	//  - For each room ID, build up an array of event IDs which represents cumulative adds/removes
+	// For each room, map cumulative event IDs to events and return. This may need to a batch SELECT based on event ID
+	// if they aren't in the event ID cache. We don't handle state deletion yet.
+	eventIDToEvent := make(map[string]types.StreamEvent)
+
+	// RoomID => A set (map[string]bool) of state event IDs which are between the two positions
+	stateNeeded := make(map[string]map[string]bool)
+
+	for rows.Next() {
+		var (
+			streamPos       types.StreamPosition
+			eventBytes      []byte
+			excludeFromSync bool
+			addIDs          pq.StringArray
+			delIDs          pq.StringArray
+		)
+		if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil {
+			return nil, nil, err
+		}
+		// Sanity check for deleted state and whine if we see it. We don't need to do anything
+		// since it'll just mark the event as not being needed.
+		if len(addIDs) < len(delIDs) {
+			log.WithFields(log.Fields{
+				"since":   oldPos,
+				"current": newPos,
+				"adds":    addIDs,
+				"dels":    delIDs,
+			}).Warn("StateBetween: ignoring deleted state")
+		}
+
+		// TODO: Handle redacted events
+		ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false)
+		if err != nil {
+			return nil, nil, err
+		}
+		needSet := stateNeeded[ev.RoomID()]
+		if needSet == nil { // make set if required
+			needSet = make(map[string]bool)
+		}
+		for _, id := range delIDs {
+			needSet[id] = false
+		}
+		for _, id := range addIDs {
+			needSet[id] = true
+		}
+		stateNeeded[ev.RoomID()] = needSet
+
+		eventIDToEvent[ev.EventID()] = types.StreamEvent{
+			Event:           ev,
+			StreamPosition:  streamPos,
+			ExcludeFromSync: excludeFromSync,
+		}
+	}
+
+	return stateNeeded, eventIDToEvent, nil
+}
+
+// MaxID returns the ID of the last inserted event in this table. 'txn' is optional. If it is not supplied,
+// then this function should only ever be used at startup, as it will race with inserting events if it is
+// done afterwards. If there are no inserted events, 0 is returned.
+func (s *outputRoomEventsStatements) selectMaxEventID(
+	ctx context.Context, txn *sql.Tx,
+) (id int64, err error) {
+	var nullableID sql.NullInt64
+	stmt := common.TxStmt(txn, s.selectMaxEventIDStmt)
+	err = stmt.QueryRowContext(ctx).Scan(&nullableID)
+	if nullableID.Valid {
+		id = nullableID.Int64
+	}
+	return
+}
+
+// InsertEvent into the output_room_events table. addState and removeState are an optional list of state event IDs. Returns the position
+// of the inserted event.
+func (s *outputRoomEventsStatements) insertEvent(
+	ctx context.Context, txn *sql.Tx,
+	event *gomatrixserverlib.Event, addState, removeState []string,
+	transactionID *api.TransactionID, excludeFromSync bool,
+) (streamPos types.StreamPosition, err error) {
+	var txnID *string
+	var sessionID *int64
+	if transactionID != nil {
+		sessionID = &transactionID.SessionID
+		txnID = &transactionID.TransactionID
+	}
+
+	// Parse content as JSON and search for an "url" key
+	containsURL := false
+	var content map[string]interface{}
+	if json.Unmarshal(event.Content(), &content) != nil {
+		// Set containsURL to true if url is present
+		_, containsURL = content["url"]
+	}
+
+	streamPos, err = s.streamIDStatements.nextStreamID(ctx, txn)
+	if err != nil {
+		return
+	}
+
+	insertStmt := common.TxStmt(txn, s.insertEventStmt)
+	selectStmt := common.TxStmt(txn, s.selectLastInsertedEventStmt)
+	_, err = insertStmt.ExecContext(
+		ctx,
+		streamPos,
+		event.RoomID(),
+		event.EventID(),
+		event.JSON(),
+		event.Type(),
+		event.Sender(),
+		containsURL,
+		pq.StringArray(addState),
+		pq.StringArray(removeState),
+		sessionID,
+		txnID,
+		excludeFromSync,
+	)
+	if err != nil {
+		return
+	}
+	err = selectStmt.QueryRowContext(ctx).Scan(&streamPos)
+	return
+}
+
+// selectRecentEvents returns the most recent events in the given room, up to a maximum of 'limit'.
+// If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude
+// from sync.
+func (s *outputRoomEventsStatements) selectRecentEvents(
+	ctx context.Context, txn *sql.Tx,
+	roomID string, fromPos, toPos types.StreamPosition, limit int,
+	chronologicalOrder bool, onlySyncEvents bool,
+) ([]types.StreamEvent, error) {
+	var stmt *sql.Stmt
+	if onlySyncEvents {
+		stmt = common.TxStmt(txn, s.selectRecentEventsForSyncStmt)
+	} else {
+		stmt = common.TxStmt(txn, s.selectRecentEventsStmt)
+	}
+
+	rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close() // nolint: errcheck
+	events, err := rowsToStreamEvents(rows)
+	if err != nil {
+		return nil, err
+	}
+	if chronologicalOrder {
+		// The events need to be returned from oldest to latest, which isn't
+		// necessary the way the SQL query returns them, so a sort is necessary to
+		// ensure the events are in the right order in the slice.
+		sort.SliceStable(events, func(i int, j int) bool {
+			return events[i].StreamPosition < events[j].StreamPosition
+		})
+	}
+	return events, nil
+}
+
+// selectEarlyEvents returns the earliest events in the given room, starting
+// from a given position, up to a maximum of 'limit'.
+func (s *outputRoomEventsStatements) selectEarlyEvents(
+	ctx context.Context, txn *sql.Tx,
+	roomID string, fromPos, toPos types.StreamPosition, limit int,
+) ([]types.StreamEvent, error) {
+	stmt := common.TxStmt(txn, s.selectEarlyEventsStmt)
+	rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close() // nolint: errcheck
+	events, err := rowsToStreamEvents(rows)
+	if err != nil {
+		return nil, err
+	}
+	// The events need to be returned from oldest to latest, which isn't
+	// necessarily the way the SQL query returns them, so a sort is necessary to
+	// ensure the events are in the right order in the slice.
+	sort.SliceStable(events, func(i int, j int) bool {
+		return events[i].StreamPosition < events[j].StreamPosition
+	})
+	return events, nil
+}
+
+// selectEvents returns the events for the given event IDs. If an event is
+// missing from the database, it will be omitted.
+func (s *outputRoomEventsStatements) selectEvents(
+	ctx context.Context, txn *sql.Tx, eventIDs []string,
+) ([]types.StreamEvent, error) {
+	var returnEvents []types.StreamEvent
+	stmt := common.TxStmt(txn, s.selectEventsStmt)
+	for _, eventID := range eventIDs {
+		rows, err := stmt.QueryContext(ctx, eventID)
+		if err != nil {
+			return nil, err
+		}
+		if streamEvents, err := rowsToStreamEvents(rows); err == nil {
+			returnEvents = append(returnEvents, streamEvents...)
+		}
+		rows.Close() // nolint: errcheck
+	}
+	return returnEvents, nil
+}
+
+func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) {
+	var result []types.StreamEvent
+	for rows.Next() {
+		var (
+			streamPos       types.StreamPosition
+			eventBytes      []byte
+			excludeFromSync bool
+			sessionID       *int64
+			txnID           *string
+			transactionID   *api.TransactionID
+		)
+		if err := rows.Scan(&streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID); err != nil {
+			return nil, err
+		}
+		// TODO: Handle redacted events
+		ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false)
+		if err != nil {
+			return nil, err
+		}
+
+		if sessionID != nil && txnID != nil {
+			transactionID = &api.TransactionID{
+				SessionID:     *sessionID,
+				TransactionID: *txnID,
+			}
+		}
+
+		result = append(result, types.StreamEvent{
+			Event:           ev,
+			StreamPosition:  streamPos,
+			TransactionID:   transactionID,
+			ExcludeFromSync: excludeFromSync,
+		})
+	}
+	return result, nil
+}
--- a/syncapi/storage/sqlite3/output_room_events_topology_table.go
+++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go
@ -0,0 +1,192 @@
+// Copyright 2018 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const outputRoomEventsTopologySchema = `
+-- Stores output room events received from the roomserver.
+CREATE TABLE IF NOT EXISTS syncapi_output_room_events_topology (
+  event_id TEXT PRIMARY KEY,
+	topological_position BIGINT NOT NULL,
+  room_id TEXT NOT NULL,
+
+	UNIQUE(topological_position, room_id)
+);
+-- The topological order will be used in events selection and ordering
+-- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, room_id);
+`
+
+const insertEventInTopologySQL = "" +
+	"INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id)" +
+	" VALUES ($1, $2, $3)" +
+	" ON CONFLICT (topological_position, room_id) DO UPDATE SET event_id = $1"
+
+const selectEventIDsInRangeASCSQL = "" +
+	"SELECT event_id FROM syncapi_output_room_events_topology" +
+	" WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" +
+	" ORDER BY topological_position ASC LIMIT $4"
+
+const selectEventIDsInRangeDESCSQL = "" +
+	"SELECT event_id FROM syncapi_output_room_events_topology" +
+	" WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" +
+	" ORDER BY topological_position DESC LIMIT $4"
+
+const selectPositionInTopologySQL = "" +
+	"SELECT topological_position FROM syncapi_output_room_events_topology" +
+	" WHERE event_id = $1"
+
+const selectMaxPositionInTopologySQL = "" +
+	"SELECT MAX(topological_position) FROM syncapi_output_room_events_topology" +
+	" WHERE room_id = $1"
+
+const selectEventIDsFromPositionSQL = "" +
+	"SELECT event_id FROM syncapi_output_room_events_topology" +
+	" WHERE room_id = $1 AND topological_position = $2"
+
+type outputRoomEventsTopologyStatements struct {
+	insertEventInTopologyStmt       *sql.Stmt
+	selectEventIDsInRangeASCStmt    *sql.Stmt
+	selectEventIDsInRangeDESCStmt   *sql.Stmt
+	selectPositionInTopologyStmt    *sql.Stmt
+	selectMaxPositionInTopologyStmt *sql.Stmt
+	selectEventIDsFromPositionStmt  *sql.Stmt
+}
+
+func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(outputRoomEventsTopologySchema)
+	if err != nil {
+		return
+	}
+	if s.insertEventInTopologyStmt, err = db.Prepare(insertEventInTopologySQL); err != nil {
+		return
+	}
+	if s.selectEventIDsInRangeASCStmt, err = db.Prepare(selectEventIDsInRangeASCSQL); err != nil {
+		return
+	}
+	if s.selectEventIDsInRangeDESCStmt, err = db.Prepare(selectEventIDsInRangeDESCSQL); err != nil {
+		return
+	}
+	if s.selectPositionInTopologyStmt, err = db.Prepare(selectPositionInTopologySQL); err != nil {
+		return
+	}
+	if s.selectMaxPositionInTopologyStmt, err = db.Prepare(selectMaxPositionInTopologySQL); err != nil {
+		return
+	}
+	if s.selectEventIDsFromPositionStmt, err = db.Prepare(selectEventIDsFromPositionSQL); err != nil {
+		return
+	}
+	return
+}
+
+// insertEventInTopology inserts the given event in the room's topology, based
+// on the event's depth.
+func (s *outputRoomEventsTopologyStatements) insertEventInTopology(
+	ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.Event,
+) (err error) {
+	stmt := common.TxStmt(txn, s.insertEventInTopologyStmt)
+	_, err = stmt.ExecContext(
+		ctx, event.EventID(), event.Depth(), event.RoomID(),
+	)
+	return
+}
+
+// selectEventIDsInRange selects the IDs of events which positions are within a
+// given range in a given room's topological order.
+// Returns an empty slice if no events match the given range.
+func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange(
+	ctx context.Context, txn *sql.Tx, roomID string,
+	fromPos, toPos types.StreamPosition,
+	limit int, chronologicalOrder bool,
+) (eventIDs []string, err error) {
+	// Decide on the selection's order according to whether chronological order
+	// is requested or not.
+	var stmt *sql.Stmt
+	if chronologicalOrder {
+		stmt = common.TxStmt(txn, s.selectEventIDsInRangeASCStmt)
+	} else {
+		stmt = common.TxStmt(txn, s.selectEventIDsInRangeDESCStmt)
+	}
+
+	// Query the event IDs.
+	rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit)
+	if err == sql.ErrNoRows {
+		// If no event matched the request, return an empty slice.
+		return []string{}, nil
+	} else if err != nil {
+		return
+	}
+
+	// Return the IDs.
+	var eventID string
+	for rows.Next() {
+		if err = rows.Scan(&eventID); err != nil {
+			return
+		}
+		eventIDs = append(eventIDs, eventID)
+	}
+
+	return
+}
+
+// selectPositionInTopology returns the position of a given event in the
+// topology of the room it belongs to.
+func (s *outputRoomEventsTopologyStatements) selectPositionInTopology(
+	ctx context.Context, txn *sql.Tx, eventID string,
+) (pos types.StreamPosition, err error) {
+	stmt := common.TxStmt(txn, s.selectPositionInTopologyStmt)
+	err = stmt.QueryRowContext(ctx, eventID).Scan(&pos)
+	return
+}
+
+func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology(
+	ctx context.Context, txn *sql.Tx, roomID string,
+) (pos types.StreamPosition, err error) {
+	stmt := common.TxStmt(txn, s.selectMaxPositionInTopologyStmt)
+	err = stmt.QueryRowContext(ctx, roomID).Scan(&pos)
+	return
+}
+
+// selectEventIDsFromPosition returns the IDs of all events that have a given
+// position in the topology of a given room.
+func (s *outputRoomEventsTopologyStatements) selectEventIDsFromPosition(
+	ctx context.Context, txn *sql.Tx, roomID string, pos types.StreamPosition,
+) (eventIDs []string, err error) {
+	// Query the event IDs.
+	stmt := common.TxStmt(txn, s.selectEventIDsFromPositionStmt)
+	rows, err := stmt.QueryContext(ctx, roomID, pos)
+	if err == sql.ErrNoRows {
+		// If no event matched the request, return an empty slice.
+		return []string{}, nil
+	} else if err != nil {
+		return
+	}
+	// Return the IDs.
+	var eventID string
+	for rows.Next() {
+		if err = rows.Scan(&eventID); err != nil {
+			return
+		}
+		eventIDs = append(eventIDs, eventID)
+	}
+	return
+}
--- a/syncapi/storage/sqlite3/stream_id_table.go
+++ b/syncapi/storage/sqlite3/stream_id_table.go
@ -0,0 +1,58 @@
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/syncapi/types"
+)
+
+const streamIDTableSchema = `
+-- Global stream ID counter, used by other tables.
+CREATE TABLE IF NOT EXISTS syncapi_stream_id (
+  stream_name TEXT NOT NULL PRIMARY KEY,
+  stream_id INT DEFAULT 0,
+
+  UNIQUE(stream_name)
+);
+INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("global", 0)
+  ON CONFLICT DO NOTHING;
+`
+
+const increaseStreamIDStmt = "" +
+	"UPDATE syncapi_stream_id SET stream_id = stream_id + 1 WHERE stream_name = $1"
+
+const selectStreamIDStmt = "" +
+	"SELECT stream_id FROM syncapi_stream_id WHERE stream_name = $1"
+
+type streamIDStatements struct {
+	increaseStreamIDStmt *sql.Stmt
+	selectStreamIDStmt   *sql.Stmt
+}
+
+func (s *streamIDStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(streamIDTableSchema)
+	if err != nil {
+		return
+	}
+	if s.increaseStreamIDStmt, err = db.Prepare(increaseStreamIDStmt); err != nil {
+		return
+	}
+	if s.selectStreamIDStmt, err = db.Prepare(selectStreamIDStmt); err != nil {
+		return
+	}
+	return
+}
+
+func (s *streamIDStatements) nextStreamID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) {
+	increaseStmt := common.TxStmt(txn, s.increaseStreamIDStmt)
+	selectStmt := common.TxStmt(txn, s.selectStreamIDStmt)
+	if _, err = increaseStmt.ExecContext(ctx, "global"); err != nil {
+		return
+	}
+	if err = selectStmt.QueryRowContext(ctx, "global").Scan(&pos); err != nil {
+		return
+	}
+	return
+}
--- a/syncapi/storage/sqlite3/syncserver.go
+++ b/syncapi/storage/sqlite3/syncserver.go
--- a/syncapi/storage/storage.go
+++ b/syncapi/storage/storage.go
@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/storage/postgres"
+	"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/dendrite/typingserver/cache"
 	"github.com/matrix-org/gomatrixserverlib"
@ -63,6 +64,8 @@ func NewSyncServerDatasource(dataSourceName string) (Database, error) {
 	switch uri.Scheme {
 	case "postgres":
 		return postgres.NewSyncServerDatasource(dataSourceName)
+	case "file":
+		return sqlite3.NewSyncServerDatasource(dataSourceName)
 	default:
 		return postgres.NewSyncServerDatasource(dataSourceName)
 	}