Implement forgetting about rooms (#1572)

* Add basic storage methods

* Add internal api handler

* Add check for forgotten room

* Add /rooms/{roomID}/forget endpoint

* Add missing rsAPI method

* Remove unused parameters

* Add passing tests

Signed-off-by: Till Faelligen <tfaelligen@gmail.com>

* Add missing file

* Add postgres migration

* Add sqlite migration

* Use Forgetter to forget room

* Remove empty line

* Update HTTP status codes

It looks like the spec calls for these to be 400, rather than 403: https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-rooms-roomid-forget

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>

roomserver/storage/sqlite3/deltas/20201028212440_add_forgotten_column.go

@@ -0,0 +1,82 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deltas
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/pressly/goose"
+)
+
+func LoadFromGoose() {
+	goose.AddMigration(UpAddForgottenColumn, DownAddForgottenColumn)
+}
+
+func LoadAddForgottenColumn(m *sqlutil.Migrations) {
+	m.AddMigration(UpAddForgottenColumn, DownAddForgottenColumn)
+}
+
+func UpAddForgottenColumn(tx *sql.Tx) error {
+	_, err := tx.Exec(`	ALTER TABLE roomserver_membership RENAME TO roomserver_membership_tmp;
+CREATE TABLE IF NOT EXISTS roomserver_membership (
+		room_nid INTEGER NOT NULL,
+		target_nid INTEGER NOT NULL,
+		sender_nid INTEGER NOT NULL DEFAULT 0,
+		membership_nid INTEGER NOT NULL DEFAULT 1,
+		event_nid INTEGER NOT NULL DEFAULT 0,
+		target_local BOOLEAN NOT NULL DEFAULT false,
+		forgotten BOOLEAN NOT NULL DEFAULT false,
+		UNIQUE (room_nid, target_nid)
+	);
+INSERT
+    INTO roomserver_membership (
+      room_nid, target_nid, sender_nid, membership_nid, event_nid, target_local
+    ) SELECT
+        room_nid, target_nid, sender_nid, membership_nid, event_nid, target_local
+    FROM roomserver_membership_tmp
+;
+DROP TABLE roomserver_membership_tmp;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownAddForgottenColumn(tx *sql.Tx) error {
+	_, err := tx.Exec(`	ALTER TABLE roomserver_membership RENAME TO roomserver_membership_tmp;
+CREATE TABLE IF NOT EXISTS roomserver_membership (
+		room_nid INTEGER NOT NULL,
+		target_nid INTEGER NOT NULL,
+		sender_nid INTEGER NOT NULL DEFAULT 0,
+		membership_nid INTEGER NOT NULL DEFAULT 1,
+		event_nid INTEGER NOT NULL DEFAULT 0,
+		target_local BOOLEAN NOT NULL DEFAULT false,
+		UNIQUE (room_nid, target_nid)
+	);
+INSERT
+    INTO roomserver_membership (
+      room_nid, target_nid, sender_nid, membership_nid, event_nid, target_local
+    ) SELECT
+        room_nid, target_nid, sender_nid, membership_nid, event_nid, target_local
+    FROM roomserver_membership_tmp
+;
+DROP TABLE roomserver_membership_tmp;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}

roomserver/storage/sqlite3/membership_table.go

@@ -36,13 +36,15 @@ const membershipSchema = `
 		membership_nid INTEGER NOT NULL DEFAULT 1,
 		event_nid INTEGER NOT NULL DEFAULT 0,
 		target_local BOOLEAN NOT NULL DEFAULT false,
+		forgotten BOOLEAN NOT NULL DEFAULT false,
 		UNIQUE (room_nid, target_nid)
 	);
 `
 
 var selectJoinedUsersSetForRoomsSQL = "" +
 	"SELECT target_nid, COUNT(room_nid) FROM roomserver_membership WHERE room_nid IN ($1) AND" +
-	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " GROUP BY target_nid"
+	" membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" +
+	" GROUP BY target_nid"
 
 // Insert a row in to membership table so that it can be locked by the
 // SELECT FOR UPDATE
@@ -52,37 +54,41 @@ const insertMembershipSQL = "" +
 	" ON CONFLICT DO NOTHING"
 
 const selectMembershipFromRoomAndTargetSQL = "" +
-	"SELECT membership_nid, event_nid FROM roomserver_membership" +
+	"SELECT membership_nid, event_nid, forgotten FROM roomserver_membership" +
 	" WHERE room_nid = $1 AND target_nid = $2"
 
 const selectMembershipsFromRoomAndMembershipSQL = "" +
 	"SELECT event_nid FROM roomserver_membership" +
-	" WHERE room_nid = $1 AND membership_nid = $2"
+	" WHERE room_nid = $1 AND membership_nid = $2 and forgotten = false"
 
 const selectLocalMembershipsFromRoomAndMembershipSQL = "" +
 	"SELECT event_nid FROM roomserver_membership" +
 	" WHERE room_nid = $1 AND membership_nid = $2" +
-	" AND target_local = true"
+	" AND target_local = true and forgotten = false"
 
 const selectMembershipsFromRoomSQL = "" +
 	"SELECT event_nid FROM roomserver_membership" +
-	" WHERE room_nid = $1"
+	" WHERE room_nid = $1 and forgotten = false"
 
 const selectLocalMembershipsFromRoomSQL = "" +
 	"SELECT event_nid FROM roomserver_membership" +
 	" WHERE room_nid = $1" +
-	" AND target_local = true"
+	" AND target_local = true and forgotten = false"
 
 const selectMembershipForUpdateSQL = "" +
 	"SELECT membership_nid FROM roomserver_membership" +
 	" WHERE room_nid = $1 AND target_nid = $2"
 
 const updateMembershipSQL = "" +
-	"UPDATE roomserver_membership SET sender_nid = $1, membership_nid = $2, event_nid = $3" +
-	" WHERE room_nid = $4 AND target_nid = $5"
+	"UPDATE roomserver_membership SET sender_nid = $1, membership_nid = $2, event_nid = $3, forgotten = $4" +
+	" WHERE room_nid = $5 AND target_nid = $6"
+
+const updateMembershipForgetRoom = "" +
+	"UPDATE roomserver_membership SET forgotten = $1" +
+	" WHERE room_nid = $2 AND target_nid = $3"
 
 const selectRoomsWithMembershipSQL = "" +
-	"SELECT room_nid FROM roomserver_membership WHERE membership_nid = $1 AND target_nid = $2"
+	"SELECT room_nid FROM roomserver_membership WHERE membership_nid = $1 AND target_nid = $2 and forgotten = false"
 
 // selectKnownUsersSQL uses a sub-select statement here to find rooms that the user is
 // joined to. Since this information is used to populate the user directory, we will
@@ -106,16 +112,13 @@ type membershipStatements struct {
 	selectRoomsWithMembershipStmt                   *sql.Stmt
 	updateMembershipStmt                            *sql.Stmt
 	selectKnownUsersStmt                            *sql.Stmt
+	updateMembershipForgetRoomStmt                  *sql.Stmt
 }
 
 func NewSqliteMembershipTable(db *sql.DB) (tables.Membership, error) {
 	s := &membershipStatements{
 		db: db,
 	}
-	_, err := db.Exec(membershipSchema)
-	if err != nil {
-		return nil, err
-	}
 
 	return s, shared.StatementList{
 		{&s.insertMembershipStmt, insertMembershipSQL},
@@ -128,9 +131,15 @@ func NewSqliteMembershipTable(db *sql.DB) (tables.Membership, error) {
 		{&s.updateMembershipStmt, updateMembershipSQL},
 		{&s.selectRoomsWithMembershipStmt, selectRoomsWithMembershipSQL},
 		{&s.selectKnownUsersStmt, selectKnownUsersSQL},
+		{&s.updateMembershipForgetRoomStmt, updateMembershipForgetRoom},
 	}.Prepare(db)
 }
 
+func (s *membershipStatements) execSchema(db *sql.DB) error {
+	_, err := db.Exec(membershipSchema)
+	return err
+}
+
 func (s *membershipStatements) InsertMembership(
 	ctx context.Context, txn *sql.Tx,
 	roomNID types.RoomNID, targetUserNID types.EventStateKeyNID,
@@ -155,10 +164,10 @@ func (s *membershipStatements) SelectMembershipForUpdate(
 func (s *membershipStatements) SelectMembershipFromRoomAndTarget(
 	ctx context.Context,
 	roomNID types.RoomNID, targetUserNID types.EventStateKeyNID,
-) (eventNID types.EventNID, membership tables.MembershipState, err error) {
+) (eventNID types.EventNID, membership tables.MembershipState, forgotten bool, err error) {
 	err = s.selectMembershipFromRoomAndTargetStmt.QueryRowContext(
 		ctx, roomNID, targetUserNID,
-	).Scan(&membership, &eventNID)
+	).Scan(&membership, &eventNID, &forgotten)
 	return
 }
 
@@ -216,13 +225,12 @@ func (s *membershipStatements) SelectMembershipsFromRoomAndMembership(
 
 func (s *membershipStatements) UpdateMembership(
 	ctx context.Context, txn *sql.Tx,
-	roomNID types.RoomNID, targetUserNID types.EventStateKeyNID,
-	senderUserNID types.EventStateKeyNID, membership tables.MembershipState,
-	eventNID types.EventNID,
+	roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, senderUserNID types.EventStateKeyNID, membership tables.MembershipState,
+	eventNID types.EventNID, forgotten bool,
 ) error {
 	stmt := sqlutil.TxStmt(txn, s.updateMembershipStmt)
 	_, err := stmt.ExecContext(
-		ctx, senderUserNID, membership, eventNID, roomNID, targetUserNID,
+		ctx, senderUserNID, membership, eventNID, forgotten, roomNID, targetUserNID,
 	)
 	return err
 }
@@ -285,3 +293,14 @@ func (s *membershipStatements) SelectKnownUsers(ctx context.Context, userID type
 	}
 	return result, rows.Err()
 }
+
+func (s *membershipStatements) UpdateForgetMembership(
+	ctx context.Context,
+	txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID,
+	forget bool,
+) error {
+	_, err := sqlutil.TxStmt(txn, s.updateMembershipForgetRoomStmt).ExecContext(
+		ctx, forget, roomNID, targetUserNID,
+	)
+	return err
+}

roomserver/storage/sqlite3/storage.go

@@ -19,127 +19,138 @@ import (
 	"context"
 	"database/sql"
 
+	_ "github.com/mattn/go-sqlite3"
+
 	"github.com/matrix-org/dendrite/internal/caching"
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/storage/shared"
-	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+	"github.com/matrix-org/dendrite/roomserver/storage/sqlite3/deltas"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
-	_ "github.com/mattn/go-sqlite3"
 )
 
 // A Database is used to store room events and stream offsets.
 type Database struct {
 	shared.Database
-	events         tables.Events
-	eventJSON      tables.EventJSON
-	eventTypes     tables.EventTypes
-	eventStateKeys tables.EventStateKeys
-	rooms          tables.Rooms
-	transactions   tables.Transactions
-	prevEvents     tables.PreviousEvents
-	invites        tables.Invites
-	membership     tables.Membership
-	db             *sql.DB
-	writer         sqlutil.Writer
 }
 
 // Open a sqlite database.
-// nolint: gocyclo
 func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) (*Database, error) {
 	var d Database
+	var db *sql.DB
 	var err error
-	if d.db, err = sqlutil.Open(dbProperties); err != nil {
+	if db, err = sqlutil.Open(dbProperties); err != nil {
 		return nil, err
 	}
-	d.writer = sqlutil.NewExclusiveWriter()
-	//d.db.Exec("PRAGMA journal_mode=WAL;")
-	//d.db.Exec("PRAGMA read_uncommitted = true;")
+
+	//db.Exec("PRAGMA journal_mode=WAL;")
+	//db.Exec("PRAGMA read_uncommitted = true;")
 
 	// FIXME: We are leaking connections somewhere. Setting this to 2 will eventually
 	// cause the roomserver to be unresponsive to new events because something will
 	// acquire the global mutex and never unlock it because it is waiting for a connection
 	// which it will never obtain.
-	d.db.SetMaxOpenConns(20)
+	db.SetMaxOpenConns(20)
 
-	d.eventStateKeys, err = NewSqliteEventStateKeysTable(d.db)
-	if err != nil {
+	// Create tables before executing migrations so we don't fail if the table is missing,
+	// and THEN prepare statements so we don't fail due to referencing new columns
+	ms := membershipStatements{}
+	if err := ms.execSchema(db); err != nil {
 		return nil, err
 	}
-	d.eventTypes, err = NewSqliteEventTypesTable(d.db)
-	if err != nil {
+	m := sqlutil.NewMigrations()
+	deltas.LoadAddForgottenColumn(m)
+	if err := m.RunDeltas(db, dbProperties); err != nil {
 		return nil, err
 	}
-	d.eventJSON, err = NewSqliteEventJSONTable(d.db)
-	if err != nil {
+	if err := d.prepare(db, cache); err != nil {
 		return nil, err
 	}
-	d.events, err = NewSqliteEventsTable(d.db)
+
+	return &d, nil
+}
+
+// nolint: gocyclo
+func (d *Database) prepare(db *sql.DB, cache caching.RoomServerCaches) error {
+	var err error
+	eventStateKeys, err := NewSqliteEventStateKeysTable(db)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	d.rooms, err = NewSqliteRoomsTable(d.db)
+	eventTypes, err := NewSqliteEventTypesTable(db)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	d.transactions, err = NewSqliteTransactionsTable(d.db)
+	eventJSON, err := NewSqliteEventJSONTable(db)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	stateBlock, err := NewSqliteStateBlockTable(d.db)
+	events, err := NewSqliteEventsTable(db)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	stateSnapshot, err := NewSqliteStateSnapshotTable(d.db)
+	rooms, err := NewSqliteRoomsTable(db)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	d.prevEvents, err = NewSqlitePrevEventsTable(d.db)
+	transactions, err := NewSqliteTransactionsTable(db)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	roomAliases, err := NewSqliteRoomAliasesTable(d.db)
+	stateBlock, err := NewSqliteStateBlockTable(db)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	d.invites, err = NewSqliteInvitesTable(d.db)
+	stateSnapshot, err := NewSqliteStateSnapshotTable(db)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	d.membership, err = NewSqliteMembershipTable(d.db)
+	prevEvents, err := NewSqlitePrevEventsTable(db)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	published, err := NewSqlitePublishedTable(d.db)
+	roomAliases, err := NewSqliteRoomAliasesTable(db)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	redactions, err := NewSqliteRedactionsTable(d.db)
+	invites, err := NewSqliteInvitesTable(db)
 	if err != nil {
-		return nil, err
+		return err
+	}
+	membership, err := NewSqliteMembershipTable(db)
+	if err != nil {
+		return err
+	}
+	published, err := NewSqlitePublishedTable(db)
+	if err != nil {
+		return err
+	}
+	redactions, err := NewSqliteRedactionsTable(db)
+	if err != nil {
+		return err
 	}
 	d.Database = shared.Database{
-		DB:                         d.db,
+		DB:                         db,
 		Cache:                      cache,
-		Writer:                     d.writer,
-		EventsTable:                d.events,
-		EventTypesTable:            d.eventTypes,
-		EventStateKeysTable:        d.eventStateKeys,
-		EventJSONTable:             d.eventJSON,
-		RoomsTable:                 d.rooms,
-		TransactionsTable:          d.transactions,
+		Writer:                     sqlutil.NewExclusiveWriter(),
+		EventsTable:                events,
+		EventTypesTable:            eventTypes,
+		EventStateKeysTable:        eventStateKeys,
+		EventJSONTable:             eventJSON,
+		RoomsTable:                 rooms,
+		TransactionsTable:          transactions,
 		StateBlockTable:            stateBlock,
 		StateSnapshotTable:         stateSnapshot,
-		PrevEventsTable:            d.prevEvents,
+		PrevEventsTable:            prevEvents,
 		RoomAliasesTable:           roomAliases,
-		InvitesTable:               d.invites,
-		MembershipTable:            d.membership,
+		InvitesTable:               invites,
+		MembershipTable:            membership,
 		PublishedTable:             published,
 		RedactionsTable:            redactions,
 		GetLatestEventsForUpdateFn: d.GetLatestEventsForUpdate,
 	}
-	return &d, nil
+	return nil
 }
 
 func (d *Database) SupportsConcurrentRoomInputs() bool {