Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver (#1174)

* Use content_value instead of membership

* Fix build

* Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver

- All public rooms paths are now handled by clientapi
- Requests to (un)publish rooms are sent to the roomserver via `PerformPublish`
  which are stored in a new `published_table.go`
- Requests for public rooms are handled in clientapi by:
    * Fetch all room IDs which are published using `QueryPublishedRooms` on the roomserver.
    * Apply pagination parameters to the slice.
    * Do a `QueryBulkStateContent` request to the currentstateserver to pull out
      required state event *content* (not entire events).
    * Aggregate and return the chunk.

Mostly but not fully implemented (DB queries on currentstateserver are missing)

* Fix pq query

* Make postgres work

* Make sqlite work

* Fix tests

* Unbreak pagination tests

* Linting

roomserver/storage/interface.go

@@ -139,4 +139,8 @@ type Database interface {
 	EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error)
 	// Look up the room version for a given room.
 	GetRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error)
+	// Publish or unpublish a room from the room directory.
+	PublishRoom(ctx context.Context, roomID string, publish bool) error
+	// Returns a list of room IDs for rooms which are published.
+	GetPublishedRooms(ctx context.Context) ([]string, error)
 }

roomserver/storage/postgres/published_table.go

@@ -0,0 +1,101 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/roomserver/storage/shared"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+)
+
+const publishedSchema = `
+-- Stores which rooms are published in the room directory
+CREATE TABLE IF NOT EXISTS roomserver_published (
+    -- The room ID of the room
+    room_id TEXT NOT NULL PRIMARY KEY,
+    -- Whether it is published or not
+    published BOOLEAN NOT NULL DEFAULT false
+);
+`
+
+const upsertPublishedSQL = "" +
+	"INSERT INTO roomserver_published (room_id, published) VALUES ($1, $2) " +
+	"ON CONFLICT (room_id) DO UPDATE SET published=$2"
+
+const selectAllPublishedSQL = "" +
+	"SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC"
+
+const selectPublishedSQL = "" +
+	"SELECT published FROM roomserver_published WHERE room_id = $1"
+
+type publishedStatements struct {
+	upsertPublishedStmt    *sql.Stmt
+	selectAllPublishedStmt *sql.Stmt
+	selectPublishedStmt    *sql.Stmt
+}
+
+func NewPostgresPublishedTable(db *sql.DB) (tables.Published, error) {
+	s := &publishedStatements{}
+	_, err := db.Exec(publishedSchema)
+	if err != nil {
+		return nil, err
+	}
+	return s, shared.StatementList{
+		{&s.upsertPublishedStmt, upsertPublishedSQL},
+		{&s.selectAllPublishedStmt, selectAllPublishedSQL},
+		{&s.selectPublishedStmt, selectPublishedSQL},
+	}.Prepare(db)
+}
+
+func (s *publishedStatements) UpsertRoomPublished(
+	ctx context.Context, roomID string, published bool,
+) (err error) {
+	_, err = s.upsertPublishedStmt.ExecContext(ctx, roomID, published)
+	return
+}
+
+func (s *publishedStatements) SelectPublishedFromRoomID(
+	ctx context.Context, roomID string,
+) (published bool, err error) {
+	err = s.selectPublishedStmt.QueryRowContext(ctx, roomID).Scan(&published)
+	if err == sql.ErrNoRows {
+		return false, nil
+	}
+	return
+}
+
+func (s *publishedStatements) SelectAllPublishedRooms(
+	ctx context.Context, published bool,
+) ([]string, error) {
+	rows, err := s.selectAllPublishedStmt.QueryContext(ctx, published)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed")
+
+	var roomIDs []string
+	for rows.Next() {
+		var roomID string
+		if err = rows.Scan(&roomID); err != nil {
+			return nil, err
+		}
+
+		roomIDs = append(roomIDs, roomID)
+	}
+	return roomIDs, rows.Err()
+}

roomserver/storage/postgres/storage.go

@@ -87,6 +87,10 @@ func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database,
 	if err != nil {
 		return nil, err
 	}
+	published, err := NewPostgresPublishedTable(db)
+	if err != nil {
+		return nil, err
+	}
 	d.Database = shared.Database{
 		DB:                  db,
 		EventTypesTable:     eventTypes,
@@ -101,6 +105,7 @@ func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database,
 		RoomAliasesTable:    roomAliases,
 		InvitesTable:        invites,
 		MembershipTable:     membership,
+		PublishedTable:      published,
 	}
 	return &d, nil
 }

roomserver/storage/shared/storage.go

@@ -26,6 +26,7 @@ type Database struct {
 	PrevEventsTable     tables.PreviousEvents
 	InvitesTable        tables.Invites
 	MembershipTable     tables.Membership
+	PublishedTable      tables.Published
 }
 
 func (d *Database) EventTypeNIDs(
@@ -420,6 +421,14 @@ func (d *Database) StoreEvent(
 	}, nil
 }
 
+func (d *Database) PublishRoom(ctx context.Context, roomID string, publish bool) error {
+	return d.PublishedTable.UpsertRoomPublished(ctx, roomID, publish)
+}
+
+func (d *Database) GetPublishedRooms(ctx context.Context) ([]string, error) {
+	return d.PublishedTable.SelectAllPublishedRooms(ctx, true)
+}
+
 func (d *Database) assignRoomNID(
 	ctx context.Context, txn *sql.Tx,
 	roomID string, roomVersion gomatrixserverlib.RoomVersion,

roomserver/storage/sqlite3/published_table.go

@@ -0,0 +1,100 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/roomserver/storage/shared"
+	"github.com/matrix-org/dendrite/roomserver/storage/tables"
+)
+
+const publishedSchema = `
+-- Stores which rooms are published in the room directory
+CREATE TABLE IF NOT EXISTS roomserver_published (
+    -- The room ID of the room
+    room_id TEXT NOT NULL PRIMARY KEY,
+    -- Whether it is published or not
+    published BOOLEAN NOT NULL DEFAULT false
+);
+`
+
+const upsertPublishedSQL = "" +
+	"INSERT OR REPLACE INTO roomserver_published (room_id, published) VALUES ($1, $2)"
+
+const selectAllPublishedSQL = "" +
+	"SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC"
+
+const selectPublishedSQL = "" +
+	"SELECT published FROM roomserver_published WHERE room_id = $1"
+
+type publishedStatements struct {
+	upsertPublishedStmt    *sql.Stmt
+	selectAllPublishedStmt *sql.Stmt
+	selectPublishedStmt    *sql.Stmt
+}
+
+func NewSqlitePublishedTable(db *sql.DB) (tables.Published, error) {
+	s := &publishedStatements{}
+	_, err := db.Exec(publishedSchema)
+	if err != nil {
+		return nil, err
+	}
+	return s, shared.StatementList{
+		{&s.upsertPublishedStmt, upsertPublishedSQL},
+		{&s.selectAllPublishedStmt, selectAllPublishedSQL},
+		{&s.selectPublishedStmt, selectPublishedSQL},
+	}.Prepare(db)
+}
+
+func (s *publishedStatements) UpsertRoomPublished(
+	ctx context.Context, roomID string, published bool,
+) (err error) {
+	_, err = s.upsertPublishedStmt.ExecContext(ctx, roomID, published)
+	return
+}
+
+func (s *publishedStatements) SelectPublishedFromRoomID(
+	ctx context.Context, roomID string,
+) (published bool, err error) {
+	err = s.selectPublishedStmt.QueryRowContext(ctx, roomID).Scan(&published)
+	if err == sql.ErrNoRows {
+		return false, nil
+	}
+	return
+}
+
+func (s *publishedStatements) SelectAllPublishedRooms(
+	ctx context.Context, published bool,
+) ([]string, error) {
+	rows, err := s.selectAllPublishedStmt.QueryContext(ctx, published)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed")
+
+	var roomIDs []string
+	for rows.Next() {
+		var roomID string
+		if err = rows.Scan(&roomID); err != nil {
+			return nil, err
+		}
+
+		roomIDs = append(roomIDs, roomID)
+	}
+	return roomIDs, rows.Err()
+}

roomserver/storage/sqlite3/storage.go

@@ -110,6 +110,10 @@ func Open(dataSourceName string) (*Database, error) {
 	if err != nil {
 		return nil, err
 	}
+	published, err := NewSqlitePublishedTable(d.db)
+	if err != nil {
+		return nil, err
+	}
 	d.Database = shared.Database{
 		DB:                  d.db,
 		EventsTable:         d.events,
@@ -124,6 +128,7 @@ func Open(dataSourceName string) (*Database, error) {
 		RoomAliasesTable:    roomAliases,
 		InvitesTable:        d.invites,
 		MembershipTable:     d.membership,
+		PublishedTable:      published,
 	}
 	return &d, nil
 }

roomserver/storage/tables/interface.go

@@ -120,3 +120,9 @@ type Membership interface {
 	SelectMembershipsFromRoomAndMembership(ctx context.Context, roomNID types.RoomNID, membership MembershipState, localOnly bool) (eventNIDs []types.EventNID, err error)
 	UpdateMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, senderUserNID types.EventStateKeyNID, membership MembershipState, eventNID types.EventNID) error
 }
+
+type Published interface {
+	UpsertRoomPublished(ctx context.Context, roomID string, published bool) (err error)
+	SelectPublishedFromRoomID(ctx context.Context, roomID string) (published bool, err error)
+	SelectAllPublishedRooms(ctx context.Context, published bool) ([]string, error)
+}