Support sqlite in addition to postgres (#869)

* Move current work into single branch * Initial massaging of clientapi etc (not working yet) * Interfaces for accounts/devices databases * Duplicate postgres package for sqlite3 (no changes made to it yet) * Some keydb, accountdb, devicedb, common partition fixes, some more syncapi tweaking * Fix accounts DB, device DB * Update naffka dependency for SQLite * Naffka SQLite * Update naffka to latest master * SQLite support for federationsender * Mostly not-bad support for SQLite in syncapi (although there are problems where lots of events get classed incorrectly as backward extremities, probably because of IN/ANY clauses that are badly supported) * Update Dockerfile -> Go 1.13.7, add build-base (as gcc and friends are needed for SQLite) * Implement GET endpoints for account_data in clientapi * Nuke filtering for now... * Revert "Implement GET endpoints for account_data in clientapi" This reverts commit 4d80dff4583d278620d9b3ed437e9fcd8d4674ee. * Implement GET endpoints for account_data in clientapi (#861) * Implement GET endpoints for account_data in clientapi * Fix accountDB parameter * Remove fmt.Println * Fix insertAccountData SQLite query * Fix accountDB storage interfaces * Add empty push rules into account data on account creation (#862) * Put SaveAccountData into the right function this time * Not sure if roomserver is better or worse now * sqlite work * Allow empty last sent ID for the first event * sqlite: room creation works * Support sending messages * Nuke fmt.println * Move QueryVariadic etc into common, other device fixes * Fix some linter issues * Fix bugs * Fix some linting errors * Fix errcheck lint errors * Make naffka use postgres as fallback, fix couple of compile errors * What on earth happened to the /rooms/{roomID}/send/{eventType} routing Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
2025-08-01 05:42:46 +00:00 · 2020-02-13 17:27:33 +00:00 · 2020-02-13 17:27:33 +00:00 · b6ea1bc67a
commit b6ea1bc67a
parent 6942ee1de0
103 changed files with 9467 additions and 710 deletions
--- a/publicroomsapi/storage/sqlite3/prepare.go
+++ b/publicroomsapi/storage/sqlite3/prepare.go
@ -0,0 +1,36 @@
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"database/sql"
+)
+
+// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement.
+type statementList []struct {
+	statement **sql.Stmt
+	sql       string
+}
+
+// prepare the SQL for each statement in the list and assign the result to the prepared statement.
+func (s statementList) prepare(db *sql.DB) (err error) {
+	for _, statement := range s {
+		if *statement.statement, err = db.Prepare(statement.sql); err != nil {
+			return
+		}
+	}
+	return
+}
--- a/publicroomsapi/storage/sqlite3/public_rooms_table.go
+++ b/publicroomsapi/storage/sqlite3/public_rooms_table.go
@ -0,0 +1,277 @@
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/publicroomsapi/types"
+)
+
+var editableAttributes = []string{
+	"aliases",
+	"canonical_alias",
+	"name",
+	"topic",
+	"world_readable",
+	"guest_can_join",
+	"avatar_url",
+	"visibility",
+}
+
+const publicRoomsSchema = `
+-- Stores all of the rooms with data needed to create the server's room directory
+CREATE TABLE IF NOT EXISTS publicroomsapi_public_rooms(
+	-- The room's ID
+	room_id TEXT NOT NULL PRIMARY KEY,
+	-- Number of joined members in the room
+	joined_members INTEGER NOT NULL DEFAULT 0,
+	-- Aliases of the room (empty array if none)
+	aliases TEXT[] NOT NULL DEFAULT '{}'::TEXT[],
+	-- Canonical alias of the room (empty string if none)
+	canonical_alias TEXT NOT NULL DEFAULT '',
+	-- Name of the room (empty string if none)
+	name TEXT NOT NULL DEFAULT '',
+	-- Topic of the room (empty string if none)
+	topic TEXT NOT NULL DEFAULT '',
+	-- Is the room world readable?
+	world_readable BOOLEAN NOT NULL DEFAULT false,
+	-- Can guest join the room?
+	guest_can_join BOOLEAN NOT NULL DEFAULT false,
+	-- URL of the room avatar (empty string if none)
+	avatar_url TEXT NOT NULL DEFAULT '',
+	-- Visibility of the room: true means the room is publicly visible, false
+	-- means the room is private
+	visibility BOOLEAN NOT NULL DEFAULT false
+);
+`
+
+const countPublicRoomsSQL = "" +
+	"SELECT COUNT(*) FROM publicroomsapi_public_rooms" +
+	" WHERE visibility = true"
+
+const selectPublicRoomsSQL = "" +
+	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
+	" FROM publicroomsapi_public_rooms WHERE visibility = true" +
+	" ORDER BY joined_members DESC" +
+	" OFFSET $1"
+
+const selectPublicRoomsWithLimitSQL = "" +
+	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
+	" FROM publicroomsapi_public_rooms WHERE visibility = true" +
+	" ORDER BY joined_members DESC" +
+	" OFFSET $1 LIMIT $2"
+
+const selectPublicRoomsWithFilterSQL = "" +
+	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
+	" FROM publicroomsapi_public_rooms" +
+	" WHERE visibility = true" +
+	" AND (LOWER(name) LIKE LOWER($1)" +
+	" OR LOWER(topic) LIKE LOWER($1)" +
+	" OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" +
+	" ORDER BY joined_members DESC" +
+	" OFFSET $2"
+
+const selectPublicRoomsWithLimitAndFilterSQL = "" +
+	"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
+	" FROM publicroomsapi_public_rooms" +
+	" WHERE visibility = true" +
+	" AND (LOWER(name) LIKE LOWER($1)" +
+	" OR LOWER(topic) LIKE LOWER($1)" +
+	" OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" +
+	" ORDER BY joined_members DESC" +
+	" OFFSET $2 LIMIT $3"
+
+const selectRoomVisibilitySQL = "" +
+	"SELECT visibility FROM publicroomsapi_public_rooms" +
+	" WHERE room_id = $1"
+
+const insertNewRoomSQL = "" +
+	"INSERT INTO publicroomsapi_public_rooms(room_id)" +
+	" VALUES ($1)"
+
+const incrementJoinedMembersInRoomSQL = "" +
+	"UPDATE publicroomsapi_public_rooms" +
+	" SET joined_members = joined_members + 1" +
+	" WHERE room_id = $1"
+
+const decrementJoinedMembersInRoomSQL = "" +
+	"UPDATE publicroomsapi_public_rooms" +
+	" SET joined_members = joined_members - 1" +
+	" WHERE room_id = $1"
+
+const updateRoomAttributeSQL = "" +
+	"UPDATE publicroomsapi_public_rooms" +
+	" SET %s = $1" +
+	" WHERE room_id = $2"
+
+type publicRoomsStatements struct {
+	countPublicRoomsStmt                    *sql.Stmt
+	selectPublicRoomsStmt                   *sql.Stmt
+	selectPublicRoomsWithLimitStmt          *sql.Stmt
+	selectPublicRoomsWithFilterStmt         *sql.Stmt
+	selectPublicRoomsWithLimitAndFilterStmt *sql.Stmt
+	selectRoomVisibilityStmt                *sql.Stmt
+	insertNewRoomStmt                       *sql.Stmt
+	incrementJoinedMembersInRoomStmt        *sql.Stmt
+	decrementJoinedMembersInRoomStmt        *sql.Stmt
+	updateRoomAttributeStmts                map[string]*sql.Stmt
+}
+
+func (s *publicRoomsStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(publicRoomsSchema)
+	if err != nil {
+		return
+	}
+
+	stmts := statementList{
+		{&s.countPublicRoomsStmt, countPublicRoomsSQL},
+		{&s.selectPublicRoomsStmt, selectPublicRoomsSQL},
+		{&s.selectPublicRoomsWithLimitStmt, selectPublicRoomsWithLimitSQL},
+		{&s.selectPublicRoomsWithFilterStmt, selectPublicRoomsWithFilterSQL},
+		{&s.selectPublicRoomsWithLimitAndFilterStmt, selectPublicRoomsWithLimitAndFilterSQL},
+		{&s.selectRoomVisibilityStmt, selectRoomVisibilitySQL},
+		{&s.insertNewRoomStmt, insertNewRoomSQL},
+		{&s.incrementJoinedMembersInRoomStmt, incrementJoinedMembersInRoomSQL},
+		{&s.decrementJoinedMembersInRoomStmt, decrementJoinedMembersInRoomSQL},
+	}
+
+	if err = stmts.prepare(db); err != nil {
+		return
+	}
+
+	s.updateRoomAttributeStmts = make(map[string]*sql.Stmt)
+	for _, editable := range editableAttributes {
+		stmt := fmt.Sprintf(updateRoomAttributeSQL, editable)
+		if s.updateRoomAttributeStmts[editable], err = db.Prepare(stmt); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, err error) {
+	err = s.countPublicRoomsStmt.QueryRowContext(ctx).Scan(&nb)
+	return
+}
+
+func (s *publicRoomsStatements) selectPublicRooms(
+	ctx context.Context, offset int64, limit int16, filter string,
+) ([]types.PublicRoom, error) {
+	var rows *sql.Rows
+	var err error
+
+	if len(filter) > 0 {
+		pattern := "%" + filter + "%"
+		if limit == 0 {
+			rows, err = s.selectPublicRoomsWithFilterStmt.QueryContext(
+				ctx, pattern, offset,
+			)
+		} else {
+			rows, err = s.selectPublicRoomsWithLimitAndFilterStmt.QueryContext(
+				ctx, pattern, offset, limit,
+			)
+		}
+	} else {
+		if limit == 0 {
+			rows, err = s.selectPublicRoomsStmt.QueryContext(ctx, offset)
+		} else {
+			rows, err = s.selectPublicRoomsWithLimitStmt.QueryContext(
+				ctx, offset, limit,
+			)
+		}
+	}
+
+	if err != nil {
+		return []types.PublicRoom{}, nil
+	}
+
+	rooms := []types.PublicRoom{}
+	for rows.Next() {
+		var r types.PublicRoom
+		var aliases pq.StringArray
+
+		err = rows.Scan(
+			&r.RoomID, &r.NumJoinedMembers, &aliases, &r.CanonicalAlias,
+			&r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL,
+		)
+		if err != nil {
+			return rooms, err
+		}
+
+		r.Aliases = aliases
+
+		rooms = append(rooms, r)
+	}
+
+	return rooms, nil
+}
+
+func (s *publicRoomsStatements) selectRoomVisibility(
+	ctx context.Context, roomID string,
+) (v bool, err error) {
+	err = s.selectRoomVisibilityStmt.QueryRowContext(ctx, roomID).Scan(&v)
+	return
+}
+
+func (s *publicRoomsStatements) insertNewRoom(
+	ctx context.Context, roomID string,
+) error {
+	_, err := s.insertNewRoomStmt.ExecContext(ctx, roomID)
+	return err
+}
+
+func (s *publicRoomsStatements) incrementJoinedMembersInRoom(
+	ctx context.Context, roomID string,
+) error {
+	_, err := s.incrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID)
+	return err
+}
+
+func (s *publicRoomsStatements) decrementJoinedMembersInRoom(
+	ctx context.Context, roomID string,
+) error {
+	_, err := s.decrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID)
+	return err
+}
+
+func (s *publicRoomsStatements) updateRoomAttribute(
+	ctx context.Context, attrName string, attrValue attributeValue, roomID string,
+) error {
+	stmt, isEditable := s.updateRoomAttributeStmts[attrName]
+
+	if !isEditable {
+		return errors.New("Cannot edit " + attrName)
+	}
+
+	var value interface{}
+	switch v := attrValue.(type) {
+	case []string:
+		value = pq.StringArray(v)
+	case bool, string:
+		value = attrValue
+	default:
+		return errors.New("Unsupported attribute type, must be bool, string or []string")
+	}
+
+	_, err := stmt.ExecContext(ctx, value, roomID)
+	return err
+}
--- a/publicroomsapi/storage/sqlite3/storage.go
+++ b/publicroomsapi/storage/sqlite3/storage.go
@ -0,0 +1,256 @@
+// Copyright 2017-2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+
+	_ "github.com/mattn/go-sqlite3"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/publicroomsapi/types"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// PublicRoomsServerDatabase represents a public rooms server database.
+type PublicRoomsServerDatabase struct {
+	db *sql.DB
+	common.PartitionOffsetStatements
+	statements publicRoomsStatements
+}
+
+type attributeValue interface{}
+
+// NewPublicRoomsServerDatabase creates a new public rooms server database.
+func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) {
+	var db *sql.DB
+	var err error
+	if db, err = sql.Open("sqlite3", dataSourceName); err != nil {
+		return nil, err
+	}
+	storage := PublicRoomsServerDatabase{
+		db: db,
+	}
+	if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil {
+		return nil, err
+	}
+	if err = storage.statements.prepare(db); err != nil {
+		return nil, err
+	}
+	return &storage, nil
+}
+
+// GetRoomVisibility returns the room visibility as a boolean: true if the room
+// is publicly visible, false if not.
+// Returns an error if the retrieval failed.
+func (d *PublicRoomsServerDatabase) GetRoomVisibility(
+	ctx context.Context, roomID string,
+) (bool, error) {
+	return d.statements.selectRoomVisibility(ctx, roomID)
+}
+
+// SetRoomVisibility updates the visibility attribute of a room. This attribute
+// must be set to true if the room is publicly visible, false if not.
+// Returns an error if the update failed.
+func (d *PublicRoomsServerDatabase) SetRoomVisibility(
+	ctx context.Context, visible bool, roomID string,
+) error {
+	return d.statements.updateRoomAttribute(ctx, "visibility", visible, roomID)
+}
+
+// CountPublicRooms returns the number of room set as publicly visible on the server.
+// Returns an error if the retrieval failed.
+func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) {
+	return d.statements.countPublicRooms(ctx)
+}
+
+// GetPublicRooms returns an array containing the local rooms set as publicly visible, ordered by their number
+// of joined members. This array can be limited by a given number of elements, and offset by a given value.
+// If the limit is 0, doesn't limit the number of results. If the offset is 0 too, the array contains all
+// the rooms set as publicly visible on the server.
+// Returns an error if the retrieval failed.
+func (d *PublicRoomsServerDatabase) GetPublicRooms(
+	ctx context.Context, offset int64, limit int16, filter string,
+) ([]types.PublicRoom, error) {
+	return d.statements.selectPublicRooms(ctx, offset, limit, filter)
+}
+
+// UpdateRoomFromEvents iterate over a slice of state events and call
+// UpdateRoomFromEvent on each of them to update the database representation of
+// the rooms updated by each event.
+// The slice of events to remove is used to update the number of joined members
+// for the room in the database.
+// If the update triggered by one of the events failed, aborts the process and
+// returns an error.
+func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(
+	ctx context.Context,
+	eventsToAdd []gomatrixserverlib.Event,
+	eventsToRemove []gomatrixserverlib.Event,
+) error {
+	for _, event := range eventsToAdd {
+		if err := d.UpdateRoomFromEvent(ctx, event); err != nil {
+			return err
+		}
+	}
+
+	for _, event := range eventsToRemove {
+		if event.Type() == "m.room.member" {
+			if err := d.updateNumJoinedUsers(ctx, event, true); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// UpdateRoomFromEvent updates the database representation of a room from a Matrix event, by
+// checking the event's type to know which attribute to change and using the event's content
+// to define the new value of the attribute.
+// If the event doesn't match with any property used to compute the public room directory,
+// does nothing.
+// If something went wrong during the process, returns an error.
+func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(
+	ctx context.Context, event gomatrixserverlib.Event,
+) error {
+	// Process the event according to its type
+	switch event.Type() {
+	case "m.room.create":
+		return d.statements.insertNewRoom(ctx, event.RoomID())
+	case "m.room.member":
+		return d.updateNumJoinedUsers(ctx, event, false)
+	case "m.room.aliases":
+		return d.updateRoomAliases(ctx, event)
+	case "m.room.canonical_alias":
+		var content common.CanonicalAliasContent
+		field := &(content.Alias)
+		attrName := "canonical_alias"
+		return d.updateStringAttribute(ctx, attrName, event, &content, field)
+	case "m.room.name":
+		var content common.NameContent
+		field := &(content.Name)
+		attrName := "name"
+		return d.updateStringAttribute(ctx, attrName, event, &content, field)
+	case "m.room.topic":
+		var content common.TopicContent
+		field := &(content.Topic)
+		attrName := "topic"
+		return d.updateStringAttribute(ctx, attrName, event, &content, field)
+	case "m.room.avatar":
+		var content common.AvatarContent
+		field := &(content.URL)
+		attrName := "avatar_url"
+		return d.updateStringAttribute(ctx, attrName, event, &content, field)
+	case "m.room.history_visibility":
+		var content common.HistoryVisibilityContent
+		field := &(content.HistoryVisibility)
+		attrName := "world_readable"
+		strForTrue := "world_readable"
+		return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue)
+	case "m.room.guest_access":
+		var content common.GuestAccessContent
+		field := &(content.GuestAccess)
+		attrName := "guest_can_join"
+		strForTrue := "can_join"
+		return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue)
+	}
+
+	// If the event type didn't match, return with no error
+	return nil
+}
+
+// updateNumJoinedUsers updates the number of joined user in the database representation
+// of a room using a given "m.room.member" Matrix event.
+// If the membership property of the event isn't "join", ignores it and returs nil.
+// If the remove parameter is set to false, increments the joined members counter in the
+// database, if set to truem decrements it.
+// Returns an error if the update failed.
+func (d *PublicRoomsServerDatabase) updateNumJoinedUsers(
+	ctx context.Context, membershipEvent gomatrixserverlib.Event, remove bool,
+) error {
+	membership, err := membershipEvent.Membership()
+	if err != nil {
+		return err
+	}
+
+	if membership != gomatrixserverlib.Join {
+		return nil
+	}
+
+	if remove {
+		return d.statements.decrementJoinedMembersInRoom(ctx, membershipEvent.RoomID())
+	}
+	return d.statements.incrementJoinedMembersInRoom(ctx, membershipEvent.RoomID())
+}
+
+// updateStringAttribute updates a given string attribute in the database
+// representation of a room using a given string data field from content of the
+// Matrix event triggering the update.
+// Returns an error if decoding the Matrix event's content or updating the attribute
+// failed.
+func (d *PublicRoomsServerDatabase) updateStringAttribute(
+	ctx context.Context, attrName string, event gomatrixserverlib.Event,
+	content interface{}, field *string,
+) error {
+	if err := json.Unmarshal(event.Content(), content); err != nil {
+		return err
+	}
+
+	return d.statements.updateRoomAttribute(ctx, attrName, *field, event.RoomID())
+}
+
+// updateBooleanAttribute updates a given boolean attribute in the database
+// representation of a room using a given string data field from content of the
+// Matrix event triggering the update.
+// The attribute is set to true if the field matches a given string, false if not.
+// Returns an error if decoding the Matrix event's content or updating the attribute
+// failed.
+func (d *PublicRoomsServerDatabase) updateBooleanAttribute(
+	ctx context.Context, attrName string, event gomatrixserverlib.Event,
+	content interface{}, field *string, strForTrue string,
+) error {
+	if err := json.Unmarshal(event.Content(), content); err != nil {
+		return err
+	}
+
+	var attrValue bool
+	if *field == strForTrue {
+		attrValue = true
+	} else {
+		attrValue = false
+	}
+
+	return d.statements.updateRoomAttribute(ctx, attrName, attrValue, event.RoomID())
+}
+
+// updateRoomAliases decodes the content of a "m.room.aliases" Matrix event and update the list of aliases of
+// a given room with it.
+// Returns an error if decoding the Matrix event or updating the list failed.
+func (d *PublicRoomsServerDatabase) updateRoomAliases(
+	ctx context.Context, aliasesEvent gomatrixserverlib.Event,
+) error {
+	var content common.AliasesContent
+	if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil {
+		return err
+	}
+
+	return d.statements.updateRoomAttribute(
+		ctx, "aliases", content.Aliases, aliasesEvent.RoomID(),
+	)
+}