User directory (#1225)

* User directory

* Fix syncapi unit test

* Make user directory only show remote users you know about from your joined rooms

* Update sytest-whitelist

* Review comments

currentstateserver/api/api.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -33,6 +34,8 @@ type CurrentStateInternalAPI interface {
 	QueryBulkStateContent(ctx context.Context, req *QueryBulkStateContentRequest, res *QueryBulkStateContentResponse) error
 	// QuerySharedUsers returns a list of users who share at least 1 room in common with the given user.
 	QuerySharedUsers(ctx context.Context, req *QuerySharedUsersRequest, res *QuerySharedUsersResponse) error
+	// QueryKnownUsers returns a list of users that we know about from our joined rooms.
+	QueryKnownUsers(ctx context.Context, req *QueryKnownUsersRequest, res *QueryKnownUsersResponse) error
 }
 
 type QuerySharedUsersRequest struct {
@@ -88,6 +91,16 @@ type QueryCurrentStateResponse struct {
 	StateEvents map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent
 }
 
+type QueryKnownUsersRequest struct {
+	UserID       string `json:"user_id"`
+	SearchString string `json:"search_string"`
+	Limit        int    `json:"limit"`
+}
+
+type QueryKnownUsersResponse struct {
+	Users []authtypes.FullyQualifiedProfile `json:"profiles"`
+}
+
 // MarshalJSON stringifies the StateKeyTuple keys so they can be sent over the wire in HTTP API mode.
 func (r *QueryCurrentStateResponse) MarshalJSON() ([]byte, error) {
 	se := make(map[string]*gomatrixserverlib.HeaderedEvent, len(r.StateEvents))

currentstateserver/internal/api.go

@@ -17,6 +17,7 @@ package internal
 import (
 	"context"
 
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/currentstateserver/api"
 	"github.com/matrix-org/dendrite/currentstateserver/storage"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -49,6 +50,19 @@ func (a *CurrentStateInternalAPI) QueryRoomsForUser(ctx context.Context, req *ap
 	return nil
 }
 
+func (a *CurrentStateInternalAPI) QueryKnownUsers(ctx context.Context, req *api.QueryKnownUsersRequest, res *api.QueryKnownUsersResponse) error {
+	users, err := a.DB.GetKnownUsers(ctx, req.UserID, req.SearchString, req.Limit)
+	if err != nil {
+		return err
+	}
+	for _, user := range users {
+		res.Users = append(res.Users, authtypes.FullyQualifiedProfile{
+			UserID: user,
+		})
+	}
+	return nil
+}
+
 func (a *CurrentStateInternalAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error {
 	events, err := a.DB.GetBulkStateContent(ctx, req.RoomIDs, req.StateTuples, req.AllowWildcards)
 	if err != nil {

currentstateserver/inthttp/client.go

@@ -30,6 +30,7 @@ const (
 	QueryRoomsForUserPath     = "/currentstateserver/queryRoomsForUser"
 	QueryBulkStateContentPath = "/currentstateserver/queryBulkStateContent"
 	QuerySharedUsersPath      = "/currentstateserver/querySharedUsers"
+	QueryKnownUsersPath       = "/currentstateserver/queryKnownUsers"
 )
 
 // NewCurrentStateAPIClient creates a CurrentStateInternalAPI implemented by talking to a HTTP POST API.
@@ -97,3 +98,13 @@ func (h *httpCurrentStateInternalAPI) QuerySharedUsers(
 	apiURL := h.apiURL + QuerySharedUsersPath
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
 }
+
+func (h *httpCurrentStateInternalAPI) QueryKnownUsers(
+	ctx context.Context, req *api.QueryKnownUsersRequest, res *api.QueryKnownUsersResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryKnownUsers")
+	defer span.Finish()
+
+	apiURL := h.apiURL + QueryKnownUsersPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+}

currentstateserver/inthttp/server.go

@@ -77,4 +77,17 @@ func AddRoutes(internalAPIMux *mux.Router, intAPI api.CurrentStateInternalAPI) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(QuerySharedUsersPath,
+		httputil.MakeInternalAPI("queryKnownUsers", func(req *http.Request) util.JSONResponse {
+			request := api.QueryKnownUsersRequest{}
+			response := api.QueryKnownUsersResponse{}
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := intAPI.QueryKnownUsers(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 }

currentstateserver/storage/interface.go

@@ -39,4 +39,6 @@ type Database interface {
 	RedactEvent(ctx context.Context, redactedEventID string, redactedBecause gomatrixserverlib.HeaderedEvent) error
 	// JoinedUsersSetInRooms returns all joined users in the rooms given, along with the count of how many times they appear.
 	JoinedUsersSetInRooms(ctx context.Context, roomIDs []string) (map[string]int, error)
+	// GetKnownUsers searches all users that userID knows about.
+	GetKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error)
 }

currentstateserver/storage/postgres/current_room_state_table.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
+	"fmt"
 
 	"github.com/lib/pq"
 	"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
@@ -81,6 +82,14 @@ const selectJoinedUsersSetForRoomsSQL = "" +
 	"SELECT state_key, COUNT(room_id) FROM currentstate_current_room_state WHERE room_id = ANY($1) AND" +
 	" type = 'm.room.member' and content_value = 'join' GROUP BY state_key"
 
+// selectKnownUsersSQL uses a sub-select statement here to find rooms that the user is
+// joined to. Since this information is used to populate the user directory, we will
+// only return users that the user would ordinarily be able to see anyway.
+const selectKnownUsersSQL = "" +
+	"SELECT DISTINCT state_key FROM currentstate_current_room_state WHERE room_id = ANY(" +
+	"  SELECT DISTINCT room_id FROM currentstate_current_room_state WHERE state_key=$1 AND TYPE='m.room.member' AND content_value='join'" +
+	") AND TYPE='m.room.member' AND content_value='join' AND state_key LIKE $2 LIMIT $3"
+
 type currentRoomStateStatements struct {
 	upsertRoomStateStmt              *sql.Stmt
 	deleteRoomStateByEventIDStmt     *sql.Stmt
@@ -90,6 +99,7 @@ type currentRoomStateStatements struct {
 	selectBulkStateContentStmt       *sql.Stmt
 	selectBulkStateContentWildStmt   *sql.Stmt
 	selectJoinedUsersSetForRoomsStmt *sql.Stmt
+	selectKnownUsersStmt             *sql.Stmt
 }
 
 func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) {
@@ -122,6 +132,9 @@ func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, erro
 	if s.selectJoinedUsersSetForRoomsStmt, err = db.Prepare(selectJoinedUsersSetForRoomsSQL); err != nil {
 		return nil, err
 	}
+	if s.selectKnownUsersStmt, err = db.Prepare(selectKnownUsersSQL); err != nil {
+		return nil, err
+	}
 	return s, nil
 }
 
@@ -295,3 +308,20 @@ func (s *currentRoomStateStatements) SelectBulkStateContent(
 	}
 	return strippedEvents, rows.Err()
 }
+
+func (s *currentRoomStateStatements) SelectKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error) {
+	rows, err := s.selectKnownUsersStmt.QueryContext(ctx, userID, fmt.Sprintf("%%%s%%", searchString), limit)
+	if err != nil {
+		return nil, err
+	}
+	result := []string{}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectKnownUsers: rows.close() failed")
+	for rows.Next() {
+		var userID string
+		if err := rows.Scan(&userID); err != nil {
+			return nil, err
+		}
+		result = append(result, userID)
+	}
+	return result, rows.Err()
+}

currentstateserver/storage/shared/storage.go

@@ -89,3 +89,7 @@ func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership
 func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs []string) (map[string]int, error) {
 	return d.CurrentRoomState.SelectJoinedUsersSetForRooms(ctx, roomIDs)
 }
+
+func (d *Database) GetKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error) {
+	return d.CurrentRoomState.SelectKnownUsers(ctx, userID, searchString, limit)
+}

currentstateserver/storage/sqlite3/current_room_state_table.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
+	"fmt"
 	"strings"
 
 	"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
@@ -69,6 +70,14 @@ const selectBulkStateContentWildSQL = "" +
 const selectJoinedUsersSetForRoomsSQL = "" +
 	"SELECT state_key, COUNT(room_id) FROM currentstate_current_room_state WHERE room_id IN ($1) AND type = 'm.room.member' and content_value = 'join' GROUP BY state_key"
 
+// selectKnownUsersSQL uses a sub-select statement here to find rooms that the user is
+// joined to. Since this information is used to populate the user directory, we will
+// only return users that the user would ordinarily be able to see anyway.
+const selectKnownUsersSQL = "" +
+	"SELECT DISTINCT state_key FROM currentstate_current_room_state WHERE room_id IN (" +
+	"  SELECT DISTINCT room_id FROM currentstate_current_room_state WHERE state_key=$1 AND TYPE='m.room.member' AND content_value='join'" +
+	") AND TYPE='m.room.member' AND content_value='join' AND state_key LIKE $2 LIMIT $3"
+
 type currentRoomStateStatements struct {
 	db                               *sql.DB
 	writer                           *sqlutil.TransactionWriter
@@ -77,6 +86,7 @@ type currentRoomStateStatements struct {
 	selectRoomIDsWithMembershipStmt  *sql.Stmt
 	selectStateEventStmt             *sql.Stmt
 	selectJoinedUsersSetForRoomsStmt *sql.Stmt
+	selectKnownUsersStmt             *sql.Stmt
 }
 
 func NewSqliteCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) {
@@ -103,6 +113,9 @@ func NewSqliteCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error)
 	if s.selectJoinedUsersSetForRoomsStmt, err = db.Prepare(selectJoinedUsersSetForRoomsSQL); err != nil {
 		return nil, err
 	}
+	if s.selectKnownUsersStmt, err = db.Prepare(selectKnownUsersSQL); err != nil {
+		return nil, err
+	}
 	return s, nil
 }
 
@@ -315,3 +328,20 @@ func (s *currentRoomStateStatements) SelectBulkStateContent(
 	}
 	return strippedEvents, rows.Err()
 }
+
+func (s *currentRoomStateStatements) SelectKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error) {
+	rows, err := s.selectKnownUsersStmt.QueryContext(ctx, userID, fmt.Sprintf("%%%s%%", searchString), limit)
+	if err != nil {
+		return nil, err
+	}
+	result := []string{}
+	defer internal.CloseAndLogIfError(ctx, rows, "SelectKnownUsers: rows.close() failed")
+	for rows.Next() {
+		var userID string
+		if err := rows.Scan(&userID); err != nil {
+			return nil, err
+		}
+		result = append(result, userID)
+	}
+	return result, rows.Err()
+}

currentstateserver/storage/tables/interface.go

@@ -39,6 +39,8 @@ type CurrentRoomState interface {
 	// SelectJoinedUsersSetForRooms returns the set of all users in the rooms who are joined to any of these rooms, along with the
 	// counts of how many rooms they are joined.
 	SelectJoinedUsersSetForRooms(ctx context.Context, roomIDs []string) (map[string]int, error)
+	// SelectKnownUsers searches all users that userID knows about.
+	SelectKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error)
 }
 
 // StrippedEvent represents a stripped event for returning extracted content values.