de-MSC-ifying space summaries (MSC2946) (#3134)

- This PR moves and refactors the
[code](https://github.com/matrix-org/dendrite/blob/main/setup/mscs/msc2946/msc2946.go)
for
[MSC2946](https://github.com/matrix-org/matrix-spec-proposals/pull/2946)
('Space Summaries') to integrate it into the rest of the codebase.
- Means space summaries are no longer hidden behind an MSC flag
- Solves #3096

Signed-off-by: Sam Wedgwood <sam@wedgwood.dev>

clientapi/routing/joinroom_test.go

@@ -35,9 +35,9 @@ func TestJoinRoomByIDOrAlias(t *testing.T) {
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		natsInstance := jetstream.NATSInstance{}
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
 		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 		asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI)
-		rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
 
 		// Create the users in the userapi
 		for _, u := range []*test.User{alice, bob, charlie} {

clientapi/routing/login_test.go

@@ -47,6 +47,7 @@ func TestLogin(t *testing.T) {
 		routers := httputil.NewRouters()
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
 		// Needed for /login
 		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 

clientapi/routing/register_test.go

@@ -415,6 +415,7 @@ func Test_register(t *testing.T) {
 
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
 		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		for _, tc := range testCases {
@@ -594,6 +595,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) {
 		natsInstance := jetstream.NATSInstance{}
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
 		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 		deviceName, deviceID := "deviceName", "deviceID"
 		expectedDisplayName := "DisplayName"
@@ -634,6 +636,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) {
 		cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
 		caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
 		rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
+		rsAPI.SetFederationAPI(nil, nil)
 		userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
 
 		expectedDisplayName := "rabbit"

clientapi/routing/room_hierarchy.go

@@ -0,0 +1,180 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+	"strconv"
+	"sync"
+
+	"github.com/google/uuid"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/roomserver/types"
+	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+	log "github.com/sirupsen/logrus"
+)
+
+// For storing pagination information for room hierarchies
+type RoomHierarchyPaginationCache struct {
+	cache map[string]roomserverAPI.RoomHierarchyWalker
+	mu    sync.Mutex
+}
+
+// Create a new, empty, pagination cache.
+func NewRoomHierarchyPaginationCache() RoomHierarchyPaginationCache {
+	return RoomHierarchyPaginationCache{
+		cache: map[string]roomserverAPI.RoomHierarchyWalker{},
+	}
+}
+
+// Get a cached page, or nil if there is no associated page in the cache.
+func (c *RoomHierarchyPaginationCache) Get(token string) *roomserverAPI.RoomHierarchyWalker {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	line, ok := c.cache[token]
+	if ok {
+		return &line
+	} else {
+		return nil
+	}
+}
+
+// Add a cache line to the pagination cache.
+func (c *RoomHierarchyPaginationCache) AddLine(line roomserverAPI.RoomHierarchyWalker) string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	token := uuid.NewString()
+	c.cache[token] = line
+	return token
+}
+
+// Query the hierarchy of a room/space
+//
+// Implements /_matrix/client/v1/rooms/{roomID}/hierarchy
+func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr string, rsAPI roomserverAPI.ClientRoomserverAPI, paginationCache *RoomHierarchyPaginationCache) util.JSONResponse {
+	parsedRoomID, err := spec.NewRoomID(roomIDStr)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: spec.InvalidParam("room is unknown/forbidden"),
+		}
+	}
+	roomID := *parsedRoomID
+
+	suggestedOnly := false // Defaults to false (spec-defined)
+	switch req.URL.Query().Get("suggested_only") {
+	case "true":
+		suggestedOnly = true
+	case "false":
+	case "": // Empty string is returned when query param is not set
+	default:
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("query parameter 'suggested_only', if set, must be 'true' or 'false'"),
+		}
+	}
+
+	limit := 1000 // Default to 1000
+	limitStr := req.URL.Query().Get("limit")
+	if limitStr != "" {
+		var maybeLimit int
+		maybeLimit, err = strconv.Atoi(limitStr)
+		if err != nil || maybeLimit < 0 {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidParam("query parameter 'limit', if set, must be a positive integer"),
+			}
+		}
+		limit = maybeLimit
+		if limit > 1000 {
+			limit = 1000 // Maximum limit of 1000
+		}
+	}
+
+	maxDepth := -1 // '-1' representing no maximum depth
+	maxDepthStr := req.URL.Query().Get("max_depth")
+	if maxDepthStr != "" {
+		var maybeMaxDepth int
+		maybeMaxDepth, err = strconv.Atoi(maxDepthStr)
+		if err != nil || maybeMaxDepth < 0 {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidParam("query parameter 'max_depth', if set, must be a positive integer"),
+			}
+		}
+		maxDepth = maybeMaxDepth
+	}
+
+	from := req.URL.Query().Get("from")
+
+	var walker roomserverAPI.RoomHierarchyWalker
+	if from == "" { // No pagination token provided, so start new hierarchy walker
+		walker = roomserverAPI.NewRoomHierarchyWalker(types.NewDeviceNotServerName(*device), roomID, suggestedOnly, maxDepth)
+	} else { // Attempt to resume cached walker
+		cachedWalker := paginationCache.Get(from)
+
+		if cachedWalker == nil || cachedWalker.SuggestedOnly != suggestedOnly || cachedWalker.MaxDepth != maxDepth {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: spec.InvalidParam("pagination not found for provided token ('from') with given 'max_depth', 'suggested_only' and room ID"),
+			}
+		}
+
+		walker = *cachedWalker
+	}
+
+	discoveredRooms, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit)
+
+	if err != nil {
+		switch err.(type) {
+		case roomserverAPI.ErrRoomUnknownOrNotAllowed:
+			util.GetLogger(req.Context()).WithError(err).Debugln("room unknown/forbidden when handling CS room hierarchy request")
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: spec.Forbidden("room is unknown/forbidden"),
+			}
+		default:
+			log.WithError(err).Errorf("failed to fetch next page of room hierarchy (CS API)")
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.Unknown("internal server error"),
+			}
+		}
+	}
+
+	nextBatch := ""
+	// nextWalker will be nil if there's no more rooms left to walk
+	if nextWalker != nil {
+		nextBatch = paginationCache.AddLine(*nextWalker)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: RoomHierarchyClientResponse{
+			Rooms:     discoveredRooms,
+			NextBatch: nextBatch,
+		},
+	}
+
+}
+
+// Success response for /_matrix/client/v1/rooms/{roomID}/hierarchy
+type RoomHierarchyClientResponse struct {
+	Rooms     []fclient.RoomHierarchyRoom `json:"rooms"`
+	NextBatch string                      `json:"next_batch,omitempty"`
+}

clientapi/routing/routing.go

@@ -288,6 +288,8 @@ func Setup(
 	// Note that 'apiversion' is chosen because it must not collide with a variable used in any of the routing!
 	v3mux := publicAPIMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter()
 
+	v1mux := publicAPIMux.PathPrefix("/v1/").Subrouter()
+
 	unstableMux := publicAPIMux.PathPrefix("/unstable").Subrouter()
 
 	v3mux.Handle("/createRoom",
@@ -505,6 +507,19 @@ func Setup(
 		}, httputil.WithAllowGuests()),
 	).Methods(http.MethodPut, http.MethodOptions)
 
+	// Defined outside of handler to persist between calls
+	// TODO: clear based on some criteria
+	roomHierarchyPaginationCache := NewRoomHierarchyPaginationCache()
+	v1mux.Handle("/rooms/{roomID}/hierarchy",
+		httputil.MakeAuthAPI("spaces", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
+			vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return QueryRoomHierarchy(req, device, vars["roomID"], rsAPI, &roomHierarchyPaginationCache)
+		}, httputil.WithAllowGuests()),
+	).Methods(http.MethodGet, http.MethodOptions)
+
 	v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
 		if r := rateLimits.Limit(req, nil); r != nil {
 			return *r