Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver (#1174)

* Use content_value instead of membership

* Fix build

* Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver

- All public rooms paths are now handled by clientapi
- Requests to (un)publish rooms are sent to the roomserver via `PerformPublish`
  which are stored in a new `published_table.go`
- Requests for public rooms are handled in clientapi by:
    * Fetch all room IDs which are published using `QueryPublishedRooms` on the roomserver.
    * Apply pagination parameters to the slice.
    * Do a `QueryBulkStateContent` request to the currentstateserver to pull out
      required state event *content* (not entire events).
    * Aggregate and return the chunk.

Mostly but not fully implemented (DB queries on currentstateserver are missing)

* Fix pq query

* Make postgres work

* Make sqlite work

* Fix tests

* Unbreak pagination tests

* Linting

federationapi/federationapi.go

@@ -16,6 +16,7 @@ package federationapi
 
 import (
 	"github.com/gorilla/mux"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
@@ -36,11 +37,12 @@ func AddPublicRoutes(
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	federationSenderAPI federationSenderAPI.FederationSenderInternalAPI,
 	eduAPI eduserverAPI.EDUServerInputAPI,
+	stateAPI currentstateAPI.CurrentStateInternalAPI,
 ) {
 
 	routing.Setup(
 		router, cfg, rsAPI,
 		eduAPI, federationSenderAPI, keyRing,
-		federation, userAPI,
+		federation, userAPI, stateAPI,
 	)
 }

federationapi/federationapi_test.go

@@ -31,7 +31,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	fsAPI := base.FederationSenderHTTPClient()
 	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
 	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
-	federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil)
+	federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil, nil)
 	httputil.SetupHTTPAPI(
 		base.BaseMux,
 		base.PublicAPIMux,

federationapi/routing/publicrooms.go

@@ -0,0 +1,178 @@
+package routing
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+type PublicRoomReq struct {
+	Since  string `json:"since,omitempty"`
+	Limit  int16  `json:"limit,omitempty"`
+	Filter filter `json:"filter,omitempty"`
+}
+
+type filter struct {
+	SearchTerms string `json:"generic_search_term,omitempty"`
+}
+
+// GetPostPublicRooms implements GET and POST /publicRooms
+func GetPostPublicRooms(req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI) util.JSONResponse {
+	var request PublicRoomReq
+	if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
+		return *fillErr
+	}
+	if request.Limit == 0 {
+		request.Limit = 50
+	}
+	response, err := publicRooms(req.Context(), request, rsAPI, stateAPI)
+	if err != nil {
+		return jsonerror.InternalServerError()
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: response,
+	}
+}
+
+func publicRooms(ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI,
+	stateAPI currentstateAPI.CurrentStateInternalAPI) (*gomatrixserverlib.RespPublicRooms, error) {
+
+	var response gomatrixserverlib.RespPublicRooms
+	var limit int16
+	var offset int64
+	limit = request.Limit
+	offset, err := strconv.ParseInt(request.Since, 10, 64)
+	// ParseInt returns 0 and an error when trying to parse an empty string
+	// In that case, we want to assign 0 so we ignore the error
+	if err != nil && len(request.Since) > 0 {
+		util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed")
+		return nil, err
+	}
+
+	var queryRes roomserverAPI.QueryPublishedRoomsResponse
+	err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
+		return nil, err
+	}
+	response.TotalRoomCountEstimate = len(queryRes.RoomIDs)
+
+	if offset > 0 {
+		response.PrevBatch = strconv.Itoa(int(offset) - 1)
+	}
+	nextIndex := int(offset) + int(limit)
+	if response.TotalRoomCountEstimate > nextIndex {
+		response.NextBatch = strconv.Itoa(nextIndex)
+	}
+
+	if offset < 0 {
+		offset = 0
+	}
+	if nextIndex > len(queryRes.RoomIDs) {
+		nextIndex = len(queryRes.RoomIDs)
+	}
+	roomIDs := queryRes.RoomIDs[offset:nextIndex]
+	response.Chunk, err = fillInRooms(ctx, roomIDs, stateAPI)
+	return &response, err
+}
+
+// fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request
+// on /publicRooms by parsing the incoming HTTP request
+// Filter is only filled for POST requests
+func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse {
+	if httpReq.Method == http.MethodGet {
+		limit, err := strconv.Atoi(httpReq.FormValue("limit"))
+		// Atoi returns 0 and an error when trying to parse an empty string
+		// In that case, we want to assign 0 so we ignore the error
+		if err != nil && len(httpReq.FormValue("limit")) > 0 {
+			util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed")
+			reqErr := jsonerror.InternalServerError()
+			return &reqErr
+		}
+		request.Limit = int16(limit)
+		request.Since = httpReq.FormValue("since")
+		return nil
+	} else if httpReq.Method == http.MethodPost {
+		return httputil.UnmarshalJSONRequest(httpReq, request)
+	}
+
+	return &util.JSONResponse{
+		Code: http.StatusMethodNotAllowed,
+		JSON: jsonerror.NotFound("Bad method"),
+	}
+}
+
+// due to lots of switches
+// nolint:gocyclo
+func fillInRooms(ctx context.Context, roomIDs []string, stateAPI currentstateAPI.CurrentStateInternalAPI) ([]gomatrixserverlib.PublicRoom, error) {
+	avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""}
+	nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""}
+	canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""}
+	topicTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.topic", StateKey: ""}
+	guestTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.guest_access", StateKey: ""}
+	visibilityTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomHistoryVisibility, StateKey: ""}
+	joinRuleTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}
+
+	var stateRes currentstateAPI.QueryBulkStateContentResponse
+	err := stateAPI.QueryBulkStateContent(ctx, &currentstateAPI.QueryBulkStateContentRequest{
+		RoomIDs:        roomIDs,
+		AllowWildcards: true,
+		StateTuples: []gomatrixserverlib.StateKeyTuple{
+			nameTuple, canonicalTuple, topicTuple, guestTuple, visibilityTuple, joinRuleTuple, avatarTuple,
+			{EventType: gomatrixserverlib.MRoomMember, StateKey: "*"},
+		},
+	}, &stateRes)
+	if err != nil {
+		util.GetLogger(ctx).WithError(err).Error("QueryBulkStateContent failed")
+		return nil, err
+	}
+	util.GetLogger(ctx).Infof("room IDs: %+v", roomIDs)
+	util.GetLogger(ctx).Infof("State res: %+v", stateRes.Rooms)
+	chunk := make([]gomatrixserverlib.PublicRoom, len(roomIDs))
+	i := 0
+	for roomID, data := range stateRes.Rooms {
+		pub := gomatrixserverlib.PublicRoom{
+			RoomID: roomID,
+		}
+		joinCount := 0
+		var joinRule, guestAccess string
+		for tuple, contentVal := range data {
+			if tuple.EventType == gomatrixserverlib.MRoomMember && contentVal == "join" {
+				joinCount++
+				continue
+			}
+			switch tuple {
+			case avatarTuple:
+				pub.AvatarURL = contentVal
+			case nameTuple:
+				pub.Name = contentVal
+			case topicTuple:
+				pub.Topic = contentVal
+			case canonicalTuple:
+				pub.CanonicalAlias = contentVal
+			case visibilityTuple:
+				pub.WorldReadable = contentVal == "world_readable"
+			// need both of these to determine whether guests can join
+			case joinRuleTuple:
+				joinRule = contentVal
+			case guestTuple:
+				guestAccess = contentVal
+			}
+		}
+		if joinRule == gomatrixserverlib.Public && guestAccess == "can_join" {
+			pub.GuestCanJoin = true
+		}
+		pub.JoinedMembersCount = joinCount
+		chunk[i] = pub
+		i++
+	}
+	return chunk, nil
+}

federationapi/routing/routing.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
 	eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
 	federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
 	"github.com/matrix-org/dendrite/internal/config"
@@ -52,6 +53,7 @@ func Setup(
 	keys gomatrixserverlib.JSONVerifier,
 	federation *gomatrixserverlib.FederationClient,
 	userAPI userapi.UserInternalAPI,
+	stateAPI currentstateAPI.CurrentStateInternalAPI,
 ) {
 	v2keysmux := publicAPIMux.PathPrefix(pathPrefixV2Keys).Subrouter()
 	v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter()
@@ -291,4 +293,10 @@ func Setup(
 			return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg)
 		},
 	)).Methods(http.MethodGet)
+
+	v1fedmux.Handle("/publicRooms",
+		httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse {
+			return GetPostPublicRooms(req, rsAPI, stateAPI)
+		}),
+	).Methods(http.MethodGet)
 }

federationapi/routing/send_test.go

@@ -111,6 +111,13 @@ func (t *testRoomserverAPI) PerformJoin(
 ) {
 }
 
+func (t *testRoomserverAPI) PerformPublish(
+	ctx context.Context,
+	req *api.PerformPublishRequest,
+	res *api.PerformPublishResponse,
+) {
+}
+
 func (t *testRoomserverAPI) PerformLeave(
 	ctx context.Context,
 	req *api.PerformLeaveRequest,
@@ -168,6 +175,14 @@ func (t *testRoomserverAPI) QueryMembershipForUser(
 	return fmt.Errorf("not implemented")
 }
 
+func (t *testRoomserverAPI) QueryPublishedRooms(
+	ctx context.Context,
+	request *api.QueryPublishedRoomsRequest,
+	response *api.QueryPublishedRoomsResponse,
+) error {
+	return fmt.Errorf("not implemented")
+}
+
 // Query a list of membership events for a room
 func (t *testRoomserverAPI) QueryMembershipsForRoom(
 	ctx context.Context,