Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver (#1174)

* Use content_value instead of membership

* Fix build

* Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver

- All public rooms paths are now handled by clientapi
- Requests to (un)publish rooms are sent to the roomserver via `PerformPublish`
  which are stored in a new `published_table.go`
- Requests for public rooms are handled in clientapi by:
    * Fetch all room IDs which are published using `QueryPublishedRooms` on the roomserver.
    * Apply pagination parameters to the slice.
    * Do a `QueryBulkStateContent` request to the currentstateserver to pull out
      required state event *content* (not entire events).
    * Aggregate and return the chunk.

Mostly but not fully implemented (DB queries on currentstateserver are missing)

* Fix pq query

* Make postgres work

* Make sqlite work

* Fix tests

* Unbreak pagination tests

* Linting

roomserver/inthttp/client.go

@@ -29,6 +29,7 @@ const (
 	RoomserverPerformJoinPath     = "/roomserver/performJoin"
 	RoomserverPerformLeavePath    = "/roomserver/performLeave"
 	RoomserverPerformBackfillPath = "/roomserver/performBackfill"
+	RoomserverPerformPublishPath  = "/roomserver/performPublish"
 
 	// Query operations
 	RoomserverQueryLatestEventsAndStatePath    = "/roomserver/queryLatestEventsAndState"
@@ -41,6 +42,7 @@ const (
 	RoomserverQueryStateAndAuthChainPath       = "/roomserver/queryStateAndAuthChain"
 	RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities"
 	RoomserverQueryRoomVersionForRoomPath      = "/roomserver/queryRoomVersionForRoom"
+	RoomserverQueryPublishedRoomsPath          = "/roomserver/queryPublishedRooms"
 )
 
 type httpRoomserverInternalAPI struct {
@@ -194,6 +196,23 @@ func (h *httpRoomserverInternalAPI) PerformLeave(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
+func (h *httpRoomserverInternalAPI) PerformPublish(
+	ctx context.Context,
+	req *api.PerformPublishRequest,
+	res *api.PerformPublishResponse,
+) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "PerformPublish")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverPerformPublishPath
+	err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
+	if err != nil {
+		res.Error = &api.PerformError{
+			Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
+		}
+	}
+}
+
 // QueryLatestEventsAndState implements RoomserverQueryAPI
 func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState(
 	ctx context.Context,
@@ -233,6 +252,18 @@ func (h *httpRoomserverInternalAPI) QueryEventsByID(
 	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
+func (h *httpRoomserverInternalAPI) QueryPublishedRooms(
+	ctx context.Context,
+	request *api.QueryPublishedRoomsRequest,
+	response *api.QueryPublishedRoomsResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublishedRooms")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + RoomserverQueryPublishedRoomsPath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
 // QueryMembershipForUser implements RoomserverQueryAPI
 func (h *httpRoomserverInternalAPI) QueryMembershipForUser(
 	ctx context.Context,

roomserver/inthttp/server.go

@@ -61,6 +61,31 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	internalAPIMux.Handle(RoomserverPerformPublishPath,
+		httputil.MakeInternalAPI("performPublish", func(req *http.Request) util.JSONResponse {
+			var request api.PerformPublishRequest
+			var response api.PerformPublishResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			r.PerformPublish(req.Context(), &request, &response)
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	internalAPIMux.Handle(
+		RoomserverQueryPublishedRoomsPath,
+		httputil.MakeInternalAPI("queryPublishedRooms", func(req *http.Request) util.JSONResponse {
+			var request api.QueryPublishedRoomsRequest
+			var response api.QueryPublishedRoomsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.QueryPublishedRooms(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 	internalAPIMux.Handle(
 		RoomserverQueryLatestEventsAndStatePath,
 		httputil.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse {