Implement /thirdparty endpoints (#2831)

Implements the following endpoints 
```
GET /_matrix/client/v3/thirdparty/protocols
GET /_matrix/client/v3/thirdparty/protocols/{protocol}
GET /_matrix/client/v3/thirdparty/location
GET /_matrix/client/v3/thirdparty/location/{protocol}
GET /_matrix/client/v3/thirdparty/user
GET /_matrix/client/v3/thirdparty/user/{protocol}
```

appservice/api/query.go

@@ -19,11 +19,13 @@ package api
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // AppServiceInternalAPI is used to query user and room alias data from application
@@ -41,6 +43,10 @@ type AppServiceInternalAPI interface {
 		req *UserIDExistsRequest,
 		resp *UserIDExistsResponse,
 	) error
+
+	Locations(ctx context.Context, req *LocationRequest, resp *LocationResponse) error
+	User(ctx context.Context, request *UserRequest, response *UserResponse) error
+	Protocols(ctx context.Context, req *ProtocolRequest, resp *ProtocolResponse) error
 }
 
 // RoomAliasExistsRequest is a request to an application service
@@ -77,6 +83,73 @@ type UserIDExistsResponse struct {
 	UserIDExists bool `json:"exists"`
 }
 
+const (
+	ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/"
+	ASUserPath     = "/_matrix/app/unstable/thirdparty/user"
+	ASLocationPath = "/_matrix/app/unstable/thirdparty/location"
+)
+
+type ProtocolRequest struct {
+	Protocol string `json:"protocol,omitempty"`
+}
+
+type ProtocolResponse struct {
+	Protocols map[string]ASProtocolResponse `json:"protocols"`
+	Exists    bool                          `json:"exists"`
+}
+
+type ASProtocolResponse struct {
+	FieldTypes     map[string]FieldType `json:"field_types,omitempty"` // NOTSPEC: field_types is required by the spec
+	Icon           string               `json:"icon"`
+	Instances      []ProtocolInstance   `json:"instances"`
+	LocationFields []string             `json:"location_fields"`
+	UserFields     []string             `json:"user_fields"`
+}
+
+type FieldType struct {
+	Placeholder string `json:"placeholder"`
+	Regexp      string `json:"regexp"`
+}
+
+type ProtocolInstance struct {
+	Description string          `json:"desc"`
+	Icon        string          `json:"icon,omitempty"`
+	NetworkID   string          `json:"network_id,omitempty"` // NOTSPEC: network_id is required by the spec
+	Fields      json.RawMessage `json:"fields,omitempty"`     // NOTSPEC: fields is required by the spec
+}
+
+type UserRequest struct {
+	Protocol string `json:"protocol"`
+	Params   string `json:"params"`
+}
+
+type UserResponse struct {
+	Users  []ASUserResponse `json:"users,omitempty"`
+	Exists bool             `json:"exists,omitempty"`
+}
+
+type ASUserResponse struct {
+	Protocol string          `json:"protocol"`
+	UserID   string          `json:"userid"`
+	Fields   json.RawMessage `json:"fields"`
+}
+
+type LocationRequest struct {
+	Protocol string `json:"protocol"`
+	Params   string `json:"params"`
+}
+
+type LocationResponse struct {
+	Locations []ASLocationResponse `json:"locations,omitempty"`
+	Exists    bool                 `json:"exists,omitempty"`
+}
+
+type ASLocationResponse struct {
+	Alias    string          `json:"alias"`
+	Protocol string          `json:"protocol"`
+	Fields   json.RawMessage `json:"fields"`
+}
+
 // RetrieveUserProfile is a wrapper that queries both the local database and
 // application services for a given user's profile
 // TODO: Remove this, it's called from federationapi and clientapi but is a pure function

appservice/appservice.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -58,8 +59,10 @@ func NewInternalAPI(
 	// Create appserivce query API with an HTTP client that will be used for all
 	// outbound and inbound requests (inbound only for the internal API)
 	appserviceQueryAPI := &query.AppServiceQueryAPI{
-		HTTPClient: client,
-		Cfg:        &base.Cfg.AppServiceAPI,
+		HTTPClient:    client,
+		Cfg:           &base.Cfg.AppServiceAPI,
+		ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{},
+		CacheMu:       sync.Mutex{},
 	}
 
 	if len(base.Cfg.Derived.ApplicationServices) == 0 {

appservice/inthttp/client.go

@@ -13,6 +13,9 @@ import (
 const (
 	AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists"
 	AppServiceUserIDExistsPath    = "/appservice/UserIDExists"
+	AppServiceLocationsPath       = "/appservice/locations"
+	AppServiceUserPath            = "/appservice/users"
+	AppServiceProtocolsPath       = "/appservice/protocols"
 )
 
 // httpAppServiceQueryAPI contains the URL to an appservice query API and a
@@ -58,3 +61,24 @@ func (h *httpAppServiceQueryAPI) UserIDExists(
 		h.httpClient, ctx, request, response,
 	)
 }
+
+func (h *httpAppServiceQueryAPI) Locations(ctx context.Context, request *api.LocationRequest, response *api.LocationResponse) error {
+	return httputil.CallInternalRPCAPI(
+		"ASLocation", h.appserviceURL+AppServiceLocationsPath,
+		h.httpClient, ctx, request, response,
+	)
+}
+
+func (h *httpAppServiceQueryAPI) User(ctx context.Context, request *api.UserRequest, response *api.UserResponse) error {
+	return httputil.CallInternalRPCAPI(
+		"ASUser", h.appserviceURL+AppServiceUserPath,
+		h.httpClient, ctx, request, response,
+	)
+}
+
+func (h *httpAppServiceQueryAPI) Protocols(ctx context.Context, request *api.ProtocolRequest, response *api.ProtocolResponse) error {
+	return httputil.CallInternalRPCAPI(
+		"ASProtocols", h.appserviceURL+AppServiceProtocolsPath,
+		h.httpClient, ctx, request, response,
+	)
+}

appservice/inthttp/server.go

@@ -2,6 +2,7 @@ package inthttp
 
 import (
 	"github.com/gorilla/mux"
+
 	"github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/internal/httputil"
 )
@@ -17,4 +18,19 @@ func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
 		AppServiceUserIDExistsPath,
 		httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
 	)
+
+	internalAPIMux.Handle(
+		AppServiceProtocolsPath,
+		httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols),
+	)
+
+	internalAPIMux.Handle(
+		AppServiceLocationsPath,
+		httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations),
+	)
+
+	internalAPIMux.Handle(
+		AppServiceUserPath,
+		httputil.MakeInternalRPCAPI("AppserviceUser", a.User),
+	)
 }

appservice/query/query.go

@@ -18,13 +18,18 @@ package query
 
 import (
 	"context"
+	"encoding/json"
+	"io"
 	"net/http"
 	"net/url"
+	"strings"
+	"sync"
+
+	"github.com/opentracing/opentracing-go"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	opentracing "github.com/opentracing/opentracing-go"
-	log "github.com/sirupsen/logrus"
 )
 
 const roomAliasExistsPath = "/rooms/"
@@ -32,8 +37,10 @@ const userIDExistsPath = "/users/"
 
 // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
 type AppServiceQueryAPI struct {
-	HTTPClient *http.Client
-	Cfg        *config.AppServiceAPI
+	HTTPClient    *http.Client
+	Cfg           *config.AppServiceAPI
+	ProtocolCache map[string]api.ASProtocolResponse
+	CacheMu       sync.Mutex
 }
 
 // RoomAliasExists performs a request to '/room/{roomAlias}' on all known
@@ -165,3 +172,178 @@ func (a *AppServiceQueryAPI) UserIDExists(
 	response.UserIDExists = false
 	return nil
 }
+
+type thirdpartyResponses interface {
+	api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse
+}
+
+func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) {
+	origURL := url
+	// try v1 and unstable appservice endpoints
+	for _, version := range []string{"v1", "unstable"} {
+		var resp *http.Response
+		var body []byte
+		asURL := strings.Replace(origURL, "unstable", version, 1)
+		resp, err = client.Get(asURL)
+		if err != nil {
+			continue
+		}
+		defer resp.Body.Close() // nolint: errcheck
+		body, err = io.ReadAll(resp.Body)
+		if err != nil {
+			continue
+		}
+		return json.Unmarshal(body, &response)
+	}
+	return err
+}
+
+func (a *AppServiceQueryAPI) Locations(
+	ctx context.Context,
+	req *api.LocationRequest,
+	resp *api.LocationResponse,
+) error {
+	params, err := url.ParseQuery(req.Params)
+	if err != nil {
+		return err
+	}
+
+	for _, as := range a.Cfg.Derived.ApplicationServices {
+		var asLocations []api.ASLocationResponse
+		params.Set("access_token", as.HSToken)
+
+		url := as.URL + api.ASLocationPath
+		if req.Protocol != "" {
+			url += "/" + req.Protocol
+		}
+
+		if err := requestDo[[]api.ASLocationResponse](a.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
+			log.WithError(err).Error("unable to get 'locations' from application service")
+			continue
+		}
+
+		resp.Locations = append(resp.Locations, asLocations...)
+	}
+
+	if len(resp.Locations) == 0 {
+		resp.Exists = false
+		return nil
+	}
+	resp.Exists = true
+	return nil
+}
+
+func (a *AppServiceQueryAPI) User(
+	ctx context.Context,
+	req *api.UserRequest,
+	resp *api.UserResponse,
+) error {
+	params, err := url.ParseQuery(req.Params)
+	if err != nil {
+		return err
+	}
+
+	for _, as := range a.Cfg.Derived.ApplicationServices {
+		var asUsers []api.ASUserResponse
+		params.Set("access_token", as.HSToken)
+
+		url := as.URL + api.ASUserPath
+		if req.Protocol != "" {
+			url += "/" + req.Protocol
+		}
+
+		if err := requestDo[[]api.ASUserResponse](a.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
+			log.WithError(err).Error("unable to get 'user' from application service")
+			continue
+		}
+
+		resp.Users = append(resp.Users, asUsers...)
+	}
+
+	if len(resp.Users) == 0 {
+		resp.Exists = false
+		return nil
+	}
+	resp.Exists = true
+	return nil
+}
+
+func (a *AppServiceQueryAPI) Protocols(
+	ctx context.Context,
+	req *api.ProtocolRequest,
+	resp *api.ProtocolResponse,
+) error {
+
+	// get a single protocol response
+	if req.Protocol != "" {
+
+		a.CacheMu.Lock()
+		defer a.CacheMu.Unlock()
+		if proto, ok := a.ProtocolCache[req.Protocol]; ok {
+			resp.Exists = true
+			resp.Protocols = map[string]api.ASProtocolResponse{
+				req.Protocol: proto,
+			}
+			return nil
+		}
+
+		response := api.ASProtocolResponse{}
+		for _, as := range a.Cfg.Derived.ApplicationServices {
+			var proto api.ASProtocolResponse
+			if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+req.Protocol, &proto); err != nil {
+				log.WithError(err).Error("unable to get 'protocol' from application service")
+				continue
+			}
+
+			if len(response.Instances) != 0 {
+				response.Instances = append(response.Instances, proto.Instances...)
+			} else {
+				response = proto
+			}
+		}
+
+		if len(response.Instances) == 0 {
+			resp.Exists = false
+			return nil
+		}
+
+		resp.Exists = true
+		resp.Protocols = map[string]api.ASProtocolResponse{
+			req.Protocol: response,
+		}
+		a.ProtocolCache[req.Protocol] = response
+		return nil
+	}
+
+	response := make(map[string]api.ASProtocolResponse, len(a.Cfg.Derived.ApplicationServices))
+
+	for _, as := range a.Cfg.Derived.ApplicationServices {
+		for _, p := range as.Protocols {
+			var proto api.ASProtocolResponse
+			if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+p, &proto); err != nil {
+				log.WithError(err).Error("unable to get 'protocol' from application service")
+				continue
+			}
+			existing, ok := response[p]
+			if !ok {
+				response[p] = proto
+				continue
+			}
+			existing.Instances = append(existing.Instances, proto.Instances...)
+			response[p] = existing
+		}
+	}
+
+	if len(response) == 0 {
+		resp.Exists = false
+		return nil
+	}
+
+	a.CacheMu.Lock()
+	defer a.CacheMu.Unlock()
+	a.ProtocolCache = response
+
+	resp.Exists = true
+	resp.Protocols = response
+	return nil
+}