hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2024-12-27 07:28:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Server key component (#1050)

Browse source 
* Server key API (works for monolith but not for polylith yet)

* Re-enable caching on server key API component

* Groundwork for HTTP APIs for server key API

* Hopefully implement HTTP for server key API

* Simplify public key request marshalling from map keys

* Update gomatrixserverlib

* go mod tidy

* Common -> internal

* remove keyring.go

* Update Docker Hub for server key API

* YAML is funny about indentation

* Wire in new server key API into hybrid monolith mode

* Create maps

* Route server key API endpoints on internal API mux

* Fix server key API URLs

* Add fetcher behaviour into server key API implementation

* Return error if we failed to fetch some keys

* Return results anyway

* Move things about a bit

* Remove unused code

* Fix comments, don't use federation sender URL in polylith mode

* Add server_key_api to sample config

* Review comments

* HTTP API to cache keys that have been requested

* Overwrite server_key_api listen in monolith hybrid mode
This commit is contained in:
Neil Alexander 2020-05-27 10:19:24 +01:00 committed by GitHub
parent 267a4d1823
commit 7d6461dd3c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 542 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
build/docker/hub/config/dendrite-config.yaml
View file

@ -110,11 +110,13 @@ listen:
room_server: "room_server:7770"
client_api: "client_api:7771"
federation_api: "federation_api:7772"
server_key_api: "server_key_api:7778"
sync_api: "sync_api:7773"
media_api: "media_api:7774"
public_rooms_api: "public_rooms_api:7775"
federation_sender: "federation_sender:7776"
edu_server: "edu_server:7777"
key_server: "key_server:7779"
# The configuration for tracing the dendrite components.
tracing:

13
build/docker/hub/docker-compose.polylith.yml
View file

@ -131,7 +131,7 @@ services:
- internal
key_server:
hostname: key_serverde
hostname: key_server
image: matrixdotorg/dendrite:keyserver
command: [
"--config=dendrite.yaml"
@ -141,6 +141,17 @@ services:
networks:
- internal
server_key_api:
hostname: server_key_api
image: matrixdotorg/dendrite:serverkeyapi
command: [
"--config=dendrite.yaml"
]
volumes:
- ./config:/etc/dendrite
networks:
- internal
networks:
internal:
attachable: true

25
build/docker/hub/images-build.sh
View file

@ -2,16 +2,17 @@
cd $(git rev-parse --show-toplevel)
docker build -f docker/hub/Dockerfile -t matrixdotorg/dendrite:latest .
docker build -f build/docker/hub/Dockerfile -t matrixdotorg/dendrite:latest .
docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:publicroomsapi --build-arg component=dendrite-public-rooms-api-server -f docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f build/docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f build/docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f build/docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f build/docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f build/docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:publicroomsapi --build-arg component=dendrite-public-rooms-api-server -f build/docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/hub/Dockerfile.component .
docker build -t matrixdotorg/dendrite:serverkeyapi --build-arg component=dendrite-server-key-api-server -f build/docker/hub/Dockerfile.component .

8
cmd/dendrite-client-api-server/main.go
View file

@ -19,7 +19,6 @@ import (
"github.com/matrix-org/dendrite/eduserver"
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/internal/basecomponent"
"github.com/matrix-org/dendrite/internal/keydb"
"github.com/matrix-org/dendrite/internal/transactions"
)
@ -31,9 +30,10 @@ func main() {
accountDB := base.CreateAccountsDB()
deviceDB := base.CreateDeviceDB()
keyDB := base.CreateKeyDB()
federation := base.CreateFederationClient()
keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
serverKeyAPI := base.CreateHTTPServerKeyAPIs()
keyRing := serverKeyAPI.KeyRing()
asQuery := base.CreateHTTPAppServiceAPIs()
rsAPI := base.CreateHTTPRoomserverAPIs()
@ -42,7 +42,7 @@ func main() {
eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
clientapi.SetupClientAPIComponent(
base, deviceDB, accountDB, federation, &keyRing,
base, deviceDB, accountDB, federation, keyRing,
rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI,
)

32
cmd/dendrite-demo-libp2p/main.go
View file

@ -37,11 +37,11 @@ import (
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/keydb"
"github.com/matrix-org/dendrite/internal/transactions"
"github.com/matrix-org/dendrite/mediaapi"
"github.com/matrix-org/dendrite/publicroomsapi"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/serverkeyapi"
"github.com/matrix-org/dendrite/syncapi"
"github.com/matrix-org/gomatrixserverlib"
@ -52,17 +52,8 @@ import (
func createKeyDB(
base *P2PDendrite,
) keydb.Database {
db, err := keydb.NewDatabase(
string(base.Base.Cfg.Database.ServerKey),
base.Base.Cfg.DbProperties(),
base.Base.Cfg.Matrix.ServerName,
base.Base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
base.Base.Cfg.Matrix.KeyID,
)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to keys db")
}
db gomatrixserverlib.KeyDatabase,
) {
mdns := mDNSListener{
host: base.LibP2P,
keydb: db,
@ -77,7 +68,6 @@ func createKeyDB(
panic(err)
}
serv.RegisterNotifee(&mdns)
return db
}
func createFederationClient(
@ -144,9 +134,15 @@ func main() {
accountDB := base.Base.CreateAccountsDB()
deviceDB := base.Base.CreateDeviceDB()
keyDB := createKeyDB(base)
federation := createFederationClient(base)
keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent(
&base.Base, federation,
)
keyRing := serverKeyAPI.KeyRing()
createKeyDB(
base, serverKeyAPI,
)
rsAPI := roomserver.SetupRoomServerComponent(
&base.Base, keyRing, federation,
@ -158,17 +154,17 @@ func main() {
&base.Base, accountDB, deviceDB, federation, rsAPI, transactions.New(),
)
fsAPI := federationsender.SetupFederationSenderComponent(
&base.Base, federation, rsAPI, &keyRing,
&base.Base, federation, rsAPI, keyRing,
)
rsAPI.SetFederationSenderAPI(fsAPI)
clientapi.SetupClientAPIComponent(
&base.Base, deviceDB, accountDB,
federation, &keyRing, rsAPI,
federation, keyRing, rsAPI,
eduInputAPI, asAPI, transactions.New(), fsAPI,
)
eduProducer := producers.NewEDUServerProducer(eduInputAPI)
federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer)
mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB)
publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub)
if err != nil {

3
cmd/dendrite-demo-libp2p/mdnslistener.go
View file

@ -21,12 +21,11 @@ import (
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/matrix-org/dendrite/internal/keydb"
"github.com/matrix-org/gomatrixserverlib"
)
type mDNSListener struct {
keydb keydb.Database
keydb gomatrixserverlib.KeyDatabase
host host.Host
}

9
cmd/dendrite-federation-api-server/main.go
View file

@ -20,7 +20,6 @@ import (
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/internal/basecomponent"
"github.com/matrix-org/dendrite/internal/keydb"
)
func main() {
@ -30,10 +29,12 @@ func main() {
accountDB := base.CreateAccountsDB()
deviceDB := base.CreateDeviceDB()
keyDB := base.CreateKeyDB()
federation := base.CreateFederationClient()
serverKeyAPI := base.CreateHTTPServerKeyAPIs()
keyRing := serverKeyAPI.KeyRing()
fsAPI := base.CreateHTTPFederationSenderAPIs()
keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
rsAPI := base.CreateHTTPRoomserverAPIs()
asAPI := base.CreateHTTPAppServiceAPIs()
@ -42,7 +43,7 @@ func main() {
eduProducer := producers.NewEDUServerProducer(eduInputAPI)
federationapi.SetupFederationAPIComponent(
base, accountDB, deviceDB, federation, &keyRing,
base, accountDB, deviceDB, federation, keyRing,
rsAPI, asAPI, fsAPI, eduProducer,
)

9
cmd/dendrite-federation-sender-server/main.go
View file

@ -17,7 +17,6 @@ package main
import (
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal/basecomponent"
"github.com/matrix-org/dendrite/internal/keydb"
)
func main() {
@ -26,11 +25,13 @@ func main() {
defer base.Close() // nolint: errcheck
federation := base.CreateFederationClient()
keyDB := base.CreateKeyDB()
keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
serverKeyAPI := base.CreateHTTPServerKeyAPIs()
keyRing := serverKeyAPI.KeyRing()
rsAPI := base.CreateHTTPRoomserverAPIs()
fsAPI := federationsender.SetupFederationSenderComponent(
base, federation, rsAPI, &keyRing,
base, federation, rsAPI, keyRing,
)
rsAPI.SetFederationSenderAPI(fsAPI)

19
cmd/dendrite-monolith-server/main.go
View file

@ -28,13 +28,13 @@ import (
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/basecomponent"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/keydb"
"github.com/matrix-org/dendrite/internal/transactions"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/mediaapi"
"github.com/matrix-org/dendrite/publicroomsapi"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/serverkeyapi"
"github.com/matrix-org/dendrite/syncapi"
"github.com/sirupsen/logrus"
@ -60,6 +60,7 @@ func main() {
cfg.Listen.EDUServer = addr
cfg.Listen.AppServiceAPI = addr
cfg.Listen.FederationSender = addr
cfg.Listen.ServerKeyAPI = addr
}
base := basecomponent.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs)
@ -67,9 +68,15 @@ func main() {
accountDB := base.CreateAccountsDB()
deviceDB := base.CreateDeviceDB()
keyDB := base.CreateKeyDB()
federation := base.CreateFederationClient()
keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent(
base, federation,
)
if base.EnableHTTPAPIs {
serverKeyAPI = base.CreateHTTPServerKeyAPIs()
}
keyRing := serverKeyAPI.KeyRing()
rsComponent := roomserver.SetupRoomServerComponent(
base, keyRing, federation,
@ -94,7 +101,7 @@ func main() {
}
fsAPI := federationsender.SetupFederationSenderComponent(
base, federation, rsAPI, &keyRing,
base, federation, rsAPI, keyRing,
)
if base.EnableHTTPAPIs {
fsAPI = base.CreateHTTPFederationSenderAPIs()
@ -103,7 +110,7 @@ func main() {
clientapi.SetupClientAPIComponent(
base, deviceDB, accountDB,
federation, &keyRing, rsAPI,
federation, keyRing, rsAPI,
eduInputAPI, asAPI, transactions.New(), fsAPI,
)
@ -111,7 +118,7 @@ func main() {
base, deviceDB, accountDB,
)
eduProducer := producers.NewEDUServerProducer(eduInputAPI)
federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer)
federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer)
mediaapi.SetupMediaAPIComponent(base, deviceDB)
publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties())
if err != nil {

6
cmd/dendrite-room-server/main.go
View file

@ -16,7 +16,6 @@ package main
import (
"github.com/matrix-org/dendrite/internal/basecomponent"
"github.com/matrix-org/dendrite/internal/keydb"
"github.com/matrix-org/dendrite/roomserver"
)
@ -24,9 +23,10 @@ func main() {
cfg := basecomponent.ParseFlags()
base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI", true)
defer base.Close() // nolint: errcheck
keyDB := base.CreateKeyDB()
federation := base.CreateFederationClient()
keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
serverKeyAPI := base.CreateHTTPServerKeyAPIs()
keyRing := serverKeyAPI.KeyRing()
fsAPI := base.CreateHTTPFederationSenderAPIs()
rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation)

32
cmd/dendrite-server-key-api-server/main.go Normal file
View file

@ -0,0 +1,32 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"github.com/matrix-org/dendrite/internal/basecomponent"
"github.com/matrix-org/dendrite/serverkeyapi"
)
func main() {
cfg := basecomponent.ParseFlags()
base := basecomponent.NewBaseDendrite(cfg, "ServerKeyAPI", true)
defer base.Close() // nolint: errcheck
federation := base.CreateFederationClient()
serverkeyapi.SetupServerKeyAPIComponent(base, federation)
base.SetupAndServeHTTP(string(base.Cfg.Bind.ServerKeyAPI), string(base.Cfg.Listen.ServerKeyAPI))
}

8
cmd/dendritejs/main.go
View file

@ -37,6 +37,7 @@ import (
"github.com/matrix-org/dendrite/publicroomsapi"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/serverkeyapi"
"github.com/matrix-org/dendrite/syncapi"
go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
"github.com/matrix-org/gomatrixserverlib"
@ -194,13 +195,16 @@ func main() {
accountDB := base.CreateAccountsDB()
deviceDB := base.CreateDeviceDB()
keyDB := base.CreateKeyDB()
federation := createFederationClient(cfg, node)
serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent(
base, federation,
)
keyRing := gomatrixserverlib.KeyRing{
KeyFetchers: []gomatrixserverlib.KeyFetcher{
&libp2pKeyFetcher{},
},
KeyDatabase: keyDB,
KeyDatabase: serverKeyAPI,
}
p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node)

1
dendrite-config.yaml
View file

@ -138,6 +138,7 @@ listen:
appservice_api: "localhost:7777"
edu_server: "localhost:7778"
key_server: "localhost:7779"
server_key_api: "localhost:7780"
# The configuration for tracing the dendrite components.
tracing:

2
go.mod
View file

@ -18,7 +18,7 @@ require (
github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b
github.com/matrix-org/gomatrixserverlib v0.0.0-20200521102632-2a81324a04ae
github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
github.com/mattn/go-sqlite3 v2.0.2+incompatible

4
go.sum
View file

@ -358,8 +358,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf
github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4=
github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b h1:nAmSc1KvQOumoRTz/LD68KyrB6Q5/6q7CmQ5Bswc2nM=
github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
github.com/matrix-org/gomatrixserverlib v0.0.0-20200521102632-2a81324a04ae h1:kFMh2aU3pMCkVCUeH57rtgm05XImbxKOHFYeUp80RCk=
github.com/matrix-org/gomatrixserverlib v0.0.0-20200521102632-2a81324a04ae/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=

40
internal/basecomponent/base.go
View file

@ -21,12 +21,8 @@ import (
"net/url"
"time"
"golang.org/x/crypto/ed25519"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/httpapis"
"github.com/matrix-org/dendrite/internal/keydb"
"github.com/matrix-org/dendrite/internal/keydb/cache"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/naffka"
@ -43,6 +39,7 @@ import (
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api"
"github.com/sirupsen/logrus"
_ "net/http/pprof"
@ -157,6 +154,20 @@ func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.Fede
return f
}
// CreateHTTPServerKeyAPIs returns ServerKeyInternalAPI for hitting the server key
// API over HTTP
func (b *BaseDendrite) CreateHTTPServerKeyAPIs() serverKeyAPI.ServerKeyInternalAPI {
f, err := serverKeyAPI.NewServerKeyInternalAPIHTTP(
b.Cfg.ServerKeyAPIURL(),
b.httpClient,
b.ImmutableCache,
)
if err != nil {
logrus.WithError(err).Panic("NewServerKeyInternalAPIHTTP failed", b.httpClient)
}
return f
}
// CreateDeviceDB creates a new instance of the device database. Should only be
// called once per component.
func (b *BaseDendrite) CreateDeviceDB() devices.Database {
@ -179,27 +190,6 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
return db
}
// CreateKeyDB creates a new instance of the key database. Should only be called
// once per component.
func (b *BaseDendrite) CreateKeyDB() keydb.Database {
db, err := keydb.NewDatabase(
string(b.Cfg.Database.ServerKey),
b.Cfg.DbProperties(),
b.Cfg.Matrix.ServerName,
b.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
b.Cfg.Matrix.KeyID,
)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to keys db")
}
cachedDB, err := cache.NewKeyDatabase(db, b.ImmutableCache)
if err != nil {
logrus.WithError(err).Panicf("failed to create key cache wrapper")
}
return cachedDB
}
// CreateFederationClient creates a new federation client. Should only be called
// once per component.
func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient {

12
internal/config/config.go
View file

@ -223,6 +223,7 @@ type Dendrite struct {
MediaAPI Address `yaml:"media_api"`
ClientAPI Address `yaml:"client_api"`
FederationAPI Address `yaml:"federation_api"`
ServerKeyAPI Address `yaml:"server_key_api"`
AppServiceAPI Address `yaml:"appservice_api"`
SyncAPI Address `yaml:"sync_api"`
RoomServer Address `yaml:"room_server"`
@ -237,6 +238,7 @@ type Dendrite struct {
MediaAPI Address `yaml:"media_api"`
ClientAPI Address `yaml:"client_api"`
FederationAPI Address `yaml:"federation_api"`
ServerKeyAPI Address `yaml:"server_key_api"`
AppServiceAPI Address `yaml:"appservice_api"`
SyncAPI Address `yaml:"sync_api"`
RoomServer Address `yaml:"room_server"`
@ -620,6 +622,7 @@ func (config *Dendrite) checkListen(configErrs *configErrors) {
checkNotEmpty(configErrs, "listen.sync_api", string(config.Listen.SyncAPI))
checkNotEmpty(configErrs, "listen.room_server", string(config.Listen.RoomServer))
checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer))
checkNotEmpty(configErrs, "listen.server_key_api", string(config.Listen.EDUServer))
}
// checkLogging verifies the parameters logging.* are valid.
@ -751,6 +754,15 @@ func (config *Dendrite) FederationSenderURL() string {
return "http://" + string(config.Listen.FederationSender)
}
// FederationSenderURL returns an HTTP URL for where the federation sender is listening.
func (config *Dendrite) ServerKeyAPIURL() string {
// Hard code the server key API server to talk HTTP for now.
// If we support HTTPS we need to think of a practical way to do certificate validation.
// People setting up servers shouldn't need to get a certificate valid for the public
// internet for an internal API.
return "http://" + string(config.Listen.ServerKeyAPI)
}
// SetupTracing configures the opentracing using the supplied configuration.
func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) {
if !config.Tracing.Enabled {

74
internal/keydb/keyring.go
View file

@ -1,74 +0,0 @@
// Copyright 2017 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package keydb
import (
"encoding/base64"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/ed25519"
)
// CreateKeyRing creates and configures a KeyRing object.
//
// It creates the necessary key fetchers and collects them into a KeyRing
// backed by the given KeyDatabase.
func CreateKeyRing(client gomatrixserverlib.Client,
keyDB gomatrixserverlib.KeyDatabase,
cfg config.KeyPerspectives) gomatrixserverlib.KeyRing {
fetchers := gomatrixserverlib.KeyRing{
KeyFetchers: []gomatrixserverlib.KeyFetcher{
&gomatrixserverlib.DirectKeyFetcher{
Client: client,
},
},
KeyDatabase: keyDB,
}
logrus.Info("Enabled direct key fetcher")
var b64e = base64.StdEncoding.WithPadding(base64.NoPadding)
for _, ps := range cfg {
perspective := &gomatrixserverlib.PerspectiveKeyFetcher{
PerspectiveServerName: ps.ServerName,
PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{},
Client: client,
}
for _, key := range ps.Keys {
rawkey, err := b64e.DecodeString(key.PublicKey)
if err != nil {
logrus.WithError(err).WithFields(logrus.Fields{
"server_name": ps.ServerName,
"public_key": key.PublicKey,
}).Warn("Couldn't parse perspective key")
continue
}
perspective.PerspectiveServerKeys[key.KeyID] = rawkey
}
fetchers.KeyFetchers = append(fetchers.KeyFetchers, perspective)
logrus.WithFields(logrus.Fields{
"server_name": ps.ServerName,
"num_public_keys": len(ps.Keys),
}).Info("Enabled perspective key fetcher")
}
return fetchers
}

113
serverkeyapi/api/api.go Normal file
View file

@ -0,0 +1,113 @@
package api
import (
"context"
"errors"
"net/http"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/gomatrixserverlib"
)
type ServerKeyInternalAPI interface {
gomatrixserverlib.KeyDatabase
KeyRing() *gomatrixserverlib.KeyRing
InputPublicKeys(
ctx context.Context,
request *InputPublicKeysRequest,
response *InputPublicKeysResponse,
) error
QueryPublicKeys(
ctx context.Context,
request *QueryPublicKeysRequest,
response *QueryPublicKeysResponse,
) error
}
// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API.
// If httpClient is nil an error is returned
func NewServerKeyInternalAPIHTTP(
serverKeyAPIURL string,
httpClient *http.Client,
immutableCache caching.ImmutableCache,
) (ServerKeyInternalAPI, error) {
if httpClient == nil {
return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is <nil>")
}
return &httpServerKeyInternalAPI{
serverKeyAPIURL: serverKeyAPIURL,
httpClient: httpClient,
immutableCache: immutableCache,
}, nil
}
type httpServerKeyInternalAPI struct {
ServerKeyInternalAPI
serverKeyAPIURL string
httpClient *http.Client
immutableCache caching.ImmutableCache
}
func (s *httpServerKeyInternalAPI) KeyRing() *gomatrixserverlib.KeyRing {
// This is a bit of a cheat - we tell gomatrixserverlib that this API is
// both the key database and the key fetcher. While this does have the
// rather unfortunate effect of preventing gomatrixserverlib from handling
// key fetchers directly, we can at least reimplement this behaviour on
// the other end of the API.
return &gomatrixserverlib.KeyRing{
KeyDatabase: s,
KeyFetchers: []gomatrixserverlib.KeyFetcher{s},
}
}
func (s *httpServerKeyInternalAPI) FetcherName() string {
return "httpServerKeyInternalAPI"
}
func (s *httpServerKeyInternalAPI) StoreKeys(
ctx context.Context,
results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult,
) error {
request := InputPublicKeysRequest{
Keys: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult),
}
response := InputPublicKeysResponse{}
for req, res := range results {
request.Keys[req] = res
s.immutableCache.StoreServerKey(req, res)
}
return s.InputPublicKeys(ctx, &request, &response)
}
func (s *httpServerKeyInternalAPI) FetchKeys(
ctx context.Context,
requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) {
result := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult)
request := QueryPublicKeysRequest{
Requests: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp),
}
response := QueryPublicKeysResponse{
Results: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult),
}
for req, ts := range requests {
if res, ok := s.immutableCache.GetServerKey(req); ok {
result[req] = res
continue
}
request.Requests[req] = ts
}
err := s.QueryPublicKeys(ctx, &request, &response)
if err != nil {
return nil, err
}
for req, res := range response.Results {
result[req] = res
s.immutableCache.StoreServerKey(req, res)
}
return result, nil
}

57
serverkeyapi/api/http.go Normal file
View file

@ -0,0 +1,57 @@
package api
import (
"context"
commonHTTP "github.com/matrix-org/dendrite/internal/http"
"github.com/matrix-org/gomatrixserverlib"
"github.com/opentracing/opentracing-go"
)
const (
// ServerKeyInputPublicKeyPath is the HTTP path for the InputPublicKeys API.
ServerKeyInputPublicKeyPath = "/serverkeyapi/inputPublicKey"
// ServerKeyQueryPublicKeyPath is the HTTP path for the QueryPublicKeys API.
ServerKeyQueryPublicKeyPath = "/serverkeyapi/queryPublicKey"
)
type InputPublicKeysRequest struct {
Keys map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"keys"`
}
type InputPublicKeysResponse struct {
}
func (h *httpServerKeyInternalAPI) InputPublicKeys(
ctx context.Context,
request *InputPublicKeysRequest,
response *InputPublicKeysResponse,
) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "InputPublicKey")
defer span.Finish()
apiURL := h.serverKeyAPIURL + ServerKeyInputPublicKeyPath
return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
}
type QueryPublicKeysRequest struct {
Requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp `json:"requests"`
}
type QueryPublicKeysResponse struct {
Results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"results"`
}
func (h *httpServerKeyInternalAPI) QueryPublicKeys(
ctx context.Context,
request *QueryPublicKeysRequest,
response *QueryPublicKeysResponse,
) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublicKey")
defer span.Finish()
apiURL := h.serverKeyAPIURL + ServerKeyQueryPublicKeyPath
return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
}

76
serverkeyapi/internal/api.go Normal file
View file

@ -0,0 +1,76 @@
package internal
import (
"context"
"fmt"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/serverkeyapi/api"
"github.com/matrix-org/gomatrixserverlib"
)
type ServerKeyAPI struct {
api.ServerKeyInternalAPI
ImmutableCache caching.ImmutableCache
OurKeyRing gomatrixserverlib.KeyRing
FedClient *gomatrixserverlib.FederationClient
}
func (s *ServerKeyAPI) KeyRing() *gomatrixserverlib.KeyRing {
// Return a real keyring - one that has the real database and real
// fetchers.
return &s.OurKeyRing
}
func (s *ServerKeyAPI) StoreKeys(
ctx context.Context,
results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult,
) error {
// Store any keys that we were given in our database.
return s.OurKeyRing.KeyDatabase.StoreKeys(ctx, results)
}
func (s *ServerKeyAPI) FetchKeys(
ctx context.Context,
requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp,
) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) {
results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{}
// First consult our local database and see if we have the requested
// keys. These might come from a cache, depending on the database
// implementation used.
if dbResults, err := s.OurKeyRing.KeyDatabase.FetchKeys(ctx, requests); err == nil {
// We successfully got some keys. Add them to the results and
// remove them from the request list.
for req, res := range dbResults {
results[req] = res
delete(requests, req)
}
}
// For any key requests that we still have outstanding, next try to
// fetch them directly. We'll go through each of the key fetchers to
// ask for the remaining keys.
for _, fetcher := range s.OurKeyRing.KeyFetchers {
if len(requests) == 0 {
break
}
if fetcherResults, err := fetcher.FetchKeys(ctx, requests); err == nil {
// We successfully got some keys. Add them to the results and
// remove them from the request list.
for req, res := range fetcherResults {
results[req] = res
delete(requests, req)
}
}
}
// If we failed to fetch any keys then we should report an error.
if len(requests) > 0 {
return results, fmt.Errorf("server key API failed to fetch %d keys", len(requests))
}
// Return the keys.
return results, nil
}
func (s *ServerKeyAPI) FetcherName() string {
return s.OurKeyRing.KeyDatabase.FetcherName()
}

60
serverkeyapi/internal/http.go Normal file
View file

@ -0,0 +1,60 @@
package internal
import (
"encoding/json"
"net/http"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/serverkeyapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
func (s *ServerKeyAPI) SetupHTTP(internalAPIMux *mux.Router) {
internalAPIMux.Handle(api.ServerKeyQueryPublicKeyPath,
internal.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse {
result := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{}
request := api.QueryPublicKeysRequest{}
response := api.QueryPublicKeysResponse{}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
lookup := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp)
for req, timestamp := range request.Requests {
if res, ok := s.ImmutableCache.GetServerKey(req); ok {
result[req] = res
continue
}
lookup[req] = timestamp
}
keys, err := s.FetchKeys(req.Context(), lookup)
if err != nil {
return util.ErrorResponse(err)
}
for req, res := range keys {
result[req] = res
}
response.Results = result
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
internalAPIMux.Handle(api.ServerKeyInputPublicKeyPath,
internal.MakeInternalAPI("inputPublicKeys", func(req *http.Request) util.JSONResponse {
request := api.InputPublicKeysRequest{}
response := api.InputPublicKeysResponse{}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
store := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult)
for req, res := range request.Keys {
store[req] = res
s.ImmutableCache.StoreServerKey(req, res)
}
if err := s.StoreKeys(req.Context(), store); err != nil {
return util.ErrorResponse(err)
}
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
}

83
serverkeyapi/serverkeyapi.go Normal file
View file

@ -0,0 +1,83 @@
package serverkeyapi
import (
"crypto/ed25519"
"encoding/base64"
"github.com/matrix-org/dendrite/internal/basecomponent"
"github.com/matrix-org/dendrite/serverkeyapi/api"
"github.com/matrix-org/dendrite/serverkeyapi/internal"
"github.com/matrix-org/dendrite/serverkeyapi/storage"
"github.com/matrix-org/dendrite/serverkeyapi/storage/cache"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
)
func SetupServerKeyAPIComponent(
base *basecomponent.BaseDendrite,
fedClient *gomatrixserverlib.FederationClient,
) api.ServerKeyInternalAPI {
innerDB, err := storage.NewDatabase(
string(base.Cfg.Database.ServerKey),
base.Cfg.DbProperties(),
base.Cfg.Matrix.ServerName,
base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
base.Cfg.Matrix.KeyID,
)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to server key database")
}
serverKeyDB, err := cache.NewKeyDatabase(innerDB, base.ImmutableCache)
if err != nil {
logrus.WithError(err).Panicf("failed to set up caching wrapper for server key database")
}
internalAPI := internal.ServerKeyAPI{
ImmutableCache: base.ImmutableCache,
FedClient: fedClient,
OurKeyRing: gomatrixserverlib.KeyRing{
KeyFetchers: []gomatrixserverlib.KeyFetcher{
&gomatrixserverlib.DirectKeyFetcher{
Client: fedClient.Client,
},
},
KeyDatabase: serverKeyDB,
},
}
var b64e = base64.StdEncoding.WithPadding(base64.NoPadding)
for _, ps := range base.Cfg.Matrix.KeyPerspectives {
perspective := &gomatrixserverlib.PerspectiveKeyFetcher{
PerspectiveServerName: ps.ServerName,
PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{},
Client: fedClient.Client,
}
for _, key := range ps.Keys {
rawkey, err := b64e.DecodeString(key.PublicKey)
if err != nil {
logrus.WithError(err).WithFields(logrus.Fields{
"server_name": ps.ServerName,
"public_key": key.PublicKey,
}).Warn("Couldn't parse perspective key")
continue
}
perspective.PerspectiveServerKeys[key.KeyID] = rawkey
}
internalAPI.OurKeyRing.KeyFetchers = append(
internalAPI.OurKeyRing.KeyFetchers,
perspective,
)
logrus.WithFields(logrus.Fields{
"server_name": ps.ServerName,
"num_public_keys": len(ps.Keys),
}).Info("Enabled perspective key fetcher")
}
internalAPI.SetupHTTP(base.InternalAPIMux)
return &internalAPI
}

5
internal/keydb/cache/keydb.go → serverkeyapi/storage/cache/keydb.go vendored
View file

@ -5,18 +5,17 @@ import (
"errors"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/keydb"
"github.com/matrix-org/gomatrixserverlib"
)
// A Database implements gomatrixserverlib.KeyDatabase and is used to store
// the public keys for other matrix servers.
type KeyDatabase struct {
inner keydb.Database
inner gomatrixserverlib.KeyDatabase
cache caching.ImmutableCache
}
func NewKeyDatabase(inner keydb.Database, cache caching.ImmutableCache) (*KeyDatabase, error) {
func NewKeyDatabase(inner gomatrixserverlib.KeyDatabase, cache caching.ImmutableCache) (*KeyDatabase, error) {
if inner == nil {
return nil, errors.New("inner database can't be nil")
}

2
internal/keydb/interface.go → serverkeyapi/storage/interface.go
View file

@ -1,4 +1,4 @@
package keydb
package storage
import (
"context"

6
internal/keydb/keydb.go → serverkeyapi/storage/keydb.go
View file

@ -14,7 +14,7 @@
// +build !wasm
package keydb
package storage
import (
"net/url"
@ -22,8 +22,8 @@ import (
"golang.org/x/crypto/ed25519"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/keydb/postgres"
"github.com/matrix-org/dendrite/internal/keydb/sqlite3"
"github.com/matrix-org/dendrite/serverkeyapi/storage/postgres"
"github.com/matrix-org/dendrite/serverkeyapi/storage/sqlite3"
"github.com/matrix-org/gomatrixserverlib"
)

8
internal/keydb/keydb_wasm.go → serverkeyapi/storage/keydb_wasm.go
View file

@ -12,7 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package keydb
// +build wasm
package storage
import (
"fmt"
@ -21,7 +23,7 @@ import (
"golang.org/x/crypto/ed25519"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/keydb/sqlite3"
"github.com/matrix-org/dendrite/serverkeyapi/storage/sqlite3"
"github.com/matrix-org/gomatrixserverlib"
)
@ -41,7 +43,7 @@ func NewDatabase(
case "postgres":
return nil, fmt.Errorf("Cannot use postgres implementation")
case "file":
return sqlite3.NewDatabase(uri.Path, serverName, serverKey, serverKeyID)
return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID)
default:
return nil, fmt.Errorf("Cannot use postgres implementation")
}

0
internal/keydb/postgres/keydb.go → serverkeyapi/storage/postgres/keydb.go
View file

0
internal/keydb/postgres/server_key_table.go → serverkeyapi/storage/postgres/server_key_table.go
View file

0
internal/keydb/sqlite3/keydb.go → serverkeyapi/storage/sqlite3/keydb.go
View file

0
internal/keydb/sqlite3/server_key_table.go → serverkeyapi/storage/sqlite3/server_key_table.go
View file