From fa1ec482a7bd35d1cb900f727edd0dcbb1d7c45c Mon Sep 17 00:00:00 2001 From: Meenal Trivedi <79007582+meenal06@users.noreply.github.com> Date: Mon, 26 Jul 2021 15:11:58 +0530 Subject: [PATCH 01/64] fix:Inviting to an unsupported room version return M_BAD_JSON instead of Incompatible_Version (#1930) * fix:Inviting to an unsupported room version return M_BAD_JSON instead of M_UNSUPPORTED_ROOM_VERSION Signed-off-by: Meenal Trivedi * fix Signed-off-by: Meenal Trivedi * fix Signed-off-by: Meenal Trivedi * feat: make requested changes Signed-off-by: Meenal Trivedi * Use error typecast from matrix-org/gomatrixserverlib#272 Co-authored-by: Neil Alexander --- federationapi/routing/invite.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 8795118e..46865965 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -40,22 +40,29 @@ func InviteV2( ) util.JSONResponse { inviteReq := gomatrixserverlib.InviteV2Request{} err := json.Unmarshal(request.Content(), &inviteReq) - switch err.(type) { + switch e := err.(type) { + case gomatrixserverlib.UnsupportedRoomVersionError: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion( + fmt.Sprintf("Room version %q is not supported by this server.", e.Version), + ), + } case gomatrixserverlib.BadJSONError: return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.BadJSON(err.Error()), } case nil: + return processInvite( + httpReq.Context(), true, inviteReq.Event(), inviteReq.RoomVersion(), inviteReq.InviteRoomState(), roomID, eventID, cfg, rsAPI, keys, + ) default: return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.NotJSON("The request body could not be decoded into an invite request. " + err.Error()), } } - return processInvite( - httpReq.Context(), true, inviteReq.Event(), inviteReq.RoomVersion(), inviteReq.InviteRoomState(), roomID, eventID, cfg, rsAPI, keys, - ) } // InviteV1 implements /_matrix/federation/v1/invite/{roomID}/{eventID} From 16bf94f2391b5334fe2ff8eae52f2f5b46951f4e Mon Sep 17 00:00:00 2001 From: kegsay Date: Mon, 26 Jul 2021 12:30:44 +0100 Subject: [PATCH 02/64] Not finding the snapshot is not fatal (#1940) --- .../postgres/deltas/2021041615092700_state_blocks_refactor.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go index 26d88e00..940a920e 100644 --- a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go +++ b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go @@ -174,6 +174,9 @@ func UpStateBlocksRefactor(tx *sql.Tx) error { err = tx.QueryRow( `SELECT event_nid FROM roomserver_events WHERE state_snapshot_nid = $1 AND event_type_nid = 1`, s.StateSnapshotNID, ).Scan(&createEventNID) + if err == sql.ErrNoRows { + continue + } if err != nil { return fmt.Errorf("cannot xref null state block with snapshot %d: %s", s.StateSnapshotNID, err) } From e3679799ea6f6387d36fb1a2f938f8ea9711ad2d Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 26 Jul 2021 12:52:11 +0100 Subject: [PATCH 03/64] Version 0.4.1 --- CHANGES.md | 20 ++++++++++++++++++++ internal/version.go | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 27356b3c..8207d484 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,25 @@ # Changelog +## Dendrite 0.4.1 (2021-07-26) + +### Features + +* Support for room version 7 has been added +* Key notary support is now more complete, allowing Dendrite to be used as a notary server for looking up signing keys +* State resolution v2 performance has been optimised further by caching the create event, power levels and join rules in memory instead of parsing them repeatedly +* The media API now handles cases where the maximum file size is configured to be less than 0 for unlimited size +* The `initial_state` in a `/createRoom` request is now respected when creating a room +* Code paths for checking if servers are joined to rooms have been optimised significantly + +### Fixes + +* A bug resulting in `cannot xref null state block with snapshot` during the new state storage migration has been fixed +* Invites are now retired correctly when rejecting an invite from a remote server which is no longer reachable +* The DNS cache `cache_lifetime` option is now handled correctly (contributed by [S7evinK](https://github.com/S7evinK)) +* Invalid events in a room join response are now dropped correctly, rather than failing the entire join +* The `prev_state` of an event will no longer be populated incorrectly to the state of the current event +* Receiving an invite to an unsupported room version will now correctly return the `M_UNSUPPORTED_ROOM_VERSION` error code instead of `M_BAD_JSON` (contributed by [meenal06](https://github.com/meenal06)) + ## Dendrite 0.4.0 (2021-07-12) ### Features diff --git a/internal/version.go b/internal/version.go index 37f0c30d..55997ffc 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 4 - VersionPatch = 0 + VersionPatch = 1 VersionTag = "" // example: "rc1" ) From 32538640db6bfbdb890729e8137151ffbbec9a28 Mon Sep 17 00:00:00 2001 From: kegsay Date: Tue, 27 Jul 2021 12:47:32 +0100 Subject: [PATCH 04/64] Key backups (1/2) : Add E2E session backup metadata tables (#1943) * Initial key backup paths and userapi API * Fix unit tests * Add key backup table * Glue REST API to database * Linting * use writer on sqlite --- clientapi/routing/key_backup.go | 173 ++++++++++++++++++ clientapi/routing/routing.go | 42 +++++ setup/mscs/msc2836/msc2836_test.go | 4 + setup/mscs/msc2946/msc2946_test.go | 4 + sytest-whitelist | 1 + userapi/api/api.go | 33 ++++ userapi/internal/api.go | 54 ++++++ userapi/inthttp/client.go | 23 +++ userapi/storage/accounts/interface.go | 6 + .../postgres/key_backup_version_table.go | 144 +++++++++++++++ userapi/storage/accounts/postgres/storage.go | 43 +++++ .../sqlite3/key_backup_version_table.go | 142 ++++++++++++++ userapi/storage/accounts/sqlite3/storage.go | 43 +++++ 13 files changed, 712 insertions(+) create mode 100644 clientapi/routing/key_backup.go create mode 100644 userapi/storage/accounts/postgres/key_backup_version_table.go create mode 100644 userapi/storage/accounts/sqlite3/key_backup_version_table.go diff --git a/clientapi/routing/key_backup.go b/clientapi/routing/key_backup.go new file mode 100644 index 00000000..0aa73624 --- /dev/null +++ b/clientapi/routing/key_backup.go @@ -0,0 +1,173 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/util" +) + +type keyBackupVersion struct { + Algorithm string `json:"algorithm"` + AuthData json.RawMessage `json:"auth_data"` +} + +type keyBackupVersionCreateResponse struct { + Version string `json:"version"` +} + +type keyBackupVersionResponse struct { + Algorithm string `json:"algorithm"` + AuthData json.RawMessage `json:"auth_data"` + Count int `json:"count"` + ETag string `json:"etag"` + Version string `json:"version"` +} + +// Create a new key backup. Request must contain a `keyBackupVersion`. Returns a `keyBackupVersionCreateResponse`. +// Implements POST /_matrix/client/r0/room_keys/version +func CreateKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device) util.JSONResponse { + var kb keyBackupVersion + resErr := httputil.UnmarshalJSONRequest(req, &kb) + if resErr != nil { + return *resErr + } + var performKeyBackupResp userapi.PerformKeyBackupResponse + userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{ + UserID: device.UserID, + Version: "", + AuthData: kb.AuthData, + Algorithm: kb.Algorithm, + }, &performKeyBackupResp) + if performKeyBackupResp.Error != "" { + if performKeyBackupResp.BadInput { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.InvalidArgumentValue(performKeyBackupResp.Error), + } + } + return util.ErrorResponse(fmt.Errorf("PerformKeyBackup: %s", performKeyBackupResp.Error)) + } + return util.JSONResponse{ + Code: 200, + JSON: keyBackupVersionCreateResponse{ + Version: performKeyBackupResp.Version, + }, + } +} + +// KeyBackupVersion returns the key backup version specified. If `version` is empty, the latest `keyBackupVersionResponse` is returned. +// Implements GET /_matrix/client/r0/room_keys/version and GET /_matrix/client/r0/room_keys/version/{version} +func KeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse { + var queryResp userapi.QueryKeyBackupResponse + userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{}, &queryResp) + if queryResp.Error != "" { + return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", queryResp.Error)) + } + if !queryResp.Exists { + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound("version not found"), + } + } + return util.JSONResponse{ + Code: 200, + JSON: keyBackupVersionResponse{ + Algorithm: queryResp.Algorithm, + AuthData: queryResp.AuthData, + Count: queryResp.Count, + ETag: queryResp.ETag, + Version: queryResp.Version, + }, + } +} + +// Modify the auth data of a key backup. Version must not be empty. Request must contain a `keyBackupVersion` +// Implements PUT /_matrix/client/r0/room_keys/version/{version} +func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse { + var kb keyBackupVersion + resErr := httputil.UnmarshalJSONRequest(req, &kb) + if resErr != nil { + return *resErr + } + var performKeyBackupResp userapi.PerformKeyBackupResponse + userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{ + UserID: device.UserID, + Version: version, + AuthData: kb.AuthData, + Algorithm: kb.Algorithm, + }, &performKeyBackupResp) + if performKeyBackupResp.Error != "" { + if performKeyBackupResp.BadInput { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.InvalidArgumentValue(performKeyBackupResp.Error), + } + } + return util.ErrorResponse(fmt.Errorf("PerformKeyBackup: %s", performKeyBackupResp.Error)) + } + if !performKeyBackupResp.Exists { + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound("backup version not found"), + } + } + // Unclear what the 200 body should be + return util.JSONResponse{ + Code: 200, + JSON: keyBackupVersionCreateResponse{ + Version: performKeyBackupResp.Version, + }, + } +} + +// Delete a version of key backup. Version must not be empty. If the key backup was previously deleted, will return 200 OK. +// Implements DELETE /_matrix/client/r0/room_keys/version/{version} +func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse { + var performKeyBackupResp userapi.PerformKeyBackupResponse + userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{ + UserID: device.UserID, + Version: version, + DeleteBackup: true, + }, &performKeyBackupResp) + if performKeyBackupResp.Error != "" { + if performKeyBackupResp.BadInput { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.InvalidArgumentValue(performKeyBackupResp.Error), + } + } + return util.ErrorResponse(fmt.Errorf("PerformKeyBackup: %s", performKeyBackupResp.Error)) + } + if !performKeyBackupResp.Exists { + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound("backup version not found"), + } + } + // Unclear what the 200 body should be + return util.JSONResponse{ + Code: 200, + JSON: keyBackupVersionCreateResponse{ + Version: performKeyBackupResp.Version, + }, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index d768247a..194ab299 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -896,6 +896,48 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) + // Key Backup Versions + r0mux.Handle("/room_keys/version/{versionID}", + httputil.MakeAuthAPI("get_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + version := req.URL.Query().Get("version") + return KeyBackupVersion(req, userAPI, device, version) + }), + ).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/room_keys/version", + httputil.MakeAuthAPI("get_latest_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return KeyBackupVersion(req, userAPI, device, "") + }), + ).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/room_keys/version/{versionID}", + httputil.MakeAuthAPI("put_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + version := req.URL.Query().Get("version") + if version == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.InvalidArgumentValue("version must be specified"), + } + } + return ModifyKeyBackupVersionAuthData(req, userAPI, device, version) + }), + ).Methods(http.MethodPut) + r0mux.Handle("/room_keys/version/{versionID}", + httputil.MakeAuthAPI("delete_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + version := req.URL.Query().Get("version") + if version == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.InvalidArgumentValue("version must be specified"), + } + } + return DeleteKeyBackupVersion(req, userAPI, device, version) + }), + ).Methods(http.MethodDelete) + r0mux.Handle("/room_keys/version", + httputil.MakeAuthAPI("post_new_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return CreateKeyBackupVersion(req, userAPI, device) + }), + ).Methods(http.MethodPost, http.MethodOptions) + // Supplying a device ID is deprecated. r0mux.Handle("/keys/upload/{deviceID}", httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index 79aaebc0..1bbb485c 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -554,6 +554,10 @@ func (u *testUserAPI) QuerySearchProfiles(ctx context.Context, req *userapi.Quer func (u *testUserAPI) QueryOpenIDToken(ctx context.Context, req *userapi.QueryOpenIDTokenRequest, res *userapi.QueryOpenIDTokenResponse) error { return nil } +func (u *testUserAPI) PerformKeyBackup(ctx context.Context, req *userapi.PerformKeyBackupRequest, res *userapi.PerformKeyBackupResponse) { +} +func (u *testUserAPI) QueryKeyBackup(ctx context.Context, req *userapi.QueryKeyBackupRequest, res *userapi.QueryKeyBackupResponse) { +} type testRoomserverAPI struct { // use a trace API as it implements method stubs so we don't need to have them here. diff --git a/setup/mscs/msc2946/msc2946_test.go b/setup/mscs/msc2946/msc2946_test.go index 96160c10..2c195d12 100644 --- a/setup/mscs/msc2946/msc2946_test.go +++ b/setup/mscs/msc2946/msc2946_test.go @@ -373,6 +373,10 @@ func (u *testUserAPI) PerformOpenIDTokenCreation(ctx context.Context, req *usera func (u *testUserAPI) QueryProfile(ctx context.Context, req *userapi.QueryProfileRequest, res *userapi.QueryProfileResponse) error { return nil } +func (u *testUserAPI) PerformKeyBackup(ctx context.Context, req *userapi.PerformKeyBackupRequest, res *userapi.PerformKeyBackupResponse) { +} +func (u *testUserAPI) QueryKeyBackup(ctx context.Context, req *userapi.QueryKeyBackupRequest, res *userapi.QueryKeyBackupResponse) { +} func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error { dev, ok := u.accessTokens[req.AccessToken] if !ok { diff --git a/sytest-whitelist b/sytest-whitelist index d90ba4fb..e2bb41da 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -540,3 +540,4 @@ Key notary server must not overwrite a valid key with a spurious result from the GET /rooms/:room_id/aliases lists aliases Only room members can list aliases of a room Users with sufficient power-level can delete other's aliases +Can create more than 10 backup versions diff --git a/userapi/api/api.go b/userapi/api/api.go index 40735012..ff10bfcd 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -33,6 +33,8 @@ type UserInternalAPI interface { PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error + PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) + QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse) QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error @@ -42,6 +44,37 @@ type UserInternalAPI interface { QueryOpenIDToken(ctx context.Context, req *QueryOpenIDTokenRequest, res *QueryOpenIDTokenResponse) error } +type PerformKeyBackupRequest struct { + UserID string + Version string // optional if modifying a key backup + AuthData json.RawMessage + Algorithm string + DeleteBackup bool // if true will delete the backup based on 'Version'. +} + +type PerformKeyBackupResponse struct { + Error string // set if there was a problem performing the request + BadInput bool // if set, the Error was due to bad input (HTTP 400) + Exists bool // set to true if the Version exists + Version string +} + +type QueryKeyBackupRequest struct { + UserID string + Version string // the version to query, if blank it means the latest +} + +type QueryKeyBackupResponse struct { + Error string + Exists bool + + Algorithm string `json:"algorithm"` + AuthData json.RawMessage `json:"auth_data"` + Count int `json:"count"` + ETag string `json:"etag"` + Version string `json:"version"` +} + // InputAccountDataRequest is the request for InputAccountData type InputAccountDataRequest struct { UserID string // required: the user to set account data for diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 21933c1c..9ff69298 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -442,3 +442,57 @@ func (a *UserInternalAPI) QueryOpenIDToken(ctx context.Context, req *api.QueryOp return nil } + +func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) { + // Delete + if req.DeleteBackup { + if req.Version == "" { + res.BadInput = true + res.Error = "must specify a version to delete" + return + } + exists, err := a.AccountDB.DeleteKeyBackup(ctx, req.UserID, req.Version) + if err != nil { + res.Error = fmt.Sprintf("failed to delete backup: %s", err) + } + res.Exists = exists + res.Version = req.Version + return + } + // Create + if req.Version == "" { + version, err := a.AccountDB.CreateKeyBackup(ctx, req.UserID, req.Algorithm, req.AuthData) + if err != nil { + res.Error = fmt.Sprintf("failed to create backup: %s", err) + } + res.Exists = err == nil + res.Version = version + return + } + // Update + err := a.AccountDB.UpdateKeyBackupAuthData(ctx, req.UserID, req.Version, req.AuthData) + if err != nil { + res.Error = fmt.Sprintf("failed to update backup: %s", err) + } + res.Version = req.Version +} + +func (a *UserInternalAPI) QueryKeyBackup(ctx context.Context, req *api.QueryKeyBackupRequest, res *api.QueryKeyBackupResponse) { + version, algorithm, authData, deleted, err := a.AccountDB.GetKeyBackup(ctx, req.UserID, req.Version) + res.Version = version + if err != nil { + if err == sql.ErrNoRows { + res.Exists = false + return + } + res.Error = fmt.Sprintf("failed to query key backup: %s", err) + return + } + res.Algorithm = algorithm + res.AuthData = authData + res.Exists = !deleted + + // TODO: + res.Count = 0 + res.ETag = "" +} diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go index 1cb5ef0a..a89d1a26 100644 --- a/userapi/inthttp/client.go +++ b/userapi/inthttp/client.go @@ -36,7 +36,9 @@ const ( PerformDeviceUpdatePath = "/userapi/performDeviceUpdate" PerformAccountDeactivationPath = "/userapi/performAccountDeactivation" PerformOpenIDTokenCreationPath = "/userapi/performOpenIDTokenCreation" + PerformKeyBackupPath = "/userapi/performKeyBackup" + QueryKeyBackupPath = "/userapi/queryKeyBackup" QueryProfilePath = "/userapi/queryProfile" QueryAccessTokenPath = "/userapi/queryAccessToken" QueryDevicesPath = "/userapi/queryDevices" @@ -225,3 +227,24 @@ func (h *httpUserInternalAPI) QueryOpenIDToken(ctx context.Context, req *api.Que apiURL := h.apiURL + QueryOpenIDTokenPath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) } + +func (h *httpUserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformKeyBackup") + defer span.Finish() + + apiURL := h.apiURL + PerformKeyBackupPath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) + if err != nil { + res.Error = err.Error() + } +} +func (h *httpUserInternalAPI) QueryKeyBackup(ctx context.Context, req *api.QueryKeyBackupRequest, res *api.QueryKeyBackupResponse) { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryKeyBackup") + defer span.Finish() + + apiURL := h.apiURL + QueryKeyBackupPath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) + if err != nil { + res.Error = err.Error() + } +} diff --git a/userapi/storage/accounts/interface.go b/userapi/storage/accounts/interface.go index 5aa61b90..88fdab48 100644 --- a/userapi/storage/accounts/interface.go +++ b/userapi/storage/accounts/interface.go @@ -54,6 +54,12 @@ type Database interface { DeactivateAccount(ctx context.Context, localpart string) (err error) CreateOpenIDToken(ctx context.Context, token, localpart string) (exp int64, err error) GetOpenIDTokenAttributes(ctx context.Context, token string) (*api.OpenIDTokenAttributes, error) + + // Key backups + CreateKeyBackup(ctx context.Context, userID, algorithm string, authData json.RawMessage) (version string, err error) + UpdateKeyBackupAuthData(ctx context.Context, userID, version string, authData json.RawMessage) (err error) + DeleteKeyBackup(ctx context.Context, userID, version string) (exists bool, err error) + GetKeyBackup(ctx context.Context, userID, version string) (versionResult, algorithm string, authData json.RawMessage, deleted bool, err error) } // Err3PIDInUse is the error returned when trying to save an association involving diff --git a/userapi/storage/accounts/postgres/key_backup_version_table.go b/userapi/storage/accounts/postgres/key_backup_version_table.go new file mode 100644 index 00000000..1b693e56 --- /dev/null +++ b/userapi/storage/accounts/postgres/key_backup_version_table.go @@ -0,0 +1,144 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strconv" +) + +const keyBackupVersionTableSchema = ` +CREATE SEQUENCE IF NOT EXISTS account_e2e_room_keys_versions_seq; + +-- the metadata for each generation of encrypted e2e session backups +CREATE TABLE IF NOT EXISTS account_e2e_room_keys_versions ( + user_id TEXT NOT NULL, + -- this means no 2 users will ever have the same version of e2e session backups which strictly + -- isn't necessary, but this is easy to do rather than SELECT MAX(version)+1. + version BIGINT DEFAULT nextval('account_e2e_room_keys_versions_seq'), + algorithm TEXT NOT NULL, + auth_data TEXT NOT NULL, + deleted SMALLINT DEFAULT 0 NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS account_e2e_room_keys_versions_idx ON account_e2e_room_keys_versions(user_id, version); +` + +const insertKeyBackupSQL = "" + + "INSERT INTO account_e2e_room_keys_versions(user_id, algorithm, auth_data) VALUES ($1, $2, $3) RETURNING version" + +const updateKeyBackupAuthDataSQL = "" + // TODO: do we need to WHERE algorithm = $3 as well? + "UPDATE account_e2e_room_keys_versions SET auth_data = $1 WHERE user_id = $2 AND version = $3" + +const deleteKeyBackupSQL = "" + + "UPDATE account_e2e_room_keys_versions SET deleted=1 WHERE user_id = $1 AND version = $2" + +const selectKeyBackupSQL = "" + + "SELECT algorithm, auth_data, deleted FROM account_e2e_room_keys_versions WHERE user_id = $1 AND version = $2" + +const selectLatestVersionSQL = "" + + "SELECT MAX(version) FROM account_e2e_room_keys_versions WHERE user_id = $1" + +type keyBackupVersionStatements struct { + insertKeyBackupStmt *sql.Stmt + updateKeyBackupAuthDataStmt *sql.Stmt + deleteKeyBackupStmt *sql.Stmt + selectKeyBackupStmt *sql.Stmt + selectLatestVersionStmt *sql.Stmt +} + +func (s *keyBackupVersionStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(keyBackupVersionTableSchema) + if err != nil { + return + } + if s.insertKeyBackupStmt, err = db.Prepare(insertKeyBackupSQL); err != nil { + return + } + if s.updateKeyBackupAuthDataStmt, err = db.Prepare(updateKeyBackupAuthDataSQL); err != nil { + return + } + if s.deleteKeyBackupStmt, err = db.Prepare(deleteKeyBackupSQL); err != nil { + return + } + if s.selectKeyBackupStmt, err = db.Prepare(selectKeyBackupSQL); err != nil { + return + } + if s.selectLatestVersionStmt, err = db.Prepare(selectLatestVersionSQL); err != nil { + return + } + return +} + +func (s *keyBackupVersionStatements) insertKeyBackup( + ctx context.Context, txn *sql.Tx, userID, algorithm string, authData json.RawMessage, +) (version string, err error) { + var versionInt int64 + err = txn.Stmt(s.insertKeyBackupStmt).QueryRowContext(ctx, userID, algorithm, string(authData)).Scan(&versionInt) + return strconv.FormatInt(versionInt, 10), err +} + +func (s *keyBackupVersionStatements) updateKeyBackupAuthData( + ctx context.Context, txn *sql.Tx, userID, version string, authData json.RawMessage, +) error { + versionInt, err := strconv.ParseInt(version, 10, 64) + if err != nil { + return fmt.Errorf("invalid version") + } + _, err = txn.Stmt(s.updateKeyBackupAuthDataStmt).ExecContext(ctx, string(authData), userID, versionInt) + return err +} + +func (s *keyBackupVersionStatements) deleteKeyBackup( + ctx context.Context, txn *sql.Tx, userID, version string, +) (bool, error) { + versionInt, err := strconv.ParseInt(version, 10, 64) + if err != nil { + return false, fmt.Errorf("invalid version") + } + result, err := txn.Stmt(s.deleteKeyBackupStmt).ExecContext(ctx, userID, versionInt) + if err != nil { + return false, err + } + ra, err := result.RowsAffected() + if err != nil { + return false, err + } + return ra == 1, nil +} + +func (s *keyBackupVersionStatements) selectKeyBackup( + ctx context.Context, txn *sql.Tx, userID, version string, +) (versionResult, algorithm string, authData json.RawMessage, deleted bool, err error) { + var versionInt int64 + if version == "" { + err = txn.Stmt(s.selectLatestVersionStmt).QueryRowContext(ctx, userID).Scan(&versionInt) + } else { + versionInt, err = strconv.ParseInt(version, 10, 64) + } + if err != nil { + return + } + versionResult = strconv.FormatInt(versionInt, 10) + var deletedInt int + var authDataStr string + err = txn.Stmt(s.selectKeyBackupStmt).QueryRowContext(ctx, userID, versionInt).Scan(&algorithm, &authDataStr, &deletedInt) + deleted = deletedInt == 1 + authData = json.RawMessage(authDataStr) + return +} diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index c5e74ed1..719e9878 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -45,6 +45,7 @@ type Database struct { accountDatas accountDataStatements threepids threepidStatements openIDTokens tokenStatements + keyBackups keyBackupVersionStatements serverName gomatrixserverlib.ServerName bcryptCost int openIDTokenLifetimeMS int64 @@ -93,6 +94,9 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err = d.openIDTokens.prepare(db, serverName); err != nil { return nil, err } + if err = d.keyBackups.prepare(db); err != nil { + return nil, err + } return d, nil } @@ -368,3 +372,42 @@ func (d *Database) GetOpenIDTokenAttributes( ) (*api.OpenIDTokenAttributes, error) { return d.openIDTokens.selectOpenIDTokenAtrributes(ctx, token) } + +func (d *Database) CreateKeyBackup( + ctx context.Context, userID, algorithm string, authData json.RawMessage, +) (version string, err error) { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + version, err = d.keyBackups.insertKeyBackup(ctx, txn, userID, algorithm, authData) + return err + }) + return +} + +func (d *Database) UpdateKeyBackupAuthData( + ctx context.Context, userID, version string, authData json.RawMessage, +) (err error) { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + return d.keyBackups.updateKeyBackupAuthData(ctx, txn, userID, version, authData) + }) + return +} + +func (d *Database) DeleteKeyBackup( + ctx context.Context, userID, version string, +) (exists bool, err error) { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + exists, err = d.keyBackups.deleteKeyBackup(ctx, txn, userID, version) + return err + }) + return +} + +func (d *Database) GetKeyBackup( + ctx context.Context, userID, version string, +) (versionResult, algorithm string, authData json.RawMessage, deleted bool, err error) { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + versionResult, algorithm, authData, deleted, err = d.keyBackups.selectKeyBackup(ctx, txn, userID, version) + return err + }) + return +} diff --git a/userapi/storage/accounts/sqlite3/key_backup_version_table.go b/userapi/storage/accounts/sqlite3/key_backup_version_table.go new file mode 100644 index 00000000..3e85705c --- /dev/null +++ b/userapi/storage/accounts/sqlite3/key_backup_version_table.go @@ -0,0 +1,142 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strconv" +) + +const keyBackupVersionTableSchema = ` +-- the metadata for each generation of encrypted e2e session backups +CREATE TABLE IF NOT EXISTS account_e2e_room_keys_versions ( + user_id TEXT NOT NULL, + -- this means no 2 users will ever have the same version of e2e session backups which strictly + -- isn't necessary, but this is easy to do rather than SELECT MAX(version)+1. + version INTEGER PRIMARY KEY AUTOINCREMENT, + algorithm TEXT NOT NULL, + auth_data TEXT NOT NULL, + deleted INTEGER DEFAULT 0 NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS account_e2e_room_keys_versions_idx ON account_e2e_room_keys_versions(user_id, version); +` + +const insertKeyBackupSQL = "" + + "INSERT INTO account_e2e_room_keys_versions(user_id, algorithm, auth_data) VALUES ($1, $2, $3) RETURNING version" + +const updateKeyBackupAuthDataSQL = "" + // TODO: do we need to WHERE algorithm = $3 as well? + "UPDATE account_e2e_room_keys_versions SET auth_data = $1 WHERE user_id = $2 AND version = $3" + +const deleteKeyBackupSQL = "" + + "UPDATE account_e2e_room_keys_versions SET deleted=1 WHERE user_id = $1 AND version = $2" + +const selectKeyBackupSQL = "" + + "SELECT algorithm, auth_data, deleted FROM account_e2e_room_keys_versions WHERE user_id = $1 AND version = $2" + +const selectLatestVersionSQL = "" + + "SELECT MAX(version) FROM account_e2e_room_keys_versions WHERE user_id = $1" + +type keyBackupVersionStatements struct { + insertKeyBackupStmt *sql.Stmt + updateKeyBackupAuthDataStmt *sql.Stmt + deleteKeyBackupStmt *sql.Stmt + selectKeyBackupStmt *sql.Stmt + selectLatestVersionStmt *sql.Stmt +} + +func (s *keyBackupVersionStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(keyBackupVersionTableSchema) + if err != nil { + return + } + if s.insertKeyBackupStmt, err = db.Prepare(insertKeyBackupSQL); err != nil { + return + } + if s.updateKeyBackupAuthDataStmt, err = db.Prepare(updateKeyBackupAuthDataSQL); err != nil { + return + } + if s.deleteKeyBackupStmt, err = db.Prepare(deleteKeyBackupSQL); err != nil { + return + } + if s.selectKeyBackupStmt, err = db.Prepare(selectKeyBackupSQL); err != nil { + return + } + if s.selectLatestVersionStmt, err = db.Prepare(selectLatestVersionSQL); err != nil { + return + } + return +} + +func (s *keyBackupVersionStatements) insertKeyBackup( + ctx context.Context, txn *sql.Tx, userID, algorithm string, authData json.RawMessage, +) (version string, err error) { + var versionInt int64 + err = txn.Stmt(s.insertKeyBackupStmt).QueryRowContext(ctx, userID, algorithm, string(authData)).Scan(&versionInt) + return strconv.FormatInt(versionInt, 10), err +} + +func (s *keyBackupVersionStatements) updateKeyBackupAuthData( + ctx context.Context, txn *sql.Tx, userID, version string, authData json.RawMessage, +) error { + versionInt, err := strconv.ParseInt(version, 10, 64) + if err != nil { + return fmt.Errorf("invalid version") + } + _, err = txn.Stmt(s.updateKeyBackupAuthDataStmt).ExecContext(ctx, string(authData), userID, versionInt) + return err +} + +func (s *keyBackupVersionStatements) deleteKeyBackup( + ctx context.Context, txn *sql.Tx, userID, version string, +) (bool, error) { + versionInt, err := strconv.ParseInt(version, 10, 64) + if err != nil { + return false, fmt.Errorf("invalid version") + } + result, err := txn.Stmt(s.deleteKeyBackupStmt).ExecContext(ctx, userID, versionInt) + if err != nil { + return false, err + } + ra, err := result.RowsAffected() + if err != nil { + return false, err + } + return ra == 1, nil +} + +func (s *keyBackupVersionStatements) selectKeyBackup( + ctx context.Context, txn *sql.Tx, userID, version string, +) (versionResult, algorithm string, authData json.RawMessage, deleted bool, err error) { + var versionInt int64 + if version == "" { + err = txn.Stmt(s.selectLatestVersionStmt).QueryRowContext(ctx, userID).Scan(&versionInt) + } else { + versionInt, err = strconv.ParseInt(version, 10, 64) + } + if err != nil { + return + } + versionResult = strconv.FormatInt(versionInt, 10) + var deletedInt int + var authDataStr string + err = txn.Stmt(s.selectKeyBackupStmt).QueryRowContext(ctx, userID, versionInt).Scan(&algorithm, &authDataStr, &deletedInt) + deleted = deletedInt == 1 + authData = json.RawMessage(authDataStr) + return +} diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index c0f7118c..99e016ff 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -43,6 +43,7 @@ type Database struct { accountDatas accountDataStatements threepids threepidStatements openIDTokens tokenStatements + keyBackups keyBackupVersionStatements serverName gomatrixserverlib.ServerName bcryptCost int openIDTokenLifetimeMS int64 @@ -97,6 +98,9 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err = d.openIDTokens.prepare(db, serverName); err != nil { return nil, err } + if err = d.keyBackups.prepare(db); err != nil { + return nil, err + } return d, nil } @@ -406,3 +410,42 @@ func (d *Database) GetOpenIDTokenAttributes( ) (*api.OpenIDTokenAttributes, error) { return d.openIDTokens.selectOpenIDTokenAtrributes(ctx, token) } + +func (d *Database) CreateKeyBackup( + ctx context.Context, userID, algorithm string, authData json.RawMessage, +) (version string, err error) { + err = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { + version, err = d.keyBackups.insertKeyBackup(ctx, txn, userID, algorithm, authData) + return err + }) + return +} + +func (d *Database) UpdateKeyBackupAuthData( + ctx context.Context, userID, version string, authData json.RawMessage, +) (err error) { + err = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { + return d.keyBackups.updateKeyBackupAuthData(ctx, txn, userID, version, authData) + }) + return +} + +func (d *Database) DeleteKeyBackup( + ctx context.Context, userID, version string, +) (exists bool, err error) { + err = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { + exists, err = d.keyBackups.deleteKeyBackup(ctx, txn, userID, version) + return err + }) + return +} + +func (d *Database) GetKeyBackup( + ctx context.Context, userID, version string, +) (versionResult, algorithm string, authData json.RawMessage, deleted bool, err error) { + err = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { + versionResult, algorithm, authData, deleted, err = d.keyBackups.selectKeyBackup(ctx, txn, userID, version) + return err + }) + return +} From a060df91e206903e4e3cbf7b7d2dabddfa0bf788 Mon Sep 17 00:00:00 2001 From: kegsay Date: Tue, 27 Jul 2021 12:47:50 +0100 Subject: [PATCH 05/64] Use db writer on sqlite account table (#1944) --- userapi/storage/accounts/sqlite3/storage.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index 99e016ff..b10f25ad 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -160,8 +160,9 @@ func (d *Database) SetPassword( if err != nil { return err } - err = d.accounts.updatePassword(ctx, localpart, hash) - return err + return d.writer.Do(nil, nil, func(txn *sql.Tx) error { + return d.accounts.updatePassword(ctx, localpart, hash) + }) } // CreateGuestAccount makes a new guest account and creates an empty profile @@ -388,7 +389,9 @@ func (d *Database) SearchProfiles(ctx context.Context, searchString string, limi // DeactivateAccount deactivates the user's account, removing all ability for the user to login again. func (d *Database) DeactivateAccount(ctx context.Context, localpart string) (err error) { - return d.accounts.deactivateAccount(ctx, localpart) + return d.writer.Do(nil, nil, func(txn *sql.Tx) error { + return d.accounts.deactivateAccount(ctx, localpart) + }) } // CreateOpenIDToken persists a new token that was issued for OpenID Connect From b3754d68fcbe9022eb0bf4f8eda7102b7c27e62d Mon Sep 17 00:00:00 2001 From: kegsay Date: Tue, 27 Jul 2021 17:08:53 +0100 Subject: [PATCH 06/64] Key Backups (2/3) : Add E2E backup key tables (#1945) * Add PUT key backup endpoints and glue them to PerformKeyBackup * Add tables for storing backup keys and glue them into the user API * Don't create tables whilst still WIPing * writer on sqlite please * Linting --- clientapi/routing/key_backup.go | 45 ++++++ clientapi/routing/routing.go | 79 +++++++++++ sytest-whitelist | 1 - userapi/api/api.go | 30 +++- userapi/internal/api.go | 57 ++++++-- userapi/storage/accounts/interface.go | 3 +- .../accounts/postgres/key_backup_table.go | 134 ++++++++++++++++++ .../postgres/key_backup_version_table.go | 34 ++++- userapi/storage/accounts/postgres/storage.go | 115 +++++++++++++-- .../accounts/sqlite3/key_backup_table.go | 134 ++++++++++++++++++ .../sqlite3/key_backup_version_table.go | 34 ++++- userapi/storage/accounts/sqlite3/storage.go | 116 +++++++++++++-- 12 files changed, 737 insertions(+), 45 deletions(-) create mode 100644 userapi/storage/accounts/postgres/key_backup_table.go create mode 100644 userapi/storage/accounts/sqlite3/key_backup_table.go diff --git a/clientapi/routing/key_backup.go b/clientapi/routing/key_backup.go index 0aa73624..dd21d482 100644 --- a/clientapi/routing/key_backup.go +++ b/clientapi/routing/key_backup.go @@ -42,6 +42,17 @@ type keyBackupVersionResponse struct { Version string `json:"version"` } +type keyBackupSessionRequest struct { + Rooms map[string]struct { + Sessions map[string]userapi.KeyBackupSession `json:"sessions"` + } `json:"rooms"` +} + +type keyBackupSessionResponse struct { + Count int64 `json:"count"` + ETag string `json:"etag"` +} + // Create a new key backup. Request must contain a `keyBackupVersion`. Returns a `keyBackupVersionCreateResponse`. // Implements POST /_matrix/client/r0/room_keys/version func CreateKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device) util.JSONResponse { @@ -171,3 +182,37 @@ func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, }, } } + +// Upload a bunch of session keys for a given `version`. +func UploadBackupKeys( + req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string, keys *keyBackupSessionRequest, +) util.JSONResponse { + var performKeyBackupResp userapi.PerformKeyBackupResponse + userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{ + UserID: device.UserID, + Version: version, + Keys: *keys, + }, &performKeyBackupResp) + if performKeyBackupResp.Error != "" { + if performKeyBackupResp.BadInput { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.InvalidArgumentValue(performKeyBackupResp.Error), + } + } + return util.ErrorResponse(fmt.Errorf("PerformKeyBackup: %s", performKeyBackupResp.Error)) + } + if !performKeyBackupResp.Exists { + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound("backup version not found"), + } + } + return util.JSONResponse{ + Code: 200, + JSON: keyBackupSessionResponse{ + Count: performKeyBackupResp.KeyCount, + ETag: performKeyBackupResp.KeyETag, + }, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 194ab299..3e0c53ee 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -938,6 +938,85 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) + // E2E Backup Keys + // Bulk room and session + r0mux.Handle("/room_keys/keys", + httputil.MakeAuthAPI("put_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + version := req.URL.Query().Get("version") + if version == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.InvalidArgumentValue("version must be specified"), + } + } + var reqBody keyBackupSessionRequest + resErr := clientutil.UnmarshalJSONRequest(req, &reqBody) + if resErr != nil { + return *resErr + } + return UploadBackupKeys(req, userAPI, device, version, &reqBody) + }), + ).Methods(http.MethodPut) + // Single room bulk session + r0mux.Handle("/room_keys/keys/{roomID}", + httputil.MakeAuthAPI("put_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + version := req.URL.Query().Get("version") + if version == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.InvalidArgumentValue("version must be specified"), + } + } + roomID := vars["roomID"] + var reqBody keyBackupSessionRequest + reqBody.Rooms[roomID] = struct { + Sessions map[string]userapi.KeyBackupSession `json:"sessions"` + }{ + Sessions: map[string]userapi.KeyBackupSession{}, + } + body := reqBody.Rooms[roomID] + resErr := clientutil.UnmarshalJSONRequest(req, &body) + if resErr != nil { + return *resErr + } + reqBody.Rooms[roomID] = body + return UploadBackupKeys(req, userAPI, device, version, &reqBody) + }), + ).Methods(http.MethodPut) + // Single room, single session + r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", + httputil.MakeAuthAPI("put_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + version := req.URL.Query().Get("version") + if version == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.InvalidArgumentValue("version must be specified"), + } + } + var reqBody userapi.KeyBackupSession + resErr := clientutil.UnmarshalJSONRequest(req, &reqBody) + if resErr != nil { + return *resErr + } + roomID := vars["roomID"] + sessionID := vars["sessionID"] + var keyReq keyBackupSessionRequest + keyReq.Rooms[roomID] = struct { + Sessions map[string]userapi.KeyBackupSession `json:"sessions"` + }{} + keyReq.Rooms[roomID].Sessions[sessionID] = reqBody + return UploadBackupKeys(req, userAPI, device, version, &keyReq) + }), + ).Methods(http.MethodPut) + // Supplying a device ID is deprecated. r0mux.Handle("/keys/upload/{deviceID}", httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { diff --git a/sytest-whitelist b/sytest-whitelist index e2bb41da..d90ba4fb 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -540,4 +540,3 @@ Key notary server must not overwrite a valid key with a spurious result from the GET /rooms/:room_id/aliases lists aliases Only room members can list aliases of a room Users with sufficient power-level can delete other's aliases -Can create more than 10 backup versions diff --git a/userapi/api/api.go b/userapi/api/api.go index ff10bfcd..7e18d72f 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -50,13 +50,39 @@ type PerformKeyBackupRequest struct { AuthData json.RawMessage Algorithm string DeleteBackup bool // if true will delete the backup based on 'Version'. + + // The keys to upload, if any. If blank, creates/updates/deletes key version metadata only. + Keys struct { + Rooms map[string]struct { + Sessions map[string]KeyBackupSession `json:"sessions"` + } `json:"rooms"` + } +} + +// KeyBackupData in https://spec.matrix.org/unstable/client-server-api/#get_matrixclientr0room_keyskeysroomidsessionid +type KeyBackupSession struct { + FirstMessageIndex int `json:"first_message_index"` + ForwardedCount int `json:"forwarded_count"` + IsVerified bool `json:"is_verified"` + SessionData json.RawMessage `json:"session_data"` +} + +// Internal KeyBackupData for passing to/from the storage layer +type InternalKeyBackupSession struct { + KeyBackupSession + RoomID string + SessionID string } type PerformKeyBackupResponse struct { Error string // set if there was a problem performing the request BadInput bool // if set, the Error was due to bad input (HTTP 400) - Exists bool // set to true if the Version exists - Version string + + Exists bool // set to true if the Version exists + Version string // the newly created version + + KeyCount int64 // only set if Keys were given in the request + KeyETag string // only set if Keys were given in the request } type QueryKeyBackupRequest struct { diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 9ff69298..27e17963 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -444,7 +444,7 @@ func (a *UserInternalAPI) QueryOpenIDToken(ctx context.Context, req *api.QueryOp } func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) { - // Delete + // Delete metadata if req.DeleteBackup { if req.Version == "" { res.BadInput = true @@ -459,7 +459,7 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform res.Version = req.Version return } - // Create + // Create metadata if req.Version == "" { version, err := a.AccountDB.CreateKeyBackup(ctx, req.UserID, req.Algorithm, req.AuthData) if err != nil { @@ -469,16 +469,55 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform res.Version = version return } - // Update - err := a.AccountDB.UpdateKeyBackupAuthData(ctx, req.UserID, req.Version, req.AuthData) - if err != nil { - res.Error = fmt.Sprintf("failed to update backup: %s", err) + // Update metadata + if len(req.Keys.Rooms) == 0 { + err := a.AccountDB.UpdateKeyBackupAuthData(ctx, req.UserID, req.Version, req.AuthData) + if err != nil { + res.Error = fmt.Sprintf("failed to update backup: %s", err) + } + res.Version = req.Version + return } - res.Version = req.Version + // Upload Keys for a specific version metadata + a.uploadBackupKeys(ctx, req, res) +} + +func (a *UserInternalAPI) uploadBackupKeys(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) { + // ensure the version metadata exists + version, _, _, _, deleted, err := a.AccountDB.GetKeyBackup(ctx, req.UserID, req.Version) + if err != nil { + res.Error = fmt.Sprintf("failed to query version: %s", err) + return + } + if deleted { + res.Error = "backup was deleted" + return + } + res.Exists = true + res.Version = version + + // map keys to a form we can upload more easily - the map ensures we have no duplicates. + var uploads []api.InternalKeyBackupSession + for roomID, data := range req.Keys.Rooms { + for sessionID, sessionData := range data.Sessions { + uploads = append(uploads, api.InternalKeyBackupSession{ + RoomID: roomID, + SessionID: sessionID, + KeyBackupSession: sessionData, + }) + } + } + count, etag, err := a.AccountDB.UpsertBackupKeys(ctx, version, req.UserID, uploads) + if err != nil { + res.Error = fmt.Sprintf("failed to upsert keys: %s", err) + return + } + res.KeyCount = count + res.KeyETag = etag } func (a *UserInternalAPI) QueryKeyBackup(ctx context.Context, req *api.QueryKeyBackupRequest, res *api.QueryKeyBackupResponse) { - version, algorithm, authData, deleted, err := a.AccountDB.GetKeyBackup(ctx, req.UserID, req.Version) + version, algorithm, authData, etag, deleted, err := a.AccountDB.GetKeyBackup(ctx, req.UserID, req.Version) res.Version = version if err != nil { if err == sql.ErrNoRows { @@ -494,5 +533,5 @@ func (a *UserInternalAPI) QueryKeyBackup(ctx context.Context, req *api.QueryKeyB // TODO: res.Count = 0 - res.ETag = "" + res.ETag = etag } diff --git a/userapi/storage/accounts/interface.go b/userapi/storage/accounts/interface.go index 88fdab48..4fd9c177 100644 --- a/userapi/storage/accounts/interface.go +++ b/userapi/storage/accounts/interface.go @@ -59,7 +59,8 @@ type Database interface { CreateKeyBackup(ctx context.Context, userID, algorithm string, authData json.RawMessage) (version string, err error) UpdateKeyBackupAuthData(ctx context.Context, userID, version string, authData json.RawMessage) (err error) DeleteKeyBackup(ctx context.Context, userID, version string) (exists bool, err error) - GetKeyBackup(ctx context.Context, userID, version string) (versionResult, algorithm string, authData json.RawMessage, deleted bool, err error) + GetKeyBackup(ctx context.Context, userID, version string) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error) + UpsertBackupKeys(ctx context.Context, version, userID string, uploads []api.InternalKeyBackupSession) (count int64, etag string, err error) } // Err3PIDInUse is the error returned when trying to save an association involving diff --git a/userapi/storage/accounts/postgres/key_backup_table.go b/userapi/storage/accounts/postgres/key_backup_table.go new file mode 100644 index 00000000..0dc5879b --- /dev/null +++ b/userapi/storage/accounts/postgres/key_backup_table.go @@ -0,0 +1,134 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + "encoding/json" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/userapi/api" +) + +const keyBackupTableSchema = ` +CREATE TABLE IF NOT EXISTS account_e2e_room_keys ( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + session_id TEXT NOT NULL, + + version TEXT NOT NULL, + first_message_index INTEGER NOT NULL, + forwarded_count INTEGER NOT NULL, + is_verified BOOLEAN NOT NULL, + session_data TEXT NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON account_e2e_room_keys(user_id, room_id, session_id); +` + +const insertBackupKeySQL = "" + + "INSERT INTO account_e2e_room_keys(user_id, room_id, session_id, version, first_message_index, forwarded_count, is_verified, session_data) " + + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8)" + +const updateBackupKeySQL = "" + + "UPDATE account_e2e_room_keys SET first_message_index=$1, forwarded_count=$2, is_verified=$3, session_data=$4 " + + "WHERE user_id=$5 AND room_id=$6 AND session_id=$7 AND version=$8" + +const countKeysSQL = "" + + "SELECT COUNT(*) FROM account_e2e_room_keys WHERE user_id = $1 AND version = $2" + +const selectKeysSQL = "" + + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + + "WHERE user_id = $1 AND version = $2" + +type keyBackupStatements struct { + insertBackupKeyStmt *sql.Stmt + updateBackupKeyStmt *sql.Stmt + countKeysStmt *sql.Stmt + selectKeysStmt *sql.Stmt +} + +// nolint:unused +func (s *keyBackupStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(keyBackupTableSchema) + if err != nil { + return + } + if s.insertBackupKeyStmt, err = db.Prepare(insertBackupKeySQL); err != nil { + return + } + if s.updateBackupKeyStmt, err = db.Prepare(updateBackupKeySQL); err != nil { + return + } + if s.countKeysStmt, err = db.Prepare(countKeysSQL); err != nil { + return + } + if s.selectKeysStmt, err = db.Prepare(selectKeysSQL); err != nil { + return + } + return +} + +func (s keyBackupStatements) countKeys( + ctx context.Context, txn *sql.Tx, userID, version string, +) (count int64, err error) { + err = txn.Stmt(s.countKeysStmt).QueryRowContext(ctx, userID, version).Scan(&count) + return +} + +func (s *keyBackupStatements) insertBackupKey( + ctx context.Context, txn *sql.Tx, userID, version string, key api.InternalKeyBackupSession, +) (err error) { + _, err = txn.Stmt(s.insertBackupKeyStmt).ExecContext( + ctx, userID, key.RoomID, key.SessionID, version, key.FirstMessageIndex, key.ForwardedCount, key.IsVerified, string(key.SessionData), + ) + return +} + +func (s *keyBackupStatements) updateBackupKey( + ctx context.Context, txn *sql.Tx, userID, version string, key api.InternalKeyBackupSession, +) (err error) { + _, err = txn.Stmt(s.updateBackupKeyStmt).ExecContext( + ctx, key.FirstMessageIndex, key.ForwardedCount, key.IsVerified, string(key.SessionData), userID, key.RoomID, key.SessionID, version, + ) + return +} + +func (s *keyBackupStatements) selectKeys( + ctx context.Context, txn *sql.Tx, userID, version string, +) (map[string]map[string]api.KeyBackupSession, error) { + result := make(map[string]map[string]api.KeyBackupSession) + rows, err := txn.Stmt(s.selectKeysStmt).QueryContext(ctx, userID, version) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectKeysStmt.Close failed") + for rows.Next() { + var key api.InternalKeyBackupSession + // room_id, session_id, first_message_index, forwarded_count, is_verified, session_data + var sessionDataStr string + if err := rows.Scan(&key.RoomID, &key.SessionID, &key.FirstMessageIndex, &key.ForwardedCount, &key.IsVerified, &sessionDataStr); err != nil { + return nil, err + } + key.SessionData = json.RawMessage(sessionDataStr) + roomData := result[key.RoomID] + if roomData == nil { + roomData = make(map[string]api.KeyBackupSession) + } + roomData[key.SessionID] = key.KeyBackupSession + result[key.RoomID] = roomData + } + return result, nil +} diff --git a/userapi/storage/accounts/postgres/key_backup_version_table.go b/userapi/storage/accounts/postgres/key_backup_version_table.go index 1b693e56..323a842d 100644 --- a/userapi/storage/accounts/postgres/key_backup_version_table.go +++ b/userapi/storage/accounts/postgres/key_backup_version_table.go @@ -33,6 +33,7 @@ CREATE TABLE IF NOT EXISTS account_e2e_room_keys_versions ( version BIGINT DEFAULT nextval('account_e2e_room_keys_versions_seq'), algorithm TEXT NOT NULL, auth_data TEXT NOT NULL, + etag TEXT NOT NULL, deleted SMALLINT DEFAULT 0 NOT NULL ); @@ -40,16 +41,19 @@ CREATE UNIQUE INDEX IF NOT EXISTS account_e2e_room_keys_versions_idx ON account_ ` const insertKeyBackupSQL = "" + - "INSERT INTO account_e2e_room_keys_versions(user_id, algorithm, auth_data) VALUES ($1, $2, $3) RETURNING version" + "INSERT INTO account_e2e_room_keys_versions(user_id, algorithm, auth_data, etag) VALUES ($1, $2, $3, $4) RETURNING version" -const updateKeyBackupAuthDataSQL = "" + // TODO: do we need to WHERE algorithm = $3 as well? +const updateKeyBackupAuthDataSQL = "" + "UPDATE account_e2e_room_keys_versions SET auth_data = $1 WHERE user_id = $2 AND version = $3" +const updateKeyBackupETagSQL = "" + + "UPDATE account_e2e_room_keys_versions SET etag = $1 WHERE user_id = $2 AND version = $3" + const deleteKeyBackupSQL = "" + "UPDATE account_e2e_room_keys_versions SET deleted=1 WHERE user_id = $1 AND version = $2" const selectKeyBackupSQL = "" + - "SELECT algorithm, auth_data, deleted FROM account_e2e_room_keys_versions WHERE user_id = $1 AND version = $2" + "SELECT algorithm, auth_data, etag, deleted FROM account_e2e_room_keys_versions WHERE user_id = $1 AND version = $2" const selectLatestVersionSQL = "" + "SELECT MAX(version) FROM account_e2e_room_keys_versions WHERE user_id = $1" @@ -60,8 +64,10 @@ type keyBackupVersionStatements struct { deleteKeyBackupStmt *sql.Stmt selectKeyBackupStmt *sql.Stmt selectLatestVersionStmt *sql.Stmt + updateKeyBackupETagStmt *sql.Stmt } +// nolint:unused func (s *keyBackupVersionStatements) prepare(db *sql.DB) (err error) { _, err = db.Exec(keyBackupVersionTableSchema) if err != nil { @@ -82,14 +88,17 @@ func (s *keyBackupVersionStatements) prepare(db *sql.DB) (err error) { if s.selectLatestVersionStmt, err = db.Prepare(selectLatestVersionSQL); err != nil { return } + if s.updateKeyBackupETagStmt, err = db.Prepare(updateKeyBackupETagSQL); err != nil { + return + } return } func (s *keyBackupVersionStatements) insertKeyBackup( - ctx context.Context, txn *sql.Tx, userID, algorithm string, authData json.RawMessage, + ctx context.Context, txn *sql.Tx, userID, algorithm string, authData json.RawMessage, etag string, ) (version string, err error) { var versionInt int64 - err = txn.Stmt(s.insertKeyBackupStmt).QueryRowContext(ctx, userID, algorithm, string(authData)).Scan(&versionInt) + err = txn.Stmt(s.insertKeyBackupStmt).QueryRowContext(ctx, userID, algorithm, string(authData), etag).Scan(&versionInt) return strconv.FormatInt(versionInt, 10), err } @@ -104,6 +113,17 @@ func (s *keyBackupVersionStatements) updateKeyBackupAuthData( return err } +func (s *keyBackupVersionStatements) updateKeyBackupETag( + ctx context.Context, txn *sql.Tx, userID, version, etag string, +) error { + versionInt, err := strconv.ParseInt(version, 10, 64) + if err != nil { + return fmt.Errorf("invalid version") + } + _, err = txn.Stmt(s.updateKeyBackupETagStmt).ExecContext(ctx, etag, userID, versionInt) + return err +} + func (s *keyBackupVersionStatements) deleteKeyBackup( ctx context.Context, txn *sql.Tx, userID, version string, ) (bool, error) { @@ -124,7 +144,7 @@ func (s *keyBackupVersionStatements) deleteKeyBackup( func (s *keyBackupVersionStatements) selectKeyBackup( ctx context.Context, txn *sql.Tx, userID, version string, -) (versionResult, algorithm string, authData json.RawMessage, deleted bool, err error) { +) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error) { var versionInt int64 if version == "" { err = txn.Stmt(s.selectLatestVersionStmt).QueryRowContext(ctx, userID).Scan(&versionInt) @@ -137,7 +157,7 @@ func (s *keyBackupVersionStatements) selectKeyBackup( versionResult = strconv.FormatInt(versionInt, 10) var deletedInt int var authDataStr string - err = txn.Stmt(s.selectKeyBackupStmt).QueryRowContext(ctx, userID, versionInt).Scan(&algorithm, &authDataStr, &deletedInt) + err = txn.Stmt(s.selectKeyBackupStmt).QueryRowContext(ctx, userID, versionInt).Scan(&algorithm, &authDataStr, &etag, &deletedInt) deleted = deletedInt == 1 authData = json.RawMessage(authDataStr) return diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index 719e9878..b07218b2 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -19,6 +19,7 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" "strconv" "time" @@ -45,7 +46,8 @@ type Database struct { accountDatas accountDataStatements threepids threepidStatements openIDTokens tokenStatements - keyBackups keyBackupVersionStatements + keyBackupVersions keyBackupVersionStatements + keyBackups keyBackupStatements serverName gomatrixserverlib.ServerName bcryptCost int openIDTokenLifetimeMS int64 @@ -94,9 +96,13 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err = d.openIDTokens.prepare(db, serverName); err != nil { return nil, err } - if err = d.keyBackups.prepare(db); err != nil { - return nil, err - } + /* + if err = d.keyBackupVersions.prepare(db); err != nil { + return nil, err + } + if err = d.keyBackups.prepare(db); err != nil { + return nil, err + } */ return d, nil } @@ -377,7 +383,7 @@ func (d *Database) CreateKeyBackup( ctx context.Context, userID, algorithm string, authData json.RawMessage, ) (version string, err error) { err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { - version, err = d.keyBackups.insertKeyBackup(ctx, txn, userID, algorithm, authData) + version, err = d.keyBackupVersions.insertKeyBackup(ctx, txn, userID, algorithm, authData, "") return err }) return @@ -387,7 +393,7 @@ func (d *Database) UpdateKeyBackupAuthData( ctx context.Context, userID, version string, authData json.RawMessage, ) (err error) { err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { - return d.keyBackups.updateKeyBackupAuthData(ctx, txn, userID, version, authData) + return d.keyBackupVersions.updateKeyBackupAuthData(ctx, txn, userID, version, authData) }) return } @@ -396,7 +402,7 @@ func (d *Database) DeleteKeyBackup( ctx context.Context, userID, version string, ) (exists bool, err error) { err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { - exists, err = d.keyBackups.deleteKeyBackup(ctx, txn, userID, version) + exists, err = d.keyBackupVersions.deleteKeyBackup(ctx, txn, userID, version) return err }) return @@ -404,10 +410,101 @@ func (d *Database) DeleteKeyBackup( func (d *Database) GetKeyBackup( ctx context.Context, userID, version string, -) (versionResult, algorithm string, authData json.RawMessage, deleted bool, err error) { +) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error) { err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { - versionResult, algorithm, authData, deleted, err = d.keyBackups.selectKeyBackup(ctx, txn, userID, version) + versionResult, algorithm, authData, etag, deleted, err = d.keyBackupVersions.selectKeyBackup(ctx, txn, userID, version) return err }) return } + +// nolint:nakedret +func (d *Database) UpsertBackupKeys( + ctx context.Context, version, userID string, uploads []api.InternalKeyBackupSession, +) (count int64, etag string, err error) { + // wrap the following logic in a txn to ensure we atomically upload keys + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + _, _, _, oldETag, deleted, err := d.keyBackupVersions.selectKeyBackup(ctx, txn, userID, version) + if err != nil { + return err + } + if deleted { + return fmt.Errorf("backup was deleted") + } + // pull out all keys for this (user_id, version) + existingKeys, err := d.keyBackups.selectKeys(ctx, txn, userID, version) + if err != nil { + return err + } + + changed := false + // loop over all the new keys (which should be smaller than the set of backed up keys) + for _, newKey := range uploads { + // if we have a matching (room_id, session_id), we may need to update the key if it meets some rules, check them. + existingRoom := existingKeys[newKey.RoomID] + if existingRoom != nil { + existingSession, ok := existingRoom[newKey.SessionID] + if ok { + if shouldReplaceRoomKey(existingSession, newKey.KeyBackupSession) { + err = d.keyBackups.updateBackupKey(ctx, txn, userID, version, newKey) + changed = true + if err != nil { + return err + } + } + // if we shouldn't replace the key we do nothing with it + continue + } + } + // if we're here, either the room or session are new, either way, we insert + err = d.keyBackups.insertBackupKey(ctx, txn, userID, version, newKey) + changed = true + if err != nil { + return err + } + } + + count, err = d.keyBackups.countKeys(ctx, txn, userID, version) + if err != nil { + return err + } + if changed { + // update the etag + var newETag string + if oldETag == "" { + newETag = "1" + } else { + oldETagInt, err := strconv.ParseInt(oldETag, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse old etag: %s", err) + } + newETag = strconv.FormatInt(oldETagInt+1, 10) + } + etag = newETag + return d.keyBackupVersions.updateKeyBackupETag(ctx, txn, userID, version, newETag) + } else { + etag = oldETag + } + return nil + }) + return +} + +// TODO FIXME XXX : This logic really shouldn't live in the storage layer, but I don't know where else is sensible which won't +// create circular import loops +func shouldReplaceRoomKey(existing, uploaded api.KeyBackupSession) bool { + // https://spec.matrix.org/unstable/client-server-api/#backup-algorithm-mmegolm_backupv1curve25519-aes-sha2 + // "if the keys have different values for is_verified, then it will keep the key that has is_verified set to true" + if uploaded.IsVerified && !existing.IsVerified { + return true + } + // "if they have the same values for is_verified, then it will keep the key with a lower first_message_index" + if uploaded.FirstMessageIndex < existing.FirstMessageIndex { + return true + } + // "and finally, is is_verified and first_message_index are equal, then it will keep the key with a lower forwarded_count" + if uploaded.ForwardedCount < existing.ForwardedCount { + return true + } + return false +} diff --git a/userapi/storage/accounts/sqlite3/key_backup_table.go b/userapi/storage/accounts/sqlite3/key_backup_table.go new file mode 100644 index 00000000..268bda93 --- /dev/null +++ b/userapi/storage/accounts/sqlite3/key_backup_table.go @@ -0,0 +1,134 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "encoding/json" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/userapi/api" +) + +const keyBackupTableSchema = ` +CREATE TABLE IF NOT EXISTS account_e2e_room_keys ( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + session_id TEXT NOT NULL, + + version TEXT NOT NULL, + first_message_index INTEGER NOT NULL, + forwarded_count INTEGER NOT NULL, + is_verified BOOLEAN NOT NULL, + session_data TEXT NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON account_e2e_room_keys(user_id, room_id, session_id); +` + +const insertBackupKeySQL = "" + + "INSERT INTO account_e2e_room_keys(user_id, room_id, session_id, version, first_message_index, forwarded_count, is_verified, session_data) " + + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8)" + +const updateBackupKeySQL = "" + + "UPDATE account_e2e_room_keys SET first_message_index=$1, forwarded_count=$2, is_verified=$3, session_data=$4 " + + "WHERE user_id=$5 AND room_id=$6 AND session_id=$7 AND version=$8" + +const countKeysSQL = "" + + "SELECT COUNT(*) FROM account_e2e_room_keys WHERE user_id = $1 AND version = $2" + +const selectKeysSQL = "" + + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + + "WHERE user_id = $1 AND version = $2" + +type keyBackupStatements struct { + insertBackupKeyStmt *sql.Stmt + updateBackupKeyStmt *sql.Stmt + countKeysStmt *sql.Stmt + selectKeysStmt *sql.Stmt +} + +// nolint:unused +func (s *keyBackupStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(keyBackupTableSchema) + if err != nil { + return + } + if s.insertBackupKeyStmt, err = db.Prepare(insertBackupKeySQL); err != nil { + return + } + if s.updateBackupKeyStmt, err = db.Prepare(updateBackupKeySQL); err != nil { + return + } + if s.countKeysStmt, err = db.Prepare(countKeysSQL); err != nil { + return + } + if s.selectKeysStmt, err = db.Prepare(selectKeysSQL); err != nil { + return + } + return +} + +func (s keyBackupStatements) countKeys( + ctx context.Context, txn *sql.Tx, userID, version string, +) (count int64, err error) { + err = txn.Stmt(s.countKeysStmt).QueryRowContext(ctx, userID, version).Scan(&count) + return +} + +func (s *keyBackupStatements) insertBackupKey( + ctx context.Context, txn *sql.Tx, userID, version string, key api.InternalKeyBackupSession, +) (err error) { + _, err = txn.Stmt(s.insertBackupKeyStmt).ExecContext( + ctx, userID, key.RoomID, key.SessionID, version, key.FirstMessageIndex, key.ForwardedCount, key.IsVerified, string(key.SessionData), + ) + return +} + +func (s *keyBackupStatements) updateBackupKey( + ctx context.Context, txn *sql.Tx, userID, version string, key api.InternalKeyBackupSession, +) (err error) { + _, err = txn.Stmt(s.updateBackupKeyStmt).ExecContext( + ctx, key.FirstMessageIndex, key.ForwardedCount, key.IsVerified, string(key.SessionData), userID, key.RoomID, key.SessionID, version, + ) + return +} + +func (s *keyBackupStatements) selectKeys( + ctx context.Context, txn *sql.Tx, userID, version string, +) (map[string]map[string]api.KeyBackupSession, error) { + result := make(map[string]map[string]api.KeyBackupSession) + rows, err := txn.Stmt(s.selectKeysStmt).QueryContext(ctx, userID, version) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectKeysStmt.Close failed") + for rows.Next() { + var key api.InternalKeyBackupSession + // room_id, session_id, first_message_index, forwarded_count, is_verified, session_data + var sessionDataStr string + if err := rows.Scan(&key.RoomID, &key.SessionID, &key.FirstMessageIndex, &key.ForwardedCount, &key.IsVerified, &sessionDataStr); err != nil { + return nil, err + } + key.SessionData = json.RawMessage(sessionDataStr) + roomData := result[key.RoomID] + if roomData == nil { + roomData = make(map[string]api.KeyBackupSession) + } + roomData[key.SessionID] = key.KeyBackupSession + result[key.RoomID] = roomData + } + return result, nil +} diff --git a/userapi/storage/accounts/sqlite3/key_backup_version_table.go b/userapi/storage/accounts/sqlite3/key_backup_version_table.go index 3e85705c..72e9b132 100644 --- a/userapi/storage/accounts/sqlite3/key_backup_version_table.go +++ b/userapi/storage/accounts/sqlite3/key_backup_version_table.go @@ -31,6 +31,7 @@ CREATE TABLE IF NOT EXISTS account_e2e_room_keys_versions ( version INTEGER PRIMARY KEY AUTOINCREMENT, algorithm TEXT NOT NULL, auth_data TEXT NOT NULL, + etag TEXT NOT NULL, deleted INTEGER DEFAULT 0 NOT NULL ); @@ -38,16 +39,19 @@ CREATE UNIQUE INDEX IF NOT EXISTS account_e2e_room_keys_versions_idx ON account_ ` const insertKeyBackupSQL = "" + - "INSERT INTO account_e2e_room_keys_versions(user_id, algorithm, auth_data) VALUES ($1, $2, $3) RETURNING version" + "INSERT INTO account_e2e_room_keys_versions(user_id, algorithm, auth_data, etag) VALUES ($1, $2, $3, $4) RETURNING version" -const updateKeyBackupAuthDataSQL = "" + // TODO: do we need to WHERE algorithm = $3 as well? +const updateKeyBackupAuthDataSQL = "" + "UPDATE account_e2e_room_keys_versions SET auth_data = $1 WHERE user_id = $2 AND version = $3" +const updateKeyBackupETagSQL = "" + + "UPDATE account_e2e_room_keys_versions SET etag = $1 WHERE user_id = $2 AND version = $3" + const deleteKeyBackupSQL = "" + "UPDATE account_e2e_room_keys_versions SET deleted=1 WHERE user_id = $1 AND version = $2" const selectKeyBackupSQL = "" + - "SELECT algorithm, auth_data, deleted FROM account_e2e_room_keys_versions WHERE user_id = $1 AND version = $2" + "SELECT algorithm, auth_data, etag, deleted FROM account_e2e_room_keys_versions WHERE user_id = $1 AND version = $2" const selectLatestVersionSQL = "" + "SELECT MAX(version) FROM account_e2e_room_keys_versions WHERE user_id = $1" @@ -58,8 +62,10 @@ type keyBackupVersionStatements struct { deleteKeyBackupStmt *sql.Stmt selectKeyBackupStmt *sql.Stmt selectLatestVersionStmt *sql.Stmt + updateKeyBackupETagStmt *sql.Stmt } +// nolint:unused func (s *keyBackupVersionStatements) prepare(db *sql.DB) (err error) { _, err = db.Exec(keyBackupVersionTableSchema) if err != nil { @@ -80,14 +86,17 @@ func (s *keyBackupVersionStatements) prepare(db *sql.DB) (err error) { if s.selectLatestVersionStmt, err = db.Prepare(selectLatestVersionSQL); err != nil { return } + if s.updateKeyBackupETagStmt, err = db.Prepare(updateKeyBackupETagSQL); err != nil { + return + } return } func (s *keyBackupVersionStatements) insertKeyBackup( - ctx context.Context, txn *sql.Tx, userID, algorithm string, authData json.RawMessage, + ctx context.Context, txn *sql.Tx, userID, algorithm string, authData json.RawMessage, etag string, ) (version string, err error) { var versionInt int64 - err = txn.Stmt(s.insertKeyBackupStmt).QueryRowContext(ctx, userID, algorithm, string(authData)).Scan(&versionInt) + err = txn.Stmt(s.insertKeyBackupStmt).QueryRowContext(ctx, userID, algorithm, string(authData), etag).Scan(&versionInt) return strconv.FormatInt(versionInt, 10), err } @@ -102,6 +111,17 @@ func (s *keyBackupVersionStatements) updateKeyBackupAuthData( return err } +func (s *keyBackupVersionStatements) updateKeyBackupETag( + ctx context.Context, txn *sql.Tx, userID, version, etag string, +) error { + versionInt, err := strconv.ParseInt(version, 10, 64) + if err != nil { + return fmt.Errorf("invalid version") + } + _, err = txn.Stmt(s.updateKeyBackupETagStmt).ExecContext(ctx, etag, userID, versionInt) + return err +} + func (s *keyBackupVersionStatements) deleteKeyBackup( ctx context.Context, txn *sql.Tx, userID, version string, ) (bool, error) { @@ -122,7 +142,7 @@ func (s *keyBackupVersionStatements) deleteKeyBackup( func (s *keyBackupVersionStatements) selectKeyBackup( ctx context.Context, txn *sql.Tx, userID, version string, -) (versionResult, algorithm string, authData json.RawMessage, deleted bool, err error) { +) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error) { var versionInt int64 if version == "" { err = txn.Stmt(s.selectLatestVersionStmt).QueryRowContext(ctx, userID).Scan(&versionInt) @@ -135,7 +155,7 @@ func (s *keyBackupVersionStatements) selectKeyBackup( versionResult = strconv.FormatInt(versionInt, 10) var deletedInt int var authDataStr string - err = txn.Stmt(s.selectKeyBackupStmt).QueryRowContext(ctx, userID, versionInt).Scan(&algorithm, &authDataStr, &deletedInt) + err = txn.Stmt(s.selectKeyBackupStmt).QueryRowContext(ctx, userID, versionInt).Scan(&algorithm, &authDataStr, &etag, &deletedInt) deleted = deletedInt == 1 authData = json.RawMessage(authDataStr) return diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index b10f25ad..4fae621f 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -19,6 +19,7 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" "strconv" "sync" "time" @@ -43,7 +44,8 @@ type Database struct { accountDatas accountDataStatements threepids threepidStatements openIDTokens tokenStatements - keyBackups keyBackupVersionStatements + keyBackupVersions keyBackupVersionStatements + keyBackups keyBackupStatements serverName gomatrixserverlib.ServerName bcryptCost int openIDTokenLifetimeMS int64 @@ -98,9 +100,13 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err = d.openIDTokens.prepare(db, serverName); err != nil { return nil, err } - if err = d.keyBackups.prepare(db); err != nil { - return nil, err - } + /* + if err = d.keyBackupVersions.prepare(db); err != nil { + return nil, err + } + if err = d.keyBackups.prepare(db); err != nil { + return nil, err + } */ return d, nil } @@ -418,7 +424,7 @@ func (d *Database) CreateKeyBackup( ctx context.Context, userID, algorithm string, authData json.RawMessage, ) (version string, err error) { err = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { - version, err = d.keyBackups.insertKeyBackup(ctx, txn, userID, algorithm, authData) + version, err = d.keyBackupVersions.insertKeyBackup(ctx, txn, userID, algorithm, authData, "") return err }) return @@ -428,7 +434,7 @@ func (d *Database) UpdateKeyBackupAuthData( ctx context.Context, userID, version string, authData json.RawMessage, ) (err error) { err = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { - return d.keyBackups.updateKeyBackupAuthData(ctx, txn, userID, version, authData) + return d.keyBackupVersions.updateKeyBackupAuthData(ctx, txn, userID, version, authData) }) return } @@ -437,7 +443,7 @@ func (d *Database) DeleteKeyBackup( ctx context.Context, userID, version string, ) (exists bool, err error) { err = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { - exists, err = d.keyBackups.deleteKeyBackup(ctx, txn, userID, version) + exists, err = d.keyBackupVersions.deleteKeyBackup(ctx, txn, userID, version) return err }) return @@ -445,10 +451,102 @@ func (d *Database) DeleteKeyBackup( func (d *Database) GetKeyBackup( ctx context.Context, userID, version string, -) (versionResult, algorithm string, authData json.RawMessage, deleted bool, err error) { +) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error) { err = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { - versionResult, algorithm, authData, deleted, err = d.keyBackups.selectKeyBackup(ctx, txn, userID, version) + versionResult, algorithm, authData, etag, deleted, err = d.keyBackupVersions.selectKeyBackup(ctx, txn, userID, version) return err }) return } + +// nolint:nakedret +func (d *Database) UpsertBackupKeys( + ctx context.Context, version, userID string, uploads []api.InternalKeyBackupSession, +) (count int64, etag string, err error) { + // wrap the following logic in a txn to ensure we atomically upload keys + err = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { + _, _, _, oldETag, deleted, err := d.keyBackupVersions.selectKeyBackup(ctx, txn, userID, version) + if err != nil { + return err + } + if deleted { + return fmt.Errorf("backup was deleted") + } + // pull out all keys for this (user_id, version) + existingKeys, err := d.keyBackups.selectKeys(ctx, txn, userID, version) + if err != nil { + return err + } + + changed := false + // loop over all the new keys (which should be smaller than the set of backed up keys) + for _, newKey := range uploads { + // if we have a matching (room_id, session_id), we may need to update the key if it meets some rules, check them. + existingRoom := existingKeys[newKey.RoomID] + if existingRoom != nil { + existingSession, ok := existingRoom[newKey.SessionID] + if ok { + if shouldReplaceRoomKey(existingSession, newKey.KeyBackupSession) { + err = d.keyBackups.updateBackupKey(ctx, txn, userID, version, newKey) + changed = true + if err != nil { + return err + } + } + // if we shouldn't replace the key we do nothing with it + continue + } + } + // if we're here, either the room or session are new, either way, we insert + err = d.keyBackups.insertBackupKey(ctx, txn, userID, version, newKey) + changed = true + if err != nil { + return err + } + } + + count, err = d.keyBackups.countKeys(ctx, txn, userID, version) + if err != nil { + return err + } + if changed { + // update the etag + var newETag string + if oldETag == "" { + newETag = "1" + } else { + oldETagInt, err := strconv.ParseInt(oldETag, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse old etag: %s", err) + } + newETag = strconv.FormatInt(oldETagInt+1, 10) + } + etag = newETag + return d.keyBackupVersions.updateKeyBackupETag(ctx, txn, userID, version, newETag) + } else { + etag = oldETag + } + + return nil + }) + return +} + +// TODO FIXME XXX : This logic really shouldn't live in the storage layer, but I don't know where else is sensible which won't +// create circular import loops +func shouldReplaceRoomKey(existing, uploaded api.KeyBackupSession) bool { + // https://spec.matrix.org/unstable/client-server-api/#backup-algorithm-mmegolm_backupv1curve25519-aes-sha2 + // "if the keys have different values for is_verified, then it will keep the key that has is_verified set to true" + if uploaded.IsVerified && !existing.IsVerified { + return true + } + // "if they have the same values for is_verified, then it will keep the key with a lower first_message_index" + if uploaded.FirstMessageIndex < existing.FirstMessageIndex { + return true + } + // "and finally, is is_verified and first_message_index are equal, then it will keep the key with a lower forwarded_count" + if uploaded.ForwardedCount < existing.ForwardedCount { + return true + } + return false +} From 32bf14a37c79a02fc5e76a7071d7494bb362be53 Mon Sep 17 00:00:00 2001 From: kegsay Date: Tue, 27 Jul 2021 19:29:32 +0100 Subject: [PATCH 07/64] Key Backups (3/3) : Implement querying keys and various bugfixes (#1946) * Add querying device keys Makes a bunch of sytests pass * Apparently only the current version supports uploading keys * Linting --- clientapi/routing/key_backup.go | 77 +++++++++++++++++- clientapi/routing/routing.go | 81 +++++++++++++------ sytest-whitelist | 10 +++ userapi/api/api.go | 25 +++++- userapi/internal/api.go | 28 +++++-- userapi/storage/accounts/interface.go | 2 + .../accounts/postgres/key_backup_table.go | 54 +++++++++++-- .../postgres/key_backup_version_table.go | 1 - userapi/storage/accounts/postgres/storage.go | 65 ++++++++------- .../accounts/sqlite3/key_backup_table.go | 54 +++++++++++-- .../sqlite3/key_backup_version_table.go | 1 - userapi/storage/accounts/sqlite3/storage.go | 65 ++++++++------- 12 files changed, 362 insertions(+), 101 deletions(-) diff --git a/clientapi/routing/key_backup.go b/clientapi/routing/key_backup.go index dd21d482..ce62a047 100644 --- a/clientapi/routing/key_backup.go +++ b/clientapi/routing/key_backup.go @@ -37,7 +37,7 @@ type keyBackupVersionCreateResponse struct { type keyBackupVersionResponse struct { Algorithm string `json:"algorithm"` AuthData json.RawMessage `json:"auth_data"` - Count int `json:"count"` + Count int64 `json:"count"` ETag string `json:"etag"` Version string `json:"version"` } @@ -89,7 +89,10 @@ func CreateKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, // Implements GET /_matrix/client/r0/room_keys/version and GET /_matrix/client/r0/room_keys/version/{version} func KeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse { var queryResp userapi.QueryKeyBackupResponse - userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{}, &queryResp) + userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{ + UserID: device.UserID, + Version: version, + }, &queryResp) if queryResp.Error != "" { return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", queryResp.Error)) } @@ -216,3 +219,73 @@ func UploadBackupKeys( }, } } + +// Get keys from a given backup version. Response returned varies depending on if roomID and sessionID are set. +func GetBackupKeys( + req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version, roomID, sessionID string, +) util.JSONResponse { + var queryResp userapi.QueryKeyBackupResponse + userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{ + UserID: device.UserID, + Version: version, + ReturnKeys: true, + KeysForRoomID: roomID, + KeysForSessionID: sessionID, + }, &queryResp) + if queryResp.Error != "" { + return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", queryResp.Error)) + } + if !queryResp.Exists { + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound("version not found"), + } + } + if sessionID != "" { + // return the key itself if it was found + roomData, ok := queryResp.Keys[roomID] + if ok { + key, ok := roomData[sessionID] + if ok { + return util.JSONResponse{ + Code: 200, + JSON: key, + } + } + } + } else if roomID != "" { + roomData, ok := queryResp.Keys[roomID] + if ok { + // wrap response in "sessions" + return util.JSONResponse{ + Code: 200, + JSON: struct { + Sessions map[string]userapi.KeyBackupSession `json:"sessions"` + }{ + Sessions: roomData, + }, + } + } + } else { + // response is the same as the upload request + var resp keyBackupSessionRequest + resp.Rooms = make(map[string]struct { + Sessions map[string]userapi.KeyBackupSession `json:"sessions"` + }) + for roomID, roomData := range queryResp.Keys { + resp.Rooms[roomID] = struct { + Sessions map[string]userapi.KeyBackupSession `json:"sessions"` + }{ + Sessions: roomData, + } + } + return util.JSONResponse{ + Code: 200, + JSON: resp, + } + } + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound("keys not found"), + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 3e0c53ee..cfc6c47f 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -896,11 +896,15 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) - // Key Backup Versions - r0mux.Handle("/room_keys/version/{versionID}", + // Key Backup Versions (Metadata) + + r0mux.Handle("/room_keys/version/{version}", httputil.MakeAuthAPI("get_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - version := req.URL.Query().Get("version") - return KeyBackupVersion(req, userAPI, device, version) + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return KeyBackupVersion(req, userAPI, device, vars["version"]) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/room_keys/version", @@ -908,28 +912,22 @@ func Setup( return KeyBackupVersion(req, userAPI, device, "") }), ).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/room_keys/version/{versionID}", + r0mux.Handle("/room_keys/version/{version}", httputil.MakeAuthAPI("put_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - version := req.URL.Query().Get("version") - if version == "" { - return util.JSONResponse{ - Code: 400, - JSON: jsonerror.InvalidArgumentValue("version must be specified"), - } + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) } - return ModifyKeyBackupVersionAuthData(req, userAPI, device, version) + return ModifyKeyBackupVersionAuthData(req, userAPI, device, vars["version"]) }), ).Methods(http.MethodPut) - r0mux.Handle("/room_keys/version/{versionID}", + r0mux.Handle("/room_keys/version/{version}", httputil.MakeAuthAPI("delete_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - version := req.URL.Query().Get("version") - if version == "" { - return util.JSONResponse{ - Code: 400, - JSON: jsonerror.InvalidArgumentValue("version must be specified"), - } + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) } - return DeleteKeyBackupVersion(req, userAPI, device, version) + return DeleteKeyBackupVersion(req, userAPI, device, vars["version"]) }), ).Methods(http.MethodDelete) r0mux.Handle("/room_keys/version", @@ -938,7 +936,8 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) - // E2E Backup Keys + // Inserting E2E Backup Keys + // Bulk room and session r0mux.Handle("/room_keys/keys", httputil.MakeAuthAPI("put_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { @@ -973,6 +972,9 @@ func Setup( } roomID := vars["roomID"] var reqBody keyBackupSessionRequest + reqBody.Rooms = make(map[string]struct { + Sessions map[string]userapi.KeyBackupSession `json:"sessions"` + }) reqBody.Rooms[roomID] = struct { Sessions map[string]userapi.KeyBackupSession `json:"sessions"` }{ @@ -989,7 +991,7 @@ func Setup( ).Methods(http.MethodPut) // Single room, single session r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", - httputil.MakeAuthAPI("put_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + httputil.MakeAuthAPI("put_backup_keys_room_session", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -1009,14 +1011,47 @@ func Setup( roomID := vars["roomID"] sessionID := vars["sessionID"] var keyReq keyBackupSessionRequest + keyReq.Rooms = make(map[string]struct { + Sessions map[string]userapi.KeyBackupSession `json:"sessions"` + }) keyReq.Rooms[roomID] = struct { Sessions map[string]userapi.KeyBackupSession `json:"sessions"` - }{} + }{ + Sessions: make(map[string]userapi.KeyBackupSession), + } keyReq.Rooms[roomID].Sessions[sessionID] = reqBody return UploadBackupKeys(req, userAPI, device, version, &keyReq) }), ).Methods(http.MethodPut) + // Querying E2E Backup Keys + + r0mux.Handle("/room_keys/keys", + httputil.MakeAuthAPI("get_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), "", "") + }), + ).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/room_keys/keys/{roomID}", + httputil.MakeAuthAPI("get_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], "") + }), + ).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", + httputil.MakeAuthAPI("get_backup_keys_room_session", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], vars["sessionID"]) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + // Deleting E2E Backup Keys + // Supplying a device ID is deprecated. r0mux.Handle("/keys/upload/{deviceID}", httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { diff --git a/sytest-whitelist b/sytest-whitelist index d90ba4fb..6bb3d770 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -540,3 +540,13 @@ Key notary server must not overwrite a valid key with a spurious result from the GET /rooms/:room_id/aliases lists aliases Only room members can list aliases of a room Users with sufficient power-level can delete other's aliases +Can create backup version +Can update backup version +Responds correctly when backup is empty +Can backup keys +Can update keys with better versions +Will not update keys with worse versions +Will not back up to an old backup version +Can create more than 10 backup versions +Can delete backup +Deleted & recreated backups are empty diff --git a/userapi/api/api.go b/userapi/api/api.go index 7e18d72f..b0d91856 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -67,6 +67,23 @@ type KeyBackupSession struct { SessionData json.RawMessage `json:"session_data"` } +func (a *KeyBackupSession) ShouldReplaceRoomKey(newKey *KeyBackupSession) bool { + // https://spec.matrix.org/unstable/client-server-api/#backup-algorithm-mmegolm_backupv1curve25519-aes-sha2 + // "if the keys have different values for is_verified, then it will keep the key that has is_verified set to true" + if newKey.IsVerified && !a.IsVerified { + return true + } + // "if they have the same values for is_verified, then it will keep the key with a lower first_message_index" + if newKey.FirstMessageIndex < a.FirstMessageIndex { + return true + } + // "and finally, is is_verified and first_message_index are equal, then it will keep the key with a lower forwarded_count" + if newKey.ForwardedCount < a.ForwardedCount { + return true + } + return false +} + // Internal KeyBackupData for passing to/from the storage layer type InternalKeyBackupSession struct { KeyBackupSession @@ -88,6 +105,10 @@ type PerformKeyBackupResponse struct { type QueryKeyBackupRequest struct { UserID string Version string // the version to query, if blank it means the latest + + ReturnKeys bool // whether to return keys in the backup response or just the metadata + KeysForRoomID string // optional string to return keys which belong to this room + KeysForSessionID string // optional string to return keys which belong to this (room, session) } type QueryKeyBackupResponse struct { @@ -96,9 +117,11 @@ type QueryKeyBackupResponse struct { Algorithm string `json:"algorithm"` AuthData json.RawMessage `json:"auth_data"` - Count int `json:"count"` + Count int64 `json:"count"` ETag string `json:"etag"` Version string `json:"version"` + + Keys map[string]map[string]KeyBackupSession // the keys if ReturnKeys=true } // InputAccountDataRequest is the request for InputAccountData diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 27e17963..a2bc8ecf 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -475,6 +475,7 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform if err != nil { res.Error = fmt.Sprintf("failed to update backup: %s", err) } + res.Exists = err == nil res.Version = req.Version return } @@ -483,8 +484,8 @@ func (a *UserInternalAPI) PerformKeyBackup(ctx context.Context, req *api.Perform } func (a *UserInternalAPI) uploadBackupKeys(ctx context.Context, req *api.PerformKeyBackupRequest, res *api.PerformKeyBackupResponse) { - // ensure the version metadata exists - version, _, _, _, deleted, err := a.AccountDB.GetKeyBackup(ctx, req.UserID, req.Version) + // you can only upload keys for the CURRENT version + version, _, _, _, deleted, err := a.AccountDB.GetKeyBackup(ctx, req.UserID, "") if err != nil { res.Error = fmt.Sprintf("failed to query version: %s", err) return @@ -493,6 +494,11 @@ func (a *UserInternalAPI) uploadBackupKeys(ctx context.Context, req *api.Perform res.Error = "backup was deleted" return } + if version != req.Version { + res.BadInput = true + res.Error = fmt.Sprintf("%s isn't the current version, %s is.", req.Version, version) + return + } res.Exists = true res.Version = version @@ -529,9 +535,21 @@ func (a *UserInternalAPI) QueryKeyBackup(ctx context.Context, req *api.QueryKeyB } res.Algorithm = algorithm res.AuthData = authData + res.ETag = etag res.Exists = !deleted - // TODO: - res.Count = 0 - res.ETag = etag + if !req.ReturnKeys { + res.Count, err = a.AccountDB.CountBackupKeys(ctx, version, req.UserID) + if err != nil { + res.Error = fmt.Sprintf("failed to count keys: %s", err) + } + return + } + + result, err := a.AccountDB.GetBackupKeys(ctx, version, req.UserID, req.KeysForRoomID, req.KeysForSessionID) + if err != nil { + res.Error = fmt.Sprintf("failed to query keys: %s", err) + return + } + res.Keys = result } diff --git a/userapi/storage/accounts/interface.go b/userapi/storage/accounts/interface.go index 4fd9c177..887f7193 100644 --- a/userapi/storage/accounts/interface.go +++ b/userapi/storage/accounts/interface.go @@ -61,6 +61,8 @@ type Database interface { DeleteKeyBackup(ctx context.Context, userID, version string) (exists bool, err error) GetKeyBackup(ctx context.Context, userID, version string) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error) UpsertBackupKeys(ctx context.Context, version, userID string, uploads []api.InternalKeyBackupSession) (count int64, etag string, err error) + GetBackupKeys(ctx context.Context, version, userID, filterRoomID, filterSessionID string) (result map[string]map[string]api.KeyBackupSession, err error) + CountBackupKeys(ctx context.Context, version, userID string) (count int64, err error) } // Err3PIDInUse is the error returned when trying to save an association involving diff --git a/userapi/storage/accounts/postgres/key_backup_table.go b/userapi/storage/accounts/postgres/key_backup_table.go index 0dc5879b..ec651826 100644 --- a/userapi/storage/accounts/postgres/key_backup_table.go +++ b/userapi/storage/accounts/postgres/key_backup_table.go @@ -35,7 +35,8 @@ CREATE TABLE IF NOT EXISTS account_e2e_room_keys ( is_verified BOOLEAN NOT NULL, session_data TEXT NOT NULL ); -CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON account_e2e_room_keys(user_id, room_id, session_id); +CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON account_e2e_room_keys(user_id, room_id, session_id, version); +CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON account_e2e_room_keys(user_id, version); ` const insertBackupKeySQL = "" + @@ -53,14 +54,23 @@ const selectKeysSQL = "" + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + "WHERE user_id = $1 AND version = $2" +const selectKeysByRoomIDSQL = "" + + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + + "WHERE user_id = $1 AND version = $2 AND room_id = $3" + +const selectKeysByRoomIDAndSessionIDSQL = "" + + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + + "WHERE user_id = $1 AND version = $2 AND room_id = $3 AND session_id = $4" + type keyBackupStatements struct { - insertBackupKeyStmt *sql.Stmt - updateBackupKeyStmt *sql.Stmt - countKeysStmt *sql.Stmt - selectKeysStmt *sql.Stmt + insertBackupKeyStmt *sql.Stmt + updateBackupKeyStmt *sql.Stmt + countKeysStmt *sql.Stmt + selectKeysStmt *sql.Stmt + selectKeysByRoomIDStmt *sql.Stmt + selectKeysByRoomIDAndSessionIDStmt *sql.Stmt } -// nolint:unused func (s *keyBackupStatements) prepare(db *sql.DB) (err error) { _, err = db.Exec(keyBackupTableSchema) if err != nil { @@ -78,6 +88,12 @@ func (s *keyBackupStatements) prepare(db *sql.DB) (err error) { if s.selectKeysStmt, err = db.Prepare(selectKeysSQL); err != nil { return } + if s.selectKeysByRoomIDStmt, err = db.Prepare(selectKeysByRoomIDSQL); err != nil { + return + } + if s.selectKeysByRoomIDAndSessionIDStmt, err = db.Prepare(selectKeysByRoomIDAndSessionIDSQL); err != nil { + return + } return } @@ -109,11 +125,35 @@ func (s *keyBackupStatements) updateBackupKey( func (s *keyBackupStatements) selectKeys( ctx context.Context, txn *sql.Tx, userID, version string, ) (map[string]map[string]api.KeyBackupSession, error) { - result := make(map[string]map[string]api.KeyBackupSession) rows, err := txn.Stmt(s.selectKeysStmt).QueryContext(ctx, userID, version) if err != nil { return nil, err } + return unpackKeys(ctx, rows) +} + +func (s *keyBackupStatements) selectKeysByRoomID( + ctx context.Context, txn *sql.Tx, userID, version, roomID string, +) (map[string]map[string]api.KeyBackupSession, error) { + rows, err := txn.Stmt(s.selectKeysByRoomIDStmt).QueryContext(ctx, userID, version, roomID) + if err != nil { + return nil, err + } + return unpackKeys(ctx, rows) +} + +func (s *keyBackupStatements) selectKeysByRoomIDAndSessionID( + ctx context.Context, txn *sql.Tx, userID, version, roomID, sessionID string, +) (map[string]map[string]api.KeyBackupSession, error) { + rows, err := txn.Stmt(s.selectKeysByRoomIDAndSessionIDStmt).QueryContext(ctx, userID, version, roomID, sessionID) + if err != nil { + return nil, err + } + return unpackKeys(ctx, rows) +} + +func unpackKeys(ctx context.Context, rows *sql.Rows) (map[string]map[string]api.KeyBackupSession, error) { + result := make(map[string]map[string]api.KeyBackupSession) defer internal.CloseAndLogIfError(ctx, rows, "selectKeysStmt.Close failed") for rows.Next() { var key api.InternalKeyBackupSession diff --git a/userapi/storage/accounts/postgres/key_backup_version_table.go b/userapi/storage/accounts/postgres/key_backup_version_table.go index 323a842d..aca575df 100644 --- a/userapi/storage/accounts/postgres/key_backup_version_table.go +++ b/userapi/storage/accounts/postgres/key_backup_version_table.go @@ -67,7 +67,6 @@ type keyBackupVersionStatements struct { updateKeyBackupETagStmt *sql.Stmt } -// nolint:unused func (s *keyBackupVersionStatements) prepare(db *sql.DB) (err error) { _, err = db.Exec(keyBackupVersionTableSchema) if err != nil { diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index b07218b2..9d6fd13a 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -96,13 +96,12 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err = d.openIDTokens.prepare(db, serverName); err != nil { return nil, err } - /* - if err = d.keyBackupVersions.prepare(db); err != nil { - return nil, err - } - if err = d.keyBackups.prepare(db); err != nil { - return nil, err - } */ + if err = d.keyBackupVersions.prepare(db); err != nil { + return nil, err + } + if err = d.keyBackups.prepare(db); err != nil { + return nil, err + } return d, nil } @@ -418,6 +417,37 @@ func (d *Database) GetKeyBackup( return } +func (d *Database) GetBackupKeys( + ctx context.Context, version, userID, filterRoomID, filterSessionID string, +) (result map[string]map[string]api.KeyBackupSession, err error) { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + if filterSessionID != "" { + result, err = d.keyBackups.selectKeysByRoomIDAndSessionID(ctx, txn, userID, version, filterRoomID, filterSessionID) + return err + } + if filterRoomID != "" { + result, err = d.keyBackups.selectKeysByRoomID(ctx, txn, userID, version, filterRoomID) + return err + } + result, err = d.keyBackups.selectKeys(ctx, txn, userID, version) + return err + }) + return +} + +func (d *Database) CountBackupKeys( + ctx context.Context, version, userID string, +) (count int64, err error) { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + count, err = d.keyBackups.countKeys(ctx, txn, userID, version) + if err != nil { + return err + } + return nil + }) + return +} + // nolint:nakedret func (d *Database) UpsertBackupKeys( ctx context.Context, version, userID string, uploads []api.InternalKeyBackupSession, @@ -445,7 +475,7 @@ func (d *Database) UpsertBackupKeys( if existingRoom != nil { existingSession, ok := existingRoom[newKey.SessionID] if ok { - if shouldReplaceRoomKey(existingSession, newKey.KeyBackupSession) { + if existingSession.ShouldReplaceRoomKey(&newKey.KeyBackupSession) { err = d.keyBackups.updateBackupKey(ctx, txn, userID, version, newKey) changed = true if err != nil { @@ -489,22 +519,3 @@ func (d *Database) UpsertBackupKeys( }) return } - -// TODO FIXME XXX : This logic really shouldn't live in the storage layer, but I don't know where else is sensible which won't -// create circular import loops -func shouldReplaceRoomKey(existing, uploaded api.KeyBackupSession) bool { - // https://spec.matrix.org/unstable/client-server-api/#backup-algorithm-mmegolm_backupv1curve25519-aes-sha2 - // "if the keys have different values for is_verified, then it will keep the key that has is_verified set to true" - if uploaded.IsVerified && !existing.IsVerified { - return true - } - // "if they have the same values for is_verified, then it will keep the key with a lower first_message_index" - if uploaded.FirstMessageIndex < existing.FirstMessageIndex { - return true - } - // "and finally, is is_verified and first_message_index are equal, then it will keep the key with a lower forwarded_count" - if uploaded.ForwardedCount < existing.ForwardedCount { - return true - } - return false -} diff --git a/userapi/storage/accounts/sqlite3/key_backup_table.go b/userapi/storage/accounts/sqlite3/key_backup_table.go index 268bda93..c1a698e6 100644 --- a/userapi/storage/accounts/sqlite3/key_backup_table.go +++ b/userapi/storage/accounts/sqlite3/key_backup_table.go @@ -35,7 +35,8 @@ CREATE TABLE IF NOT EXISTS account_e2e_room_keys ( is_verified BOOLEAN NOT NULL, session_data TEXT NOT NULL ); -CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON account_e2e_room_keys(user_id, room_id, session_id); +CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON account_e2e_room_keys(user_id, room_id, session_id, version); +CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON account_e2e_room_keys(user_id, version); ` const insertBackupKeySQL = "" + @@ -53,14 +54,23 @@ const selectKeysSQL = "" + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + "WHERE user_id = $1 AND version = $2" +const selectKeysByRoomIDSQL = "" + + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + + "WHERE user_id = $1 AND version = $2 AND room_id = $3" + +const selectKeysByRoomIDAndSessionIDSQL = "" + + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + + "WHERE user_id = $1 AND version = $2 AND room_id = $3 AND session_id = $4" + type keyBackupStatements struct { - insertBackupKeyStmt *sql.Stmt - updateBackupKeyStmt *sql.Stmt - countKeysStmt *sql.Stmt - selectKeysStmt *sql.Stmt + insertBackupKeyStmt *sql.Stmt + updateBackupKeyStmt *sql.Stmt + countKeysStmt *sql.Stmt + selectKeysStmt *sql.Stmt + selectKeysByRoomIDStmt *sql.Stmt + selectKeysByRoomIDAndSessionIDStmt *sql.Stmt } -// nolint:unused func (s *keyBackupStatements) prepare(db *sql.DB) (err error) { _, err = db.Exec(keyBackupTableSchema) if err != nil { @@ -78,6 +88,12 @@ func (s *keyBackupStatements) prepare(db *sql.DB) (err error) { if s.selectKeysStmt, err = db.Prepare(selectKeysSQL); err != nil { return } + if s.selectKeysByRoomIDStmt, err = db.Prepare(selectKeysByRoomIDSQL); err != nil { + return + } + if s.selectKeysByRoomIDAndSessionIDStmt, err = db.Prepare(selectKeysByRoomIDAndSessionIDSQL); err != nil { + return + } return } @@ -109,11 +125,35 @@ func (s *keyBackupStatements) updateBackupKey( func (s *keyBackupStatements) selectKeys( ctx context.Context, txn *sql.Tx, userID, version string, ) (map[string]map[string]api.KeyBackupSession, error) { - result := make(map[string]map[string]api.KeyBackupSession) rows, err := txn.Stmt(s.selectKeysStmt).QueryContext(ctx, userID, version) if err != nil { return nil, err } + return unpackKeys(ctx, rows) +} + +func (s *keyBackupStatements) selectKeysByRoomID( + ctx context.Context, txn *sql.Tx, userID, version, roomID string, +) (map[string]map[string]api.KeyBackupSession, error) { + rows, err := txn.Stmt(s.selectKeysByRoomIDStmt).QueryContext(ctx, userID, version, roomID) + if err != nil { + return nil, err + } + return unpackKeys(ctx, rows) +} + +func (s *keyBackupStatements) selectKeysByRoomIDAndSessionID( + ctx context.Context, txn *sql.Tx, userID, version, roomID, sessionID string, +) (map[string]map[string]api.KeyBackupSession, error) { + rows, err := txn.Stmt(s.selectKeysByRoomIDAndSessionIDStmt).QueryContext(ctx, userID, version, roomID, sessionID) + if err != nil { + return nil, err + } + return unpackKeys(ctx, rows) +} + +func unpackKeys(ctx context.Context, rows *sql.Rows) (map[string]map[string]api.KeyBackupSession, error) { + result := make(map[string]map[string]api.KeyBackupSession) defer internal.CloseAndLogIfError(ctx, rows, "selectKeysStmt.Close failed") for rows.Next() { var key api.InternalKeyBackupSession diff --git a/userapi/storage/accounts/sqlite3/key_backup_version_table.go b/userapi/storage/accounts/sqlite3/key_backup_version_table.go index 72e9b132..9a58fee7 100644 --- a/userapi/storage/accounts/sqlite3/key_backup_version_table.go +++ b/userapi/storage/accounts/sqlite3/key_backup_version_table.go @@ -65,7 +65,6 @@ type keyBackupVersionStatements struct { updateKeyBackupETagStmt *sql.Stmt } -// nolint:unused func (s *keyBackupVersionStatements) prepare(db *sql.DB) (err error) { _, err = db.Exec(keyBackupVersionTableSchema) if err != nil { diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index 4fae621f..728ae901 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -100,13 +100,12 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err = d.openIDTokens.prepare(db, serverName); err != nil { return nil, err } - /* - if err = d.keyBackupVersions.prepare(db); err != nil { - return nil, err - } - if err = d.keyBackups.prepare(db); err != nil { - return nil, err - } */ + if err = d.keyBackupVersions.prepare(db); err != nil { + return nil, err + } + if err = d.keyBackups.prepare(db); err != nil { + return nil, err + } return d, nil } @@ -459,6 +458,37 @@ func (d *Database) GetKeyBackup( return } +func (d *Database) GetBackupKeys( + ctx context.Context, version, userID, filterRoomID, filterSessionID string, +) (result map[string]map[string]api.KeyBackupSession, err error) { + err = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { + if filterSessionID != "" { + result, err = d.keyBackups.selectKeysByRoomIDAndSessionID(ctx, txn, userID, version, filterRoomID, filterSessionID) + return err + } + if filterRoomID != "" { + result, err = d.keyBackups.selectKeysByRoomID(ctx, txn, userID, version, filterRoomID) + return err + } + result, err = d.keyBackups.selectKeys(ctx, txn, userID, version) + return err + }) + return +} + +func (d *Database) CountBackupKeys( + ctx context.Context, version, userID string, +) (count int64, err error) { + err = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { + count, err = d.keyBackups.countKeys(ctx, txn, userID, version) + if err != nil { + return err + } + return nil + }) + return +} + // nolint:nakedret func (d *Database) UpsertBackupKeys( ctx context.Context, version, userID string, uploads []api.InternalKeyBackupSession, @@ -486,7 +516,7 @@ func (d *Database) UpsertBackupKeys( if existingRoom != nil { existingSession, ok := existingRoom[newKey.SessionID] if ok { - if shouldReplaceRoomKey(existingSession, newKey.KeyBackupSession) { + if existingSession.ShouldReplaceRoomKey(&newKey.KeyBackupSession) { err = d.keyBackups.updateBackupKey(ctx, txn, userID, version, newKey) changed = true if err != nil { @@ -531,22 +561,3 @@ func (d *Database) UpsertBackupKeys( }) return } - -// TODO FIXME XXX : This logic really shouldn't live in the storage layer, but I don't know where else is sensible which won't -// create circular import loops -func shouldReplaceRoomKey(existing, uploaded api.KeyBackupSession) bool { - // https://spec.matrix.org/unstable/client-server-api/#backup-algorithm-mmegolm_backupv1curve25519-aes-sha2 - // "if the keys have different values for is_verified, then it will keep the key that has is_verified set to true" - if uploaded.IsVerified && !existing.IsVerified { - return true - } - // "if they have the same values for is_verified, then it will keep the key with a lower first_message_index" - if uploaded.FirstMessageIndex < existing.FirstMessageIndex { - return true - } - // "and finally, is is_verified and first_message_index are equal, then it will keep the key with a lower forwarded_count" - if uploaded.ForwardedCount < existing.ForwardedCount { - return true - } - return false -} From 3e01a88a0cb5b22dfa5247e855ba2f5f7c97d8c2 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 27 Jul 2021 21:34:40 +0100 Subject: [PATCH 08/64] Update to neilalexander/utp@54ae7b1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 30e3c97b..fbad10d6 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb github.com/morikuni/aec v1.0.0 // indirect - github.com/neilalexander/utp v0.1.1-0.20210720104546-52626cdf31b2 + github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 github.com/opentracing/opentracing-go v1.2.0 diff --git a/go.sum b/go.sum index a090dbc3..ff6090a5 100644 --- a/go.sum +++ b/go.sum @@ -1184,8 +1184,8 @@ github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/neilalexander/utp v0.1.1-0.20210622132614-ee9a34a30488/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8= -github.com/neilalexander/utp v0.1.1-0.20210720104546-52626cdf31b2 h1:txJOiDxsypF8RbzbcyOD3ovip+uUWNZE/Zo7qLdARZQ= -github.com/neilalexander/utp v0.1.1-0.20210720104546-52626cdf31b2/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8= +github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ= +github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3hXzpHDl6GcEZeLXAhlH9Csdww= From 9e4618000e0347741eac1279bf6c94c3b9980785 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 28 Jul 2021 10:25:45 +0100 Subject: [PATCH 09/64] Alias key backup endpoints onto /unstable, fix key backup bugs (#1947) * Default /unstable requests to stable endpoints if not overridden specifically with a custom route * Rewrite URL * Try something different * Fix routing manually * Fix selectLatestVersionSQL * Don't return 0 if no backup version exists * Log more useful error * fix up replace keys check * Don't enforce uniqueness on e2e_room_keys_versions_idx Co-authored-by: kegsay --- clientapi/routing/routing.go | 292 +++++++++--------- userapi/api/api.go | 10 +- .../accounts/postgres/key_backup_table.go | 2 +- .../postgres/key_backup_version_table.go | 17 +- userapi/storage/accounts/postgres/storage.go | 4 +- .../accounts/sqlite3/key_backup_table.go | 2 +- .../sqlite3/key_backup_version_table.go | 17 +- userapi/storage/accounts/sqlite3/storage.go | 4 +- 8 files changed, 187 insertions(+), 161 deletions(-) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index cfc6c47f..541c7628 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -898,157 +898,171 @@ func Setup( // Key Backup Versions (Metadata) - r0mux.Handle("/room_keys/version/{version}", - httputil.MakeAuthAPI("get_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return KeyBackupVersion(req, userAPI, device, vars["version"]) - }), - ).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/room_keys/version", - httputil.MakeAuthAPI("get_latest_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return KeyBackupVersion(req, userAPI, device, "") - }), - ).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/room_keys/version/{version}", - httputil.MakeAuthAPI("put_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return ModifyKeyBackupVersionAuthData(req, userAPI, device, vars["version"]) - }), - ).Methods(http.MethodPut) - r0mux.Handle("/room_keys/version/{version}", - httputil.MakeAuthAPI("delete_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return DeleteKeyBackupVersion(req, userAPI, device, vars["version"]) - }), - ).Methods(http.MethodDelete) - r0mux.Handle("/room_keys/version", - httputil.MakeAuthAPI("post_new_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return CreateKeyBackupVersion(req, userAPI, device) - }), - ).Methods(http.MethodPost, http.MethodOptions) + getBackupKeysVersion := httputil.MakeAuthAPI("get_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return KeyBackupVersion(req, userAPI, device, vars["version"]) + }) + + getLatestBackupKeysVersion := httputil.MakeAuthAPI("get_latest_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return KeyBackupVersion(req, userAPI, device, "") + }) + + putBackupKeysVersion := httputil.MakeAuthAPI("put_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return ModifyKeyBackupVersionAuthData(req, userAPI, device, vars["version"]) + }) + + deleteBackupKeysVersion := httputil.MakeAuthAPI("delete_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return DeleteKeyBackupVersion(req, userAPI, device, vars["version"]) + }) + + postNewBackupKeysVersion := httputil.MakeAuthAPI("post_new_backup_keys_version", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return CreateKeyBackupVersion(req, userAPI, device) + }) + + r0mux.Handle("/room_keys/version/{version}", getBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/room_keys/version", getLatestBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/room_keys/version/{version}", putBackupKeysVersion).Methods(http.MethodPut) + r0mux.Handle("/room_keys/version/{version}", deleteBackupKeysVersion).Methods(http.MethodDelete) + r0mux.Handle("/room_keys/version", postNewBackupKeysVersion).Methods(http.MethodPost, http.MethodOptions) + + unstableMux.Handle("/room_keys/version/{version}", getBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions) + unstableMux.Handle("/room_keys/version", getLatestBackupKeysVersion).Methods(http.MethodGet, http.MethodOptions) + unstableMux.Handle("/room_keys/version/{version}", putBackupKeysVersion).Methods(http.MethodPut) + unstableMux.Handle("/room_keys/version/{version}", deleteBackupKeysVersion).Methods(http.MethodDelete) + unstableMux.Handle("/room_keys/version", postNewBackupKeysVersion).Methods(http.MethodPost, http.MethodOptions) // Inserting E2E Backup Keys // Bulk room and session - r0mux.Handle("/room_keys/keys", - httputil.MakeAuthAPI("put_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - version := req.URL.Query().Get("version") - if version == "" { - return util.JSONResponse{ - Code: 400, - JSON: jsonerror.InvalidArgumentValue("version must be specified"), - } + putBackupKeys := httputil.MakeAuthAPI("put_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + version := req.URL.Query().Get("version") + if version == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.InvalidArgumentValue("version must be specified"), } - var reqBody keyBackupSessionRequest - resErr := clientutil.UnmarshalJSONRequest(req, &reqBody) - if resErr != nil { - return *resErr - } - return UploadBackupKeys(req, userAPI, device, version, &reqBody) - }), - ).Methods(http.MethodPut) + } + var reqBody keyBackupSessionRequest + resErr := clientutil.UnmarshalJSONRequest(req, &reqBody) + if resErr != nil { + return *resErr + } + return UploadBackupKeys(req, userAPI, device, version, &reqBody) + }) + // Single room bulk session - r0mux.Handle("/room_keys/keys/{roomID}", - httputil.MakeAuthAPI("put_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) + putBackupKeysRoom := httputil.MakeAuthAPI("put_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + version := req.URL.Query().Get("version") + if version == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.InvalidArgumentValue("version must be specified"), } - version := req.URL.Query().Get("version") - if version == "" { - return util.JSONResponse{ - Code: 400, - JSON: jsonerror.InvalidArgumentValue("version must be specified"), - } - } - roomID := vars["roomID"] - var reqBody keyBackupSessionRequest - reqBody.Rooms = make(map[string]struct { - Sessions map[string]userapi.KeyBackupSession `json:"sessions"` - }) - reqBody.Rooms[roomID] = struct { - Sessions map[string]userapi.KeyBackupSession `json:"sessions"` - }{ - Sessions: map[string]userapi.KeyBackupSession{}, - } - body := reqBody.Rooms[roomID] - resErr := clientutil.UnmarshalJSONRequest(req, &body) - if resErr != nil { - return *resErr - } - reqBody.Rooms[roomID] = body - return UploadBackupKeys(req, userAPI, device, version, &reqBody) - }), - ).Methods(http.MethodPut) + } + roomID := vars["roomID"] + var reqBody keyBackupSessionRequest + reqBody.Rooms = make(map[string]struct { + Sessions map[string]userapi.KeyBackupSession `json:"sessions"` + }) + reqBody.Rooms[roomID] = struct { + Sessions map[string]userapi.KeyBackupSession `json:"sessions"` + }{ + Sessions: map[string]userapi.KeyBackupSession{}, + } + body := reqBody.Rooms[roomID] + resErr := clientutil.UnmarshalJSONRequest(req, &body) + if resErr != nil { + return *resErr + } + reqBody.Rooms[roomID] = body + return UploadBackupKeys(req, userAPI, device, version, &reqBody) + }) + // Single room, single session - r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", - httputil.MakeAuthAPI("put_backup_keys_room_session", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) + putBackupKeysRoomSession := httputil.MakeAuthAPI("put_backup_keys_room_session", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + version := req.URL.Query().Get("version") + if version == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.InvalidArgumentValue("version must be specified"), } - version := req.URL.Query().Get("version") - if version == "" { - return util.JSONResponse{ - Code: 400, - JSON: jsonerror.InvalidArgumentValue("version must be specified"), - } - } - var reqBody userapi.KeyBackupSession - resErr := clientutil.UnmarshalJSONRequest(req, &reqBody) - if resErr != nil { - return *resErr - } - roomID := vars["roomID"] - sessionID := vars["sessionID"] - var keyReq keyBackupSessionRequest - keyReq.Rooms = make(map[string]struct { - Sessions map[string]userapi.KeyBackupSession `json:"sessions"` - }) - keyReq.Rooms[roomID] = struct { - Sessions map[string]userapi.KeyBackupSession `json:"sessions"` - }{ - Sessions: make(map[string]userapi.KeyBackupSession), - } - keyReq.Rooms[roomID].Sessions[sessionID] = reqBody - return UploadBackupKeys(req, userAPI, device, version, &keyReq) - }), - ).Methods(http.MethodPut) + } + var reqBody userapi.KeyBackupSession + resErr := clientutil.UnmarshalJSONRequest(req, &reqBody) + if resErr != nil { + return *resErr + } + roomID := vars["roomID"] + sessionID := vars["sessionID"] + var keyReq keyBackupSessionRequest + keyReq.Rooms = make(map[string]struct { + Sessions map[string]userapi.KeyBackupSession `json:"sessions"` + }) + keyReq.Rooms[roomID] = struct { + Sessions map[string]userapi.KeyBackupSession `json:"sessions"` + }{ + Sessions: make(map[string]userapi.KeyBackupSession), + } + keyReq.Rooms[roomID].Sessions[sessionID] = reqBody + return UploadBackupKeys(req, userAPI, device, version, &keyReq) + }) + + r0mux.Handle("/room_keys/keys", putBackupKeys).Methods(http.MethodPut) + r0mux.Handle("/room_keys/keys/{roomID}", putBackupKeysRoom).Methods(http.MethodPut) + r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", putBackupKeysRoomSession).Methods(http.MethodPut) + + unstableMux.Handle("/room_keys/keys", putBackupKeys).Methods(http.MethodPut) + unstableMux.Handle("/room_keys/keys/{roomID}", putBackupKeysRoom).Methods(http.MethodPut) + unstableMux.Handle("/room_keys/keys/{roomID}/{sessionID}", putBackupKeysRoomSession).Methods(http.MethodPut) // Querying E2E Backup Keys - r0mux.Handle("/room_keys/keys", - httputil.MakeAuthAPI("get_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), "", "") - }), - ).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/room_keys/keys/{roomID}", - httputil.MakeAuthAPI("get_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], "") - }), - ).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", - httputil.MakeAuthAPI("get_backup_keys_room_session", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], vars["sessionID"]) - }), - ).Methods(http.MethodGet, http.MethodOptions) + getBackupKeys := httputil.MakeAuthAPI("get_backup_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), "", "") + }) + + getBackupKeysRoom := httputil.MakeAuthAPI("get_backup_keys_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], "") + }) + + getBackupKeysRoomSession := httputil.MakeAuthAPI("get_backup_keys_room_session", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetBackupKeys(req, userAPI, device, req.URL.Query().Get("version"), vars["roomID"], vars["sessionID"]) + }) + + r0mux.Handle("/room_keys/keys", getBackupKeys).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/room_keys/keys/{roomID}", getBackupKeysRoom).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/room_keys/keys/{roomID}/{sessionID}", getBackupKeysRoomSession).Methods(http.MethodGet, http.MethodOptions) + + unstableMux.Handle("/room_keys/keys", getBackupKeys).Methods(http.MethodGet, http.MethodOptions) + unstableMux.Handle("/room_keys/keys/{roomID}", getBackupKeysRoom).Methods(http.MethodGet, http.MethodOptions) + unstableMux.Handle("/room_keys/keys/{roomID}/{sessionID}", getBackupKeysRoomSession).Methods(http.MethodGet, http.MethodOptions) // Deleting E2E Backup Keys diff --git a/userapi/api/api.go b/userapi/api/api.go index b0d91856..75d06dd6 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -72,13 +72,11 @@ func (a *KeyBackupSession) ShouldReplaceRoomKey(newKey *KeyBackupSession) bool { // "if the keys have different values for is_verified, then it will keep the key that has is_verified set to true" if newKey.IsVerified && !a.IsVerified { return true - } - // "if they have the same values for is_verified, then it will keep the key with a lower first_message_index" - if newKey.FirstMessageIndex < a.FirstMessageIndex { + } else if newKey.FirstMessageIndex < a.FirstMessageIndex { + // "if they have the same values for is_verified, then it will keep the key with a lower first_message_index" return true - } - // "and finally, is is_verified and first_message_index are equal, then it will keep the key with a lower forwarded_count" - if newKey.ForwardedCount < a.ForwardedCount { + } else if newKey.ForwardedCount < a.ForwardedCount { + // "and finally, is is_verified and first_message_index are equal, then it will keep the key with a lower forwarded_count" return true } return false diff --git a/userapi/storage/accounts/postgres/key_backup_table.go b/userapi/storage/accounts/postgres/key_backup_table.go index ec651826..0a2a2655 100644 --- a/userapi/storage/accounts/postgres/key_backup_table.go +++ b/userapi/storage/accounts/postgres/key_backup_table.go @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS account_e2e_room_keys ( session_data TEXT NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON account_e2e_room_keys(user_id, room_id, session_id, version); -CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON account_e2e_room_keys(user_id, version); +CREATE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON account_e2e_room_keys(user_id, version); ` const insertBackupKeySQL = "" + diff --git a/userapi/storage/accounts/postgres/key_backup_version_table.go b/userapi/storage/accounts/postgres/key_backup_version_table.go index aca575df..51a462b3 100644 --- a/userapi/storage/accounts/postgres/key_backup_version_table.go +++ b/userapi/storage/accounts/postgres/key_backup_version_table.go @@ -146,12 +146,19 @@ func (s *keyBackupVersionStatements) selectKeyBackup( ) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error) { var versionInt int64 if version == "" { - err = txn.Stmt(s.selectLatestVersionStmt).QueryRowContext(ctx, userID).Scan(&versionInt) + var v *int64 // allows nulls + if err = txn.Stmt(s.selectLatestVersionStmt).QueryRowContext(ctx, userID).Scan(&v); err != nil { + return + } + if v == nil { + err = sql.ErrNoRows + return + } + versionInt = *v } else { - versionInt, err = strconv.ParseInt(version, 10, 64) - } - if err != nil { - return + if versionInt, err = strconv.ParseInt(version, 10, 64); err != nil { + return + } } versionResult = strconv.FormatInt(versionInt, 10) var deletedInt int diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index 9d6fd13a..6bddbfc3 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -479,7 +479,7 @@ func (d *Database) UpsertBackupKeys( err = d.keyBackups.updateBackupKey(ctx, txn, userID, version, newKey) changed = true if err != nil { - return err + return fmt.Errorf("d.keyBackups.updateBackupKey: %w", err) } } // if we shouldn't replace the key we do nothing with it @@ -490,7 +490,7 @@ func (d *Database) UpsertBackupKeys( err = d.keyBackups.insertBackupKey(ctx, txn, userID, version, newKey) changed = true if err != nil { - return err + return fmt.Errorf("d.keyBackups.insertBackupKey: %w", err) } } diff --git a/userapi/storage/accounts/sqlite3/key_backup_table.go b/userapi/storage/accounts/sqlite3/key_backup_table.go index c1a698e6..67509351 100644 --- a/userapi/storage/accounts/sqlite3/key_backup_table.go +++ b/userapi/storage/accounts/sqlite3/key_backup_table.go @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS account_e2e_room_keys ( session_data TEXT NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON account_e2e_room_keys(user_id, room_id, session_id, version); -CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON account_e2e_room_keys(user_id, version); +CREATE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON account_e2e_room_keys(user_id, version); ` const insertBackupKeySQL = "" + diff --git a/userapi/storage/accounts/sqlite3/key_backup_version_table.go b/userapi/storage/accounts/sqlite3/key_backup_version_table.go index 9a58fee7..a9e7bf5d 100644 --- a/userapi/storage/accounts/sqlite3/key_backup_version_table.go +++ b/userapi/storage/accounts/sqlite3/key_backup_version_table.go @@ -144,12 +144,19 @@ func (s *keyBackupVersionStatements) selectKeyBackup( ) (versionResult, algorithm string, authData json.RawMessage, etag string, deleted bool, err error) { var versionInt int64 if version == "" { - err = txn.Stmt(s.selectLatestVersionStmt).QueryRowContext(ctx, userID).Scan(&versionInt) + var v *int64 // allows nulls + if err = txn.Stmt(s.selectLatestVersionStmt).QueryRowContext(ctx, userID).Scan(&v); err != nil { + return + } + if v == nil { + err = sql.ErrNoRows + return + } + versionInt = *v } else { - versionInt, err = strconv.ParseInt(version, 10, 64) - } - if err != nil { - return + if versionInt, err = strconv.ParseInt(version, 10, 64); err != nil { + return + } } versionResult = strconv.FormatInt(versionInt, 10) var deletedInt int diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index 728ae901..d752e3db 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -520,7 +520,7 @@ func (d *Database) UpsertBackupKeys( err = d.keyBackups.updateBackupKey(ctx, txn, userID, version, newKey) changed = true if err != nil { - return err + return fmt.Errorf("d.keyBackups.updateBackupKey: %w", err) } } // if we shouldn't replace the key we do nothing with it @@ -531,7 +531,7 @@ func (d *Database) UpsertBackupKeys( err = d.keyBackups.insertBackupKey(ctx, txn, userID, version, newKey) changed = true if err != nil { - return err + return fmt.Errorf("d.keyBackups.insertBackupKey: %w", err) } } From ed4097825bc65f2332bcdc975ed201841221ff7c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 28 Jul 2021 18:30:04 +0100 Subject: [PATCH 10/64] Factor out StatementList to `sqlutil` and use it in `userapi` It helps with the boilerplate. --- internal/sqlutil/sql.go | 16 ++++++++++ .../storage/postgres/event_json_table.go | 4 +-- .../postgres/event_state_keys_table.go | 3 +- .../storage/postgres/event_types_table.go | 3 +- roomserver/storage/postgres/events_table.go | 3 +- roomserver/storage/postgres/invite_table.go | 3 +- .../storage/postgres/membership_table.go | 3 +- .../storage/postgres/previous_events_table.go | 3 +- .../storage/postgres/published_table.go | 3 +- .../storage/postgres/redactions_table.go | 3 +- .../storage/postgres/room_aliases_table.go | 3 +- roomserver/storage/postgres/rooms_table.go | 3 +- .../storage/postgres/state_block_table.go | 4 +-- .../storage/postgres/state_snapshot_table.go | 3 +- .../storage/postgres/transactions_table.go | 4 +-- .../storage/sqlite3/event_json_table.go | 3 +- .../storage/sqlite3/event_state_keys_table.go | 3 +- .../storage/sqlite3/event_types_table.go | 3 +- roomserver/storage/sqlite3/events_table.go | 3 +- roomserver/storage/sqlite3/invite_table.go | 3 +- .../storage/sqlite3/membership_table.go | 3 +- .../storage/sqlite3/previous_events_table.go | 3 +- roomserver/storage/sqlite3/published_table.go | 3 +- .../storage/sqlite3/redactions_table.go | 3 +- .../storage/sqlite3/room_aliases_table.go | 3 +- roomserver/storage/sqlite3/rooms_table.go | 3 +- .../storage/sqlite3/state_block_table.go | 3 +- .../storage/sqlite3/state_snapshot_table.go | 3 +- .../storage/sqlite3/transactions_table.go | 3 +- .../accounts/postgres/account_data_table.go | 15 ++++------ .../accounts/postgres/accounts_table.go | 27 +++++------------ .../accounts/postgres/key_backup_table.go | 28 ++++++------------ .../postgres/key_backup_version_table.go | 29 +++++++------------ .../storage/accounts/postgres/openid_table.go | 11 +++---- .../accounts/postgres/profile_table.go | 23 +++++---------- .../accounts/postgres/threepid_table.go | 20 ++++--------- .../accounts/sqlite3/account_data_table.go | 15 ++++------ .../accounts/sqlite3/accounts_table.go | 27 +++++------------ .../accounts/sqlite3/key_backup_table.go | 28 ++++++------------ .../sqlite3/key_backup_version_table.go | 29 +++++++------------ .../storage/accounts/sqlite3/openid_table.go | 11 +++---- .../storage/accounts/sqlite3/profile_table.go | 23 +++++---------- .../accounts/sqlite3/threepid_table.go | 20 ++++--------- 43 files changed, 145 insertions(+), 264 deletions(-) diff --git a/internal/sqlutil/sql.go b/internal/sqlutil/sql.go index 6cf17bea..98cc396a 100644 --- a/internal/sqlutil/sql.go +++ b/internal/sqlutil/sql.go @@ -152,3 +152,19 @@ func RunLimitedVariablesQuery(ctx context.Context, query string, qp QueryProvide } return nil } + +// StatementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement. +type StatementList []struct { + Statement **sql.Stmt + SQL string +} + +// Prepare the SQL for each statement in the list and assign the result to the prepared statement. +func (s StatementList) Prepare(db *sql.DB) (err error) { + for _, statement := range s { + if *statement.Statement, err = db.Prepare(statement.SQL); err != nil { + return + } + } + return +} diff --git a/roomserver/storage/postgres/event_json_table.go b/roomserver/storage/postgres/event_json_table.go index e0976b12..32e45782 100644 --- a/roomserver/storage/postgres/event_json_table.go +++ b/roomserver/storage/postgres/event_json_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/roomserver/storage/shared" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -67,7 +67,7 @@ func createEventJSONTable(db *sql.DB) error { func prepareEventJSONTable(db *sql.DB) (tables.EventJSON, error) { s := &eventJSONStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertEventJSONStmt, insertEventJSONSQL}, {&s.bulkSelectEventJSONStmt, bulkSelectEventJSONSQL}, }.Prepare(db) diff --git a/roomserver/storage/postgres/event_state_keys_table.go b/roomserver/storage/postgres/event_state_keys_table.go index 61682356..3a7cf03e 100644 --- a/roomserver/storage/postgres/event_state_keys_table.go +++ b/roomserver/storage/postgres/event_state_keys_table.go @@ -22,7 +22,6 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -85,7 +84,7 @@ func createEventStateKeysTable(db *sql.DB) error { func prepareEventStateKeysTable(db *sql.DB) (tables.EventStateKeys, error) { s := &eventStateKeyStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertEventStateKeyNIDStmt, insertEventStateKeyNIDSQL}, {&s.selectEventStateKeyNIDStmt, selectEventStateKeyNIDSQL}, {&s.bulkSelectEventStateKeyNIDStmt, bulkSelectEventStateKeyNIDSQL}, diff --git a/roomserver/storage/postgres/event_types_table.go b/roomserver/storage/postgres/event_types_table.go index f4257850..e558072a 100644 --- a/roomserver/storage/postgres/event_types_table.go +++ b/roomserver/storage/postgres/event_types_table.go @@ -22,7 +22,6 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -108,7 +107,7 @@ func createEventTypesTable(db *sql.DB) error { func prepareEventTypesTable(db *sql.DB) (tables.EventTypes, error) { s := &eventTypeStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertEventTypeNIDStmt, insertEventTypeNIDSQL}, {&s.selectEventTypeNIDStmt, selectEventTypeNIDSQL}, {&s.bulkSelectEventTypeNIDStmt, bulkSelectEventTypeNIDSQL}, diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index 88c82083..c549fb65 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -24,7 +24,6 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -160,7 +159,7 @@ func createEventsTable(db *sql.DB) error { func prepareEventsTable(db *sql.DB) (tables.Events, error) { s := &eventStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertEventStmt, insertEventSQL}, {&s.selectEventStmt, selectEventSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index 0a2183e2..344302c8 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -21,7 +21,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -90,7 +89,7 @@ func createInvitesTable(db *sql.DB) error { func prepareInvitesTable(db *sql.DB) (tables.Invites, error) { s := &inviteStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertInviteEventStmt, insertInviteEventSQL}, {&s.selectInviteActiveForUserInRoomStmt, selectInviteActiveForUserInRoomSQL}, {&s.updateInviteRetiredStmt, updateInviteRetiredSQL}, diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index b4a27900..b0d906c8 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -23,7 +23,6 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -168,7 +167,7 @@ func createMembershipTable(db *sql.DB) error { func prepareMembershipTable(db *sql.DB) (tables.Membership, error) { s := &membershipStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertMembershipStmt, insertMembershipSQL}, {&s.selectMembershipForUpdateStmt, selectMembershipForUpdateSQL}, {&s.selectMembershipFromRoomAndTargetStmt, selectMembershipFromRoomAndTargetSQL}, diff --git a/roomserver/storage/postgres/previous_events_table.go b/roomserver/storage/postgres/previous_events_table.go index 4a93c3d6..bd4e853e 100644 --- a/roomserver/storage/postgres/previous_events_table.go +++ b/roomserver/storage/postgres/previous_events_table.go @@ -20,7 +20,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -73,7 +72,7 @@ func createPrevEventsTable(db *sql.DB) error { func preparePrevEventsTable(db *sql.DB) (tables.PreviousEvents, error) { s := &previousEventStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertPreviousEventStmt, insertPreviousEventSQL}, {&s.selectPreviousEventExistsStmt, selectPreviousEventExistsSQL}, }.Prepare(db) diff --git a/roomserver/storage/postgres/published_table.go b/roomserver/storage/postgres/published_table.go index c180576e..8deb6844 100644 --- a/roomserver/storage/postgres/published_table.go +++ b/roomserver/storage/postgres/published_table.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -58,7 +57,7 @@ func createPublishedTable(db *sql.DB) error { func preparePublishedTable(db *sql.DB) (tables.Published, error) { s := &publishedStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.upsertPublishedStmt, upsertPublishedSQL}, {&s.selectAllPublishedStmt, selectAllPublishedSQL}, {&s.selectPublishedStmt, selectPublishedSQL}, diff --git a/roomserver/storage/postgres/redactions_table.go b/roomserver/storage/postgres/redactions_table.go index 3741d5f6..5614f2bd 100644 --- a/roomserver/storage/postgres/redactions_table.go +++ b/roomserver/storage/postgres/redactions_table.go @@ -19,7 +19,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -68,7 +67,7 @@ func createRedactionsTable(db *sql.DB) error { func prepareRedactionsTable(db *sql.DB) (tables.Redactions, error) { s := &redactionStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertRedactionStmt, insertRedactionSQL}, {&s.selectRedactionInfoByRedactionEventIDStmt, selectRedactionInfoByRedactionEventIDSQL}, {&s.selectRedactionInfoByEventBeingRedactedStmt, selectRedactionInfoByEventBeingRedactedSQL}, diff --git a/roomserver/storage/postgres/room_aliases_table.go b/roomserver/storage/postgres/room_aliases_table.go index c808813e..031825fe 100644 --- a/roomserver/storage/postgres/room_aliases_table.go +++ b/roomserver/storage/postgres/room_aliases_table.go @@ -21,7 +21,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -70,7 +69,7 @@ func createRoomAliasesTable(db *sql.DB) error { func prepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) { s := &roomAliasesStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertRoomAliasStmt, insertRoomAliasSQL}, {&s.selectRoomIDFromAliasStmt, selectRoomIDFromAliasSQL}, {&s.selectAliasesFromRoomIDStmt, selectAliasesFromRoomIDSQL}, diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index f2b39fe5..ba8eb671 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -22,7 +22,6 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -104,7 +103,7 @@ func createRoomsTable(db *sql.DB) error { func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) { s := &roomStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertRoomNIDStmt, insertRoomNIDSQL}, {&s.selectRoomNIDStmt, selectRoomNIDSQL}, {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go index 7ae987d6..27d85e83 100644 --- a/roomserver/storage/postgres/state_block_table.go +++ b/roomserver/storage/postgres/state_block_table.go @@ -23,7 +23,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/roomserver/storage/shared" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" @@ -79,7 +79,7 @@ func createStateBlockTable(db *sql.DB) error { func prepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) { s := &stateBlockStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertStateDataStmt, insertStateDataSQL}, {&s.bulkSelectStateBlockEntriesStmt, bulkSelectStateBlockEntriesSQL}, }.Prepare(db) diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go index 15e14e2e..4fc0fa48 100644 --- a/roomserver/storage/postgres/state_snapshot_table.go +++ b/roomserver/storage/postgres/state_snapshot_table.go @@ -22,7 +22,6 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" @@ -86,7 +85,7 @@ func createStateSnapshotTable(db *sql.DB) error { func prepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) { s := &stateSnapshotStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertStateStmt, insertStateSQL}, {&s.bulkSelectStateBlockNIDsStmt, bulkSelectStateBlockNIDsSQL}, }.Prepare(db) diff --git a/roomserver/storage/postgres/transactions_table.go b/roomserver/storage/postgres/transactions_table.go index cada0d8a..af023b74 100644 --- a/roomserver/storage/postgres/transactions_table.go +++ b/roomserver/storage/postgres/transactions_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/roomserver/storage/shared" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -62,7 +62,7 @@ func createTransactionsTable(db *sql.DB) error { func prepareTransactionsTable(db *sql.DB) (tables.Transactions, error) { s := &transactionStatements{} - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertTransactionStmt, insertTransactionSQL}, {&s.selectTransactionEventIDStmt, selectTransactionEventIDSQL}, }.Prepare(db) diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index 29d54b83..53b21929 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -22,7 +22,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -63,7 +62,7 @@ func prepareEventJSONTable(db *sql.DB) (tables.EventJSON, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertEventJSONStmt, insertEventJSONSQL}, {&s.bulkSelectEventJSONStmt, bulkSelectEventJSONSQL}, }.Prepare(db) diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index d430e553..62fbce2d 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -22,7 +22,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -80,7 +79,7 @@ func prepareEventStateKeysTable(db *sql.DB) (tables.EventStateKeys, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertEventStateKeyNIDStmt, insertEventStateKeyNIDSQL}, {&s.selectEventStateKeyNIDStmt, selectEventStateKeyNIDSQL}, {&s.bulkSelectEventStateKeyNIDStmt, bulkSelectEventStateKeyNIDSQL}, diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index 694f4e21..22df3fb2 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -95,7 +94,7 @@ func prepareEventTypesTable(db *sql.DB) (tables.EventTypes, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertEventTypeNIDStmt, insertEventTypeNIDSQL}, {&s.insertEventTypeNIDResultStmt, insertEventTypeNIDResultSQL}, {&s.selectEventTypeNIDStmt, selectEventTypeNIDSQL}, diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index e964770d..a28d95fa 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -25,7 +25,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -131,7 +130,7 @@ func prepareEventsTable(db *sql.DB) (tables.Events, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertEventStmt, insertEventSQL}, {&s.selectEventStmt, selectEventSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index e1aa1ebd..c1d7347a 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -21,7 +21,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -80,7 +79,7 @@ func prepareInvitesTable(db *sql.DB) (tables.Invites, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertInviteEventStmt, insertInviteEventSQL}, {&s.selectInviteActiveForUserInRoomStmt, selectInviteActiveForUserInRoomSQL}, {&s.updateInviteRetiredStmt, updateInviteRetiredSQL}, diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 911a2516..2e58431d 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -146,7 +145,7 @@ func prepareMembershipTable(db *sql.DB) (tables.Membership, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertMembershipStmt, insertMembershipSQL}, {&s.selectMembershipForUpdateStmt, selectMembershipForUpdateSQL}, {&s.selectMembershipFromRoomAndTargetStmt, selectMembershipFromRoomAndTargetSQL}, diff --git a/roomserver/storage/sqlite3/previous_events_table.go b/roomserver/storage/sqlite3/previous_events_table.go index 3cb52767..7304bf0d 100644 --- a/roomserver/storage/sqlite3/previous_events_table.go +++ b/roomserver/storage/sqlite3/previous_events_table.go @@ -22,7 +22,6 @@ import ( "strings" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -81,7 +80,7 @@ func preparePrevEventsTable(db *sql.DB) (tables.PreviousEvents, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertPreviousEventStmt, insertPreviousEventSQL}, {&s.selectPreviousEventNIDsStmt, selectPreviousEventNIDsSQL}, {&s.selectPreviousEventExistsStmt, selectPreviousEventExistsSQL}, diff --git a/roomserver/storage/sqlite3/published_table.go b/roomserver/storage/sqlite3/published_table.go index 6d9d9135..b07c0ac4 100644 --- a/roomserver/storage/sqlite3/published_table.go +++ b/roomserver/storage/sqlite3/published_table.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -60,7 +59,7 @@ func preparePublishedTable(db *sql.DB) (tables.Published, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.upsertPublishedStmt, upsertPublishedSQL}, {&s.selectAllPublishedStmt, selectAllPublishedSQL}, {&s.selectPublishedStmt, selectPublishedSQL}, diff --git a/roomserver/storage/sqlite3/redactions_table.go b/roomserver/storage/sqlite3/redactions_table.go index b3498182..aed190b1 100644 --- a/roomserver/storage/sqlite3/redactions_table.go +++ b/roomserver/storage/sqlite3/redactions_table.go @@ -19,7 +19,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -69,7 +68,7 @@ func prepareRedactionsTable(db *sql.DB) (tables.Redactions, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertRedactionStmt, insertRedactionSQL}, {&s.selectRedactionInfoByRedactionEventIDStmt, selectRedactionInfoByRedactionEventIDSQL}, {&s.selectRedactionInfoByEventBeingRedactedStmt, selectRedactionInfoByEventBeingRedactedSQL}, diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index 5215fa6f..323945b8 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -21,7 +21,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -74,7 +73,7 @@ func prepareRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertRoomAliasStmt, insertRoomAliasSQL}, {&s.selectRoomIDFromAliasStmt, selectRoomIDFromAliasSQL}, {&s.selectAliasesFromRoomIDStmt, selectAliasesFromRoomIDSQL}, diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 534a870c..2dfb830d 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -96,7 +95,7 @@ func prepareRoomsTable(db *sql.DB) (tables.Rooms, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertRoomNIDStmt, insertRoomNIDSQL}, {&s.selectRoomNIDStmt, selectRoomNIDSQL}, {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index 5cb21e91..a472437a 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -25,7 +25,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" @@ -75,7 +74,7 @@ func prepareStateBlockTable(db *sql.DB) (tables.StateBlock, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertStateDataStmt, insertStateDataSQL}, {&s.bulkSelectStateBlockEntriesStmt, bulkSelectStateBlockEntriesSQL}, }.Prepare(db) diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 95cae99e..ad623d65 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" @@ -79,7 +78,7 @@ func prepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertStateStmt, insertStateSQL}, {&s.bulkSelectStateBlockNIDsStmt, bulkSelectStateBlockNIDsSQL}, }.Prepare(db) diff --git a/roomserver/storage/sqlite3/transactions_table.go b/roomserver/storage/sqlite3/transactions_table.go index e7471d7b..1fb0a831 100644 --- a/roomserver/storage/sqlite3/transactions_table.go +++ b/roomserver/storage/sqlite3/transactions_table.go @@ -20,7 +20,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -59,7 +58,7 @@ func prepareTransactionsTable(db *sql.DB) (tables.Transactions, error) { db: db, } - return s, shared.StatementList{ + return s, sqlutil.StatementList{ {&s.insertTransactionStmt, insertTransactionSQL}, {&s.selectTransactionEventIDStmt, selectTransactionEventIDSQL}, }.Prepare(db) diff --git a/userapi/storage/accounts/postgres/account_data_table.go b/userapi/storage/accounts/postgres/account_data_table.go index 09eb2611..8ba890e7 100644 --- a/userapi/storage/accounts/postgres/account_data_table.go +++ b/userapi/storage/accounts/postgres/account_data_table.go @@ -61,16 +61,11 @@ func (s *accountDataStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil { - return - } - if s.selectAccountDataStmt, err = db.Prepare(selectAccountDataSQL); err != nil { - return - } - if s.selectAccountDataByTypeStmt, err = db.Prepare(selectAccountDataByTypeSQL); err != nil { - return - } - return + return sqlutil.StatementList{ + {&s.insertAccountDataStmt, insertAccountDataSQL}, + {&s.selectAccountDataStmt, selectAccountDataSQL}, + {&s.selectAccountDataByTypeStmt, selectAccountDataByTypeSQL}, + }.Prepare(db) } func (s *accountDataStatements) insertAccountData( diff --git a/userapi/storage/accounts/postgres/accounts_table.go b/userapi/storage/accounts/postgres/accounts_table.go index 4eaa5b58..b57aa901 100644 --- a/userapi/storage/accounts/postgres/accounts_table.go +++ b/userapi/storage/accounts/postgres/accounts_table.go @@ -81,26 +81,15 @@ func (s *accountsStatements) execSchema(db *sql.DB) error { } func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { - if s.insertAccountStmt, err = db.Prepare(insertAccountSQL); err != nil { - return - } - if s.updatePasswordStmt, err = db.Prepare(updatePasswordSQL); err != nil { - return - } - if s.deactivateAccountStmt, err = db.Prepare(deactivateAccountSQL); err != nil { - return - } - if s.selectAccountByLocalpartStmt, err = db.Prepare(selectAccountByLocalpartSQL); err != nil { - return - } - if s.selectPasswordHashStmt, err = db.Prepare(selectPasswordHashSQL); err != nil { - return - } - if s.selectNewNumericLocalpartStmt, err = db.Prepare(selectNewNumericLocalpartSQL); err != nil { - return - } s.serverName = server - return + return sqlutil.StatementList{ + {&s.insertAccountStmt, insertAccountSQL}, + {&s.updatePasswordStmt, updatePasswordSQL}, + {&s.deactivateAccountStmt, deactivateAccountSQL}, + {&s.selectAccountByLocalpartStmt, selectAccountByLocalpartSQL}, + {&s.selectPasswordHashStmt, selectPasswordHashSQL}, + {&s.selectNewNumericLocalpartStmt, selectNewNumericLocalpartSQL}, + }.Prepare(db) } // insertAccount creates a new account. 'hash' should be the password hash for this account. If it is missing, diff --git a/userapi/storage/accounts/postgres/key_backup_table.go b/userapi/storage/accounts/postgres/key_backup_table.go index 0a2a2655..c1402d4d 100644 --- a/userapi/storage/accounts/postgres/key_backup_table.go +++ b/userapi/storage/accounts/postgres/key_backup_table.go @@ -20,6 +20,7 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" ) @@ -76,25 +77,14 @@ func (s *keyBackupStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.insertBackupKeyStmt, err = db.Prepare(insertBackupKeySQL); err != nil { - return - } - if s.updateBackupKeyStmt, err = db.Prepare(updateBackupKeySQL); err != nil { - return - } - if s.countKeysStmt, err = db.Prepare(countKeysSQL); err != nil { - return - } - if s.selectKeysStmt, err = db.Prepare(selectKeysSQL); err != nil { - return - } - if s.selectKeysByRoomIDStmt, err = db.Prepare(selectKeysByRoomIDSQL); err != nil { - return - } - if s.selectKeysByRoomIDAndSessionIDStmt, err = db.Prepare(selectKeysByRoomIDAndSessionIDSQL); err != nil { - return - } - return + return sqlutil.StatementList{ + {&s.insertBackupKeyStmt, insertBackupKeySQL}, + {&s.updateBackupKeyStmt, updateBackupKeySQL}, + {&s.countKeysStmt, countKeysSQL}, + {&s.selectKeysStmt, selectKeysSQL}, + {&s.selectKeysByRoomIDStmt, selectKeysByRoomIDSQL}, + {&s.selectKeysByRoomIDAndSessionIDStmt, selectKeysByRoomIDAndSessionIDSQL}, + }.Prepare(db) } func (s keyBackupStatements) countKeys( diff --git a/userapi/storage/accounts/postgres/key_backup_version_table.go b/userapi/storage/accounts/postgres/key_backup_version_table.go index 51a462b3..d73447b4 100644 --- a/userapi/storage/accounts/postgres/key_backup_version_table.go +++ b/userapi/storage/accounts/postgres/key_backup_version_table.go @@ -20,6 +20,8 @@ import ( "encoding/json" "fmt" "strconv" + + "github.com/matrix-org/dendrite/internal/sqlutil" ) const keyBackupVersionTableSchema = ` @@ -72,25 +74,14 @@ func (s *keyBackupVersionStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.insertKeyBackupStmt, err = db.Prepare(insertKeyBackupSQL); err != nil { - return - } - if s.updateKeyBackupAuthDataStmt, err = db.Prepare(updateKeyBackupAuthDataSQL); err != nil { - return - } - if s.deleteKeyBackupStmt, err = db.Prepare(deleteKeyBackupSQL); err != nil { - return - } - if s.selectKeyBackupStmt, err = db.Prepare(selectKeyBackupSQL); err != nil { - return - } - if s.selectLatestVersionStmt, err = db.Prepare(selectLatestVersionSQL); err != nil { - return - } - if s.updateKeyBackupETagStmt, err = db.Prepare(updateKeyBackupETagSQL); err != nil { - return - } - return + return sqlutil.StatementList{ + {&s.insertKeyBackupStmt, insertKeyBackupSQL}, + {&s.updateKeyBackupAuthDataStmt, updateKeyBackupAuthDataSQL}, + {&s.deleteKeyBackupStmt, deleteKeyBackupSQL}, + {&s.selectKeyBackupStmt, selectKeyBackupSQL}, + {&s.selectLatestVersionStmt, selectLatestVersionSQL}, + {&s.updateKeyBackupETagStmt, updateKeyBackupETagSQL}, + }.Prepare(db) } func (s *keyBackupVersionStatements) insertKeyBackup( diff --git a/userapi/storage/accounts/postgres/openid_table.go b/userapi/storage/accounts/postgres/openid_table.go index 86c19705..190d141b 100644 --- a/userapi/storage/accounts/postgres/openid_table.go +++ b/userapi/storage/accounts/postgres/openid_table.go @@ -39,14 +39,11 @@ func (s *tokenStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerNam if err != nil { return } - if s.insertTokenStmt, err = db.Prepare(insertTokenSQL); err != nil { - return - } - if s.selectTokenStmt, err = db.Prepare(selectTokenSQL); err != nil { - return - } s.serverName = server - return + return sqlutil.StatementList{ + {&s.insertTokenStmt, insertTokenSQL}, + {&s.selectTokenStmt, selectTokenSQL}, + }.Prepare(db) } // insertToken inserts a new OpenID Connect token to the DB. diff --git a/userapi/storage/accounts/postgres/profile_table.go b/userapi/storage/accounts/postgres/profile_table.go index 45d802f1..9313864b 100644 --- a/userapi/storage/accounts/postgres/profile_table.go +++ b/userapi/storage/accounts/postgres/profile_table.go @@ -64,22 +64,13 @@ func (s *profilesStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.insertProfileStmt, err = db.Prepare(insertProfileSQL); err != nil { - return - } - if s.selectProfileByLocalpartStmt, err = db.Prepare(selectProfileByLocalpartSQL); err != nil { - return - } - if s.setAvatarURLStmt, err = db.Prepare(setAvatarURLSQL); err != nil { - return - } - if s.setDisplayNameStmt, err = db.Prepare(setDisplayNameSQL); err != nil { - return - } - if s.selectProfilesBySearchStmt, err = db.Prepare(selectProfilesBySearchSQL); err != nil { - return - } - return + return sqlutil.StatementList{ + {&s.insertProfileStmt, insertProfileSQL}, + {&s.selectProfileByLocalpartStmt, selectProfileByLocalpartSQL}, + {&s.setAvatarURLStmt, setAvatarURLSQL}, + {&s.setDisplayNameStmt, setDisplayNameSQL}, + {&s.selectProfilesBySearchStmt, selectProfilesBySearchSQL}, + }.Prepare(db) } func (s *profilesStatements) insertProfile( diff --git a/userapi/storage/accounts/postgres/threepid_table.go b/userapi/storage/accounts/postgres/threepid_table.go index 7de96350..9280fc87 100644 --- a/userapi/storage/accounts/postgres/threepid_table.go +++ b/userapi/storage/accounts/postgres/threepid_table.go @@ -63,20 +63,12 @@ func (s *threepidStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.selectLocalpartForThreePIDStmt, err = db.Prepare(selectLocalpartForThreePIDSQL); err != nil { - return - } - if s.selectThreePIDsForLocalpartStmt, err = db.Prepare(selectThreePIDsForLocalpartSQL); err != nil { - return - } - if s.insertThreePIDStmt, err = db.Prepare(insertThreePIDSQL); err != nil { - return - } - if s.deleteThreePIDStmt, err = db.Prepare(deleteThreePIDSQL); err != nil { - return - } - - return + return sqlutil.StatementList{ + {&s.selectLocalpartForThreePIDStmt, selectLocalpartForThreePIDSQL}, + {&s.selectThreePIDsForLocalpartStmt, selectThreePIDsForLocalpartSQL}, + {&s.insertThreePIDStmt, insertThreePIDSQL}, + {&s.deleteThreePIDStmt, deleteThreePIDSQL}, + }.Prepare(db) } func (s *threepidStatements) selectLocalpartForThreePID( diff --git a/userapi/storage/accounts/sqlite3/account_data_table.go b/userapi/storage/accounts/sqlite3/account_data_table.go index 870a3706..871f996e 100644 --- a/userapi/storage/accounts/sqlite3/account_data_table.go +++ b/userapi/storage/accounts/sqlite3/account_data_table.go @@ -62,16 +62,11 @@ func (s *accountDataStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil { - return - } - if s.selectAccountDataStmt, err = db.Prepare(selectAccountDataSQL); err != nil { - return - } - if s.selectAccountDataByTypeStmt, err = db.Prepare(selectAccountDataByTypeSQL); err != nil { - return - } - return + return sqlutil.StatementList{ + {&s.insertAccountDataStmt, insertAccountDataSQL}, + {&s.selectAccountDataStmt, selectAccountDataSQL}, + {&s.selectAccountDataByTypeStmt, selectAccountDataByTypeSQL}, + }.Prepare(db) } func (s *accountDataStatements) insertAccountData( diff --git a/userapi/storage/accounts/sqlite3/accounts_table.go b/userapi/storage/accounts/sqlite3/accounts_table.go index 50f07237..8a7c8fba 100644 --- a/userapi/storage/accounts/sqlite3/accounts_table.go +++ b/userapi/storage/accounts/sqlite3/accounts_table.go @@ -81,26 +81,15 @@ func (s *accountsStatements) execSchema(db *sql.DB) error { func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { s.db = db - if s.insertAccountStmt, err = db.Prepare(insertAccountSQL); err != nil { - return - } - if s.updatePasswordStmt, err = db.Prepare(updatePasswordSQL); err != nil { - return - } - if s.deactivateAccountStmt, err = db.Prepare(deactivateAccountSQL); err != nil { - return - } - if s.selectAccountByLocalpartStmt, err = db.Prepare(selectAccountByLocalpartSQL); err != nil { - return - } - if s.selectPasswordHashStmt, err = db.Prepare(selectPasswordHashSQL); err != nil { - return - } - if s.selectNewNumericLocalpartStmt, err = db.Prepare(selectNewNumericLocalpartSQL); err != nil { - return - } s.serverName = server - return + return sqlutil.StatementList{ + {&s.insertAccountStmt, insertAccountSQL}, + {&s.updatePasswordStmt, updatePasswordSQL}, + {&s.deactivateAccountStmt, deactivateAccountSQL}, + {&s.selectAccountByLocalpartStmt, selectAccountByLocalpartSQL}, + {&s.selectPasswordHashStmt, selectPasswordHashSQL}, + {&s.selectNewNumericLocalpartStmt, selectNewNumericLocalpartSQL}, + }.Prepare(db) } // insertAccount creates a new account. 'hash' should be the password hash for this account. If it is missing, diff --git a/userapi/storage/accounts/sqlite3/key_backup_table.go b/userapi/storage/accounts/sqlite3/key_backup_table.go index 67509351..837d38cf 100644 --- a/userapi/storage/accounts/sqlite3/key_backup_table.go +++ b/userapi/storage/accounts/sqlite3/key_backup_table.go @@ -20,6 +20,7 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" ) @@ -76,25 +77,14 @@ func (s *keyBackupStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.insertBackupKeyStmt, err = db.Prepare(insertBackupKeySQL); err != nil { - return - } - if s.updateBackupKeyStmt, err = db.Prepare(updateBackupKeySQL); err != nil { - return - } - if s.countKeysStmt, err = db.Prepare(countKeysSQL); err != nil { - return - } - if s.selectKeysStmt, err = db.Prepare(selectKeysSQL); err != nil { - return - } - if s.selectKeysByRoomIDStmt, err = db.Prepare(selectKeysByRoomIDSQL); err != nil { - return - } - if s.selectKeysByRoomIDAndSessionIDStmt, err = db.Prepare(selectKeysByRoomIDAndSessionIDSQL); err != nil { - return - } - return + return sqlutil.StatementList{ + {&s.insertBackupKeyStmt, insertBackupKeySQL}, + {&s.updateBackupKeyStmt, updateBackupKeySQL}, + {&s.countKeysStmt, countKeysSQL}, + {&s.selectKeysStmt, selectKeysSQL}, + {&s.selectKeysByRoomIDStmt, selectKeysByRoomIDSQL}, + {&s.selectKeysByRoomIDAndSessionIDStmt, selectKeysByRoomIDAndSessionIDSQL}, + }.Prepare(db) } func (s keyBackupStatements) countKeys( diff --git a/userapi/storage/accounts/sqlite3/key_backup_version_table.go b/userapi/storage/accounts/sqlite3/key_backup_version_table.go index a9e7bf5d..4211ed0f 100644 --- a/userapi/storage/accounts/sqlite3/key_backup_version_table.go +++ b/userapi/storage/accounts/sqlite3/key_backup_version_table.go @@ -20,6 +20,8 @@ import ( "encoding/json" "fmt" "strconv" + + "github.com/matrix-org/dendrite/internal/sqlutil" ) const keyBackupVersionTableSchema = ` @@ -70,25 +72,14 @@ func (s *keyBackupVersionStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.insertKeyBackupStmt, err = db.Prepare(insertKeyBackupSQL); err != nil { - return - } - if s.updateKeyBackupAuthDataStmt, err = db.Prepare(updateKeyBackupAuthDataSQL); err != nil { - return - } - if s.deleteKeyBackupStmt, err = db.Prepare(deleteKeyBackupSQL); err != nil { - return - } - if s.selectKeyBackupStmt, err = db.Prepare(selectKeyBackupSQL); err != nil { - return - } - if s.selectLatestVersionStmt, err = db.Prepare(selectLatestVersionSQL); err != nil { - return - } - if s.updateKeyBackupETagStmt, err = db.Prepare(updateKeyBackupETagSQL); err != nil { - return - } - return + return sqlutil.StatementList{ + {&s.insertKeyBackupStmt, insertKeyBackupSQL}, + {&s.updateKeyBackupAuthDataStmt, updateKeyBackupAuthDataSQL}, + {&s.deleteKeyBackupStmt, deleteKeyBackupSQL}, + {&s.selectKeyBackupStmt, selectKeyBackupSQL}, + {&s.selectLatestVersionStmt, selectLatestVersionSQL}, + {&s.updateKeyBackupETagStmt, updateKeyBackupETagSQL}, + }.Prepare(db) } func (s *keyBackupVersionStatements) insertKeyBackup( diff --git a/userapi/storage/accounts/sqlite3/openid_table.go b/userapi/storage/accounts/sqlite3/openid_table.go index 80b9dd4c..98c0488b 100644 --- a/userapi/storage/accounts/sqlite3/openid_table.go +++ b/userapi/storage/accounts/sqlite3/openid_table.go @@ -41,14 +41,11 @@ func (s *tokenStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerNam if err != nil { return err } - if s.insertTokenStmt, err = db.Prepare(insertTokenSQL); err != nil { - return - } - if s.selectTokenStmt, err = db.Prepare(selectTokenSQL); err != nil { - return - } s.serverName = server - return + return sqlutil.StatementList{ + {&s.insertTokenStmt, insertTokenSQL}, + {&s.selectTokenStmt, selectTokenSQL}, + }.Prepare(db) } // insertToken inserts a new OpenID Connect token to the DB. diff --git a/userapi/storage/accounts/sqlite3/profile_table.go b/userapi/storage/accounts/sqlite3/profile_table.go index a67e892f..a92e9566 100644 --- a/userapi/storage/accounts/sqlite3/profile_table.go +++ b/userapi/storage/accounts/sqlite3/profile_table.go @@ -66,22 +66,13 @@ func (s *profilesStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.insertProfileStmt, err = db.Prepare(insertProfileSQL); err != nil { - return - } - if s.selectProfileByLocalpartStmt, err = db.Prepare(selectProfileByLocalpartSQL); err != nil { - return - } - if s.setAvatarURLStmt, err = db.Prepare(setAvatarURLSQL); err != nil { - return - } - if s.setDisplayNameStmt, err = db.Prepare(setDisplayNameSQL); err != nil { - return - } - if s.selectProfilesBySearchStmt, err = db.Prepare(selectProfilesBySearchSQL); err != nil { - return - } - return + return sqlutil.StatementList{ + {&s.insertProfileStmt, insertProfileSQL}, + {&s.selectProfileByLocalpartStmt, selectProfileByLocalpartSQL}, + {&s.setAvatarURLStmt, setAvatarURLSQL}, + {&s.setDisplayNameStmt, setDisplayNameSQL}, + {&s.selectProfilesBySearchStmt, selectProfilesBySearchSQL}, + }.Prepare(db) } func (s *profilesStatements) insertProfile( diff --git a/userapi/storage/accounts/sqlite3/threepid_table.go b/userapi/storage/accounts/sqlite3/threepid_table.go index 43112d38..9dc0e2d2 100644 --- a/userapi/storage/accounts/sqlite3/threepid_table.go +++ b/userapi/storage/accounts/sqlite3/threepid_table.go @@ -66,20 +66,12 @@ func (s *threepidStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.selectLocalpartForThreePIDStmt, err = db.Prepare(selectLocalpartForThreePIDSQL); err != nil { - return - } - if s.selectThreePIDsForLocalpartStmt, err = db.Prepare(selectThreePIDsForLocalpartSQL); err != nil { - return - } - if s.insertThreePIDStmt, err = db.Prepare(insertThreePIDSQL); err != nil { - return - } - if s.deleteThreePIDStmt, err = db.Prepare(deleteThreePIDSQL); err != nil { - return - } - - return + return sqlutil.StatementList{ + {&s.selectLocalpartForThreePIDStmt, selectLocalpartForThreePIDSQL}, + {&s.selectThreePIDsForLocalpartStmt, selectThreePIDsForLocalpartSQL}, + {&s.insertThreePIDStmt, insertThreePIDSQL}, + {&s.deleteThreePIDStmt, deleteThreePIDSQL}, + }.Prepare(db) } func (s *threepidStatements) selectLocalpartForThreePID( From a22ab3eee6f2d6e41f0345e930c61c86e2e61ad7 Mon Sep 17 00:00:00 2001 From: kegsay Date: Mon, 2 Aug 2021 13:11:23 +0100 Subject: [PATCH 11/64] Read the right username field when performing account deactivation (#1954) `Login` has 2 username fields, and we were always checking the deprecated one. Instead, check both. --- clientapi/routing/deactivate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clientapi/routing/deactivate.go b/clientapi/routing/deactivate.go index effe3769..9e3b0bfb 100644 --- a/clientapi/routing/deactivate.go +++ b/clientapi/routing/deactivate.go @@ -33,7 +33,7 @@ func Deactivate( return *errRes } - localpart, _, err := gomatrixserverlib.SplitID('@', login.User) + localpart, _, err := gomatrixserverlib.SplitID('@', login.Username()) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") return jsonerror.InternalServerError() From a81c499f655edd5bdf7d73cc5a49020c25863edc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 3 Aug 2021 11:15:17 +0100 Subject: [PATCH 12/64] Update dependencies (#1956) --- go.mod | 26 +++--- go.sum | 245 +++++++++++++++++++-------------------------------------- 2 files changed, 92 insertions(+), 179 deletions(-) diff --git a/go.mod b/go.mod index fbad10d6..4f42e37b 100644 --- a/go.mod +++ b/go.mod @@ -5,19 +5,19 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect github.com/Masterminds/semver/v3 v3.1.1 - github.com/Shopify/sarama v1.28.0 + github.com/Shopify/sarama v1.29.1 github.com/codeclysm/extract v2.2.0+incompatible github.com/containerd/containerd v1.5.2 // indirect github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 - github.com/getsentry/sentry-go v0.10.0 + github.com/getsentry/sentry-go v0.11.0 github.com/gologme/log v1.2.0 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 github.com/h2non/filetype v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 github.com/juju/testing v0.0.0-20210324180055-18c50b0c2098 // indirect - github.com/lib/pq v1.9.0 + github.com/lib/pq v1.10.2 github.com/libp2p/go-libp2p v0.13.0 github.com/libp2p/go-libp2p-circuit v0.4.0 github.com/libp2p/go-libp2p-core v0.8.3 @@ -26,8 +26,8 @@ require ( github.com/libp2p/go-libp2p-kad-dht v0.11.1 github.com/libp2p/go-libp2p-pubsub v0.4.1 github.com/libp2p/go-libp2p-record v0.1.3 - github.com/lucas-clemente/quic-go v0.19.3 - github.com/matrix-org/dugong v0.0.0-20180820122854-51a565b5666b + github.com/lucas-clemente/quic-go v0.22.0 + github.com/matrix-org/dugong v0.0.0-20210603171012-8379174dca81 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 @@ -35,7 +35,7 @@ require ( github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 - github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb + github.com/mattn/go-sqlite3 v1.14.8 github.com/morikuni/aec v1.0.0 // indirect github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 @@ -44,20 +44,20 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/pressly/goose v2.7.0+incompatible - github.com/prometheus/client_golang v1.9.0 + github.com/prometheus/client_golang v1.11.0 github.com/sirupsen/logrus v1.8.1 github.com/tidwall/gjson v1.8.1 github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/sjson v1.1.7 - github.com/uber/jaeger-client-go v2.25.0+incompatible - github.com/uber/jaeger-lib v2.4.0+incompatible + github.com/uber/jaeger-client-go v2.29.1+incompatible + github.com/uber/jaeger-lib v2.4.1+incompatible github.com/yggdrasil-network/yggdrasil-go v0.4.1-0.20210715083903-52309d094c00 - go.uber.org/atomic v1.7.0 + go.uber.org/atomic v1.9.0 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 - golang.org/x/mobile v0.0.0-20210220033013-bdb1ca9a1e08 - golang.org/x/net v0.0.0-20210610132358-84b48f89b13b + golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554 + golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect - golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 + golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b gopkg.in/h2non/bimg.v1 v1.1.5 gopkg.in/yaml.v2 v2.4.0 nhooyr.io/websocket v1.8.7 diff --git a/go.sum b/go.sum index ff6090a5..1723bcfe 100644 --- a/go.sum +++ b/go.sum @@ -58,7 +58,6 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bEn0jTI6LJU0mpw= github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= @@ -90,17 +89,14 @@ github.com/RyanCarrier/dijkstra v1.0.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEk github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= -github.com/Shopify/sarama v1.28.0 h1:lOi3SfE6OcFlW9Trgtked2aHNZ2BIG/d6Do+PEUAqqM= -github.com/Shopify/sarama v1.28.0/go.mod h1:j/2xTrU39dlzBmsxF1eQ2/DdWrxyBCl6pzz7a81o/ZY= +github.com/Shopify/sarama v1.29.1 h1:wBAacXbYVLmWieEA/0X/JagDdCZ8NVFOfS6l6+2u5S0= +github.com/Shopify/sarama v1.29.1/go.mod h1:mdtqvCSg8JOxk8PmpTNGyo6wzd4BMm4QXSfDnTXmgkE= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/albertorestifo/dijkstra v0.0.0-20160910063646-aba76f725f72/go.mod h1:o+JdB7VetTHjLhU0N57x18B9voDBQe0paApdEAEoEfw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -124,18 +120,9 @@ github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K github.com/anacrolix/utp v0.1.0/go.mod h1:MDwc+vsGEq7RMw6lr2GKOEqjWny5hO5OZXRVNaBJ2Dk= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/benbjohnson/clock v1.0.2/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.0.3 h1:vkLuvpK4fmtSCuo60+yC63p7y0BmQ8gm5ZXGuBCJyXg= @@ -174,8 +161,6 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -192,11 +177,9 @@ github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmE github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codeclysm/extract v2.2.0+incompatible h1:q3wyckoA30bhUSiwdQezMqVhwd8+WGE64/GL//LtUhI= github.com/codeclysm/extract v2.2.0+incompatible/go.mod h1:2nhFMPHiU9At61hz+12bfrlpXSUrOnK+wR+KlGO4Uks= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= @@ -347,19 +330,16 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -378,8 +358,6 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= @@ -390,8 +368,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= -github.com/getsentry/sentry-go v0.10.0 h1:6gwY+66NHKqyZrdi6O2jGdo7wGdo9b3B69E01NFgT5g= -github.com/getsentry/sentry-go v0.10.0/go.mod h1:kELm/9iCblqUYh+ZRML7PNdCvEuw24wBvJPYyi86cws= +github.com/getsentry/sentry-go v0.11.0 h1:qro8uttJGvNAMr5CLcFI9CHR0aDzXl0Vs3Pmw/oTPg8= +github.com/getsentry/sentry-go v0.11.0/go.mod h1:KBQIxiZAetw62Cj8Ri964vAEWVdgfaUCn30Q3bCvANo= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= @@ -412,7 +390,7 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -434,8 +412,9 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= @@ -446,11 +425,9 @@ github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblf github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -461,7 +438,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -471,8 +447,9 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -487,12 +464,14 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw= -github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gologme/log v1.2.0 h1:Ya5Ip/KD6FX7uH0S31QO87nCCSucKtF44TLbTtO7V4c= github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= @@ -507,8 +486,9 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -535,11 +515,8 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= @@ -564,42 +541,27 @@ github.com/h2non/filetype v1.1.1 h1:xvOwnXKAckvtLWsN398qS9QhlxlnVXBjXBydK2/UFB4= github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= @@ -610,7 +572,6 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -691,16 +652,15 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -753,9 +713,9 @@ github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8= +github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -779,8 +739,8 @@ github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvf github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8= -github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2 h1:7cWK5cdA5x72jX0g8iLrQWm5TRJZ6CzGdPEhWj7plWU= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= @@ -992,15 +952,12 @@ github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux/v2 v2.0.0 h1:vSGhAy5u6iHBq11ZDcyHH4Blcf9xlBhT4WQDoOE90LU= github.com/libp2p/go-yamux/v2 v2.0.0/go.mod h1:NVWira5+sVUIU6tu1JWvaRn1dRnG+cawOJiflsAM+7U= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lucas-clemente/quic-go v0.19.3 h1:eCDQqvGBB+kCTkA0XrAFtNe81FMa0/fn4QSoeAbmiF4= -github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= +github.com/lucas-clemente/quic-go v0.22.0 h1:o8NIiHaavjoHe6z8Bqm6fw7g0YIP6AFKMYer+oNxInA= +github.com/lucas-clemente/quic-go v0.22.0/go.mod h1:vF5M1XqhBAHgbjKcJOXY3JZz3GP0T3FQhz/uyOUS38Q= github.com/lunixbochs/vtclean v0.0.0-20160125035106-4fbf7632a2c6/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ= github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -1010,16 +967,19 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= -github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc= -github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= -github.com/marten-seemann/qtls-go1-15 v0.1.1 h1:LIH6K34bPVttyXnUWixk0bzH6/N07VxbSabxn5A5gZQ= -github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-15 v0.1.5 h1:Ci4EIUN6Rlb+D6GmLdej/bCQ4nPYNtVXQB+xjiXE1nk= +github.com/marten-seemann/qtls-go1-15 v0.1.5/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-16 v0.1.4 h1:xbHbOGGhrenVtII6Co8akhLEdrawwB2iHl5yhJRpnco= +github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= +github.com/marten-seemann/qtls-go1-17 v0.1.0-rc.1 h1:/rpmWuGvceLwwWuaKPdjpR4JJEUH0tq64/I3hvzaNLM= +github.com/marten-seemann/qtls-go1-17 v0.1.0-rc.1/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8= github.com/masterzen/azure-sdk-for-go v3.2.0-beta.0.20161014135628-ee4f0065d00c+incompatible/go.mod h1:mf8fjOu33zCqxUjuiU3I8S1lJMyEAlH+0F2+M5xl3hE= github.com/masterzen/simplexml v0.0.0-20160608183007-4572e39b1ab9/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc= github.com/masterzen/winrm v0.0.0-20161014151040-7a535cd943fc/go.mod h1:CfZSN7zwz5gJiFhZJz49Uzk7mEBHIceWmbFmYx7Hf7E= github.com/masterzen/xmlpath v0.0.0-20140218185901-13f4951698ad/go.mod h1:A0zPC53iKKKcXYxr4ROjpQRQ5FgJXtelNdSmHHuq/tY= -github.com/matrix-org/dugong v0.0.0-20180820122854-51a565b5666b h1:xpcmnpfUImRC4O2SAS/dmTcJENDXvGmLUzey76V1R3Q= -github.com/matrix-org/dugong v0.0.0-20180820122854-51a565b5666b/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= +github.com/matrix-org/dugong v0.0.0-20210603171012-8379174dca81 h1:lHNIqMZ77Y++RbHuR4eSehyv6KUKb2sJ8Gdfro4P5f4= +github.com/matrix-org/dugong v0.0.0-20210603171012-8379174dca81/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 h1:eqE5OnGx9ZMWmrRbD3KF/3KtTunw0iQulI7YxOIdxo4= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4/go.mod h1:3WluEZ9QXSwU30tWYqktnpC1x9mwZKx1r8uAv8Iq+a4= github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d h1:mGhPVaTht5NViFN/UpdrIlRApmH2FWcVaKUH5MdBKiY= @@ -1042,7 +1002,6 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.0-20160806122752-66b8e73f3f5c/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -1056,8 +1015,8 @@ github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRC github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-sqlite3 v1.14.2/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= -github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb h1:ax2vG2unlxsjwS7PMRo4FECIfAdQLowd6ejWYwPQhBo= -github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU= +github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -1067,7 +1026,6 @@ github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= @@ -1082,13 +1040,7 @@ github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= @@ -1172,11 +1124,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= @@ -1192,10 +1141,9 @@ github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3h github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6/go.mod h1:E26fwEtRNigBfFfHDWsklmo0T7Ixbg0XXgck+Hq4O9k= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -1208,8 +1156,10 @@ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1218,9 +1168,9 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= +github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -1243,29 +1193,17 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.m github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -1276,47 +1214,38 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/pressly/goose v2.7.0+incompatible h1:PWejVEv07LCerQEzMMeAtjuyCKbyprZ/LBa6K5P0OCQ= github.com/pressly/goose v2.7.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU= -github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= -github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -1328,7 +1257,6 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -1338,14 +1266,11 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= @@ -1386,7 +1311,6 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= @@ -1412,9 +1336,6 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -1453,10 +1374,10 @@ github.com/tidwall/sjson v1.1.7/go.mod h1:w/yG+ezBeTdUxiKs5NcPicO9diP38nk96QBAbI github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= -github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ= -github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/uber/jaeger-client-go v2.29.1+incompatible h1:R9ec3zO3sGpzs0abd43Y+fBZRJ9uiH6lXyR/+u6brW4= +github.com/uber/jaeger-client-go v2.29.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= +github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= @@ -1497,7 +1418,9 @@ github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/scram v1.0.3/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= @@ -1512,18 +1435,16 @@ github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDf github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= @@ -1533,20 +1454,18 @@ go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.0.0 h1:qsup4IcBdlmsnGfqyLl4Ntn3C2XCCuKAE7DwHpScyUo= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= @@ -1559,7 +1478,6 @@ golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1583,12 +1501,12 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1618,30 +1536,27 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20210220033013-bdb1ca9a1e08 h1:h+GZ3ubjuWaQjGe8owMGcmMVCqs0xYJtRG5y2bpHaqU= -golang.org/x/mobile v0.0.0-20210220033013-bdb1ca9a1e08/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= +golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554 h1:3In5TnfvnuXTF/uflgpYxSCEGP2NdYT37KsPh3VjZYU= +golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554/go.mod h1:jFTmtFYCV0MFtXBU+J5V/+5AUeVS0ON/0WkE/KSrl6E= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1675,11 +1590,14 @@ golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b h1:k+E048sYJHyVnsr1GDrRZWQ32D2C7lWs9JRc0bel53A= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 h1:4CSI6oo7cOjJKajidEljs9h+uP0rRZBPPPhcCbj5mw8= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1697,17 +1615,15 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1745,7 +1661,6 @@ golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1776,27 +1691,29 @@ golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210611083646-a4fc73990273/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7-0.20210503195748-5c7c50ebbd4f h1:yQJrRE0hDxDFmZLlRaw+3vusO4fwNHgHIjUOMO7bHYI= golang.org/x/text v0.3.7-0.20210503195748-5c7c50ebbd4f/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -1842,8 +1759,6 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1853,8 +1768,11 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1867,7 +1785,6 @@ google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+ google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1896,7 +1813,6 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1919,11 +1835,9 @@ google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmE google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= @@ -1945,8 +1859,10 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1963,7 +1879,6 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v1 v1.0.0-20161222125816-442357a80af5/go.mod h1:u0ALmqvLRxLI95fkdCEWrE6mhWYZW1aMOJHp5YXLHTg= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= @@ -1994,7 +1909,6 @@ gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFab gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170712054546-1be3d31502d6/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -2062,6 +1976,5 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= From d4aa311c744ff59c5c92713e34e3ddb375841c3c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 3 Aug 2021 11:17:27 +0100 Subject: [PATCH 13/64] Update containerd dependency --- go.mod | 2 +- go.sum | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4f42e37b..724d8bb7 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/Shopify/sarama v1.29.1 github.com/codeclysm/extract v2.2.0+incompatible - github.com/containerd/containerd v1.5.2 // indirect + github.com/containerd/containerd v1.5.5 // indirect github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 github.com/getsentry/sentry-go v0.11.0 diff --git a/go.sum b/go.sum index 1723bcfe..7eb82d5d 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,7 @@ github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg3 github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -134,6 +135,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= @@ -167,6 +169,7 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= @@ -177,6 +180,7 @@ github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmE github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -214,8 +218,9 @@ github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7 github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= -github.com/containerd/containerd v1.5.2 h1:MG/Bg1pbmMb61j3wHCFWPxESXHieiKr2xG64px/k8zQ= -github.com/containerd/containerd v1.5.2/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.5 h1:q1gxsZsGZ8ddVe98yO6pR21b5xQSMiR61lD0W96pgQo= +github.com/containerd/containerd v1.5.5/go.mod h1:oSTh0QpT1w6jYcGmbiSbxv9OSQYaa88mPyWIuU79zyo= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -280,6 +285,7 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -425,6 +431,7 @@ github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblf github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -1185,14 +1192,17 @@ github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59P github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= +github.com/opencontainers/runc v1.0.1/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= @@ -1698,6 +1708,7 @@ golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210611083646-a4fc73990273/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From e3df6129537427ae7dae52a4d04dfc8822d8c2ab Mon Sep 17 00:00:00 2001 From: kegsay Date: Tue, 3 Aug 2021 11:23:25 +0100 Subject: [PATCH 14/64] Add tracing to user API (#1948) Use the trace version in tests so we can just implement the required API functions. --- cmd/dendrite-monolith-server/main.go | 6 ++ federationapi/routing/send_test.go | 141 +-------------------------- setup/mscs/msc2836/msc2836_test.go | 50 +--------- setup/mscs/msc2946/msc2946_test.go | 50 +--------- userapi/api/api_trace.go | 125 ++++++++++++++++++++++++ 5 files changed, 136 insertions(+), 236 deletions(-) create mode 100644 userapi/api/api_trace.go diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 3785371a..9974fe66 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -30,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/setup/mscs" "github.com/matrix-org/dendrite/signingkeyserver" "github.com/matrix-org/dendrite/userapi" + uapi "github.com/matrix-org/dendrite/userapi/api" "github.com/sirupsen/logrus" _ "github.com/mattn/go-sqlite3" @@ -114,6 +115,11 @@ func main() { keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI) keyAPI.SetUserAPI(userAPI) + if traceInternal { + userAPI = &uapi.UserInternalAPITrace{ + Impl: userAPI, + } + } eduInputAPI := eduserver.NewInternalAPI( base, cache.New(), userAPI, diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 0da06aa9..5b5af9c4 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -158,32 +158,6 @@ func (t *testRoomserverAPI) QueryEventsByID( return nil } -// Query the membership event for an user for a room. -func (t *testRoomserverAPI) QueryMembershipForUser( - ctx context.Context, - request *api.QueryMembershipForUserRequest, - response *api.QueryMembershipForUserResponse, -) error { - return fmt.Errorf("not implemented") -} - -func (t *testRoomserverAPI) QueryPublishedRooms( - ctx context.Context, - request *api.QueryPublishedRoomsRequest, - response *api.QueryPublishedRoomsResponse, -) error { - return fmt.Errorf("not implemented") -} - -// Query a list of membership events for a room -func (t *testRoomserverAPI) QueryMembershipsForRoom( - ctx context.Context, - request *api.QueryMembershipsForRoomRequest, - response *api.QueryMembershipsForRoomResponse, -) error { - return fmt.Errorf("not implemented") -} - // Query if a server is joined to a room func (t *testRoomserverAPI) QueryServerJoinedToRoom( ctx context.Context, @@ -195,53 +169,6 @@ func (t *testRoomserverAPI) QueryServerJoinedToRoom( return nil } -// Query whether a server is allowed to see an event -func (t *testRoomserverAPI) QueryServerAllowedToSeeEvent( - ctx context.Context, - request *api.QueryServerAllowedToSeeEventRequest, - response *api.QueryServerAllowedToSeeEventResponse, -) error { - return fmt.Errorf("not implemented") -} - -// Query missing events for a room from roomserver -func (t *testRoomserverAPI) QueryMissingEvents( - ctx context.Context, - request *api.QueryMissingEventsRequest, - response *api.QueryMissingEventsResponse, -) error { - return fmt.Errorf("not implemented") -} - -// Query to get state and auth chain for a (potentially hypothetical) event. -// Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate -// the state and auth chain to return. -func (t *testRoomserverAPI) QueryStateAndAuthChain( - ctx context.Context, - request *api.QueryStateAndAuthChainRequest, - response *api.QueryStateAndAuthChainResponse, -) error { - return fmt.Errorf("not implemented") -} - -// Query a given amount (or less) of events prior to a given set of events. -func (t *testRoomserverAPI) PerformBackfill( - ctx context.Context, - request *api.PerformBackfillRequest, - response *api.PerformBackfillResponse, -) error { - return fmt.Errorf("not implemented") -} - -// Asks for the default room version as preferred by the server. -func (t *testRoomserverAPI) QueryRoomVersionCapabilities( - ctx context.Context, - request *api.QueryRoomVersionCapabilitiesRequest, - response *api.QueryRoomVersionCapabilitiesResponse, -) error { - return fmt.Errorf("not implemented") -} - // Asks for the room version for a given room. func (t *testRoomserverAPI) QueryRoomVersionForRoom( ctx context.Context, @@ -252,72 +179,10 @@ func (t *testRoomserverAPI) QueryRoomVersionForRoom( return nil } -// Set a room alias -func (t *testRoomserverAPI) SetRoomAlias( - ctx context.Context, - req *api.SetRoomAliasRequest, - response *api.SetRoomAliasResponse, +func (t *testRoomserverAPI) QueryServerBannedFromRoom( + ctx context.Context, req *api.QueryServerBannedFromRoomRequest, res *api.QueryServerBannedFromRoomResponse, ) error { - return fmt.Errorf("not implemented") -} - -// Get the room ID for an alias -func (t *testRoomserverAPI) GetRoomIDForAlias( - ctx context.Context, - req *api.GetRoomIDForAliasRequest, - response *api.GetRoomIDForAliasResponse, -) error { - return fmt.Errorf("not implemented") -} - -// Get all known aliases for a room ID -func (t *testRoomserverAPI) GetAliasesForRoomID( - ctx context.Context, - req *api.GetAliasesForRoomIDRequest, - response *api.GetAliasesForRoomIDResponse, -) error { - return fmt.Errorf("not implemented") -} - -// Get the user ID of the creator of an alias -func (t *testRoomserverAPI) GetCreatorIDForAlias( - ctx context.Context, - req *api.GetCreatorIDForAliasRequest, - response *api.GetCreatorIDForAliasResponse, -) error { - return fmt.Errorf("not implemented") -} - -// Remove a room alias -func (t *testRoomserverAPI) RemoveRoomAlias( - ctx context.Context, - req *api.RemoveRoomAliasRequest, - response *api.RemoveRoomAliasResponse, -) error { - return fmt.Errorf("not implemented") -} - -func (t *testRoomserverAPI) QueryCurrentState(ctx context.Context, req *api.QueryCurrentStateRequest, res *api.QueryCurrentStateResponse) error { - return nil -} - -func (t *testRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForUserRequest, res *api.QueryRoomsForUserResponse) error { - return fmt.Errorf("not implemented") -} - -func (t *testRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error { - return fmt.Errorf("not implemented") -} - -func (t *testRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse) error { - return fmt.Errorf("not implemented") -} - -func (t *testRoomserverAPI) QueryKnownUsers(ctx context.Context, req *api.QueryKnownUsersRequest, res *api.QueryKnownUsersResponse) error { - return fmt.Errorf("not implemented") -} - -func (t *testRoomserverAPI) QueryServerBannedFromRoom(ctx context.Context, req *api.QueryServerBannedFromRoomRequest, res *api.QueryServerBannedFromRoomResponse) error { + res.Banned = false return nil } diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index 1bbb485c..51fde691 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -497,39 +497,10 @@ func assertUnsignedChildren(t *testing.T, ev gomatrixserverlib.ClientEvent, relT } type testUserAPI struct { + userapi.UserInternalAPITrace accessTokens map[string]userapi.Device } -func (u *testUserAPI) InputAccountData(ctx context.Context, req *userapi.InputAccountDataRequest, res *userapi.InputAccountDataResponse) error { - return nil -} -func (u *testUserAPI) PerformAccountCreation(ctx context.Context, req *userapi.PerformAccountCreationRequest, res *userapi.PerformAccountCreationResponse) error { - return nil -} -func (u *testUserAPI) PerformPasswordUpdate(ctx context.Context, req *userapi.PerformPasswordUpdateRequest, res *userapi.PerformPasswordUpdateResponse) error { - return nil -} -func (u *testUserAPI) PerformDeviceCreation(ctx context.Context, req *userapi.PerformDeviceCreationRequest, res *userapi.PerformDeviceCreationResponse) error { - return nil -} -func (u *testUserAPI) PerformDeviceDeletion(ctx context.Context, req *userapi.PerformDeviceDeletionRequest, res *userapi.PerformDeviceDeletionResponse) error { - return nil -} -func (u *testUserAPI) PerformDeviceUpdate(ctx context.Context, req *userapi.PerformDeviceUpdateRequest, res *userapi.PerformDeviceUpdateResponse) error { - return nil -} -func (u *testUserAPI) PerformLastSeenUpdate(ctx context.Context, req *userapi.PerformLastSeenUpdateRequest, res *userapi.PerformLastSeenUpdateResponse) error { - return nil -} -func (u *testUserAPI) PerformAccountDeactivation(ctx context.Context, req *userapi.PerformAccountDeactivationRequest, res *userapi.PerformAccountDeactivationResponse) error { - return nil -} -func (u *testUserAPI) PerformOpenIDTokenCreation(ctx context.Context, req *userapi.PerformOpenIDTokenCreationRequest, res *userapi.PerformOpenIDTokenCreationResponse) error { - return nil -} -func (u *testUserAPI) QueryProfile(ctx context.Context, req *userapi.QueryProfileRequest, res *userapi.QueryProfileResponse) error { - return nil -} func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error { dev, ok := u.accessTokens[req.AccessToken] if !ok { @@ -539,25 +510,6 @@ func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAc res.Device = &dev return nil } -func (u *testUserAPI) QueryDevices(ctx context.Context, req *userapi.QueryDevicesRequest, res *userapi.QueryDevicesResponse) error { - return nil -} -func (u *testUserAPI) QueryAccountData(ctx context.Context, req *userapi.QueryAccountDataRequest, res *userapi.QueryAccountDataResponse) error { - return nil -} -func (u *testUserAPI) QueryDeviceInfos(ctx context.Context, req *userapi.QueryDeviceInfosRequest, res *userapi.QueryDeviceInfosResponse) error { - return nil -} -func (u *testUserAPI) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error { - return nil -} -func (u *testUserAPI) QueryOpenIDToken(ctx context.Context, req *userapi.QueryOpenIDTokenRequest, res *userapi.QueryOpenIDTokenResponse) error { - return nil -} -func (u *testUserAPI) PerformKeyBackup(ctx context.Context, req *userapi.PerformKeyBackupRequest, res *userapi.PerformKeyBackupResponse) { -} -func (u *testUserAPI) QueryKeyBackup(ctx context.Context, req *userapi.QueryKeyBackupRequest, res *userapi.QueryKeyBackupResponse) { -} type testRoomserverAPI struct { // use a trace API as it implements method stubs so we don't need to have them here. diff --git a/setup/mscs/msc2946/msc2946_test.go b/setup/mscs/msc2946/msc2946_test.go index 2c195d12..c362d9fb 100644 --- a/setup/mscs/msc2946/msc2946_test.go +++ b/setup/mscs/msc2946/msc2946_test.go @@ -340,43 +340,10 @@ func postSpaces(t *testing.T, expectCode int, accessToken, roomID string, req *g } type testUserAPI struct { + userapi.UserInternalAPITrace accessTokens map[string]userapi.Device } -func (u *testUserAPI) InputAccountData(ctx context.Context, req *userapi.InputAccountDataRequest, res *userapi.InputAccountDataResponse) error { - return nil -} -func (u *testUserAPI) PerformAccountCreation(ctx context.Context, req *userapi.PerformAccountCreationRequest, res *userapi.PerformAccountCreationResponse) error { - return nil -} -func (u *testUserAPI) PerformPasswordUpdate(ctx context.Context, req *userapi.PerformPasswordUpdateRequest, res *userapi.PerformPasswordUpdateResponse) error { - return nil -} -func (u *testUserAPI) PerformDeviceCreation(ctx context.Context, req *userapi.PerformDeviceCreationRequest, res *userapi.PerformDeviceCreationResponse) error { - return nil -} -func (u *testUserAPI) PerformDeviceDeletion(ctx context.Context, req *userapi.PerformDeviceDeletionRequest, res *userapi.PerformDeviceDeletionResponse) error { - return nil -} -func (u *testUserAPI) PerformDeviceUpdate(ctx context.Context, req *userapi.PerformDeviceUpdateRequest, res *userapi.PerformDeviceUpdateResponse) error { - return nil -} -func (u *testUserAPI) PerformLastSeenUpdate(ctx context.Context, req *userapi.PerformLastSeenUpdateRequest, res *userapi.PerformLastSeenUpdateResponse) error { - return nil -} -func (u *testUserAPI) PerformAccountDeactivation(ctx context.Context, req *userapi.PerformAccountDeactivationRequest, res *userapi.PerformAccountDeactivationResponse) error { - return nil -} -func (u *testUserAPI) PerformOpenIDTokenCreation(ctx context.Context, req *userapi.PerformOpenIDTokenCreationRequest, res *userapi.PerformOpenIDTokenCreationResponse) error { - return nil -} -func (u *testUserAPI) QueryProfile(ctx context.Context, req *userapi.QueryProfileRequest, res *userapi.QueryProfileResponse) error { - return nil -} -func (u *testUserAPI) PerformKeyBackup(ctx context.Context, req *userapi.PerformKeyBackupRequest, res *userapi.PerformKeyBackupResponse) { -} -func (u *testUserAPI) QueryKeyBackup(ctx context.Context, req *userapi.QueryKeyBackupRequest, res *userapi.QueryKeyBackupResponse) { -} func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAccessTokenRequest, res *userapi.QueryAccessTokenResponse) error { dev, ok := u.accessTokens[req.AccessToken] if !ok { @@ -386,21 +353,6 @@ func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAc res.Device = &dev return nil } -func (u *testUserAPI) QueryDevices(ctx context.Context, req *userapi.QueryDevicesRequest, res *userapi.QueryDevicesResponse) error { - return nil -} -func (u *testUserAPI) QueryAccountData(ctx context.Context, req *userapi.QueryAccountDataRequest, res *userapi.QueryAccountDataResponse) error { - return nil -} -func (u *testUserAPI) QueryDeviceInfos(ctx context.Context, req *userapi.QueryDeviceInfosRequest, res *userapi.QueryDeviceInfosResponse) error { - return nil -} -func (u *testUserAPI) QuerySearchProfiles(ctx context.Context, req *userapi.QuerySearchProfilesRequest, res *userapi.QuerySearchProfilesResponse) error { - return nil -} -func (u *testUserAPI) QueryOpenIDToken(ctx context.Context, req *userapi.QueryOpenIDTokenRequest, res *userapi.QueryOpenIDTokenResponse) error { - return nil -} type testRoomserverAPI struct { // use a trace API as it implements method stubs so we don't need to have them here. diff --git a/userapi/api/api_trace.go b/userapi/api/api_trace.go new file mode 100644 index 00000000..84dcb309 --- /dev/null +++ b/userapi/api/api_trace.go @@ -0,0 +1,125 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/matrix-org/util" +) + +// UserInternalAPITrace wraps a RoomserverInternalAPI and logs the +// complete request/response/error +type UserInternalAPITrace struct { + Impl UserInternalAPI +} + +func (t *UserInternalAPITrace) InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error { + err := t.Impl.InputAccountData(ctx, req, res) + util.GetLogger(ctx).Infof("InputAccountData req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error { + err := t.Impl.PerformAccountCreation(ctx, req, res) + util.GetLogger(ctx).Infof("PerformAccountCreation req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) PerformPasswordUpdate(ctx context.Context, req *PerformPasswordUpdateRequest, res *PerformPasswordUpdateResponse) error { + err := t.Impl.PerformPasswordUpdate(ctx, req, res) + util.GetLogger(ctx).Infof("PerformPasswordUpdate req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *UserInternalAPITrace) PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error { + err := t.Impl.PerformDeviceCreation(ctx, req, res) + util.GetLogger(ctx).Infof("PerformDeviceCreation req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) PerformDeviceDeletion(ctx context.Context, req *PerformDeviceDeletionRequest, res *PerformDeviceDeletionResponse) error { + err := t.Impl.PerformDeviceDeletion(ctx, req, res) + util.GetLogger(ctx).Infof("PerformDeviceDeletion req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) PerformLastSeenUpdate(ctx context.Context, req *PerformLastSeenUpdateRequest, res *PerformLastSeenUpdateResponse) error { + err := t.Impl.PerformLastSeenUpdate(ctx, req, res) + util.GetLogger(ctx).Infof("PerformLastSeenUpdate req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error { + err := t.Impl.PerformDeviceUpdate(ctx, req, res) + util.GetLogger(ctx).Infof("PerformDeviceUpdate req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error { + err := t.Impl.PerformAccountDeactivation(ctx, req, res) + util.GetLogger(ctx).Infof("PerformAccountDeactivation req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error { + err := t.Impl.PerformOpenIDTokenCreation(ctx, req, res) + util.GetLogger(ctx).Infof("PerformOpenIDTokenCreation req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) { + t.Impl.PerformKeyBackup(ctx, req, res) +} +func (t *UserInternalAPITrace) QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse) { + t.Impl.QueryKeyBackup(ctx, req, res) +} +func (t *UserInternalAPITrace) QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error { + err := t.Impl.QueryProfile(ctx, req, res) + util.GetLogger(ctx).Infof("QueryProfile req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error { + err := t.Impl.QueryAccessToken(ctx, req, res) + util.GetLogger(ctx).Infof("QueryAccessToken req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error { + err := t.Impl.QueryDevices(ctx, req, res) + util.GetLogger(ctx).Infof("QueryDevices req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error { + err := t.Impl.QueryAccountData(ctx, req, res) + util.GetLogger(ctx).Infof("QueryAccountData req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) QueryDeviceInfos(ctx context.Context, req *QueryDeviceInfosRequest, res *QueryDeviceInfosResponse) error { + err := t.Impl.QueryDeviceInfos(ctx, req, res) + util.GetLogger(ctx).Infof("QueryDeviceInfos req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error { + err := t.Impl.QuerySearchProfiles(ctx, req, res) + util.GetLogger(ctx).Infof("QuerySearchProfiles req=%+v res=%+v", js(req), js(res)) + return err +} +func (t *UserInternalAPITrace) QueryOpenIDToken(ctx context.Context, req *QueryOpenIDTokenRequest, res *QueryOpenIDTokenResponse) error { + err := t.Impl.QueryOpenIDToken(ctx, req, res) + util.GetLogger(ctx).Infof("QueryOpenIDToken req=%+v res=%+v", js(req), js(res)) + return err +} + +func js(thing interface{}) string { + b, err := json.Marshal(thing) + if err != nil { + return fmt.Sprintf("Marshal error:%s", err) + } + return string(b) +} From da101469faab6667df83b859b782b65c84d8631e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 3 Aug 2021 11:35:29 +0100 Subject: [PATCH 15/64] Minimum version Go 1.15 (#1957) --- README.md | 2 +- cmd/dendrite-demo-yggdrasil/README.md | 2 +- docs/INSTALL.md | 2 +- go.mod | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8c793841..d8d6b2b6 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j ## Requirements -To build Dendrite, you will need Go 1.13 or later. +To build Dendrite, you will need Go 1.15 or later. For a usable federating Dendrite deployment, you will also need: - A domain name (or subdomain) diff --git a/cmd/dendrite-demo-yggdrasil/README.md b/cmd/dendrite-demo-yggdrasil/README.md index 33df7e60..c471cef2 100644 --- a/cmd/dendrite-demo-yggdrasil/README.md +++ b/cmd/dendrite-demo-yggdrasil/README.md @@ -1,6 +1,6 @@ # Yggdrasil Demo -This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.14 or later. +This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.15 or later. To run the homeserver, start at the root of the Dendrite repository and run: diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 352c73ff..1752d7e8 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -25,7 +25,7 @@ use in production environments just yet! Dendrite requires: -* Go 1.14 or higher +* Go 1.15 or higher * Postgres 9.6 or higher (if using Postgres databases, not needed for SQLite) If you want to run a polylith deployment, you also need: diff --git a/go.mod b/go.mod index 724d8bb7..a0491c26 100644 --- a/go.mod +++ b/go.mod @@ -63,4 +63,4 @@ require ( nhooyr.io/websocket v1.8.7 ) -go 1.14 +go 1.15 From ed04eed4411596a9ad83c322b89a7cbb49bf3b0f Mon Sep 17 00:00:00 2001 From: kegsay Date: Wed, 4 Aug 2021 17:08:17 +0100 Subject: [PATCH 16/64] Fix sqlite migration issues (#1960) * Do not store 'null' in the database for empty JSON arrays This can cause issues, though it should be noted that the majority of the time this will marshal/unmarshal just fine, see https://play.golang.org/p/Doe2NZUgv7Q * bugfix: sqlite migration should handle create events as having no 'before' snapshot The state snapshot for any given event in the roomserver represents the state _before_ the event. For the create event, this is nothing, so the state snapshot nid should be 0. In some cases this wasn't happening, resulting in a nice mix of possible options including: - A state snapshot without any state blocks `[]` or `null`. - A state snapshot with a single state block with a single event, the create event, causing a circular loop. This is incorrect as it represents the state before the event, not after. * Add state key check --- .../2021041615092700_state_blocks_refactor.go | 14 ++++++++++++++ roomserver/storage/sqlite3/events_table.go | 3 +++ roomserver/storage/sqlite3/state_block_table.go | 2 +- roomserver/storage/sqlite3/state_snapshot_table.go | 3 +++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go b/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go index 42edbbc6..8d033174 100644 --- a/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go +++ b/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go @@ -93,6 +93,20 @@ func UpStateBlocksRefactor(tx *sql.Tx) error { } var newblocks types.StateBlockNIDs + if len(blocks) == 0 { + // some m.room.create events have a state snapshot but no state blocks at all which makes + // sense as there is no state before creation. The correct form should be to give the event + // in question a state snapshot NID of 0 to indicate 'no snapshot'. + // If we don't do this, we'll fail the assertions later on which try to ensure we didn't forget + // any snapshots. + _, err = tx.Exec( + `UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE event_type_nid = $1 AND event_state_key_nid = $2 AND state_snapshot_nid = $3`, + types.MRoomCreateNID, types.EmptyStateKeyNID, snapshot, + ) + if err != nil { + return fmt.Errorf("resetting create events snapshots to 0 errored: %s", err) + } + } for _, block := range blocks { if err = func() error { blockrows, berr := tx.Query(`SELECT event_nid FROM _roomserver_state_block WHERE state_block_nid = $1`, block) diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index a28d95fa..b7fe7ee4 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -571,6 +571,9 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs( } func eventNIDsAsArray(eventNIDs []types.EventNID) string { + if eventNIDs == nil { + eventNIDs = []types.EventNID{} // don't store 'null' in the DB + } b, _ := json.Marshal(eventNIDs) return string(b) } diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index a472437a..58b0b5dc 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -86,7 +86,7 @@ func (s *stateBlockStatements) BulkInsertStateData( entries types.StateEntries, ) (id types.StateBlockNID, err error) { entries = entries[:util.SortAndUnique(entries)] - var nids types.EventNIDs + nids := types.EventNIDs{} // zero slice to not store 'null' in the DB for _, e := range entries { nids = append(nids, e.EventNID) } diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index ad623d65..040d99ae 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -87,6 +87,9 @@ func prepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) { func (s *stateSnapshotStatements) InsertState( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, stateBlockNIDs types.StateBlockNIDs, ) (stateNID types.StateSnapshotNID, err error) { + if stateBlockNIDs == nil { + stateBlockNIDs = []types.StateBlockNID{} // zero slice to not store 'null' in the DB + } stateBlockNIDs = stateBlockNIDs[:util.SortAndUnique(stateBlockNIDs)] stateBlockNIDsJSON, err := json.Marshal(stateBlockNIDs) if err != nil { From 7a9a2547b337acaa01a911a150871665799147a7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 4 Aug 2021 17:31:18 +0100 Subject: [PATCH 17/64] Cross-signing storage code (#1959) --- go.mod | 2 +- go.sum | 4 +- keyserver/storage/interface.go | 10 ++ .../postgres/cross_signing_keys_table.go | 102 +++++++++++++++++ .../postgres/cross_signing_sigs_table.go | 103 ++++++++++++++++++ keyserver/storage/postgres/storage.go | 18 ++- keyserver/storage/shared/storage.go | 42 +++++++ .../sqlite3/cross_signing_keys_table.go | 101 +++++++++++++++++ .../sqlite3/cross_signing_sigs_table.go | 102 +++++++++++++++++ keyserver/storage/sqlite3/storage.go | 18 ++- keyserver/storage/tables/interface.go | 11 ++ keyserver/types/storage.go | 39 +++++++ 12 files changed, 545 insertions(+), 7 deletions(-) create mode 100644 keyserver/storage/postgres/cross_signing_keys_table.go create mode 100644 keyserver/storage/postgres/cross_signing_sigs_table.go create mode 100644 keyserver/storage/sqlite3/cross_signing_keys_table.go create mode 100644 keyserver/storage/sqlite3/cross_signing_sigs_table.go create mode 100644 keyserver/types/storage.go diff --git a/go.mod b/go.mod index a0491c26..154bf8ca 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20210722110442-5061d6986876 + github.com/matrix-org/gomatrixserverlib v0.0.0-20210802144451-bec8d2252d83 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index 7eb82d5d..5c614c45 100644 --- a/go.sum +++ b/go.sum @@ -994,8 +994,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210722110442-5061d6986876 h1:6ypwCtgRLK0v/hGWvnd847+KTo9BSkP9N0A4qSniP4E= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210722110442-5061d6986876/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210802144451-bec8d2252d83 h1:fkUmeKj/U5TnWXTsJnVjEL0FQiVhf1r9WL4VWI00q2k= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210802144451-bec8d2252d83/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b h1:5X5vdWQ13xrNkJVqaJHPsrt7rKkMJH5iac0EtfOuxSg= diff --git a/keyserver/storage/interface.go b/keyserver/storage/interface.go index 0ec62f56..756dc32a 100644 --- a/keyserver/storage/interface.go +++ b/keyserver/storage/interface.go @@ -18,11 +18,15 @@ import ( "context" "encoding/json" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/keyserver/types" "github.com/matrix-org/gomatrixserverlib" ) type Database interface { + internal.PartitionStorer + // ExistingOneTimeKeys returns a map of keyIDWithAlgorithm to key JSON for the given parameters. If no keys exist with this combination // of user/device/key/algorithm 4-uple then it is omitted from the map. Returns an error when failing to communicate with the database. ExistingOneTimeKeys(ctx context.Context, userID, deviceID string, keyIDsWithAlgorithms []string) (map[string]json.RawMessage, error) @@ -73,4 +77,10 @@ type Database interface { // MarkDeviceListStale sets the stale bit for this user to isStale. MarkDeviceListStale(ctx context.Context, userID string, isStale bool) error + + CrossSigningKeysForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error) + CrossSigningSigsForTarget(ctx context.Context, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) + + StoreCrossSigningKeysForUser(ctx context.Context, userID string, keyMap types.CrossSigningKeyMap) error + StoreCrossSigningSigsForTarget(ctx context.Context, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error } diff --git a/keyserver/storage/postgres/cross_signing_keys_table.go b/keyserver/storage/postgres/cross_signing_keys_table.go new file mode 100644 index 00000000..1022157e --- /dev/null +++ b/keyserver/storage/postgres/cross_signing_keys_table.go @@ -0,0 +1,102 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/keyserver/types" + "github.com/matrix-org/gomatrixserverlib" +) + +var crossSigningKeysSchema = ` +CREATE TABLE IF NOT EXISTS keyserver_cross_signing_keys ( + user_id TEXT NOT NULL, + key_type SMALLINT NOT NULL, + key_data TEXT NOT NULL, + PRIMARY KEY (user_id, key_type) +); +` + +const selectCrossSigningKeysForUserSQL = "" + + "SELECT key_type, key_data FROM keyserver_cross_signing_keys" + + " WHERE user_id = $1" + +const upsertCrossSigningKeysForUserSQL = "" + + "INSERT INTO keyserver_cross_signing_keys (user_id, key_type, key_data)" + + " VALUES($1, $2, $3)" + + " ON CONFLICT (user_id, key_type) DO UPDATE SET key_data = $3" + +type crossSigningKeysStatements struct { + db *sql.DB + selectCrossSigningKeysForUserStmt *sql.Stmt + upsertCrossSigningKeysForUserStmt *sql.Stmt +} + +func NewPostgresCrossSigningKeysTable(db *sql.DB) (tables.CrossSigningKeys, error) { + s := &crossSigningKeysStatements{ + db: db, + } + _, err := db.Exec(crossSigningKeysSchema) + if err != nil { + return nil, err + } + return s, sqlutil.StatementList{ + {&s.selectCrossSigningKeysForUserStmt, selectCrossSigningKeysForUserSQL}, + {&s.upsertCrossSigningKeysForUserStmt, upsertCrossSigningKeysForUserSQL}, + }.Prepare(db) +} + +func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( + ctx context.Context, txn *sql.Tx, userID string, +) (r types.CrossSigningKeyMap, err error) { + rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningKeysForUserStmt).QueryContext(ctx, userID) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectCrossSigningKeysForUserStmt: rows.close() failed") + r = types.CrossSigningKeyMap{} + for rows.Next() { + var keyTypeInt int16 + var keyData gomatrixserverlib.Base64Bytes + if err := rows.Scan(&keyTypeInt, &keyData); err != nil { + return nil, err + } + keyType, ok := types.KeyTypeIntToPurpose[keyTypeInt] + if !ok { + return nil, fmt.Errorf("unknown key purpose int %d", keyTypeInt) + } + r[keyType] = keyData + } + return +} + +func (s *crossSigningKeysStatements) UpsertCrossSigningKeysForUser( + ctx context.Context, txn *sql.Tx, userID string, keyType gomatrixserverlib.CrossSigningKeyPurpose, keyData gomatrixserverlib.Base64Bytes, +) error { + keyTypeInt, ok := types.KeyTypePurposeToInt[keyType] + if !ok { + return fmt.Errorf("unknown key purpose %q", keyType) + } + if _, err := sqlutil.TxStmt(txn, s.upsertCrossSigningKeysForUserStmt).ExecContext(ctx, userID, keyTypeInt, keyData); err != nil { + return fmt.Errorf("s.upsertCrossSigningKeysForUserStmt: %w", err) + } + return nil +} diff --git a/keyserver/storage/postgres/cross_signing_sigs_table.go b/keyserver/storage/postgres/cross_signing_sigs_table.go new file mode 100644 index 00000000..677e7a48 --- /dev/null +++ b/keyserver/storage/postgres/cross_signing_sigs_table.go @@ -0,0 +1,103 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/keyserver/types" + "github.com/matrix-org/gomatrixserverlib" +) + +var crossSigningSigsSchema = ` +CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs ( + origin_user_id TEXT NOT NULL, + origin_key_id TEXT NOT NULL, + target_user_id TEXT NOT NULL, + target_key_id TEXT NOT NULL, + signature TEXT NOT NULL, + PRIMARY KEY (origin_user_id, target_user_id, target_key_id) +); +` + +const selectCrossSigningSigsForTargetSQL = "" + + "SELECT origin_user_id, origin_key_id, signature FROM keyserver_cross_signing_sigs" + + " WHERE target_user_id = $1 AND target_key_id = $2" + +const upsertCrossSigningSigsForTargetSQL = "" + + "INSERT INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" + + " VALUES($1, $2, $3, $4, $5)" + + " ON CONFLICT (origin_user_id, target_user_id, target_key_id) DO UPDATE SET (origin_key_id, signature) = ($2, $5)" + +type crossSigningSigsStatements struct { + db *sql.DB + selectCrossSigningSigsForTargetStmt *sql.Stmt + upsertCrossSigningSigsForTargetStmt *sql.Stmt +} + +func NewPostgresCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, error) { + s := &crossSigningSigsStatements{ + db: db, + } + _, err := db.Exec(crossSigningSigsSchema) + if err != nil { + return nil, err + } + return s, sqlutil.StatementList{ + {&s.selectCrossSigningSigsForTargetStmt, selectCrossSigningSigsForTargetSQL}, + {&s.upsertCrossSigningSigsForTargetStmt, upsertCrossSigningSigsForTargetSQL}, + }.Prepare(db) +} + +func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( + ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID, +) (r types.CrossSigningSigMap, err error) { + rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, targetUserID, targetKeyID) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectCrossSigningSigsForTargetStmt: rows.close() failed") + r = types.CrossSigningSigMap{} + for rows.Next() { + var userID string + var keyID gomatrixserverlib.KeyID + var signature gomatrixserverlib.Base64Bytes + if err := rows.Scan(&userID, &keyID, &signature); err != nil { + return nil, err + } + if _, ok := r[userID]; !ok { + r[userID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} + } + r[userID][keyID] = signature + } + return +} + +func (s *crossSigningSigsStatements) UpsertCrossSigningSigsForTarget( + ctx context.Context, txn *sql.Tx, + originUserID string, originKeyID gomatrixserverlib.KeyID, + targetUserID string, targetKeyID gomatrixserverlib.KeyID, + signature gomatrixserverlib.Base64Bytes, +) error { + if _, err := sqlutil.TxStmt(txn, s.upsertCrossSigningSigsForTargetStmt).ExecContext(ctx, originUserID, originKeyID, targetUserID, targetKeyID, signature); err != nil { + return fmt.Errorf("s.upsertCrossSigningSigsForTargetStmt: %w", err) + } + return nil +} diff --git a/keyserver/storage/postgres/storage.go b/keyserver/storage/postgres/storage.go index cb16ffaa..52f3a7f6 100644 --- a/keyserver/storage/postgres/storage.go +++ b/keyserver/storage/postgres/storage.go @@ -43,12 +43,26 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) if err != nil { return nil, err } - return &shared.Database{ + csk, err := NewPostgresCrossSigningKeysTable(db) + if err != nil { + return nil, err + } + css, err := NewPostgresCrossSigningSigsTable(db) + if err != nil { + return nil, err + } + d := &shared.Database{ DB: db, Writer: sqlutil.NewDummyWriter(), OneTimeKeysTable: otk, DeviceKeysTable: dk, KeyChangesTable: kc, StaleDeviceListsTable: sdl, - }, nil + CrossSigningKeysTable: csk, + CrossSigningSigsTable: css, + } + if err = d.PartitionOffsetStatements.Prepare(db, d.Writer, "keyserver"); err != nil { + return nil, err + } + return d, nil } diff --git a/keyserver/storage/shared/storage.go b/keyserver/storage/shared/storage.go index de757f29..76724295 100644 --- a/keyserver/storage/shared/storage.go +++ b/keyserver/storage/shared/storage.go @@ -18,10 +18,12 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/keyserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -32,6 +34,9 @@ type Database struct { DeviceKeysTable tables.DeviceKeys KeyChangesTable tables.KeyChanges StaleDeviceListsTable tables.StaleDeviceLists + CrossSigningKeysTable tables.CrossSigningKeys + CrossSigningSigsTable tables.CrossSigningSigs + sqlutil.PartitionOffsetStatements } func (d *Database) ExistingOneTimeKeys(ctx context.Context, userID, deviceID string, keyIDsWithAlgorithms []string) (map[string]json.RawMessage, error) { @@ -152,3 +157,40 @@ func (d *Database) MarkDeviceListStale(ctx context.Context, userID string, isSta return d.StaleDeviceListsTable.InsertStaleDeviceList(ctx, userID, isStale) }) } + +// CrossSigningKeysForUser returns the latest known cross-signing keys for a user, if any. +func (d *Database) CrossSigningKeysForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error) { + return d.CrossSigningKeysTable.SelectCrossSigningKeysForUser(ctx, nil, userID) +} + +// CrossSigningSigsForTarget returns the signatures for a given user's key ID, if any. +func (d *Database) CrossSigningSigsForTarget(ctx context.Context, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) { + return d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, targetUserID, targetKeyID) +} + +// StoreCrossSigningKeysForUser stores the latest known cross-signing keys for a user. +func (d *Database) StoreCrossSigningKeysForUser(ctx context.Context, userID string, keyMap types.CrossSigningKeyMap) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + for keyType, keyData := range keyMap { + if err := d.CrossSigningKeysTable.UpsertCrossSigningKeysForUser(ctx, txn, userID, keyType, keyData); err != nil { + return fmt.Errorf("d.CrossSigningKeysTable.InsertCrossSigningKeysForUser: %w", err) + } + } + return nil + }) +} + +// StoreCrossSigningSigsForTarget stores a signature for a target user ID and key/dvice. +func (d *Database) StoreCrossSigningSigsForTarget( + ctx context.Context, + originUserID string, originKeyID gomatrixserverlib.KeyID, + targetUserID string, targetKeyID gomatrixserverlib.KeyID, + signature gomatrixserverlib.Base64Bytes, +) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + if err := d.CrossSigningSigsTable.UpsertCrossSigningSigsForTarget(ctx, nil, originUserID, originKeyID, targetUserID, targetKeyID, signature); err != nil { + return fmt.Errorf("d.CrossSigningSigsTable.InsertCrossSigningSigsForTarget: %w", err) + } + return nil + }) +} diff --git a/keyserver/storage/sqlite3/cross_signing_keys_table.go b/keyserver/storage/sqlite3/cross_signing_keys_table.go new file mode 100644 index 00000000..e103d988 --- /dev/null +++ b/keyserver/storage/sqlite3/cross_signing_keys_table.go @@ -0,0 +1,101 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/keyserver/types" + "github.com/matrix-org/gomatrixserverlib" +) + +var crossSigningKeysSchema = ` +CREATE TABLE IF NOT EXISTS keyserver_cross_signing_keys ( + user_id TEXT NOT NULL, + key_type INTEGER NOT NULL, + key_data TEXT NOT NULL, + PRIMARY KEY (user_id, key_type) +); +` + +const selectCrossSigningKeysForUserSQL = "" + + "SELECT key_type, key_data FROM keyserver_cross_signing_keys" + + " WHERE user_id = $1" + +const upsertCrossSigningKeysForUserSQL = "" + + "INSERT OR REPLACE INTO keyserver_cross_signing_keys (user_id, key_type, key_data)" + + " VALUES($1, $2, $3)" + +type crossSigningKeysStatements struct { + db *sql.DB + selectCrossSigningKeysForUserStmt *sql.Stmt + upsertCrossSigningKeysForUserStmt *sql.Stmt +} + +func NewSqliteCrossSigningKeysTable(db *sql.DB) (tables.CrossSigningKeys, error) { + s := &crossSigningKeysStatements{ + db: db, + } + _, err := db.Exec(crossSigningKeysSchema) + if err != nil { + return nil, err + } + return s, sqlutil.StatementList{ + {&s.selectCrossSigningKeysForUserStmt, selectCrossSigningKeysForUserSQL}, + {&s.upsertCrossSigningKeysForUserStmt, upsertCrossSigningKeysForUserSQL}, + }.Prepare(db) +} + +func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( + ctx context.Context, txn *sql.Tx, userID string, +) (r types.CrossSigningKeyMap, err error) { + rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningKeysForUserStmt).QueryContext(ctx, userID) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectCrossSigningKeysForUserStmt: rows.close() failed") + r = types.CrossSigningKeyMap{} + for rows.Next() { + var keyTypeInt int16 + var keyData gomatrixserverlib.Base64Bytes + if err := rows.Scan(&keyTypeInt, &keyData); err != nil { + return nil, err + } + keyType, ok := types.KeyTypeIntToPurpose[keyTypeInt] + if !ok { + return nil, fmt.Errorf("unknown key purpose int %d", keyTypeInt) + } + r[keyType] = keyData + } + return +} + +func (s *crossSigningKeysStatements) UpsertCrossSigningKeysForUser( + ctx context.Context, txn *sql.Tx, userID string, keyType gomatrixserverlib.CrossSigningKeyPurpose, keyData gomatrixserverlib.Base64Bytes, +) error { + keyTypeInt, ok := types.KeyTypePurposeToInt[keyType] + if !ok { + return fmt.Errorf("unknown key purpose %q", keyType) + } + if _, err := sqlutil.TxStmt(txn, s.upsertCrossSigningKeysForUserStmt).ExecContext(ctx, userID, keyTypeInt, keyData); err != nil { + return fmt.Errorf("s.upsertCrossSigningKeysForUserStmt: %w", err) + } + return nil +} diff --git a/keyserver/storage/sqlite3/cross_signing_sigs_table.go b/keyserver/storage/sqlite3/cross_signing_sigs_table.go new file mode 100644 index 00000000..aa702583 --- /dev/null +++ b/keyserver/storage/sqlite3/cross_signing_sigs_table.go @@ -0,0 +1,102 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/keyserver/types" + "github.com/matrix-org/gomatrixserverlib" +) + +var crossSigningSigsSchema = ` +CREATE TABLE IF NOT EXISTS keyserver_cross_signing_sigs ( + origin_user_id TEXT NOT NULL, + origin_key_id TEXT NOT NULL, + target_user_id TEXT NOT NULL, + target_key_id TEXT NOT NULL, + signature TEXT NOT NULL, + PRIMARY KEY (origin_user_id, target_user_id, target_key_id) +); +` + +const selectCrossSigningSigsForTargetSQL = "" + + "SELECT origin_user_id, origin_key_id, signature FROM keyserver_cross_signing_sigs" + + " WHERE target_user_id = $1 AND target_key_id = $2" + +const upsertCrossSigningSigsForTargetSQL = "" + + "INSERT OR REPLACE INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" + + " VALUES($1, $2, $3, $4, $5)" + +type crossSigningSigsStatements struct { + db *sql.DB + selectCrossSigningSigsForTargetStmt *sql.Stmt + upsertCrossSigningSigsForTargetStmt *sql.Stmt +} + +func NewSqliteCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, error) { + s := &crossSigningSigsStatements{ + db: db, + } + _, err := db.Exec(crossSigningSigsSchema) + if err != nil { + return nil, err + } + return s, sqlutil.StatementList{ + {&s.selectCrossSigningSigsForTargetStmt, selectCrossSigningSigsForTargetSQL}, + {&s.upsertCrossSigningSigsForTargetStmt, upsertCrossSigningSigsForTargetSQL}, + }.Prepare(db) +} + +func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( + ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID, +) (r types.CrossSigningSigMap, err error) { + rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, targetUserID, targetKeyID) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectCrossSigningSigsForTargetStmt: rows.close() failed") + r = types.CrossSigningSigMap{} + for rows.Next() { + var userID string + var keyID gomatrixserverlib.KeyID + var signature gomatrixserverlib.Base64Bytes + if err := rows.Scan(&userID, &keyID, &signature); err != nil { + return nil, err + } + if _, ok := r[userID]; !ok { + r[userID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} + } + r[userID][keyID] = signature + } + return +} + +func (s *crossSigningSigsStatements) UpsertCrossSigningSigsForTarget( + ctx context.Context, txn *sql.Tx, + originUserID string, originKeyID gomatrixserverlib.KeyID, + targetUserID string, targetKeyID gomatrixserverlib.KeyID, + signature gomatrixserverlib.Base64Bytes, +) error { + if _, err := sqlutil.TxStmt(txn, s.upsertCrossSigningSigsForTargetStmt).ExecContext(ctx, originUserID, originKeyID, targetUserID, targetKeyID, signature); err != nil { + return fmt.Errorf("s.upsertCrossSigningSigsForTargetStmt: %w", err) + } + return nil +} diff --git a/keyserver/storage/sqlite3/storage.go b/keyserver/storage/sqlite3/storage.go index ca1e7560..ee1746cd 100644 --- a/keyserver/storage/sqlite3/storage.go +++ b/keyserver/storage/sqlite3/storage.go @@ -41,12 +41,26 @@ func NewDatabase(dbProperties *config.DatabaseOptions) (*shared.Database, error) if err != nil { return nil, err } - return &shared.Database{ + csk, err := NewSqliteCrossSigningKeysTable(db) + if err != nil { + return nil, err + } + css, err := NewSqliteCrossSigningSigsTable(db) + if err != nil { + return nil, err + } + d := &shared.Database{ DB: db, Writer: sqlutil.NewExclusiveWriter(), OneTimeKeysTable: otk, DeviceKeysTable: dk, KeyChangesTable: kc, StaleDeviceListsTable: sdl, - }, nil + CrossSigningKeysTable: csk, + CrossSigningSigsTable: css, + } + if err = d.PartitionOffsetStatements.Prepare(db, d.Writer, "keyserver"); err != nil { + return nil, err + } + return d, nil } diff --git a/keyserver/storage/tables/interface.go b/keyserver/storage/tables/interface.go index b70c9bce..0649b680 100644 --- a/keyserver/storage/tables/interface.go +++ b/keyserver/storage/tables/interface.go @@ -20,6 +20,7 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/keyserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -52,3 +53,13 @@ type StaleDeviceLists interface { InsertStaleDeviceList(ctx context.Context, userID string, isStale bool) error SelectUserIDsWithStaleDeviceLists(ctx context.Context, domains []gomatrixserverlib.ServerName) ([]string, error) } + +type CrossSigningKeys interface { + SelectCrossSigningKeysForUser(ctx context.Context, txn *sql.Tx, userID string) (r types.CrossSigningKeyMap, err error) + UpsertCrossSigningKeysForUser(ctx context.Context, txn *sql.Tx, userID string, keyType gomatrixserverlib.CrossSigningKeyPurpose, keyData gomatrixserverlib.Base64Bytes) error +} + +type CrossSigningSigs interface { + SelectCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (r types.CrossSigningSigMap, err error) + UpsertCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error +} diff --git a/keyserver/types/storage.go b/keyserver/types/storage.go new file mode 100644 index 00000000..3480ec65 --- /dev/null +++ b/keyserver/types/storage.go @@ -0,0 +1,39 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import "github.com/matrix-org/gomatrixserverlib" + +// KeyTypePurposeToInt maps a purpose to an integer, which is used in the +// database to reduce the amount of space taken up by this column. +var KeyTypePurposeToInt = map[gomatrixserverlib.CrossSigningKeyPurpose]int16{ + gomatrixserverlib.CrossSigningKeyPurposeMaster: 1, + gomatrixserverlib.CrossSigningKeyPurposeSelfSigning: 2, + gomatrixserverlib.CrossSigningKeyPurposeUserSigning: 3, +} + +// KeyTypeIntToPurpose maps an integer to a purpose, which is used in the +// database to reduce the amount of space taken up by this column. +var KeyTypeIntToPurpose = map[int16]gomatrixserverlib.CrossSigningKeyPurpose{ + 1: gomatrixserverlib.CrossSigningKeyPurposeMaster, + 2: gomatrixserverlib.CrossSigningKeyPurposeSelfSigning, + 3: gomatrixserverlib.CrossSigningKeyPurposeUserSigning, +} + +// Map of purpose -> public key +type CrossSigningKeyMap map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.Base64Bytes + +// Map of user ID -> key ID -> signature +type CrossSigningSigMap map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes From 4cc8b28b7f341c21a0bf7a261d6b49e8276e1e99 Mon Sep 17 00:00:00 2001 From: kegsay Date: Wed, 4 Aug 2021 17:48:23 +0100 Subject: [PATCH 18/64] Ensure all create events have a snapshot NID of 0 (#1961) Fixes #1924 for postgres users, though the underlying cause of why they aren't 0 in the first place is unresolved. --- .../deltas/2021041615092700_state_blocks_refactor.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go index 940a920e..6b72de51 100644 --- a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go +++ b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go @@ -99,6 +99,18 @@ func UpStateBlocksRefactor(tx *sql.Tx) error { return fmt.Errorf("tx.Exec (create snapshots table): %w", err) } logrus.Warn("New tables created...") + // some m.room.create events have a state snapshot but no state blocks at all which makes + // sense as there is no state before creation. The correct form should be to give the event + // in question a state snapshot NID of 0 to indicate 'no snapshot'. + // If we don't do this, we'll fail the assertions later on which try to ensure we didn't forget + // any snapshots. + _, err = tx.Exec( + `UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE event_type_nid = $1 AND event_state_key_nid = $2`, + types.MRoomCreateNID, types.EmptyStateKeyNID, + ) + if err != nil { + return fmt.Errorf("resetting create events snapshots to 0 errored: %s", err) + } batchsize := 100 for batchoffset := 0; batchoffset < snapshotcount; batchoffset += batchsize { From eb0efa46361ca2d80050ef871d0a8f20652398af Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 4 Aug 2021 17:56:29 +0100 Subject: [PATCH 19/64] Cross-signing groundwork (#1953) * Cross-signing groundwork * Update to matrix-org/gomatrixserverlib#274 * Fix gobind builds, which stops unit tests in CI from yelling * Some changes from review comments * Fix build by passing in UIA * Update to matrix-org/gomatrixserverlib@bec8d22 * Process master/self-signing keys from devices call * nolint * Enum-ify the key type in the database * Process self-signing key too * Fix sanity check in device list updater * Fix check * Fix sytest, hopefully * Fix build --- build/gobind-pinecone/monolith.go | 2 +- build/gobind-yggdrasil/monolith.go | 2 +- clientapi/auth/password.go | 4 +- clientapi/auth/user_interactive.go | 4 +- clientapi/jsonerror/jsonerror.go | 12 + clientapi/routing/key_crosssigning.go | 125 ++++++ clientapi/routing/keys.go | 10 +- clientapi/routing/routing.go | 22 +- cmd/dendrite-demo-libp2p/main.go | 2 +- cmd/dendrite-demo-pinecone/main.go | 2 +- cmd/dendrite-demo-yggdrasil/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 2 +- .../personalities/keyserver.go | 2 +- cmd/dendritejs-pinecone/main.go | 2 +- cmd/dendritejs/main.go | 2 +- keyserver/api/api.go | 33 +- keyserver/consumers/eduserver.go | 61 +++ keyserver/internal/cross_signing.go | 389 ++++++++++++++++++ keyserver/internal/device_list_update.go | 27 +- keyserver/internal/device_list_update_test.go | 13 +- keyserver/internal/internal.go | 83 +++- keyserver/inthttp/client.go | 50 ++- keyserver/inthttp/server.go | 22 + keyserver/keyserver.go | 29 +- setup/config/config_kafka.go | 1 + syncapi/internal/keychange_test.go | 4 + sytest-whitelist | 3 + 27 files changed, 860 insertions(+), 50 deletions(-) create mode 100644 clientapi/routing/key_crosssigning.go create mode 100644 keyserver/consumers/eduserver.go create mode 100644 keyserver/internal/cross_signing.go diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index e30057ed..20228404 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -299,7 +299,7 @@ func (m *DendriteMonolith) Start() { base, federation, rsAPI, keyRing, true, ) - keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI) + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) m.userAPI = userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI) keyAPI.SetUserAPI(m.userAPI) diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index d5aedeac..6fc5f244 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -119,7 +119,7 @@ func (m *DendriteMonolith) Start() { base, federation, rsAPI, keyRing, true, ) - keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation) + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI) keyAPI.SetUserAPI(userAPI) diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index bf4a9536..a66e2fe7 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -52,7 +52,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, if username == "" { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, - JSON: jsonerror.BadJSON("'user' must be supplied."), + JSON: jsonerror.BadJSON("A username must be supplied."), } } localpart, err := userutil.ParseUsernameParam(username, &t.Config.Matrix.ServerName) @@ -68,7 +68,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, // but that would leak the existence of the user. return nil, &util.JSONResponse{ Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"), + JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."), } } return &r.Login, nil diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go index 9e0db68e..30469fc4 100644 --- a/clientapi/auth/user_interactive.go +++ b/clientapi/auth/user_interactive.go @@ -220,7 +220,7 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device * if !ok { return nil, &util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("unknown auth.type: " + authType), + JSON: jsonerror.BadJSON("Unknown auth.type: " + authType), } } @@ -231,7 +231,7 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device * if !u.IsSingleStageFlow(authType) { return nil, &util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.Unknown("missing or unknown auth.session"), + JSON: jsonerror.Unknown("The auth.session is missing or unknown."), } } } diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go index 7f8f264b..c42b25be 100644 --- a/clientapi/jsonerror/jsonerror.go +++ b/clientapi/jsonerror/jsonerror.go @@ -125,6 +125,18 @@ func GuestAccessForbidden(msg string) *MatrixError { return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg} } +// InvalidSignature is an error which is returned when the client tries +// to upload invalid signatures. +func InvalidSignature(msg string) *MatrixError { + return &MatrixError{"M_INVALID_SIGNATURE", msg} +} + +// MissingParam is an error that is returned when a parameter was incorrect, +// traditionally with cross-signing. +func MissingParam(msg string) *MatrixError { + return &MatrixError{"M_MISSING_PARAM", msg} +} + type IncompatibleRoomVersionError struct { RoomVersion string `json:"room_version"` Error string `json:"error"` diff --git a/clientapi/routing/key_crosssigning.go b/clientapi/routing/key_crosssigning.go new file mode 100644 index 00000000..3c103fd7 --- /dev/null +++ b/clientapi/routing/key_crosssigning.go @@ -0,0 +1,125 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "encoding/json" + "io/ioutil" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/auth" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/setup/config" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" + "github.com/matrix-org/util" +) + +func UploadCrossSigningDeviceKeys( + req *http.Request, userInteractiveAuth *auth.UserInteractive, + keyserverAPI api.KeyInternalAPI, device *userapi.Device, + accountDB accounts.Database, cfg *config.ClientAPI, +) util.JSONResponse { + uploadReq := &api.PerformUploadDeviceKeysRequest{} + uploadRes := &api.PerformUploadDeviceKeysResponse{} + + ctx := req.Context() + defer req.Body.Close() // nolint:errcheck + bodyBytes, err := ioutil.ReadAll(req.Body) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()), + } + } + + if _, err := userInteractiveAuth.Verify(ctx, bodyBytes, device); err != nil { + return *err + } + + if err = json.Unmarshal(bodyBytes, &uploadReq); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("The request body could not be unmarshalled: " + err.Error()), + } + } + + uploadReq.UserID = device.UserID + keyserverAPI.PerformUploadDeviceKeys(req.Context(), uploadReq, uploadRes) + + if err := uploadRes.Error; err != nil { + switch { + case err.IsInvalidSignature: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidSignature(err.Error()), + } + case err.IsMissingParam: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingParam(err.Error()), + } + default: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.Unknown(err.Error()), + } + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} + +func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.KeyInternalAPI, device *userapi.Device) util.JSONResponse { + uploadReq := &api.PerformUploadDeviceSignaturesRequest{} + uploadRes := &api.PerformUploadDeviceSignaturesResponse{} + + if err := httputil.UnmarshalJSONRequest(req, &uploadReq.Signatures); err != nil { + return *err + } + + uploadReq.UserID = device.UserID + keyserverAPI.PerformUploadDeviceSignatures(req.Context(), uploadReq, uploadRes) + + if err := uploadRes.Error; err != nil { + switch { + case err.IsInvalidSignature: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidSignature(err.Error()), + } + case err.IsMissingParam: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingParam(err.Error()), + } + default: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.Unknown(err.Error()), + } + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/keys.go b/clientapi/routing/keys.go index e2233642..2d65ac35 100644 --- a/clientapi/routing/keys.go +++ b/clientapi/routing/keys.go @@ -100,7 +100,7 @@ func (r *queryKeysRequest) GetTimeout() time.Duration { return time.Duration(r.Timeout) * time.Millisecond } -func QueryKeys(req *http.Request, keyAPI api.KeyInternalAPI) util.JSONResponse { +func QueryKeys(req *http.Request, keyAPI api.KeyInternalAPI, device *userapi.Device) util.JSONResponse { var r queryKeysRequest resErr := httputil.UnmarshalJSONRequest(req, &r) if resErr != nil { @@ -108,6 +108,7 @@ func QueryKeys(req *http.Request, keyAPI api.KeyInternalAPI) util.JSONResponse { } queryRes := api.QueryKeysResponse{} keyAPI.QueryKeys(req.Context(), &api.QueryKeysRequest{ + UserID: device.UserID, UserToDevices: r.DeviceKeys, Timeout: r.GetTimeout(), // TODO: Token? @@ -115,8 +116,11 @@ func QueryKeys(req *http.Request, keyAPI api.KeyInternalAPI) util.JSONResponse { return util.JSONResponse{ Code: 200, JSON: map[string]interface{}{ - "device_keys": queryRes.DeviceKeys, - "failures": queryRes.Failures, + "device_keys": queryRes.DeviceKeys, + "master_keys": queryRes.MasterKeys, + "self_signing_keys": queryRes.SelfSigningKeys, + "user_signing_keys": queryRes.UserSigningKeys, + "failures": queryRes.Failures, }, } } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 541c7628..c6be8939 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -64,7 +64,9 @@ func Setup( rateLimits := newRateLimits(&cfg.RateLimiting) userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg) - unstableFeatures := make(map[string]bool) + unstableFeatures := map[string]bool{ + //"org.matrix.e2e_cross_signing": true, + } for _, msc := range cfg.MSCs.MSCs { unstableFeatures["org.matrix."+msc] = true } @@ -1066,6 +1068,22 @@ func Setup( // Deleting E2E Backup Keys + // Cross-signing device keys + + postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, keyAPI, device, accountDB, cfg) + }) + + postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return UploadCrossSigningDeviceSignatures(req, keyAPI, device) + }) + + r0mux.Handle("/keys/device_signing/upload", postDeviceSigningKeys).Methods(http.MethodPost, http.MethodOptions) + r0mux.Handle("/keys/signatures/upload", postDeviceSigningSignatures).Methods(http.MethodPost, http.MethodOptions) + + unstableMux.Handle("/keys/device_signing/upload", postDeviceSigningKeys).Methods(http.MethodPost, http.MethodOptions) + unstableMux.Handle("/keys/signatures/upload", postDeviceSigningSignatures).Methods(http.MethodPost, http.MethodOptions) + // Supplying a device ID is deprecated. r0mux.Handle("/keys/upload/{deviceID}", httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { @@ -1079,7 +1097,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/keys/query", httputil.MakeAuthAPI("keys_query", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return QueryKeys(req, keyAPI) + return QueryKeys(req, keyAPI, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/keys/claim", diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 1c511ce7..4caa617f 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -147,7 +147,7 @@ func main() { accountDB := base.Base.CreateAccountsDB() federation := createFederationClient(base) - keyAPI := keyserver.NewInternalAPI(&base.Base.Cfg.KeyServer, federation) + keyAPI := keyserver.NewInternalAPI(&base.Base, &base.Base.Cfg.KeyServer, federation) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI) keyAPI.SetUserAPI(userAPI) diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index d763f080..1933c5ad 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -179,7 +179,7 @@ func main() { base, federation, rsAPI, keyRing, true, ) - keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI) + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI) keyAPI.SetUserAPI(userAPI) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index fc33a41f..dbdb9a76 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -102,7 +102,7 @@ func main() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation) + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI) keyAPI.SetUserAPI(userAPI) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 9974fe66..bb313e57 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -112,7 +112,7 @@ func main() { // This is different to rsAPI which can be the http client which doesn't need this dependency rsImpl.SetFederationSenderAPI(fsAPI) - keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI) + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI) keyAPI.SetUserAPI(userAPI) if traceInternal { diff --git a/cmd/dendrite-polylith-multi/personalities/keyserver.go b/cmd/dendrite-polylith-multi/personalities/keyserver.go index d7fc9f4f..8a99d779 100644 --- a/cmd/dendrite-polylith-multi/personalities/keyserver.go +++ b/cmd/dendrite-polylith-multi/personalities/keyserver.go @@ -22,7 +22,7 @@ import ( func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) { fsAPI := base.FederationSenderHTTPClient() - intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI) + intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) intAPI.SetUserAPI(base.UserAPIClient()) keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI) diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index 0201b291..b44c609c 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -184,7 +184,7 @@ func startup() { accountDB := base.CreateAccountsDB() federation := conn.CreateFederationClient(base, pSessions) - keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation) + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI) keyAPI.SetUserAPI(userAPI) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index d5a845ae..d8fc8b83 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -192,7 +192,7 @@ func main() { accountDB := base.CreateAccountsDB() federation := createFederationClient(cfg, node) - keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation) + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI) keyAPI.SetUserAPI(userAPI) diff --git a/keyserver/api/api.go b/keyserver/api/api.go index 5cb287bc..f46a9ee2 100644 --- a/keyserver/api/api.go +++ b/keyserver/api/api.go @@ -32,6 +32,8 @@ type KeyInternalAPI interface { PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse) // PerformClaimKeys claims one-time keys for use in pre-key messages PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) + PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse) + PerformUploadDeviceSignatures(ctx context.Context, req *PerformUploadDeviceSignaturesRequest, res *PerformUploadDeviceSignaturesResponse) QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse) QueryKeyChanges(ctx context.Context, req *QueryKeyChangesRequest, res *QueryKeyChangesResponse) QueryOneTimeKeys(ctx context.Context, req *QueryOneTimeKeysRequest, res *QueryOneTimeKeysResponse) @@ -40,7 +42,9 @@ type KeyInternalAPI interface { // KeyError is returned if there was a problem performing/querying the server type KeyError struct { - Err string + Err string `json:"error"` + IsInvalidSignature bool `json:"is_invalid_signature,omitempty"` // M_INVALID_SIGNATURE + IsMissingParam bool `json:"is_missing_param,omitempty"` // M_MISSING_PARAM } func (k *KeyError) Error() string { @@ -151,7 +155,30 @@ type PerformClaimKeysResponse struct { Error *KeyError } +type PerformUploadDeviceKeysRequest struct { + gomatrixserverlib.CrossSigningKeys + // The user that uploaded the key, should be populated by the clientapi. + UserID string `json:"user_id"` +} + +type PerformUploadDeviceKeysResponse struct { + Error *KeyError +} + +type PerformUploadDeviceSignaturesRequest struct { + Signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice + // The user that uploaded the sig, should be populated by the clientapi. + UserID string `json:"user_id"` +} + +type PerformUploadDeviceSignaturesResponse struct { + Error *KeyError +} + type QueryKeysRequest struct { + // The user ID asking for the keys, e.g. if from a client API request. + // Will not be populated if the key request came from federation. + UserID string // Maps user IDs to a list of devices UserToDevices map[string][]string Timeout time.Duration @@ -162,6 +189,10 @@ type QueryKeysResponse struct { Failures map[string]interface{} // Map of user_id to device_id to device_key DeviceKeys map[string]map[string]json.RawMessage + // Maps of user_id to cross signing key + MasterKeys map[string]gomatrixserverlib.CrossSigningKey + SelfSigningKeys map[string]gomatrixserverlib.CrossSigningKey + UserSigningKeys map[string]gomatrixserverlib.CrossSigningKey // Set if there was a fatal error processing this query Error *KeyError } diff --git a/keyserver/consumers/eduserver.go b/keyserver/consumers/eduserver.go new file mode 100644 index 00000000..d764950b --- /dev/null +++ b/keyserver/consumers/eduserver.go @@ -0,0 +1,61 @@ +package consumers + +import ( + "fmt" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/keyserver/storage" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" + + "github.com/Shopify/sarama" +) + +type OutputSigningKeyUpdateConsumer struct { + eduServerConsumer *internal.ContinualConsumer + keyDB storage.Database + keyAPI api.KeyInternalAPI + serverName string +} + +func NewOutputSigningKeyUpdateConsumer( + process *process.ProcessContext, + cfg *config.Dendrite, + kafkaConsumer sarama.Consumer, + keyDB storage.Database, + keyAPI api.KeyInternalAPI, +) *OutputSigningKeyUpdateConsumer { + consumer := internal.ContinualConsumer{ + Process: process, + ComponentName: "keyserver/eduserver", + Topic: cfg.Global.Kafka.TopicFor(config.TopicOutputSigningKeyUpdate), + Consumer: kafkaConsumer, + PartitionStore: keyDB, + } + s := &OutputSigningKeyUpdateConsumer{ + eduServerConsumer: &consumer, + keyDB: keyDB, + keyAPI: keyAPI, + serverName: string(cfg.Global.ServerName), + } + consumer.ProcessMessage = s.onMessage + + return s +} + +func (s *OutputSigningKeyUpdateConsumer) Start() error { + return s.eduServerConsumer.Start() +} + +func (s *OutputSigningKeyUpdateConsumer) onMessage(msg *sarama.ConsumerMessage) error { + /* + var output eduapi.OutputSigningKeyUpdate + if err := json.Unmarshal(msg.Value, &output); err != nil { + log.WithError(err).Errorf("eduserver output log: message parse failure") + return nil + } + return nil + */ + return fmt.Errorf("TODO") +} diff --git a/keyserver/internal/cross_signing.go b/keyserver/internal/cross_signing.go new file mode 100644 index 00000000..802d02b3 --- /dev/null +++ b/keyserver/internal/cross_signing.go @@ -0,0 +1,389 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "fmt" + "strings" + + "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/keyserver/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" +) + +func sanityCheckKey(key gomatrixserverlib.CrossSigningKey, userID string, purpose gomatrixserverlib.CrossSigningKeyPurpose) error { + // Is there exactly one key? + if len(key.Keys) != 1 { + return fmt.Errorf("should contain exactly one key") + } + + // Does the key ID match the key value? Iterates exactly once + for keyID, keyData := range key.Keys { + b64 := keyData.Encode() + tokens := strings.Split(string(keyID), ":") + if len(tokens) != 2 { + return fmt.Errorf("key ID is incorrectly formatted") + } + if tokens[1] != b64 { + return fmt.Errorf("key ID isn't correct") + } + } + + // Does the key claim to be from the right user? + if userID != key.UserID { + return fmt.Errorf("key has a user ID mismatch") + } + + // Does the key contain the correct purpose? + useful := false + for _, usage := range key.Usage { + if usage == purpose { + useful = true + break + } + } + if !useful { + return fmt.Errorf("key does not contain correct usage purpose") + } + + return nil +} + +// nolint:gocyclo +func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) { + var masterKey gomatrixserverlib.Base64Bytes + hasMasterKey := false + + if len(req.MasterKey.Keys) > 0 { + if err := sanityCheckKey(req.MasterKey, req.UserID, gomatrixserverlib.CrossSigningKeyPurposeMaster); err != nil { + res.Error = &api.KeyError{ + Err: "Master key sanity check failed: " + err.Error(), + } + return + } + hasMasterKey = true + for _, keyData := range req.MasterKey.Keys { // iterates once, because sanityCheckKey requires one key + masterKey = keyData + } + } + + if len(req.SelfSigningKey.Keys) > 0 { + if err := sanityCheckKey(req.SelfSigningKey, req.UserID, gomatrixserverlib.CrossSigningKeyPurposeSelfSigning); err != nil { + res.Error = &api.KeyError{ + Err: "Self-signing key sanity check failed: " + err.Error(), + } + return + } + } + + if len(req.UserSigningKey.Keys) > 0 { + if err := sanityCheckKey(req.UserSigningKey, req.UserID, gomatrixserverlib.CrossSigningKeyPurposeUserSigning); err != nil { + res.Error = &api.KeyError{ + Err: "User-signing key sanity check failed: " + err.Error(), + } + return + } + } + + // If the user hasn't given a new master key, then let's go and get their + // existing keys from the database. + if !hasMasterKey { + existingKeys, err := a.DB.CrossSigningKeysForUser(ctx, req.UserID) + if err != nil { + res.Error = &api.KeyError{ + Err: "Retrieving cross-signing keys from database failed: " + err.Error(), + } + return + } + + masterKey, hasMasterKey = existingKeys[gomatrixserverlib.CrossSigningKeyPurposeMaster] + } + + // If the user isn't a local user and we haven't successfully found a key + // through any local means then ask over federation. + if !hasMasterKey { + _, host, err := gomatrixserverlib.SplitID('@', req.UserID) + if err != nil { + res.Error = &api.KeyError{ + Err: "Retrieving cross-signing keys from federation failed: " + err.Error(), + } + return + } + keys, err := a.FedClient.QueryKeys(ctx, host, map[string][]string{ + req.UserID: {}, + }) + if err != nil { + res.Error = &api.KeyError{ + Err: "Retrieving cross-signing keys from federation failed: " + err.Error(), + } + return + } + switch k := keys.MasterKeys[req.UserID].CrossSigningBody.(type) { + case *gomatrixserverlib.CrossSigningKey: + if err := sanityCheckKey(*k, req.UserID, gomatrixserverlib.CrossSigningKeyPurposeMaster); err != nil { + res.Error = &api.KeyError{ + Err: "Master key sanity check failed: " + err.Error(), + } + return + } + default: + res.Error = &api.KeyError{ + Err: "Unexpected type for master key retrieved from federation", + } + return + } + } + + // If we still don't have a master key at this point then there's nothing else + // we can do - we've checked both the request and the database. + if !hasMasterKey { + res.Error = &api.KeyError{ + Err: "No master key was found, either in the database or in the request!", + IsMissingParam: true, + } + return + } + + // The key ID is basically the key itself. + masterKeyID := gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", masterKey.Encode())) + + // Work out which things we need to verify the signatures for. + toVerify := make(map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, 3) + toStore := types.CrossSigningKeyMap{} + if len(req.MasterKey.Keys) > 0 { + toVerify[gomatrixserverlib.CrossSigningKeyPurposeMaster] = req.MasterKey + } + if len(req.SelfSigningKey.Keys) > 0 { + toVerify[gomatrixserverlib.CrossSigningKeyPurposeSelfSigning] = req.SelfSigningKey + } + if len(req.UserSigningKey.Keys) > 0 { + toVerify[gomatrixserverlib.CrossSigningKeyPurposeUserSigning] = req.UserSigningKey + } + for purpose, key := range toVerify { + // Collect together the key IDs we need to verify with. This will include + // all of the key IDs specified in the signatures. We don't do this for + // the master key because we have no means to verify the signatures - we + // instead just need to store them. + if purpose != gomatrixserverlib.CrossSigningKeyPurposeMaster { + // Marshal the specific key back into JSON so that we can verify the + // signature of it. + keyJSON, err := json.Marshal(key) + if err != nil { + res.Error = &api.KeyError{ + Err: fmt.Sprintf("The JSON of the key section is invalid: %s", err.Error()), + } + return + } + + // Now check if the subkey is signed by the master key. + if err := gomatrixserverlib.VerifyJSON(req.UserID, masterKeyID, ed25519.PublicKey(masterKey), keyJSON); err != nil { + res.Error = &api.KeyError{ + Err: fmt.Sprintf("The %q sub-key failed master key signature verification: %s", purpose, err.Error()), + IsInvalidSignature: true, + } + return + } + } + + // If we've reached this point then all the signatures are valid so + // add the key to the list of keys to store. + for _, keyData := range key.Keys { // iterates once, see sanityCheckKey + toStore[purpose] = keyData + } + } + + if err := a.DB.StoreCrossSigningKeysForUser(ctx, req.UserID, toStore); err != nil { + res.Error = &api.KeyError{ + Err: fmt.Sprintf("a.DB.StoreCrossSigningKeysForUser: %s", err), + } + } +} + +func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req *api.PerformUploadDeviceSignaturesRequest, res *api.PerformUploadDeviceSignaturesResponse) { + selfSignatures := map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} + otherSignatures := map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} + + for userID, forUserID := range req.Signatures { + for keyID, keyOrDevice := range forUserID { + switch key := keyOrDevice.CrossSigningBody.(type) { + case *gomatrixserverlib.CrossSigningKey: + if key.UserID == req.UserID { + if _, ok := selfSignatures[userID]; !ok { + selfSignatures[userID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} + } + selfSignatures[userID][keyID] = keyOrDevice + } else { + if _, ok := otherSignatures[userID]; !ok { + otherSignatures[userID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} + } + otherSignatures[userID][keyID] = keyOrDevice + } + + case *gomatrixserverlib.DeviceKeys: + if key.UserID == req.UserID { + if _, ok := selfSignatures[userID]; !ok { + selfSignatures[userID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} + } + selfSignatures[userID][keyID] = keyOrDevice + } else { + if _, ok := otherSignatures[userID]; !ok { + otherSignatures[userID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} + } + otherSignatures[userID][keyID] = keyOrDevice + } + + default: + continue + } + } + } + + if err := a.processSelfSignatures(ctx, req.UserID, selfSignatures); err != nil { + res.Error = &api.KeyError{ + Err: fmt.Sprintf("a.processSelfSignatures: %s", err), + } + return + } + + if err := a.processOtherSignatures(ctx, req.UserID, otherSignatures); err != nil { + res.Error = &api.KeyError{ + Err: fmt.Sprintf("a.processOtherSignatures: %s", err), + } + return + } +} + +func (a *KeyInternalAPI) processSelfSignatures( + ctx context.Context, _ string, + signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice, +) error { + // Here we will process: + // * The user signing their own devices using their self-signing key + // * The user signing their master key using one of their devices + + for targetUserID, forTargetUserID := range signatures { + for targetKeyID, signature := range forTargetUserID { + switch sig := signature.CrossSigningBody.(type) { + case *gomatrixserverlib.CrossSigningKey: + for originUserID, forOriginUserID := range sig.Signatures { + for originKeyID, originSig := range forOriginUserID { + if err := a.DB.StoreCrossSigningSigsForTarget( + ctx, originUserID, originKeyID, targetUserID, targetKeyID, originSig, + ); err != nil { + return fmt.Errorf("a.DB.StoreCrossSigningKeysForTarget: %w", err) + } + } + } + + case *gomatrixserverlib.DeviceKeys: + for originUserID, forOriginUserID := range sig.Signatures { + for originKeyID, originSig := range forOriginUserID { + if err := a.DB.StoreCrossSigningSigsForTarget( + ctx, originUserID, originKeyID, targetUserID, targetKeyID, originSig, + ); err != nil { + return fmt.Errorf("a.DB.StoreCrossSigningKeysForTarget: %w", err) + } + } + } + + default: + return fmt.Errorf("unexpected type assertion") + } + } + } + + return nil +} + +func (a *KeyInternalAPI) processOtherSignatures( + ctx context.Context, userID string, + signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice, +) error { + // Here we will process: + // * A user signing someone else's master keys using their user-signing keys + + return nil +} + +func (a *KeyInternalAPI) crossSigningKeysFromDatabase( + ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse, +) { + for userID := range req.UserToDevices { + keys, err := a.DB.CrossSigningKeysForUser(ctx, userID) + if err != nil { + logrus.WithError(err).Errorf("Failed to get cross-signing keys for user %q", userID) + continue + } + + for keyType, keyData := range keys { + b64 := keyData.Encode() + keyID := gomatrixserverlib.KeyID("ed25519:" + b64) + key := gomatrixserverlib.CrossSigningKey{ + UserID: userID, + Usage: []gomatrixserverlib.CrossSigningKeyPurpose{ + keyType, + }, + Keys: map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{ + keyID: keyData, + }, + } + + sigs, err := a.DB.CrossSigningSigsForTarget(ctx, userID, keyID) + if err != nil { + logrus.WithError(err).Errorf("Failed to get cross-signing signatures for user %q key %q", userID, keyID) + continue + } + + appendSignature := func(originUserID string, originKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) { + if key.Signatures == nil { + key.Signatures = types.CrossSigningSigMap{} + } + if _, ok := key.Signatures[originUserID]; !ok { + key.Signatures[originUserID] = make(map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes) + } + key.Signatures[originUserID][originKeyID] = signature + } + + for originUserID, forOrigin := range sigs { + for originKeyID, signature := range forOrigin { + switch { + case req.UserID != "" && originUserID == req.UserID: + // Include signatures that we created + appendSignature(originUserID, originKeyID, signature) + case originUserID == userID: + // Include signatures that were created by the person whose key + // we are processing + appendSignature(originUserID, originKeyID, signature) + } + } + } + + switch keyType { + case gomatrixserverlib.CrossSigningKeyPurposeMaster: + res.MasterKeys[userID] = key + + case gomatrixserverlib.CrossSigningKeyPurposeSelfSigning: + res.SelfSigningKeys[userID] = key + + case gomatrixserverlib.CrossSigningKeyPurposeUserSigning: + res.UserSigningKeys[userID] = key + } + } + } +} diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index 47bfb72c..91d4b53d 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -82,6 +82,7 @@ type DeviceListUpdater struct { mu *sync.Mutex // protects UserIDToMutex db DeviceListUpdaterDatabase + api DeviceListUpdaterAPI producer KeyChangeProducer fedClient fedsenderapi.FederationClient workerChans []chan gomatrixserverlib.ServerName @@ -114,6 +115,10 @@ type DeviceListUpdaterDatabase interface { DeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage) error } +type DeviceListUpdaterAPI interface { + PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) +} + // KeyChangeProducer is the interface for producers.KeyChange useful for testing. type KeyChangeProducer interface { ProduceKeyChanges(keys []api.DeviceMessage) error @@ -121,13 +126,14 @@ type KeyChangeProducer interface { // NewDeviceListUpdater creates a new updater which fetches fresh device lists when they go stale. func NewDeviceListUpdater( - db DeviceListUpdaterDatabase, producer KeyChangeProducer, fedClient fedsenderapi.FederationClient, - numWorkers int, + db DeviceListUpdaterDatabase, api DeviceListUpdaterAPI, producer KeyChangeProducer, + fedClient fedsenderapi.FederationClient, numWorkers int, ) *DeviceListUpdater { return &DeviceListUpdater{ userIDToMutex: make(map[string]*sync.Mutex), mu: &sync.Mutex{}, db: db, + api: api, producer: producer, fedClient: fedClient, workerChans: make([]chan gomatrixserverlib.ServerName, numWorkers), @@ -367,6 +373,23 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam } continue } + if res.MasterKey != nil || res.SelfSigningKey != nil { + uploadReq := &api.PerformUploadDeviceKeysRequest{ + UserID: userID, + } + uploadRes := &api.PerformUploadDeviceKeysResponse{} + if res.MasterKey != nil { + if err = sanityCheckKey(*res.MasterKey, userID, gomatrixserverlib.CrossSigningKeyPurposeMaster); err == nil { + uploadReq.MasterKey = *res.MasterKey + } + } + if res.SelfSigningKey != nil { + if err = sanityCheckKey(*res.SelfSigningKey, userID, gomatrixserverlib.CrossSigningKeyPurposeSelfSigning); err == nil { + uploadReq.SelfSigningKey = *res.SelfSigningKey + } + } + u.api.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes) + } err = u.updateDeviceList(&res) if err != nil { logger.WithError(err).WithField("user_id", userID).Error("fetched device list but failed to store/emit it") diff --git a/keyserver/internal/device_list_update_test.go b/keyserver/internal/device_list_update_test.go index eab2a78d..7c170de2 100644 --- a/keyserver/internal/device_list_update_test.go +++ b/keyserver/internal/device_list_update_test.go @@ -95,6 +95,13 @@ func (d *mockDeviceListUpdaterDatabase) DeviceKeysJSON(ctx context.Context, keys return nil } +type mockDeviceListUpdaterAPI struct { +} + +func (d *mockDeviceListUpdaterAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) { + +} + type roundTripper struct { fn func(*http.Request) (*http.Response, error) } @@ -122,8 +129,9 @@ func TestUpdateHavePrevID(t *testing.T) { return true }, } + ap := &mockDeviceListUpdaterAPI{} producer := &mockKeyChangeProducer{} - updater := NewDeviceListUpdater(db, producer, nil, 1) + updater := NewDeviceListUpdater(db, ap, producer, nil, 1) event := gomatrixserverlib.DeviceListUpdateEvent{ DeviceDisplayName: "Foo Bar", Deleted: false, @@ -166,6 +174,7 @@ func TestUpdateNoPrevID(t *testing.T) { return false }, } + ap := &mockDeviceListUpdaterAPI{} producer := &mockKeyChangeProducer{} remoteUserID := "@alice:example.somewhere" var wg sync.WaitGroup @@ -193,7 +202,7 @@ func TestUpdateNoPrevID(t *testing.T) { `)), }, nil }) - updater := NewDeviceListUpdater(db, producer, fedClient, 2) + updater := NewDeviceListUpdater(db, ap, producer, fedClient, 2) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index f53a0761..c5711e73 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -221,9 +221,17 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse) { res.DeviceKeys = make(map[string]map[string]json.RawMessage) + res.MasterKeys = make(map[string]gomatrixserverlib.CrossSigningKey) + res.SelfSigningKeys = make(map[string]gomatrixserverlib.CrossSigningKey) + res.UserSigningKeys = make(map[string]gomatrixserverlib.CrossSigningKey) res.Failures = make(map[string]interface{}) + + // get cross-signing keys from the database + a.crossSigningKeysFromDatabase(ctx, req, res) + // make a map from domain to device keys domainToDeviceKeys := make(map[string]map[string][]string) + domainToCrossSigningKeys := make(map[string]map[string]struct{}) for userID, deviceIDs := range req.UserToDevices { _, serverName, err := gomatrixserverlib.SplitID('@', userID) if err != nil { @@ -274,16 +282,30 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques domainToDeviceKeys[domain] = make(map[string][]string) domainToDeviceKeys[domain][userID] = append(domainToDeviceKeys[domain][userID], deviceIDs...) } + // work out if our cross-signing request for this user was + // satisfied, if not add them to the list of things to fetch + if _, ok := res.MasterKeys[userID]; !ok { + if _, ok := domainToCrossSigningKeys[domain]; !ok { + domainToCrossSigningKeys[domain] = make(map[string]struct{}) + } + domainToCrossSigningKeys[domain][userID] = struct{}{} + } + if _, ok := res.SelfSigningKeys[userID]; !ok { + if _, ok := domainToCrossSigningKeys[domain]; !ok { + domainToCrossSigningKeys[domain] = make(map[string]struct{}) + } + domainToCrossSigningKeys[domain][userID] = struct{}{} + } } // attempt to satisfy key queries from the local database first as we should get device updates pushed to us domainToDeviceKeys = a.remoteKeysFromDatabase(ctx, res, domainToDeviceKeys) - if len(domainToDeviceKeys) == 0 { + if len(domainToDeviceKeys) == 0 && len(domainToCrossSigningKeys) == 0 { return // nothing to query } // perform key queries for remote devices - a.queryRemoteKeys(ctx, req.Timeout, res, domainToDeviceKeys) + a.queryRemoteKeys(ctx, req.Timeout, res, domainToDeviceKeys, domainToCrossSigningKeys) } func (a *KeyInternalAPI) remoteKeysFromDatabase( @@ -313,18 +335,30 @@ func (a *KeyInternalAPI) remoteKeysFromDatabase( } func (a *KeyInternalAPI) queryRemoteKeys( - ctx context.Context, timeout time.Duration, res *api.QueryKeysResponse, domainToDeviceKeys map[string]map[string][]string, + ctx context.Context, timeout time.Duration, res *api.QueryKeysResponse, + domainToDeviceKeys map[string]map[string][]string, domainToCrossSigningKeys map[string]map[string]struct{}, ) { resultCh := make(chan *gomatrixserverlib.RespQueryKeys, len(domainToDeviceKeys)) // allows us to wait until all federation servers have been poked var wg sync.WaitGroup - wg.Add(len(domainToDeviceKeys)) // mutex for writing directly to res (e.g failures) var respMu sync.Mutex + domains := map[string]struct{}{} + for domain := range domainToDeviceKeys { + domains[domain] = struct{}{} + } + for domain := range domainToCrossSigningKeys { + domains[domain] = struct{}{} + } + wg.Add(len(domains)) + // fan out - for domain, deviceKeys := range domainToDeviceKeys { - go a.queryRemoteKeysOnServer(ctx, domain, deviceKeys, &wg, &respMu, timeout, resultCh, res) + for domain := range domains { + go a.queryRemoteKeysOnServer( + ctx, domain, domainToDeviceKeys[domain], domainToCrossSigningKeys[domain], + &wg, &respMu, timeout, resultCh, res, + ) } // Close the result channel when the goroutines have quit so the for .. range exits @@ -344,12 +378,29 @@ func (a *KeyInternalAPI) queryRemoteKeys( res.DeviceKeys[userID][deviceID] = keyJSON } } + + for userID, body := range result.MasterKeys { + switch b := body.CrossSigningBody.(type) { + case *gomatrixserverlib.CrossSigningKey: + res.MasterKeys[userID] = *b + } + } + + for userID, body := range result.SelfSigningKeys { + switch b := body.CrossSigningBody.(type) { + case *gomatrixserverlib.CrossSigningKey: + res.SelfSigningKeys[userID] = *b + } + } + + // TODO: do we want to persist these somewhere now + // that we have fetched them? } } func (a *KeyInternalAPI) queryRemoteKeysOnServer( - ctx context.Context, serverName string, devKeys map[string][]string, wg *sync.WaitGroup, - respMu *sync.Mutex, timeout time.Duration, resultCh chan<- *gomatrixserverlib.RespQueryKeys, + ctx context.Context, serverName string, devKeys map[string][]string, crossSigningKeys map[string]struct{}, + wg *sync.WaitGroup, respMu *sync.Mutex, timeout time.Duration, resultCh chan<- *gomatrixserverlib.RespQueryKeys, res *api.QueryKeysResponse, ) { defer wg.Done() @@ -358,14 +409,24 @@ func (a *KeyInternalAPI) queryRemoteKeysOnServer( // for users who we do not have any knowledge about, try to start doing device list updates for them // by hitting /users/devices - otherwise fallback to /keys/query which has nicer bulk properties but // lack a stream ID. - var userIDsForAllDevices []string + userIDsForAllDevices := map[string]struct{}{} for userID, deviceIDs := range devKeys { if len(deviceIDs) == 0 { - userIDsForAllDevices = append(userIDsForAllDevices, userID) + userIDsForAllDevices[userID] = struct{}{} delete(devKeys, userID) } } - for _, userID := range userIDsForAllDevices { + // for cross-signing keys, it's probably easier just to hit /keys/query if we aren't already doing + // a device list update, so we'll populate those back into the /keys/query list if not + for userID := range crossSigningKeys { + if devKeys == nil { + devKeys = map[string][]string{} + } + if _, ok := userIDsForAllDevices[userID]; !ok { + devKeys[userID] = []string{} + } + } + for userID := range userIDsForAllDevices { err := a.Updater.ManualUpdate(context.Background(), gomatrixserverlib.ServerName(serverName), userID) if err != nil { logrus.WithFields(logrus.Fields{ diff --git a/keyserver/inthttp/client.go b/keyserver/inthttp/client.go index 98200022..b685ae1a 100644 --- a/keyserver/inthttp/client.go +++ b/keyserver/inthttp/client.go @@ -27,13 +27,15 @@ import ( // HTTP paths for the internal HTTP APIs const ( - InputDeviceListUpdatePath = "/keyserver/inputDeviceListUpdate" - PerformUploadKeysPath = "/keyserver/performUploadKeys" - PerformClaimKeysPath = "/keyserver/performClaimKeys" - QueryKeysPath = "/keyserver/queryKeys" - QueryKeyChangesPath = "/keyserver/queryKeyChanges" - QueryOneTimeKeysPath = "/keyserver/queryOneTimeKeys" - QueryDeviceMessagesPath = "/keyserver/queryDeviceMessages" + InputDeviceListUpdatePath = "/keyserver/inputDeviceListUpdate" + PerformUploadKeysPath = "/keyserver/performUploadKeys" + PerformClaimKeysPath = "/keyserver/performClaimKeys" + PerformUploadDeviceKeysPath = "/keyserver/performUploadDeviceKeys" + PerformUploadDeviceSignaturesPath = "/keyserver/performUploadDeviceSignatures" + QueryKeysPath = "/keyserver/queryKeys" + QueryKeyChangesPath = "/keyserver/queryKeyChanges" + QueryOneTimeKeysPath = "/keyserver/queryOneTimeKeys" + QueryDeviceMessagesPath = "/keyserver/queryDeviceMessages" ) // NewKeyServerClient creates a KeyInternalAPI implemented by talking to a HTTP POST API. @@ -175,3 +177,37 @@ func (h *httpKeyInternalAPI) QueryKeyChanges( } } } + +func (h *httpKeyInternalAPI) PerformUploadDeviceKeys( + ctx context.Context, + request *api.PerformUploadDeviceKeysRequest, + response *api.PerformUploadDeviceKeysResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformUploadDeviceKeys") + defer span.Finish() + + apiURL := h.apiURL + PerformUploadDeviceKeysPath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.Error = &api.KeyError{ + Err: err.Error(), + } + } +} + +func (h *httpKeyInternalAPI) PerformUploadDeviceSignatures( + ctx context.Context, + request *api.PerformUploadDeviceSignaturesRequest, + response *api.PerformUploadDeviceSignaturesResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformUploadDeviceSignatures") + defer span.Finish() + + apiURL := h.apiURL + PerformUploadDeviceSignaturesPath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.Error = &api.KeyError{ + Err: err.Error(), + } + } +} diff --git a/keyserver/inthttp/server.go b/keyserver/inthttp/server.go index 7dfaed2e..ab096c15 100644 --- a/keyserver/inthttp/server.go +++ b/keyserver/inthttp/server.go @@ -58,6 +58,28 @@ func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(PerformUploadDeviceKeysPath, + httputil.MakeInternalAPI("performUploadDeviceKeys", func(req *http.Request) util.JSONResponse { + request := api.PerformUploadDeviceKeysRequest{} + response := api.PerformUploadDeviceKeysResponse{} + if err := json.NewDecoder(req.Body).Decode(&request.CrossSigningKeys); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + s.PerformUploadDeviceKeys(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(PerformUploadDeviceSignaturesPath, + httputil.MakeInternalAPI("performUploadDeviceSignatures", func(req *http.Request) util.JSONResponse { + request := api.PerformUploadDeviceSignaturesRequest{} + response := api.PerformUploadDeviceSignaturesResponse{} + if err := json.NewDecoder(req.Body).Decode(&request.Signatures); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + s.PerformUploadDeviceSignatures(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle(QueryKeysPath, httputil.MakeInternalAPI("queryKeys", func(req *http.Request) util.JSONResponse { request := api.QueryKeysRequest{} diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 7e8fc2e0..fcfe24de 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -18,10 +18,12 @@ import ( "github.com/gorilla/mux" fedsenderapi "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/keyserver/consumers" "github.com/matrix-org/dendrite/keyserver/internal" "github.com/matrix-org/dendrite/keyserver/inthttp" "github.com/matrix-org/dendrite/keyserver/producers" "github.com/matrix-org/dendrite/keyserver/storage" + "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/kafka" "github.com/sirupsen/logrus" @@ -36,9 +38,9 @@ func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) { // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - cfg *config.KeyServer, fedClient fedsenderapi.FederationClient, + base *setup.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.FederationClient, ) api.KeyInternalAPI { - _, producer := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka) + consumer, producer := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka) db, err := storage.NewDatabase(&cfg.Database) if err != nil { @@ -49,17 +51,26 @@ func NewInternalAPI( Producer: producer, DB: db, } - updater := internal.NewDeviceListUpdater(db, keyChangeProducer, fedClient, 8) // 8 workers TODO: configurable + ap := &internal.KeyInternalAPI{ + DB: db, + ThisServer: cfg.Matrix.ServerName, + FedClient: fedClient, + Producer: keyChangeProducer, + } + updater := internal.NewDeviceListUpdater(db, ap, keyChangeProducer, fedClient, 8) // 8 workers TODO: configurable + ap.Updater = updater go func() { if err := updater.Start(); err != nil { logrus.WithError(err).Panicf("failed to start device list updater") } }() - return &internal.KeyInternalAPI{ - DB: db, - ThisServer: cfg.Matrix.ServerName, - FedClient: fedClient, - Producer: keyChangeProducer, - Updater: updater, + + keyconsumer := consumers.NewOutputSigningKeyUpdateConsumer( + base.ProcessContext, base.Cfg, consumer, db, ap, + ) + if err := keyconsumer.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start keyserver EDU server consumer") } + + return ap } diff --git a/setup/config/config_kafka.go b/setup/config/config_kafka.go index 36191428..15b3ad71 100644 --- a/setup/config/config_kafka.go +++ b/setup/config/config_kafka.go @@ -10,6 +10,7 @@ const ( TopicOutputRoomEvent = "OutputRoomEvent" TopicOutputClientData = "OutputClientData" TopicOutputReceiptEvent = "OutputReceiptEvent" + TopicOutputSigningKeyUpdate = "OutputSigningKeyUpdate" ) type Kafka struct { diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index 44c4a4dd..38be34f6 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -33,6 +33,10 @@ func (k *mockKeyAPI) SetUserAPI(i userapi.UserInternalAPI) {} // PerformClaimKeys claims one-time keys for use in pre-key messages func (k *mockKeyAPI) PerformClaimKeys(ctx context.Context, req *keyapi.PerformClaimKeysRequest, res *keyapi.PerformClaimKeysResponse) { } +func (k *mockKeyAPI) PerformUploadDeviceKeys(ctx context.Context, req *keyapi.PerformUploadDeviceKeysRequest, res *keyapi.PerformUploadDeviceKeysResponse) { +} +func (k *mockKeyAPI) PerformUploadDeviceSignatures(ctx context.Context, req *keyapi.PerformUploadDeviceSignaturesRequest, res *keyapi.PerformUploadDeviceSignaturesResponse) { +} func (k *mockKeyAPI) QueryKeys(ctx context.Context, req *keyapi.QueryKeysRequest, res *keyapi.QueryKeysResponse) { } func (k *mockKeyAPI) QueryKeyChanges(ctx context.Context, req *keyapi.QueryKeyChangesRequest, res *keyapi.QueryKeyChangesResponse) { diff --git a/sytest-whitelist b/sytest-whitelist index 6bb3d770..27109e60 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -550,3 +550,6 @@ Will not back up to an old backup version Can create more than 10 backup versions Can delete backup Deleted & recreated backups are empty +Can upload self-signing keys +Fails to upload self-signing keys with no auth +Fails to upload self-signing key without master key From 8e5a0139b5eb3c02c8c2e65b2a6444616e813978 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 5 Aug 2021 11:48:22 +0100 Subject: [PATCH 20/64] Update to matrix-org/gomatrixserverlib@03e40fa --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 154bf8ca..39f6020c 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20210802144451-bec8d2252d83 + github.com/matrix-org/gomatrixserverlib v0.0.0-20210805104751-03e40fa2c0e1 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index 5c614c45..df5e2b85 100644 --- a/go.sum +++ b/go.sum @@ -994,8 +994,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210802144451-bec8d2252d83 h1:fkUmeKj/U5TnWXTsJnVjEL0FQiVhf1r9WL4VWI00q2k= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210802144451-bec8d2252d83/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210805104751-03e40fa2c0e1 h1:ExS9AmOWYSZ4DR7W7Eqm7glP/g/u/CTVLAu9blAnve4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210805104751-03e40fa2c0e1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b h1:5X5vdWQ13xrNkJVqaJHPsrt7rKkMJH5iac0EtfOuxSg= From e95b1fd2383d29dbce8831f5588e7e3df064af31 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 6 Aug 2021 10:13:35 +0100 Subject: [PATCH 21/64] Cross-signing validation for self-sigs, expose signatures over `/user/keys/query` and `/user/devices/{userId}` (#1962) * Enable unstable feature again * Try to verify when a device signs a key * Try to verify when a key signs a device * It's the self-signing key, not the master key * Fix error * Try to verify master key uploads * Actually we can't guarantee we can do that so nevermind * Add signatures into /devices/list request * Fix nil pointer * Reprioritise map creation * Don't skip devices that don't have signatures * Add some debug logging * Fix logic error in QuerySignatures * Fix bugs * Expose master and self-signing keys on /devices/list hopefully * maps are tedious * Expose signatures via /keys/query * Upload signatures when uploading keys * Fixes * Disable the feature again --- federationapi/routing/devices.go | 30 ++++ keyserver/api/api.go | 20 +++ keyserver/internal/cross_signing.go | 219 ++++++++++++++++++++++++---- keyserver/internal/internal.go | 34 ++++- keyserver/inthttp/client.go | 18 +++ keyserver/inthttp/server.go | 11 ++ keyserver/storage/interface.go | 3 +- keyserver/storage/shared/storage.go | 41 +++++- syncapi/internal/keychange_test.go | 2 + 9 files changed, 341 insertions(+), 37 deletions(-) diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index 07862451..b397c546 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -37,12 +37,28 @@ func GetUserDevices( return jsonerror.InternalServerError() } + sigReq := &keyapi.QuerySignaturesRequest{ + TargetIDs: map[string][]gomatrixserverlib.KeyID{}, + } + sigRes := &keyapi.QuerySignaturesResponse{} + for _, dev := range res.Devices { + sigReq.TargetIDs[userID] = append(sigReq.TargetIDs[userID], gomatrixserverlib.KeyID(dev.DeviceID)) + } + keyAPI.QuerySignatures(req.Context(), sigReq, sigRes) + response := gomatrixserverlib.RespUserDevices{ UserID: userID, StreamID: res.StreamID, Devices: []gomatrixserverlib.RespUserDevice{}, } + if masterKey, ok := sigRes.MasterKeys[userID]; ok { + response.MasterKey = &masterKey + } + if selfSigningKey, ok := sigRes.SelfSigningKeys[userID]; ok { + response.SelfSigningKey = &selfSigningKey + } + for _, dev := range res.Devices { var key gomatrixserverlib.RespUserDeviceKeys err := json.Unmarshal(dev.DeviceKeys.KeyJSON, &key) @@ -56,6 +72,20 @@ func GetUserDevices( DisplayName: dev.DisplayName, Keys: key, } + + if targetUser, ok := sigRes.Signatures[userID]; ok { + if targetKey, ok := targetUser[gomatrixserverlib.KeyID(dev.DeviceID)]; ok { + for sourceUserID, forSourceUser := range targetKey { + for sourceKeyID, sourceKey := range forSourceUser { + if _, ok := device.Keys.Signatures[sourceUserID]; !ok { + device.Keys.Signatures[sourceUserID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} + } + device.Keys.Signatures[sourceUserID][sourceKeyID] = sourceKey + } + } + } + } + response.Devices = append(response.Devices, device) } diff --git a/keyserver/api/api.go b/keyserver/api/api.go index f46a9ee2..520562e7 100644 --- a/keyserver/api/api.go +++ b/keyserver/api/api.go @@ -20,6 +20,7 @@ import ( "strings" "time" + "github.com/matrix-org/dendrite/keyserver/types" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -38,6 +39,7 @@ type KeyInternalAPI interface { QueryKeyChanges(ctx context.Context, req *QueryKeyChangesRequest, res *QueryKeyChangesResponse) QueryOneTimeKeys(ctx context.Context, req *QueryOneTimeKeysRequest, res *QueryOneTimeKeysResponse) QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse) + QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse) } // KeyError is returned if there was a problem performing/querying the server @@ -242,6 +244,24 @@ type QueryDeviceMessagesResponse struct { Error *KeyError } +type QuerySignaturesRequest struct { + // A map of target user ID -> target key/device IDs to retrieve signatures for + TargetIDs map[string][]gomatrixserverlib.KeyID `json:"target_ids"` +} + +type QuerySignaturesResponse struct { + // A map of target user ID -> target key/device ID -> origin user ID -> origin key/device ID -> signatures + Signatures map[string]map[gomatrixserverlib.KeyID]types.CrossSigningSigMap + // A map of target user ID -> cross-signing master key + MasterKeys map[string]gomatrixserverlib.CrossSigningKey + // A map of target user ID -> cross-signing self-signing key + SelfSigningKeys map[string]gomatrixserverlib.CrossSigningKey + // A map of target user ID -> cross-signing user-signing key + UserSigningKeys map[string]gomatrixserverlib.CrossSigningKey + // The request error, if any + Error *KeyError +} + type InputDeviceListUpdateRequest struct { Event gomatrixserverlib.DeviceListUpdateEvent } diff --git a/keyserver/internal/cross_signing.go b/keyserver/internal/cross_signing.go index 802d02b3..141a4eb2 100644 --- a/keyserver/internal/cross_signing.go +++ b/keyserver/internal/cross_signing.go @@ -17,6 +17,7 @@ package internal import ( "context" "crypto/ed25519" + "database/sql" "encoding/json" "fmt" "strings" @@ -104,7 +105,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P // If the user hasn't given a new master key, then let's go and get their // existing keys from the database. if !hasMasterKey { - existingKeys, err := a.DB.CrossSigningKeysForUser(ctx, req.UserID) + existingKeys, err := a.DB.CrossSigningKeysDataForUser(ctx, req.UserID) if err != nil { res.Error = &api.KeyError{ Err: "Retrieving cross-signing keys from database failed: " + err.Error(), @@ -177,21 +178,24 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P } for purpose, key := range toVerify { // Collect together the key IDs we need to verify with. This will include - // all of the key IDs specified in the signatures. We don't do this for - // the master key because we have no means to verify the signatures - we - // instead just need to store them. - if purpose != gomatrixserverlib.CrossSigningKeyPurposeMaster { - // Marshal the specific key back into JSON so that we can verify the - // signature of it. - keyJSON, err := json.Marshal(key) - if err != nil { - res.Error = &api.KeyError{ - Err: fmt.Sprintf("The JSON of the key section is invalid: %s", err.Error()), - } - return + // all of the key IDs specified in the signatures. + keyJSON, err := json.Marshal(key) + if err != nil { + res.Error = &api.KeyError{ + Err: fmt.Sprintf("The JSON of the key section is invalid: %s", err.Error()), } + return + } - // Now check if the subkey is signed by the master key. + switch purpose { + case gomatrixserverlib.CrossSigningKeyPurposeMaster: + // The master key might have a signature attached to it from the + // previous key, or from a device key, but there's no real need + // to verify it. Clients will perform key checks when the master + // key changes. + + default: + // Sub-keys should be signed by the master key. if err := gomatrixserverlib.VerifyJSON(req.UserID, masterKeyID, ed25519.PublicKey(masterKey), keyJSON); err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("The %q sub-key failed master key signature verification: %s", purpose, err.Error()), @@ -212,10 +216,44 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P res.Error = &api.KeyError{ Err: fmt.Sprintf("a.DB.StoreCrossSigningKeysForUser: %s", err), } + return + } + + // Now upload any signatures that were included with the keys. + for _, key := range toVerify { + var targetKeyID gomatrixserverlib.KeyID + for targetKey := range key.Keys { // iterates once, see sanityCheckKey + targetKeyID = targetKey + } + for sigUserID, forSigUserID := range key.Signatures { + if sigUserID != req.UserID { + continue + } + for sigKeyID, sigBytes := range forSigUserID { + if err := a.DB.StoreCrossSigningSigsForTarget(ctx, sigUserID, sigKeyID, req.UserID, targetKeyID, sigBytes); err != nil { + res.Error = &api.KeyError{ + Err: fmt.Sprintf("a.DB.StoreCrossSigningSigsForTarget: %s", err), + } + return + } + } + } } } func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req *api.PerformUploadDeviceSignaturesRequest, res *api.PerformUploadDeviceSignaturesResponse) { + // Before we do anything, we need the master and self-signing keys for this user. + // Then we can verify the signatures make sense. + queryReq := &api.QueryKeysRequest{ + UserID: req.UserID, + UserToDevices: map[string][]string{}, + } + queryRes := &api.QueryKeysResponse{} + for userID := range req.Signatures { + queryReq.UserToDevices[userID] = []string{} + } + a.QueryKeys(ctx, queryReq, queryRes) + selfSignatures := map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} otherSignatures := map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} @@ -254,14 +292,14 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req } } - if err := a.processSelfSignatures(ctx, req.UserID, selfSignatures); err != nil { + if err := a.processSelfSignatures(ctx, req.UserID, queryRes, selfSignatures); err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.processSelfSignatures: %s", err), } return } - if err := a.processOtherSignatures(ctx, req.UserID, otherSignatures); err != nil { + if err := a.processOtherSignatures(ctx, req.UserID, queryRes, otherSignatures); err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.processOtherSignatures: %s", err), } @@ -270,7 +308,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req } func (a *KeyInternalAPI) processSelfSignatures( - ctx context.Context, _ string, + ctx context.Context, _ string, queryRes *api.QueryKeysResponse, signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice, ) error { // Here we will process: @@ -281,8 +319,39 @@ func (a *KeyInternalAPI) processSelfSignatures( for targetKeyID, signature := range forTargetUserID { switch sig := signature.CrossSigningBody.(type) { case *gomatrixserverlib.CrossSigningKey: + // The user is signing their master key with one of their devices + // The QueryKeys response should contain the device key hopefully. + // First we need to marshal the blob back into JSON so we can verify + // it. + j, err := json.Marshal(sig) + if err != nil { + return fmt.Errorf("json.Marshal: %w", err) + } + for originUserID, forOriginUserID := range sig.Signatures { + originDeviceKeys, ok := queryRes.DeviceKeys[originUserID] + if !ok { + return fmt.Errorf("missing device keys for user %q", originUserID) + } + for originKeyID, originSig := range forOriginUserID { + originDeviceKeyID := gomatrixserverlib.KeyID("ed25519:" + originKeyID) + + var originKey gomatrixserverlib.DeviceKeys + if err := json.Unmarshal(originDeviceKeys[string(originKeyID)], &originKey); err != nil { + return fmt.Errorf("json.Unmarshal: %w", err) + } + + originSigningKey, ok := originKey.Keys[originDeviceKeyID] + if !ok { + return fmt.Errorf("missing origin signing key %q", originDeviceKeyID) + } + originSigningKeyPublic := ed25519.PublicKey(originSigningKey) + + if err := gomatrixserverlib.VerifyJSON(originUserID, originDeviceKeyID, originSigningKeyPublic, j); err != nil { + return fmt.Errorf("gomatrixserverlib.VerifyJSON: %w", err) + } + if err := a.DB.StoreCrossSigningSigsForTarget( ctx, originUserID, originKeyID, targetUserID, targetKeyID, originSig, ); err != nil { @@ -292,8 +361,35 @@ func (a *KeyInternalAPI) processSelfSignatures( } case *gomatrixserverlib.DeviceKeys: + // The user is signing one of their devices with their self-signing key + // The QueryKeys response should contain the master key hopefully. + // First we need to marshal the blob back into JSON so we can verify + // it. + j, err := json.Marshal(sig) + if err != nil { + return fmt.Errorf("json.Marshal: %w", err) + } + for originUserID, forOriginUserID := range sig.Signatures { for originKeyID, originSig := range forOriginUserID { + originSelfSigningKeys, ok := queryRes.SelfSigningKeys[originUserID] + if !ok { + return fmt.Errorf("missing self-signing key for user %q", originUserID) + } + + var originSelfSigningKeyID gomatrixserverlib.KeyID + var originSelfSigningKey gomatrixserverlib.Base64Bytes + for keyID, key := range originSelfSigningKeys.Keys { + originSelfSigningKeyID, originSelfSigningKey = keyID, key + break + } + + originSelfSigningKeyPublic := ed25519.PublicKey(originSelfSigningKey) + + if err := gomatrixserverlib.VerifyJSON(originUserID, originSelfSigningKeyID, originSelfSigningKeyPublic, j); err != nil { + return fmt.Errorf("gomatrixserverlib.VerifyJSON: %w", err) + } + if err := a.DB.StoreCrossSigningSigsForTarget( ctx, originUserID, originKeyID, targetUserID, targetKeyID, originSig, ); err != nil { @@ -312,7 +408,7 @@ func (a *KeyInternalAPI) processSelfSignatures( } func (a *KeyInternalAPI) processOtherSignatures( - ctx context.Context, userID string, + ctx context.Context, userID string, queryRes *api.QueryKeysResponse, signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice, ) error { // Here we will process: @@ -331,20 +427,14 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase( continue } - for keyType, keyData := range keys { - b64 := keyData.Encode() - keyID := gomatrixserverlib.KeyID("ed25519:" + b64) - key := gomatrixserverlib.CrossSigningKey{ - UserID: userID, - Usage: []gomatrixserverlib.CrossSigningKeyPurpose{ - keyType, - }, - Keys: map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{ - keyID: keyData, - }, + for keyType, key := range keys { + var keyID gomatrixserverlib.KeyID + for id := range key.Keys { + keyID = id + break } - sigs, err := a.DB.CrossSigningSigsForTarget(ctx, userID, keyID) + sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, userID, keyID) if err != nil { logrus.WithError(err).Errorf("Failed to get cross-signing signatures for user %q key %q", userID, keyID) continue @@ -360,7 +450,7 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase( key.Signatures[originUserID][originKeyID] = signature } - for originUserID, forOrigin := range sigs { + for originUserID, forOrigin := range sigMap { for originKeyID, signature := range forOrigin { switch { case req.UserID != "" && originUserID == req.UserID: @@ -387,3 +477,70 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase( } } } + +func (a *KeyInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySignaturesRequest, res *api.QuerySignaturesResponse) { + for targetUserID, forTargetUser := range req.TargetIDs { + for _, targetKeyID := range forTargetUser { + keyMap, err := a.DB.CrossSigningKeysForUser(ctx, targetUserID) + if err != nil { + if err == sql.ErrNoRows { + continue + } + res.Error = &api.KeyError{ + Err: fmt.Sprintf("a.DB.CrossSigningKeysForUser: %s", err), + } + } + + for targetPurpose, targetKey := range keyMap { + switch targetPurpose { + case gomatrixserverlib.CrossSigningKeyPurposeMaster: + if res.MasterKeys == nil { + res.MasterKeys = map[string]gomatrixserverlib.CrossSigningKey{} + } + res.MasterKeys[targetUserID] = targetKey + + case gomatrixserverlib.CrossSigningKeyPurposeSelfSigning: + if res.SelfSigningKeys == nil { + res.SelfSigningKeys = map[string]gomatrixserverlib.CrossSigningKey{} + } + res.SelfSigningKeys[targetUserID] = targetKey + + case gomatrixserverlib.CrossSigningKeyPurposeUserSigning: + if res.UserSigningKeys == nil { + res.UserSigningKeys = map[string]gomatrixserverlib.CrossSigningKey{} + } + res.UserSigningKeys[targetUserID] = targetKey + } + } + + sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, targetUserID, targetKeyID) + if err != nil { + if err == sql.ErrNoRows { + continue + } + res.Error = &api.KeyError{ + Err: fmt.Sprintf("a.DB.CrossSigningSigsForTarget: %s", err), + } + return + } + + for sourceUserID, forSourceUser := range sigMap { + for sourceKeyID, sourceSig := range forSourceUser { + if res.Signatures == nil { + res.Signatures = map[string]map[gomatrixserverlib.KeyID]types.CrossSigningSigMap{} + } + if _, ok := res.Signatures[targetUserID]; !ok { + res.Signatures[targetUserID] = map[gomatrixserverlib.KeyID]types.CrossSigningSigMap{} + } + if _, ok := res.Signatures[targetUserID][targetKeyID]; !ok { + res.Signatures[targetUserID][targetKeyID] = types.CrossSigningSigMap{} + } + if _, ok := res.Signatures[targetUserID][targetKeyID][sourceUserID]; !ok { + res.Signatures[targetUserID][targetKeyID][sourceUserID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} + } + res.Signatures[targetUserID][targetKeyID][sourceUserID][sourceKeyID] = sourceSig + } + } + } + } +} diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index c5711e73..28638c29 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -300,12 +300,38 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques // attempt to satisfy key queries from the local database first as we should get device updates pushed to us domainToDeviceKeys = a.remoteKeysFromDatabase(ctx, res, domainToDeviceKeys) - if len(domainToDeviceKeys) == 0 && len(domainToCrossSigningKeys) == 0 { - return // nothing to query + if len(domainToDeviceKeys) > 0 || len(domainToCrossSigningKeys) > 0 { + // perform key queries for remote devices + a.queryRemoteKeys(ctx, req.Timeout, res, domainToDeviceKeys, domainToCrossSigningKeys) } - // perform key queries for remote devices - a.queryRemoteKeys(ctx, req.Timeout, res, domainToDeviceKeys, domainToCrossSigningKeys) + // Finally, append signatures that we know about + // TODO: This is horrible because we need to round-trip the signature from + // JSON, add the signatures and marshal it again, for some reason? + for userID, forUserID := range res.DeviceKeys { + for keyID, key := range forUserID { + sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, userID, gomatrixserverlib.KeyID(keyID)) + if err != nil { + logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed") + continue + } + if len(sigMap) == 0 { + continue + } + var deviceKey gomatrixserverlib.DeviceKeys + if err = json.Unmarshal(key, &deviceKey); err != nil { + continue + } + for sourceUserID, forSourceUser := range sigMap { + for sourceKeyID, sourceSig := range forSourceUser { + deviceKey.Signatures[sourceUserID][sourceKeyID] = sourceSig + } + } + if js, err := json.Marshal(deviceKey); err == nil { + res.DeviceKeys[userID][keyID] = js + } + } + } } func (a *KeyInternalAPI) remoteKeysFromDatabase( diff --git a/keyserver/inthttp/client.go b/keyserver/inthttp/client.go index b685ae1a..15870571 100644 --- a/keyserver/inthttp/client.go +++ b/keyserver/inthttp/client.go @@ -36,6 +36,7 @@ const ( QueryKeyChangesPath = "/keyserver/queryKeyChanges" QueryOneTimeKeysPath = "/keyserver/queryOneTimeKeys" QueryDeviceMessagesPath = "/keyserver/queryDeviceMessages" + QuerySignaturesPath = "/keyserver/querySignatures" ) // NewKeyServerClient creates a KeyInternalAPI implemented by talking to a HTTP POST API. @@ -211,3 +212,20 @@ func (h *httpKeyInternalAPI) PerformUploadDeviceSignatures( } } } + +func (h *httpKeyInternalAPI) QuerySignatures( + ctx context.Context, + request *api.QuerySignaturesRequest, + response *api.QuerySignaturesResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "QuerySignatures") + defer span.Finish() + + apiURL := h.apiURL + QuerySignaturesPath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.Error = &api.KeyError{ + Err: err.Error(), + } + } +} diff --git a/keyserver/inthttp/server.go b/keyserver/inthttp/server.go index ab096c15..ac70e3e5 100644 --- a/keyserver/inthttp/server.go +++ b/keyserver/inthttp/server.go @@ -124,4 +124,15 @@ func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(QuerySignaturesPath, + httputil.MakeInternalAPI("querySignatures", func(req *http.Request) util.JSONResponse { + request := api.QuerySignaturesRequest{} + response := api.QuerySignaturesResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + s.QuerySignatures(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } diff --git a/keyserver/storage/interface.go b/keyserver/storage/interface.go index 756dc32a..b9db81ad 100644 --- a/keyserver/storage/interface.go +++ b/keyserver/storage/interface.go @@ -78,7 +78,8 @@ type Database interface { // MarkDeviceListStale sets the stale bit for this user to isStale. MarkDeviceListStale(ctx context.Context, userID string, isStale bool) error - CrossSigningKeysForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error) + CrossSigningKeysForUser(ctx context.Context, userID string) (map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, error) + CrossSigningKeysDataForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error) CrossSigningSigsForTarget(ctx context.Context, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) StoreCrossSigningKeysForUser(ctx context.Context, userID string, keyMap types.CrossSigningKeyMap) error diff --git a/keyserver/storage/shared/storage.go b/keyserver/storage/shared/storage.go index 76724295..64ce53ef 100644 --- a/keyserver/storage/shared/storage.go +++ b/keyserver/storage/shared/storage.go @@ -159,7 +159,46 @@ func (d *Database) MarkDeviceListStale(ctx context.Context, userID string, isSta } // CrossSigningKeysForUser returns the latest known cross-signing keys for a user, if any. -func (d *Database) CrossSigningKeysForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error) { +func (d *Database) CrossSigningKeysForUser(ctx context.Context, userID string) (map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, error) { + keyMap, err := d.CrossSigningKeysTable.SelectCrossSigningKeysForUser(ctx, nil, userID) + if err != nil { + return nil, fmt.Errorf("d.CrossSigningKeysTable.SelectCrossSigningKeysForUser: %w", err) + } + results := map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey{} + for purpose, key := range keyMap { + keyID := gomatrixserverlib.KeyID("ed25519:" + key.Encode()) + result := gomatrixserverlib.CrossSigningKey{ + UserID: userID, + Usage: []gomatrixserverlib.CrossSigningKeyPurpose{purpose}, + Keys: map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{ + keyID: key, + }, + } + sigMap, err := d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, userID, keyID) + if err != nil { + continue + } + for sigUserID, forSigUserID := range sigMap { + if userID != sigUserID { + continue + } + if result.Signatures == nil { + result.Signatures = map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} + } + if _, ok := result.Signatures[sigUserID]; !ok { + result.Signatures[sigUserID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} + } + for sigKeyID, sigBytes := range forSigUserID { + result.Signatures[sigUserID][sigKeyID] = sigBytes + } + } + results[purpose] = result + } + return results, nil +} + +// CrossSigningKeysForUser returns the latest known cross-signing keys for a user, if any. +func (d *Database) CrossSigningKeysDataForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error) { return d.CrossSigningKeysTable.SelectCrossSigningKeysForUser(ctx, nil, userID) } diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index 38be34f6..0c567a96 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -49,6 +49,8 @@ func (k *mockKeyAPI) QueryDeviceMessages(ctx context.Context, req *keyapi.QueryD } func (k *mockKeyAPI) InputDeviceListUpdate(ctx context.Context, req *keyapi.InputDeviceListUpdateRequest, res *keyapi.InputDeviceListUpdateResponse) { +} +func (k *mockKeyAPI) QuerySignatures(ctx context.Context, req *keyapi.QuerySignaturesRequest, res *keyapi.QuerySignaturesResponse) { } type mockRoomserverAPI struct { From b1377d991a1d4fd71cf275d96da343ca558993b7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 9 Aug 2021 14:35:24 +0100 Subject: [PATCH 22/64] Cross-signing signature handling (#1965) * Handle other signatures * Decorate key ID properly * Match by key IDs * Tweaks * Fixes * Fix /user/keys/query bug, review comments, update sytest-whitelist * Various wtweaks * Fix wiring for keyserver in API mode * Additional fixes --- cmd/dendrite-monolith-server/main.go | 5 + federationapi/routing/devices.go | 7 +- federationapi/routing/keys.go | 10 +- go.mod | 2 +- go.sum | 4 +- keyserver/api/api.go | 4 +- keyserver/internal/cross_signing.go | 186 +++++++++++++++------------ keyserver/internal/internal.go | 24 ++-- keyserver/inthttp/server.go | 4 +- sytest-whitelist | 1 + 10 files changed, 145 insertions(+), 102 deletions(-) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index bb313e57..4d8e2ee1 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -120,6 +120,11 @@ func main() { Impl: userAPI, } } + // needs to be after the SetUserAPI call above + if base.UseHTTPAPIs { + keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI) + keyAPI = base.KeyServerHTTPClient() + } eduInputAPI := eduserver.NewInternalAPI( base, cache.New(), userAPI, diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index b397c546..4cd19996 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -38,12 +38,11 @@ func GetUserDevices( } sigReq := &keyapi.QuerySignaturesRequest{ - TargetIDs: map[string][]gomatrixserverlib.KeyID{}, + TargetIDs: map[string][]gomatrixserverlib.KeyID{ + userID: {}, + }, } sigRes := &keyapi.QuerySignaturesResponse{} - for _, dev := range res.Devices { - sigReq.TargetIDs[userID] = append(sigReq.TargetIDs[userID], gomatrixserverlib.KeyID(dev.DeviceID)) - } keyAPI.QuerySignatures(req.Context(), sigReq, sigRes) response := gomatrixserverlib.RespUserDevices{ diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index d73161e9..bba3272b 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -71,8 +71,14 @@ func QueryDeviceKeys( return util.JSONResponse{ Code: 200, JSON: struct { - DeviceKeys interface{} `json:"device_keys"` - }{queryRes.DeviceKeys}, + DeviceKeys interface{} `json:"device_keys"` + MasterKeys interface{} `json:"master_keys"` + SelfSigningKeys interface{} `json:"self_signing_keys"` + }{ + queryRes.DeviceKeys, + queryRes.MasterKeys, + queryRes.SelfSigningKeys, + }, } } diff --git a/go.mod b/go.mod index 39f6020c..e5d15756 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20210805104751-03e40fa2c0e1 + github.com/matrix-org/gomatrixserverlib v0.0.0-20210809130922-d9c3f400582b github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index df5e2b85..d848988d 100644 --- a/go.sum +++ b/go.sum @@ -994,8 +994,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210805104751-03e40fa2c0e1 h1:ExS9AmOWYSZ4DR7W7Eqm7glP/g/u/CTVLAu9blAnve4= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210805104751-03e40fa2c0e1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210809130922-d9c3f400582b h1:8St1B8QmlvMLsOmGqW3++0akUs0250IAi+AGcr5faxw= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210809130922-d9c3f400582b/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b h1:5X5vdWQ13xrNkJVqaJHPsrt7rKkMJH5iac0EtfOuxSg= diff --git a/keyserver/api/api.go b/keyserver/api/api.go index 520562e7..aa6df96f 100644 --- a/keyserver/api/api.go +++ b/keyserver/api/api.go @@ -160,7 +160,7 @@ type PerformClaimKeysResponse struct { type PerformUploadDeviceKeysRequest struct { gomatrixserverlib.CrossSigningKeys // The user that uploaded the key, should be populated by the clientapi. - UserID string `json:"user_id"` + UserID string } type PerformUploadDeviceKeysResponse struct { @@ -170,7 +170,7 @@ type PerformUploadDeviceKeysResponse struct { type PerformUploadDeviceSignaturesRequest struct { Signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice // The user that uploaded the sig, should be populated by the clientapi. - UserID string `json:"user_id"` + UserID string } type PerformUploadDeviceSignaturesResponse struct { diff --git a/keyserver/internal/cross_signing.go b/keyserver/internal/cross_signing.go index 141a4eb2..4009dd45 100644 --- a/keyserver/internal/cross_signing.go +++ b/keyserver/internal/cross_signing.go @@ -15,6 +15,7 @@ package internal import ( + "bytes" "context" "crypto/ed25519" "database/sql" @@ -78,8 +79,8 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P } return } - hasMasterKey = true for _, keyData := range req.MasterKey.Keys { // iterates once, because sanityCheckKey requires one key + hasMasterKey = true masterKey = keyData } } @@ -116,46 +117,11 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P masterKey, hasMasterKey = existingKeys[gomatrixserverlib.CrossSigningKeyPurposeMaster] } - // If the user isn't a local user and we haven't successfully found a key - // through any local means then ask over federation. - if !hasMasterKey { - _, host, err := gomatrixserverlib.SplitID('@', req.UserID) - if err != nil { - res.Error = &api.KeyError{ - Err: "Retrieving cross-signing keys from federation failed: " + err.Error(), - } - return - } - keys, err := a.FedClient.QueryKeys(ctx, host, map[string][]string{ - req.UserID: {}, - }) - if err != nil { - res.Error = &api.KeyError{ - Err: "Retrieving cross-signing keys from federation failed: " + err.Error(), - } - return - } - switch k := keys.MasterKeys[req.UserID].CrossSigningBody.(type) { - case *gomatrixserverlib.CrossSigningKey: - if err := sanityCheckKey(*k, req.UserID, gomatrixserverlib.CrossSigningKeyPurposeMaster); err != nil { - res.Error = &api.KeyError{ - Err: "Master key sanity check failed: " + err.Error(), - } - return - } - default: - res.Error = &api.KeyError{ - Err: "Unexpected type for master key retrieved from federation", - } - return - } - } - // If we still don't have a master key at this point then there's nothing else // we can do - we've checked both the request and the database. if !hasMasterKey { res.Error = &api.KeyError{ - Err: "No master key was found, either in the database or in the request!", + Err: "No master key was found either in the database or in the request!", IsMissingParam: true, } return @@ -176,6 +142,15 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P if len(req.UserSigningKey.Keys) > 0 { toVerify[gomatrixserverlib.CrossSigningKeyPurposeUserSigning] = req.UserSigningKey } + + if len(toVerify) == 0 { + res.Error = &api.KeyError{ + Err: "No supplied keys available for verification", + IsMissingParam: true, + } + return + } + for purpose, key := range toVerify { // Collect together the key IDs we need to verify with. This will include // all of the key IDs specified in the signatures. @@ -212,6 +187,14 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P } } + if len(toStore) == 0 { + res.Error = &api.KeyError{ + Err: "No supplied keys passed verification", + IsMissingParam: true, + } + return + } + if err := a.DB.StoreCrossSigningKeysForUser(ctx, req.UserID, toStore); err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.DB.StoreCrossSigningKeysForUser: %s", err), @@ -257,6 +240,8 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req selfSignatures := map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} otherSignatures := map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice{} + // Sort signatures into two groups: one where people have signed their own + // keys and one where people have signed someone elses for userID, forUserID := range req.Signatures { for keyID, keyOrDevice := range forUserID { switch key := keyOrDevice.CrossSigningBody.(type) { @@ -335,20 +320,18 @@ func (a *KeyInternalAPI) processSelfSignatures( } for originKeyID, originSig := range forOriginUserID { - originDeviceKeyID := gomatrixserverlib.KeyID("ed25519:" + originKeyID) - var originKey gomatrixserverlib.DeviceKeys if err := json.Unmarshal(originDeviceKeys[string(originKeyID)], &originKey); err != nil { return fmt.Errorf("json.Unmarshal: %w", err) } - originSigningKey, ok := originKey.Keys[originDeviceKeyID] + originSigningKey, ok := originKey.Keys[originKeyID] if !ok { - return fmt.Errorf("missing origin signing key %q", originDeviceKeyID) + return fmt.Errorf("missing origin signing key %q", originKeyID) } originSigningKeyPublic := ed25519.PublicKey(originSigningKey) - if err := gomatrixserverlib.VerifyJSON(originUserID, originDeviceKeyID, originSigningKeyPublic, j); err != nil { + if err := gomatrixserverlib.VerifyJSON(originUserID, originKeyID, originSigningKeyPublic, j); err != nil { return fmt.Errorf("gomatrixserverlib.VerifyJSON: %w", err) } @@ -414,6 +397,56 @@ func (a *KeyInternalAPI) processOtherSignatures( // Here we will process: // * A user signing someone else's master keys using their user-signing keys + for targetUserID, forTargetUserID := range signatures { + for _, signature := range forTargetUserID { + switch sig := signature.CrossSigningBody.(type) { + case *gomatrixserverlib.CrossSigningKey: + // Find the local copy of the master key. We'll use this to be + // sure that the supplied stanza matches the key that we think it + // should be. + masterKey, ok := queryRes.MasterKeys[targetUserID] + if !ok { + return fmt.Errorf("failed to find master key for user %q", targetUserID) + } + + // For each key ID, write the signatures. Maybe there'll be more + // than one algorithm in the future so it's best not to focus on + // everything being ed25519:. + for targetKeyID, suppliedKeyData := range sig.Keys { + // The master key will be supplied in the request, but we should + // make sure that it matches what we think the master key should + // actually be. + localKeyData, lok := masterKey.Keys[targetKeyID] + if !lok { + return fmt.Errorf("uploaded master key %q for user %q doesn't match local copy", targetKeyID, targetUserID) + } else if !bytes.Equal(suppliedKeyData, localKeyData) { + return fmt.Errorf("uploaded master key %q for user %q doesn't match local copy", targetKeyID, targetUserID) + } + + // We only care about the signatures from the uploading user, so + // we will ignore anything that didn't originate from them. + userSigs, ok := sig.Signatures[userID] + if !ok { + return fmt.Errorf("there are no signatures on master key %q from uploading user %q", targetKeyID, userID) + } + + for originKeyID, originSig := range userSigs { + if err := a.DB.StoreCrossSigningSigsForTarget( + ctx, userID, originKeyID, targetUserID, targetKeyID, originSig, + ); err != nil { + return fmt.Errorf("a.DB.StoreCrossSigningKeysForTarget: %w", err) + } + } + } + + default: + // Users should only be signing another person's master key, + // so if we're here, it's probably because it's actually a + // gomatrixserverlib.DeviceKeys, which doesn't make sense. + } + } + } + return nil } @@ -435,7 +468,7 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase( } sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, userID, keyID) - if err != nil { + if err != nil && err != sql.ErrNoRows { logrus.WithError(err).Errorf("Failed to get cross-signing signatures for user %q key %q", userID, keyID) continue } @@ -480,44 +513,39 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase( func (a *KeyInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySignaturesRequest, res *api.QuerySignaturesResponse) { for targetUserID, forTargetUser := range req.TargetIDs { + keyMap, err := a.DB.CrossSigningKeysForUser(ctx, targetUserID) + if err != nil && err != sql.ErrNoRows { + res.Error = &api.KeyError{ + Err: fmt.Sprintf("a.DB.CrossSigningKeysForUser: %s", err), + } + continue + } + + for targetPurpose, targetKey := range keyMap { + switch targetPurpose { + case gomatrixserverlib.CrossSigningKeyPurposeMaster: + if res.MasterKeys == nil { + res.MasterKeys = map[string]gomatrixserverlib.CrossSigningKey{} + } + res.MasterKeys[targetUserID] = targetKey + + case gomatrixserverlib.CrossSigningKeyPurposeSelfSigning: + if res.SelfSigningKeys == nil { + res.SelfSigningKeys = map[string]gomatrixserverlib.CrossSigningKey{} + } + res.SelfSigningKeys[targetUserID] = targetKey + + case gomatrixserverlib.CrossSigningKeyPurposeUserSigning: + if res.UserSigningKeys == nil { + res.UserSigningKeys = map[string]gomatrixserverlib.CrossSigningKey{} + } + res.UserSigningKeys[targetUserID] = targetKey + } + } + for _, targetKeyID := range forTargetUser { - keyMap, err := a.DB.CrossSigningKeysForUser(ctx, targetUserID) - if err != nil { - if err == sql.ErrNoRows { - continue - } - res.Error = &api.KeyError{ - Err: fmt.Sprintf("a.DB.CrossSigningKeysForUser: %s", err), - } - } - - for targetPurpose, targetKey := range keyMap { - switch targetPurpose { - case gomatrixserverlib.CrossSigningKeyPurposeMaster: - if res.MasterKeys == nil { - res.MasterKeys = map[string]gomatrixserverlib.CrossSigningKey{} - } - res.MasterKeys[targetUserID] = targetKey - - case gomatrixserverlib.CrossSigningKeyPurposeSelfSigning: - if res.SelfSigningKeys == nil { - res.SelfSigningKeys = map[string]gomatrixserverlib.CrossSigningKey{} - } - res.SelfSigningKeys[targetUserID] = targetKey - - case gomatrixserverlib.CrossSigningKeyPurposeUserSigning: - if res.UserSigningKeys == nil { - res.UserSigningKeys = map[string]gomatrixserverlib.CrossSigningKey{} - } - res.UserSigningKeys[targetUserID] = targetKey - } - } - sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, targetUserID, targetKeyID) - if err != nil { - if err == sql.ErrNoRows { - continue - } + if err != nil && err != sql.ErrNoRows { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.DB.CrossSigningSigsForTarget: %s", err), } diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index 28638c29..de269911 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -372,9 +372,15 @@ func (a *KeyInternalAPI) queryRemoteKeys( domains := map[string]struct{}{} for domain := range domainToDeviceKeys { + if domain == string(a.ThisServer) { + continue + } domains[domain] = struct{}{} } for domain := range domainToCrossSigningKeys { + if domain == string(a.ThisServer) { + continue + } domains[domain] = struct{}{} } wg.Add(len(domains)) @@ -406,17 +412,11 @@ func (a *KeyInternalAPI) queryRemoteKeys( } for userID, body := range result.MasterKeys { - switch b := body.CrossSigningBody.(type) { - case *gomatrixserverlib.CrossSigningKey: - res.MasterKeys[userID] = *b - } + res.MasterKeys[userID] = body } for userID, body := range result.SelfSigningKeys { - switch b := body.CrossSigningBody.(type) { - case *gomatrixserverlib.CrossSigningKey: - res.SelfSigningKeys[userID] = *b - } + res.SelfSigningKeys[userID] = body } // TODO: do we want to persist these somewhere now @@ -430,8 +430,12 @@ func (a *KeyInternalAPI) queryRemoteKeysOnServer( res *api.QueryKeysResponse, ) { defer wg.Done() - fedCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() + fedCtx := ctx + if timeout > 0 { + var cancel context.CancelFunc + fedCtx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } // for users who we do not have any knowledge about, try to start doing device list updates for them // by hitting /users/devices - otherwise fallback to /keys/query which has nicer bulk properties but // lack a stream ID. diff --git a/keyserver/inthttp/server.go b/keyserver/inthttp/server.go index ac70e3e5..475544a5 100644 --- a/keyserver/inthttp/server.go +++ b/keyserver/inthttp/server.go @@ -62,7 +62,7 @@ func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI) { httputil.MakeInternalAPI("performUploadDeviceKeys", func(req *http.Request) util.JSONResponse { request := api.PerformUploadDeviceKeysRequest{} response := api.PerformUploadDeviceKeysResponse{} - if err := json.NewDecoder(req.Body).Decode(&request.CrossSigningKeys); err != nil { + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } s.PerformUploadDeviceKeys(req.Context(), &request, &response) @@ -73,7 +73,7 @@ func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI) { httputil.MakeInternalAPI("performUploadDeviceSignatures", func(req *http.Request) util.JSONResponse { request := api.PerformUploadDeviceSignaturesRequest{} response := api.PerformUploadDeviceSignaturesResponse{} - if err := json.NewDecoder(req.Body).Decode(&request.Signatures); err != nil { + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } s.PerformUploadDeviceSignatures(req.Context(), &request, &response) diff --git a/sytest-whitelist b/sytest-whitelist index 27109e60..d2f2a1c7 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -553,3 +553,4 @@ Deleted & recreated backups are empty Can upload self-signing keys Fails to upload self-signing keys with no auth Fails to upload self-signing key without master key +can fetch self-signing keys over federation From 125ea75b2419aa5ebf58506e3fbd03a90eb06d68 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 11 Aug 2021 09:44:14 +0100 Subject: [PATCH 23/64] Add type field to DeviceMessage, allow fields to be nullable (#1969) --- keyserver/api/api.go | 12 ++++++++++-- keyserver/internal/device_list_update.go | 9 ++++++--- keyserver/internal/device_list_update_test.go | 6 ++++-- keyserver/internal/internal.go | 3 ++- keyserver/storage/postgres/device_keys_table.go | 6 +++++- keyserver/storage/sqlite3/device_keys_table.go | 7 ++++++- keyserver/storage/storage_test.go | 12 ++++++++---- 7 files changed, 41 insertions(+), 14 deletions(-) diff --git a/keyserver/api/api.go b/keyserver/api/api.go index aa6df96f..490f0e41 100644 --- a/keyserver/api/api.go +++ b/keyserver/api/api.go @@ -53,9 +53,17 @@ func (k *KeyError) Error() string { return k.Err } +type DeviceMessageType int + +const ( + TypeDeviceKeyUpdate DeviceMessageType = iota + TypeCrossSigningUpdate +) + // DeviceMessage represents the message produced into Kafka by the key server. type DeviceMessage struct { - DeviceKeys + Type DeviceMessageType `json:"Type,omitempty"` + *DeviceKeys `json:"DeviceKeys,omitempty"` // A monotonically increasing number which represents device changes for this user. StreamID int } @@ -76,7 +84,7 @@ type DeviceKeys struct { // WithStreamID returns a copy of this device message with the given stream ID func (k *DeviceKeys) WithStreamID(streamID int) DeviceMessage { return DeviceMessage{ - DeviceKeys: *k, + DeviceKeys: k, StreamID: streamID, } } diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index 91d4b53d..1f7c6e2a 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -231,7 +231,8 @@ func (u *DeviceListUpdater) update(ctx context.Context, event gomatrixserverlib. } keys := []api.DeviceMessage{ { - DeviceKeys: api.DeviceKeys{ + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ DeviceID: event.DeviceID, DisplayName: event.DeviceDisplayName, KeyJSON: k, @@ -417,8 +418,9 @@ func (u *DeviceListUpdater) updateDeviceList(res *gomatrixserverlib.RespUserDevi continue } keys[i] = api.DeviceMessage{ + Type: api.TypeDeviceKeyUpdate, StreamID: res.StreamID, - DeviceKeys: api.DeviceKeys{ + DeviceKeys: &api.DeviceKeys{ DeviceID: device.DeviceID, DisplayName: device.DisplayName, UserID: res.UserID, @@ -426,7 +428,8 @@ func (u *DeviceListUpdater) updateDeviceList(res *gomatrixserverlib.RespUserDevi }, } existingKeys[i] = api.DeviceMessage{ - DeviceKeys: api.DeviceKeys{ + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ UserID: res.UserID, DeviceID: device.DeviceID, }, diff --git a/keyserver/internal/device_list_update_test.go b/keyserver/internal/device_list_update_test.go index 7c170de2..164be6be 100644 --- a/keyserver/internal/device_list_update_test.go +++ b/keyserver/internal/device_list_update_test.go @@ -146,8 +146,9 @@ func TestUpdateHavePrevID(t *testing.T) { t.Fatalf("Update returned an error: %s", err) } want := api.DeviceMessage{ + Type: api.TypeDeviceKeyUpdate, StreamID: event.StreamID, - DeviceKeys: api.DeviceKeys{ + DeviceKeys: &api.DeviceKeys{ DeviceID: event.DeviceID, DisplayName: event.DeviceDisplayName, KeyJSON: event.Keys, @@ -224,8 +225,9 @@ func TestUpdateNoPrevID(t *testing.T) { // wait a bit for db to be updated... time.Sleep(100 * time.Millisecond) want := api.DeviceMessage{ + Type: api.TypeDeviceKeyUpdate, StreamID: 5, - DeviceKeys: api.DeviceKeys{ + DeviceKeys: &api.DeviceKeys{ DeviceID: "JLAFKJWSCS", DisplayName: "Mobile Phone", UserID: remoteUserID, diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index de269911..47eda179 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -573,7 +573,8 @@ func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Per existingKeys := make([]api.DeviceMessage, len(keysToStore)) for i := range keysToStore { existingKeys[i] = api.DeviceMessage{ - DeviceKeys: api.DeviceKeys{ + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ UserID: keysToStore[i].UserID, DeviceID: keysToStore[i].DeviceID, }, diff --git a/keyserver/storage/postgres/device_keys_table.go b/keyserver/storage/postgres/device_keys_table.go index 95064fc8..e5f68fd0 100644 --- a/keyserver/storage/postgres/device_keys_table.go +++ b/keyserver/storage/postgres/device_keys_table.go @@ -114,6 +114,7 @@ func (s *deviceKeysStatements) SelectDeviceKeysJSON(ctx context.Context, keys [] return err } // this will be '' when there is no device + keys[i].Type = api.TypeDeviceKeyUpdate keys[i].KeyJSON = []byte(keyJSONStr) keys[i].StreamID = streamID if displayName.Valid { @@ -179,7 +180,10 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID } var result []api.DeviceMessage for rows.Next() { - var dk api.DeviceMessage + dk := api.DeviceMessage{ + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{}, + } dk.UserID = userID var keyJSON string var streamID int diff --git a/keyserver/storage/sqlite3/device_keys_table.go b/keyserver/storage/sqlite3/device_keys_table.go index 9112fc6e..ca7ed9cf 100644 --- a/keyserver/storage/sqlite3/device_keys_table.go +++ b/keyserver/storage/sqlite3/device_keys_table.go @@ -113,7 +113,11 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID defer internal.CloseAndLogIfError(ctx, rows, "selectBatchDeviceKeysStmt: rows.close() failed") var result []api.DeviceMessage for rows.Next() { - var dk api.DeviceMessage + dk := api.DeviceMessage{ + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{}, + } + dk.Type = api.TypeDeviceKeyUpdate dk.UserID = userID var keyJSON string var streamID int @@ -144,6 +148,7 @@ func (s *deviceKeysStatements) SelectDeviceKeysJSON(ctx context.Context, keys [] return err } // this will be '' when there is no device + keys[i].Type = api.TypeDeviceKeyUpdate keys[i].KeyJSON = []byte(keyJSONStr) keys[i].StreamID = streamID if displayName.Valid { diff --git a/keyserver/storage/storage_test.go b/keyserver/storage/storage_test.go index afdb086d..4e0a8af1 100644 --- a/keyserver/storage/storage_test.go +++ b/keyserver/storage/storage_test.go @@ -105,7 +105,8 @@ func TestDeviceKeysStreamIDGeneration(t *testing.T) { bob := "@bob:TestDeviceKeysStreamIDGeneration" msgs := []api.DeviceMessage{ { - DeviceKeys: api.DeviceKeys{ + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ DeviceID: "AAA", UserID: alice, KeyJSON: []byte(`{"key":"v1"}`), @@ -113,7 +114,8 @@ func TestDeviceKeysStreamIDGeneration(t *testing.T) { // StreamID: 1 }, { - DeviceKeys: api.DeviceKeys{ + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ DeviceID: "AAA", UserID: bob, KeyJSON: []byte(`{"key":"v1"}`), @@ -121,7 +123,8 @@ func TestDeviceKeysStreamIDGeneration(t *testing.T) { // StreamID: 1 as this is a different user }, { - DeviceKeys: api.DeviceKeys{ + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ DeviceID: "another_device", UserID: alice, KeyJSON: []byte(`{"key":"v1"}`), @@ -143,7 +146,8 @@ func TestDeviceKeysStreamIDGeneration(t *testing.T) { // updating a device sets the next stream ID for that user msgs = []api.DeviceMessage{ { - DeviceKeys: api.DeviceKeys{ + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ DeviceID: "AAA", UserID: alice, KeyJSON: []byte(`{"key":"v2"}`), From c2b9ab7470627d49c986472cb9131ea2f33e3366 Mon Sep 17 00:00:00 2001 From: Devon Mizelle Date: Mon, 16 Aug 2021 09:19:35 -0400 Subject: [PATCH 24/64] Fix ineffectual error assignment (#1976) Was working on another PR and noticed that golangci-lint was failing locally on `ineffassign` Signed-off-by: Devon Mizelle --- appservice/query/query.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/appservice/query/query.go b/appservice/query/query.go index 9f6c79a8..dacd3caa 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -51,6 +51,10 @@ func (a *AppServiceQueryAPI) RoomAliasExists( if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) { // The full path to the rooms API, includes hs token URL, err := url.Parse(appservice.URL + roomAliasExistsPath) + if err != nil { + return err + } + URL.Path += request.Alias apiURL := URL.String() + "?access_token=" + appservice.HSToken @@ -114,6 +118,9 @@ func (a *AppServiceQueryAPI) UserIDExists( if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) { // The full path to the rooms API, includes hs token URL, err := url.Parse(appservice.URL + userIDExistsPath) + if err != nil { + return err + } URL.Path += request.UserID apiURL := URL.String() + "?access_token=" + appservice.HSToken From 8a4b90b7dd3b5e2b25dc79679f2ef7c10485ebb0 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Tue, 17 Aug 2021 11:39:09 +0200 Subject: [PATCH 25/64] Update golangci-lint (#1978) * Update golangci-lint * Use unconditional strings.TrimSuffix * Add error checks Signed-off-by: Till Faelligen * Disable lint typecheck * Fix ineffectual error assignment (#1976) Was working on another PR and noticed that golangci-lint was failing locally on `ineffassign` Signed-off-by: Devon Mizelle * Revert "Disable lint typecheck" This reverts commit 3c76f219d3cb43e4ee9a9c319abd0b8879972cfc. Co-authored-by: Devon Mizelle --- build/scripts/find-lint.sh | 2 +- clientapi/routing/routing.go | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/build/scripts/find-lint.sh b/build/scripts/find-lint.sh index 4ab5e4de..af87e14d 100755 --- a/build/scripts/find-lint.sh +++ b/build/scripts/find-lint.sh @@ -25,7 +25,7 @@ echo "Installing golangci-lint..." # Make a backup of go.{mod,sum} first cp go.mod go.mod.bak && cp go.sum go.sum.bak -go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1 +go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.41.1 # Run linting echo "Looking for lint..." diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index c6be8939..d282306f 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -291,10 +291,7 @@ func Setup( return util.ErrorResponse(err) } // If there's a trailing slash, remove it - eventType := vars["type"] - if strings.HasSuffix(eventType, "/") { - eventType = eventType[:len(eventType)-1] - } + eventType := strings.TrimSuffix(vars["type"], "/") eventFormat := req.URL.Query().Get("format") == "event" return OnIncomingStateTypeRequest(req.Context(), device, rsAPI, vars["roomID"], eventType, "", eventFormat) })).Methods(http.MethodGet, http.MethodOptions) @@ -315,11 +312,7 @@ func Setup( return util.ErrorResponse(err) } emptyString := "" - eventType := vars["eventType"] - // If there's a trailing slash, remove it - if strings.HasSuffix(eventType, "/") { - eventType = eventType[:len(eventType)-1] - } + eventType := strings.TrimSuffix(vars["eventType"], "/") return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, nil) }), ).Methods(http.MethodPut, http.MethodOptions) From ff21675c5b1269b40d4c9174c65b77d3d38a0137 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 17 Aug 2021 13:44:30 +0100 Subject: [PATCH 26/64] Cross-signing fixes, notifications via sync, federation (#1974) * Initial work on signing key update EDUs * Fix build * Produce/consume EDUs * Producer logging * Only produce key change notifications for local users * Better naming * Try to notify sync * Enable feature * Use key change topic * Don't bother verifying signatures, validate key lengths if we can, notifier fixes * Copyright notices * Remove tests from whitelist until matrix-org/sytest#1117 * Some review comment fixes * Update to matrix-org/gomatrixserverlib@f9416ac * Remove unneeded parameter --- clientapi/jsonerror/jsonerror.go | 6 + clientapi/routing/key_crosssigning.go | 10 + clientapi/routing/routing.go | 2 +- eduserver/api/input.go | 12 ++ eduserver/api/output.go | 36 +--- eduserver/api/types.go | 59 ++++++ eduserver/eduserver.go | 1 + eduserver/input/input.go | 33 +++ eduserver/inthttp/client.go | 20 +- eduserver/inthttp/server.go | 13 ++ federationapi/routing/send.go | 16 ++ federationapi/routing/send_test.go | 8 + federationsender/consumers/keychange.go | 57 ++++++ go.mod | 2 +- go.sum | 4 +- keyserver/api/api.go | 7 +- keyserver/consumers/cross_signing.go | 112 ++++++++++ keyserver/consumers/eduserver.go | 61 ------ keyserver/internal/cross_signing.go | 258 +++++++++++------------- keyserver/keyserver.go | 2 +- keyserver/producers/keychange.go | 34 ++++ setup/config/config_kafka.go | 1 - syncapi/consumers/keychange.go | 54 ++++- sytest-whitelist | 2 + 24 files changed, 556 insertions(+), 254 deletions(-) create mode 100644 eduserver/api/types.go create mode 100644 keyserver/consumers/cross_signing.go delete mode 100644 keyserver/consumers/eduserver.go diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go index c42b25be..7accde5f 100644 --- a/clientapi/jsonerror/jsonerror.go +++ b/clientapi/jsonerror/jsonerror.go @@ -131,6 +131,12 @@ func InvalidSignature(msg string) *MatrixError { return &MatrixError{"M_INVALID_SIGNATURE", msg} } +// InvalidParam is an error that is returned when a parameter was invalid, +// traditionally with cross-signing. +func InvalidParam(msg string) *MatrixError { + return &MatrixError{"M_INVALID_PARAM", msg} +} + // MissingParam is an error that is returned when a parameter was incorrect, // traditionally with cross-signing. func MissingParam(msg string) *MatrixError { diff --git a/clientapi/routing/key_crosssigning.go b/clientapi/routing/key_crosssigning.go index 3c103fd7..756598db 100644 --- a/clientapi/routing/key_crosssigning.go +++ b/clientapi/routing/key_crosssigning.go @@ -73,6 +73,11 @@ func UploadCrossSigningDeviceKeys( Code: http.StatusBadRequest, JSON: jsonerror.MissingParam(err.Error()), } + case err.IsInvalidParam: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidParam(err.Error()), + } default: return util.JSONResponse{ Code: http.StatusBadRequest, @@ -110,6 +115,11 @@ func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.KeyI Code: http.StatusBadRequest, JSON: jsonerror.MissingParam(err.Error()), } + case err.IsInvalidParam: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidParam(err.Error()), + } default: return util.JSONResponse{ Code: http.StatusBadRequest, diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index d282306f..30ecc292 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -65,7 +65,7 @@ func Setup( userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg) unstableFeatures := map[string]bool{ - //"org.matrix.e2e_cross_signing": true, + "org.matrix.e2e_cross_signing": true, } for _, msc := range cfg.MSCs.MSCs { unstableFeatures["org.matrix."+msc] = true diff --git a/eduserver/api/input.go b/eduserver/api/input.go index f8599e1c..2fa253f4 100644 --- a/eduserver/api/input.go +++ b/eduserver/api/input.go @@ -75,6 +75,12 @@ type InputReceiptEventRequest struct { // InputReceiptEventResponse is a response to InputReceiptEventRequest type InputReceiptEventResponse struct{} +type InputCrossSigningKeyUpdateRequest struct { + CrossSigningKeyUpdate `json:"signing_keys"` +} + +type InputCrossSigningKeyUpdateResponse struct{} + // EDUServerInputAPI is used to write events to the typing server. type EDUServerInputAPI interface { InputTypingEvent( @@ -94,4 +100,10 @@ type EDUServerInputAPI interface { request *InputReceiptEventRequest, response *InputReceiptEventResponse, ) error + + InputCrossSigningKeyUpdate( + ctx context.Context, + request *InputCrossSigningKeyUpdateRequest, + response *InputCrossSigningKeyUpdateResponse, + ) error } diff --git a/eduserver/api/output.go b/eduserver/api/output.go index 650458a2..c6de4e01 100644 --- a/eduserver/api/output.go +++ b/eduserver/api/output.go @@ -33,14 +33,6 @@ type OutputTypingEvent struct { ExpireTime *time.Time } -// TypingEvent represents a matrix edu event of type 'm.typing'. -type TypingEvent struct { - Type string `json:"type"` - RoomID string `json:"room_id"` - UserID string `json:"user_id"` - Typing bool `json:"typing"` -} - // OutputSendToDeviceEvent is an entry in the send-to-device output kafka log. // This contains the full event content, along with the user ID and device ID // to which it is destined. @@ -50,14 +42,6 @@ type OutputSendToDeviceEvent struct { gomatrixserverlib.SendToDeviceEvent } -type ReceiptEvent struct { - UserID string `json:"user_id"` - RoomID string `json:"room_id"` - EventID string `json:"event_id"` - Type string `json:"type"` - Timestamp gomatrixserverlib.Timestamp `json:"timestamp"` -} - // OutputReceiptEvent is an entry in the receipt output kafka log type OutputReceiptEvent struct { UserID string `json:"user_id"` @@ -67,21 +51,7 @@ type OutputReceiptEvent struct { Timestamp gomatrixserverlib.Timestamp `json:"timestamp"` } -// Helper structs for receipts json creation -type ReceiptMRead struct { - User map[string]ReceiptTS `json:"m.read"` -} - -type ReceiptTS struct { - TS gomatrixserverlib.Timestamp `json:"ts"` -} - -// FederationSender output -type FederationReceiptMRead struct { - User map[string]FederationReceiptData `json:"m.read"` -} - -type FederationReceiptData struct { - Data ReceiptTS `json:"data"` - EventIDs []string `json:"event_ids"` +// OutputCrossSigningKeyUpdate is an entry in the signing key update output kafka log +type OutputCrossSigningKeyUpdate struct { + CrossSigningKeyUpdate `json:"signing_keys"` } diff --git a/eduserver/api/types.go b/eduserver/api/types.go new file mode 100644 index 00000000..a207580f --- /dev/null +++ b/eduserver/api/types.go @@ -0,0 +1,59 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import "github.com/matrix-org/gomatrixserverlib" + +const ( + MSigningKeyUpdate = "m.signing_key_update" +) + +type TypingEvent struct { + Type string `json:"type"` + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + Typing bool `json:"typing"` +} + +type ReceiptEvent struct { + UserID string `json:"user_id"` + RoomID string `json:"room_id"` + EventID string `json:"event_id"` + Type string `json:"type"` + Timestamp gomatrixserverlib.Timestamp `json:"timestamp"` +} + +type FederationReceiptMRead struct { + User map[string]FederationReceiptData `json:"m.read"` +} + +type FederationReceiptData struct { + Data ReceiptTS `json:"data"` + EventIDs []string `json:"event_ids"` +} + +type ReceiptMRead struct { + User map[string]ReceiptTS `json:"m.read"` +} + +type ReceiptTS struct { + TS gomatrixserverlib.Timestamp `json:"ts"` +} + +type CrossSigningKeyUpdate struct { + MasterKey *gomatrixserverlib.CrossSigningKey `json:"master_key,omitempty"` + SelfSigningKey *gomatrixserverlib.CrossSigningKey `json:"self_signing_key,omitempty"` + UserID string `json:"user_id"` +} diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index 7cc40510..7875e27f 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -52,6 +52,7 @@ func NewInternalAPI( OutputTypingEventTopic: cfg.Matrix.Kafka.TopicFor(config.TopicOutputTypingEvent), OutputSendToDeviceEventTopic: cfg.Matrix.Kafka.TopicFor(config.TopicOutputSendToDeviceEvent), OutputReceiptEventTopic: cfg.Matrix.Kafka.TopicFor(config.TopicOutputReceiptEvent), + OutputKeyChangeEventTopic: cfg.Matrix.Kafka.TopicFor(config.TopicOutputKeyChangeEvent), ServerName: cfg.Matrix.ServerName, } } diff --git a/eduserver/input/input.go b/eduserver/input/input.go index c54fb9de..bdc24374 100644 --- a/eduserver/input/input.go +++ b/eduserver/input/input.go @@ -24,6 +24,7 @@ import ( "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" + keyapi "github.com/matrix-org/dendrite/keyserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -39,6 +40,8 @@ type EDUServerInputAPI struct { OutputSendToDeviceEventTopic string // The kafka topic to output new receipt events to OutputReceiptEventTopic string + // The kafka topic to output new key change events to + OutputKeyChangeEventTopic string // kafka producer Producer sarama.SyncProducer // Internal user query API @@ -77,6 +80,36 @@ func (t *EDUServerInputAPI) InputSendToDeviceEvent( return t.sendToDeviceEvent(ise) } +// InputCrossSigningKeyUpdate implements api.EDUServerInputAPI +func (t *EDUServerInputAPI) InputCrossSigningKeyUpdate( + ctx context.Context, + request *api.InputCrossSigningKeyUpdateRequest, + response *api.InputCrossSigningKeyUpdateResponse, +) error { + eventJSON, err := json.Marshal(&keyapi.DeviceMessage{ + Type: keyapi.TypeCrossSigningUpdate, + OutputCrossSigningKeyUpdate: &api.OutputCrossSigningKeyUpdate{ + CrossSigningKeyUpdate: request.CrossSigningKeyUpdate, + }, + }) + if err != nil { + return err + } + + logrus.WithFields(logrus.Fields{ + "user_id": request.UserID, + }).Infof("Producing to topic '%s'", t.OutputKeyChangeEventTopic) + + m := &sarama.ProducerMessage{ + Topic: string(t.OutputKeyChangeEventTopic), + Key: sarama.StringEncoder(request.UserID), + Value: sarama.ByteEncoder(eventJSON), + } + + _, _, err = t.Producer.SendMessage(m) + return err +} + func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error { ev := &api.TypingEvent{ Type: gomatrixserverlib.MTyping, diff --git a/eduserver/inthttp/client.go b/eduserver/inthttp/client.go index 0690ed82..9a6f483c 100644 --- a/eduserver/inthttp/client.go +++ b/eduserver/inthttp/client.go @@ -12,9 +12,10 @@ import ( // HTTP paths for the internal HTTP APIs const ( - EDUServerInputTypingEventPath = "/eduserver/input" - EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice" - EDUServerInputReceiptEventPath = "/eduserver/receipt" + EDUServerInputTypingEventPath = "/eduserver/input" + EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice" + EDUServerInputReceiptEventPath = "/eduserver/receipt" + EDUServerInputCrossSigningKeyUpdatePath = "/eduserver/crossSigningKeyUpdate" ) // NewEDUServerClient creates a EDUServerInputAPI implemented by talking to a HTTP POST API. @@ -68,3 +69,16 @@ func (h *httpEDUServerInputAPI) InputReceiptEvent( apiURL := h.eduServerURL + EDUServerInputReceiptEventPath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } + +// InputCrossSigningKeyUpdate implements EDUServerInputAPI +func (h *httpEDUServerInputAPI) InputCrossSigningKeyUpdate( + ctx context.Context, + request *api.InputCrossSigningKeyUpdateRequest, + response *api.InputCrossSigningKeyUpdateResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "InputCrossSigningKeyUpdate") + defer span.Finish() + + apiURL := h.eduServerURL + EDUServerInputCrossSigningKeyUpdatePath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/eduserver/inthttp/server.go b/eduserver/inthttp/server.go index a3494375..a50ca84f 100644 --- a/eduserver/inthttp/server.go +++ b/eduserver/inthttp/server.go @@ -51,4 +51,17 @@ func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(EDUServerInputCrossSigningKeyUpdatePath, + httputil.MakeInternalAPI("inputCrossSigningKeyUpdate", func(req *http.Request) util.JSONResponse { + var request api.InputCrossSigningKeyUpdateRequest + var response api.InputCrossSigningKeyUpdateResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := t.InputCrossSigningKeyUpdate(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 5f214e0f..2b282826 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -502,6 +502,22 @@ func (t *txnReq) processEDUs(ctx context.Context) { } } } + case eduserverAPI.MSigningKeyUpdate: + var updatePayload eduserverAPI.CrossSigningKeyUpdate + if err := json.Unmarshal(e.Content, &updatePayload); err != nil { + util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ + "user_id": updatePayload.UserID, + }).Error("Failed to send signing key update to edu server") + continue + } + inputReq := &eduserverAPI.InputCrossSigningKeyUpdateRequest{ + CrossSigningKeyUpdate: updatePayload, + } + inputRes := &eduserverAPI.InputCrossSigningKeyUpdateResponse{} + if err := t.eduAPI.InputCrossSigningKeyUpdate(ctx, inputReq, inputRes); err != nil { + util.GetLogger(ctx).WithError(err).Error("Failed to unmarshal cross-signing update") + continue + } default: util.GetLogger(ctx).WithField("type", e.Type).Debug("Unhandled EDU") } diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 5b5af9c4..70288461 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -84,6 +84,14 @@ func (o *testEDUProducer) InputReceiptEvent( return nil } +func (o *testEDUProducer) InputCrossSigningKeyUpdate( + ctx context.Context, + request *eduAPI.InputCrossSigningKeyUpdateRequest, + response *eduAPI.InputCrossSigningKeyUpdateResponse, +) error { + return nil +} + type testRoomserverAPI struct { api.RoomserverInternalAPITrace inputRoomEvents []api.InputRoomEvent diff --git a/federationsender/consumers/keychange.go b/federationsender/consumers/keychange.go index 9e146390..675328df 100644 --- a/federationsender/consumers/keychange.go +++ b/federationsender/consumers/keychange.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/Shopify/sarama" + eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/internal" @@ -28,6 +29,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" ) @@ -83,6 +85,17 @@ func (t *KeyChangeConsumer) onMessage(msg *sarama.ConsumerMessage) error { log.WithError(err).Errorf("failed to read device message from key change topic") return nil } + switch m.Type { + case api.TypeCrossSigningUpdate: + return t.onCrossSigningMessage(m) + case api.TypeDeviceKeyUpdate: + fallthrough + default: + return t.onDeviceKeyMessage(m) + } +} + +func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) error { logger := log.WithField("user_id", m.UserID) // only send key change events which originated from us @@ -133,6 +146,50 @@ func (t *KeyChangeConsumer) onMessage(msg *sarama.ConsumerMessage) error { return t.queues.SendEDU(edu, t.serverName, destinations) } +func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) error { + output := m.CrossSigningKeyUpdate + _, host, err := gomatrixserverlib.SplitID('@', output.UserID) + if err != nil { + logrus.WithError(err).Errorf("fedsender key change consumer: user ID parse failure") + return nil + } + if host != gomatrixserverlib.ServerName(t.serverName) { + // Ignore any messages that didn't originate locally, otherwise we'll + // end up parroting information we received from other servers. + return nil + } + logger := log.WithField("user_id", output.UserID) + + var queryRes roomserverAPI.QueryRoomsForUserResponse + err = t.rsAPI.QueryRoomsForUser(context.Background(), &roomserverAPI.QueryRoomsForUserRequest{ + UserID: output.UserID, + WantMembership: "join", + }, &queryRes) + if err != nil { + logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined rooms for user") + return nil + } + // send this key change to all servers who share rooms with this user. + destinations, err := t.db.GetJoinedHostsForRooms(context.Background(), queryRes.RoomIDs) + if err != nil { + logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in") + return nil + } + + // Pack the EDU and marshal it + edu := &gomatrixserverlib.EDU{ + Type: eduserverAPI.MSigningKeyUpdate, + Origin: string(t.serverName), + } + if edu.Content, err = json.Marshal(output); err != nil { + logger.WithError(err).Error("fedsender key change consumer: failed to marshal output, dropping") + return nil + } + + logger.Infof("Sending cross-signing update message to %q", destinations) + return t.queues.SendEDU(edu, t.serverName, destinations) +} + func prevID(streamID int) []int { if streamID <= 1 { return nil diff --git a/go.mod b/go.mod index e5d15756..a1832854 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20210809130922-d9c3f400582b + github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index d848988d..7f3be4c0 100644 --- a/go.sum +++ b/go.sum @@ -994,8 +994,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210809130922-d9c3f400582b h1:8St1B8QmlvMLsOmGqW3++0akUs0250IAi+AGcr5faxw= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210809130922-d9c3f400582b/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 h1:b8cyR4aYv9Lmf1lKgASJ+PFSp/GBv8ZFgb/O42ZXLGA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b h1:5X5vdWQ13xrNkJVqaJHPsrt7rKkMJH5iac0EtfOuxSg= diff --git a/keyserver/api/api.go b/keyserver/api/api.go index 490f0e41..40120236 100644 --- a/keyserver/api/api.go +++ b/keyserver/api/api.go @@ -20,6 +20,7 @@ import ( "strings" "time" + eduapi "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/keyserver/types" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -47,6 +48,7 @@ type KeyError struct { Err string `json:"error"` IsInvalidSignature bool `json:"is_invalid_signature,omitempty"` // M_INVALID_SIGNATURE IsMissingParam bool `json:"is_missing_param,omitempty"` // M_MISSING_PARAM + IsInvalidParam bool `json:"is_invalid_param,omitempty"` // M_INVALID_PARAM } func (k *KeyError) Error() string { @@ -62,8 +64,9 @@ const ( // DeviceMessage represents the message produced into Kafka by the key server. type DeviceMessage struct { - Type DeviceMessageType `json:"Type,omitempty"` - *DeviceKeys `json:"DeviceKeys,omitempty"` + Type DeviceMessageType `json:"Type,omitempty"` + *DeviceKeys `json:"DeviceKeys,omitempty"` + *eduapi.OutputCrossSigningKeyUpdate `json:"CrossSigningKeyUpdate,omitempty"` // A monotonically increasing number which represents device changes for this user. StreamID int } diff --git a/keyserver/consumers/cross_signing.go b/keyserver/consumers/cross_signing.go new file mode 100644 index 00000000..f9973ec9 --- /dev/null +++ b/keyserver/consumers/cross_signing.go @@ -0,0 +1,112 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "encoding/json" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/keyserver/storage" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" + + "github.com/Shopify/sarama" +) + +type OutputCrossSigningKeyUpdateConsumer struct { + eduServerConsumer *internal.ContinualConsumer + keyDB storage.Database + keyAPI api.KeyInternalAPI + serverName string +} + +func NewOutputCrossSigningKeyUpdateConsumer( + process *process.ProcessContext, + cfg *config.Dendrite, + kafkaConsumer sarama.Consumer, + keyDB storage.Database, + keyAPI api.KeyInternalAPI, +) *OutputCrossSigningKeyUpdateConsumer { + // The keyserver both produces and consumes on the TopicOutputKeyChangeEvent + // topic. We will only produce events where the UserID matches our server name, + // and we will only consume events where the UserID does NOT match our server + // name (because the update came from a remote server). + consumer := internal.ContinualConsumer{ + Process: process, + ComponentName: "keyserver/keyserver", + Topic: cfg.Global.Kafka.TopicFor(config.TopicOutputKeyChangeEvent), + Consumer: kafkaConsumer, + PartitionStore: keyDB, + } + s := &OutputCrossSigningKeyUpdateConsumer{ + eduServerConsumer: &consumer, + keyDB: keyDB, + keyAPI: keyAPI, + serverName: string(cfg.Global.ServerName), + } + consumer.ProcessMessage = s.onMessage + + return s +} + +func (s *OutputCrossSigningKeyUpdateConsumer) Start() error { + return s.eduServerConsumer.Start() +} + +// onMessage is called in response to a message received on the +// key change events topic from the key server. +func (t *OutputCrossSigningKeyUpdateConsumer) onMessage(msg *sarama.ConsumerMessage) error { + var m api.DeviceMessage + if err := json.Unmarshal(msg.Value, &m); err != nil { + logrus.WithError(err).Errorf("failed to read device message from key change topic") + return nil + } + switch m.Type { + case api.TypeCrossSigningUpdate: + return t.onCrossSigningMessage(m) + default: + return nil + } +} + +func (s *OutputCrossSigningKeyUpdateConsumer) onCrossSigningMessage(m api.DeviceMessage) error { + output := m.CrossSigningKeyUpdate + _, host, err := gomatrixserverlib.SplitID('@', output.UserID) + if err != nil { + logrus.WithError(err).Errorf("eduserver output log: user ID parse failure") + return nil + } + if host == gomatrixserverlib.ServerName(s.serverName) { + // Ignore any messages that contain information about our own users, as + // they already originated from this server. + return nil + } + uploadReq := &api.PerformUploadDeviceKeysRequest{ + UserID: output.UserID, + } + if output.MasterKey != nil { + uploadReq.MasterKey = *output.MasterKey + } + if output.SelfSigningKey != nil { + uploadReq.SelfSigningKey = *output.SelfSigningKey + } + uploadRes := &api.PerformUploadDeviceKeysResponse{} + s.keyAPI.PerformUploadDeviceKeys(context.TODO(), uploadReq, uploadRes) + return uploadRes.Error +} diff --git a/keyserver/consumers/eduserver.go b/keyserver/consumers/eduserver.go deleted file mode 100644 index d764950b..00000000 --- a/keyserver/consumers/eduserver.go +++ /dev/null @@ -1,61 +0,0 @@ -package consumers - -import ( - "fmt" - - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/storage" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/setup/process" - - "github.com/Shopify/sarama" -) - -type OutputSigningKeyUpdateConsumer struct { - eduServerConsumer *internal.ContinualConsumer - keyDB storage.Database - keyAPI api.KeyInternalAPI - serverName string -} - -func NewOutputSigningKeyUpdateConsumer( - process *process.ProcessContext, - cfg *config.Dendrite, - kafkaConsumer sarama.Consumer, - keyDB storage.Database, - keyAPI api.KeyInternalAPI, -) *OutputSigningKeyUpdateConsumer { - consumer := internal.ContinualConsumer{ - Process: process, - ComponentName: "keyserver/eduserver", - Topic: cfg.Global.Kafka.TopicFor(config.TopicOutputSigningKeyUpdate), - Consumer: kafkaConsumer, - PartitionStore: keyDB, - } - s := &OutputSigningKeyUpdateConsumer{ - eduServerConsumer: &consumer, - keyDB: keyDB, - keyAPI: keyAPI, - serverName: string(cfg.Global.ServerName), - } - consumer.ProcessMessage = s.onMessage - - return s -} - -func (s *OutputSigningKeyUpdateConsumer) Start() error { - return s.eduServerConsumer.Start() -} - -func (s *OutputSigningKeyUpdateConsumer) onMessage(msg *sarama.ConsumerMessage) error { - /* - var output eduapi.OutputSigningKeyUpdate - if err := json.Unmarshal(msg.Value, &output); err != nil { - log.WithError(err).Errorf("eduserver output log: message parse failure") - return nil - } - return nil - */ - return fmt.Errorf("TODO") -} diff --git a/keyserver/internal/cross_signing.go b/keyserver/internal/cross_signing.go index 4009dd45..1e1871b8 100644 --- a/keyserver/internal/cross_signing.go +++ b/keyserver/internal/cross_signing.go @@ -19,14 +19,15 @@ import ( "context" "crypto/ed25519" "database/sql" - "encoding/json" "fmt" "strings" + eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/keyserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" + "golang.org/x/crypto/curve25519" ) func sanityCheckKey(key gomatrixserverlib.CrossSigningKey, userID string, purpose gomatrixserverlib.CrossSigningKeyPurpose) error { @@ -45,6 +46,41 @@ func sanityCheckKey(key gomatrixserverlib.CrossSigningKey, userID string, purpos if tokens[1] != b64 { return fmt.Errorf("key ID isn't correct") } + switch tokens[0] { + case "ed25519": + if len(keyData) != ed25519.PublicKeySize { + return fmt.Errorf("ed25519 key is not the correct length") + } + case "curve25519": + if len(keyData) != curve25519.PointSize { + return fmt.Errorf("curve25519 key is not the correct length") + } + default: + // We can't enforce the key length to be correct for an + // algorithm that we don't recognise, so instead we'll + // just make sure that it isn't incredibly excessive. + if l := len(keyData); l > 4096 { + return fmt.Errorf("unknown key type is too long (%d bytes)", l) + } + } + } + + // Check to see if the signatures make sense + for _, forOriginUser := range key.Signatures { + for originKeyID, originSignature := range forOriginUser { + switch strings.SplitN(string(originKeyID), ":", 1)[0] { + case "ed25519": + if len(originSignature) != ed25519.SignatureSize { + return fmt.Errorf("ed25519 signature is not the correct length") + } + case "curve25519": + return fmt.Errorf("curve25519 signatures are impossible") + default: + if l := len(originSignature); l > 4096 { + return fmt.Errorf("unknown signature type is too long (%d bytes)", l) + } + } + } } // Does the key claim to be from the right user? @@ -69,42 +105,68 @@ func sanityCheckKey(key gomatrixserverlib.CrossSigningKey, userID string, purpos // nolint:gocyclo func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) { - var masterKey gomatrixserverlib.Base64Bytes + // Find the keys to store. + byPurpose := map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey{} + toStore := types.CrossSigningKeyMap{} hasMasterKey := false if len(req.MasterKey.Keys) > 0 { if err := sanityCheckKey(req.MasterKey, req.UserID, gomatrixserverlib.CrossSigningKeyPurposeMaster); err != nil { res.Error = &api.KeyError{ - Err: "Master key sanity check failed: " + err.Error(), + Err: "Master key sanity check failed: " + err.Error(), + IsInvalidParam: true, } return } - for _, keyData := range req.MasterKey.Keys { // iterates once, because sanityCheckKey requires one key - hasMasterKey = true - masterKey = keyData + + byPurpose[gomatrixserverlib.CrossSigningKeyPurposeMaster] = req.MasterKey + for _, key := range req.MasterKey.Keys { // iterates once, see sanityCheckKey + toStore[gomatrixserverlib.CrossSigningKeyPurposeMaster] = key } + hasMasterKey = true } if len(req.SelfSigningKey.Keys) > 0 { if err := sanityCheckKey(req.SelfSigningKey, req.UserID, gomatrixserverlib.CrossSigningKeyPurposeSelfSigning); err != nil { res.Error = &api.KeyError{ - Err: "Self-signing key sanity check failed: " + err.Error(), + Err: "Self-signing key sanity check failed: " + err.Error(), + IsInvalidParam: true, } return } + + byPurpose[gomatrixserverlib.CrossSigningKeyPurposeSelfSigning] = req.SelfSigningKey + for _, key := range req.SelfSigningKey.Keys { // iterates once, see sanityCheckKey + toStore[gomatrixserverlib.CrossSigningKeyPurposeSelfSigning] = key + } } if len(req.UserSigningKey.Keys) > 0 { if err := sanityCheckKey(req.UserSigningKey, req.UserID, gomatrixserverlib.CrossSigningKeyPurposeUserSigning); err != nil { res.Error = &api.KeyError{ - Err: "User-signing key sanity check failed: " + err.Error(), + Err: "User-signing key sanity check failed: " + err.Error(), + IsInvalidParam: true, } return } + + byPurpose[gomatrixserverlib.CrossSigningKeyPurposeUserSigning] = req.UserSigningKey + for _, key := range req.UserSigningKey.Keys { // iterates once, see sanityCheckKey + toStore[gomatrixserverlib.CrossSigningKeyPurposeUserSigning] = key + } } - // If the user hasn't given a new master key, then let's go and get their - // existing keys from the database. + // If there's nothing to do then stop here. + if len(toStore) == 0 { + res.Error = &api.KeyError{ + Err: "No keys were supplied in the request", + IsMissingParam: true, + } + return + } + + // We can't have a self-signing or user-signing key without a master + // key, so make sure we have one of those. if !hasMasterKey { existingKeys, err := a.DB.CrossSigningKeysDataForUser(ctx, req.UserID) if err != nil { @@ -114,87 +176,20 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P return } - masterKey, hasMasterKey = existingKeys[gomatrixserverlib.CrossSigningKeyPurposeMaster] + _, hasMasterKey = existingKeys[gomatrixserverlib.CrossSigningKeyPurposeMaster] } - // If we still don't have a master key at this point then there's nothing else - // we can do - we've checked both the request and the database. + // If we still can't find a master key for the user then stop the upload. + // This satisfies the "Fails to upload self-signing key without master key" test. if !hasMasterKey { res.Error = &api.KeyError{ - Err: "No master key was found either in the database or in the request!", - IsMissingParam: true, - } - return - } - - // The key ID is basically the key itself. - masterKeyID := gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", masterKey.Encode())) - - // Work out which things we need to verify the signatures for. - toVerify := make(map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, 3) - toStore := types.CrossSigningKeyMap{} - if len(req.MasterKey.Keys) > 0 { - toVerify[gomatrixserverlib.CrossSigningKeyPurposeMaster] = req.MasterKey - } - if len(req.SelfSigningKey.Keys) > 0 { - toVerify[gomatrixserverlib.CrossSigningKeyPurposeSelfSigning] = req.SelfSigningKey - } - if len(req.UserSigningKey.Keys) > 0 { - toVerify[gomatrixserverlib.CrossSigningKeyPurposeUserSigning] = req.UserSigningKey - } - - if len(toVerify) == 0 { - res.Error = &api.KeyError{ - Err: "No supplied keys available for verification", - IsMissingParam: true, - } - return - } - - for purpose, key := range toVerify { - // Collect together the key IDs we need to verify with. This will include - // all of the key IDs specified in the signatures. - keyJSON, err := json.Marshal(key) - if err != nil { - res.Error = &api.KeyError{ - Err: fmt.Sprintf("The JSON of the key section is invalid: %s", err.Error()), - } - return - } - - switch purpose { - case gomatrixserverlib.CrossSigningKeyPurposeMaster: - // The master key might have a signature attached to it from the - // previous key, or from a device key, but there's no real need - // to verify it. Clients will perform key checks when the master - // key changes. - - default: - // Sub-keys should be signed by the master key. - if err := gomatrixserverlib.VerifyJSON(req.UserID, masterKeyID, ed25519.PublicKey(masterKey), keyJSON); err != nil { - res.Error = &api.KeyError{ - Err: fmt.Sprintf("The %q sub-key failed master key signature verification: %s", purpose, err.Error()), - IsInvalidSignature: true, - } - return - } - } - - // If we've reached this point then all the signatures are valid so - // add the key to the list of keys to store. - for _, keyData := range key.Keys { // iterates once, see sanityCheckKey - toStore[purpose] = keyData - } - } - - if len(toStore) == 0 { - res.Error = &api.KeyError{ - Err: "No supplied keys passed verification", + Err: "No master key was found", IsMissingParam: true, } return } + // Store the keys. if err := a.DB.StoreCrossSigningKeysForUser(ctx, req.UserID, toStore); err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.DB.StoreCrossSigningKeysForUser: %s", err), @@ -203,7 +198,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P } // Now upload any signatures that were included with the keys. - for _, key := range toVerify { + for _, key := range byPurpose { var targetKeyID gomatrixserverlib.KeyID for targetKey := range key.Keys { // iterates once, see sanityCheckKey targetKeyID = targetKey @@ -222,6 +217,28 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P } } } + + // Finally, generate a notification that we updated the keys. + if _, host, err := gomatrixserverlib.SplitID('@', req.UserID); err == nil && host == a.ThisServer { + update := eduserverAPI.CrossSigningKeyUpdate{ + UserID: req.UserID, + } + if mk, ok := byPurpose[gomatrixserverlib.CrossSigningKeyPurposeMaster]; ok { + update.MasterKey = &mk + } + if ssk, ok := byPurpose[gomatrixserverlib.CrossSigningKeyPurposeSelfSigning]; ok { + update.SelfSigningKey = &ssk + } + if update.MasterKey == nil && update.SelfSigningKey == nil { + return + } + if err := a.Producer.ProduceSigningKeyUpdate(update); err != nil { + res.Error = &api.KeyError{ + Err: fmt.Sprintf("a.Producer.ProduceSigningKeyUpdate: %s", err), + } + return + } + } } func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req *api.PerformUploadDeviceSignaturesRequest, res *api.PerformUploadDeviceSignaturesResponse) { @@ -277,7 +294,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req } } - if err := a.processSelfSignatures(ctx, req.UserID, queryRes, selfSignatures); err != nil { + if err := a.processSelfSignatures(ctx, selfSignatures); err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.processSelfSignatures: %s", err), } @@ -290,10 +307,25 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req } return } + + // Finally, generate a notification that we updated the signatures. + for userID := range req.Signatures { + if _, host, err := gomatrixserverlib.SplitID('@', userID); err == nil && host == a.ThisServer { + update := eduserverAPI.CrossSigningKeyUpdate{ + UserID: userID, + } + if err := a.Producer.ProduceSigningKeyUpdate(update); err != nil { + res.Error = &api.KeyError{ + Err: fmt.Sprintf("a.Producer.ProduceSigningKeyUpdate: %s", err), + } + return + } + } + } } func (a *KeyInternalAPI) processSelfSignatures( - ctx context.Context, _ string, queryRes *api.QueryKeysResponse, + ctx context.Context, signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice, ) error { // Here we will process: @@ -304,37 +336,8 @@ func (a *KeyInternalAPI) processSelfSignatures( for targetKeyID, signature := range forTargetUserID { switch sig := signature.CrossSigningBody.(type) { case *gomatrixserverlib.CrossSigningKey: - // The user is signing their master key with one of their devices - // The QueryKeys response should contain the device key hopefully. - // First we need to marshal the blob back into JSON so we can verify - // it. - j, err := json.Marshal(sig) - if err != nil { - return fmt.Errorf("json.Marshal: %w", err) - } - for originUserID, forOriginUserID := range sig.Signatures { - originDeviceKeys, ok := queryRes.DeviceKeys[originUserID] - if !ok { - return fmt.Errorf("missing device keys for user %q", originUserID) - } - for originKeyID, originSig := range forOriginUserID { - var originKey gomatrixserverlib.DeviceKeys - if err := json.Unmarshal(originDeviceKeys[string(originKeyID)], &originKey); err != nil { - return fmt.Errorf("json.Unmarshal: %w", err) - } - - originSigningKey, ok := originKey.Keys[originKeyID] - if !ok { - return fmt.Errorf("missing origin signing key %q", originKeyID) - } - originSigningKeyPublic := ed25519.PublicKey(originSigningKey) - - if err := gomatrixserverlib.VerifyJSON(originUserID, originKeyID, originSigningKeyPublic, j); err != nil { - return fmt.Errorf("gomatrixserverlib.VerifyJSON: %w", err) - } - if err := a.DB.StoreCrossSigningSigsForTarget( ctx, originUserID, originKeyID, targetUserID, targetKeyID, originSig, ); err != nil { @@ -344,35 +347,8 @@ func (a *KeyInternalAPI) processSelfSignatures( } case *gomatrixserverlib.DeviceKeys: - // The user is signing one of their devices with their self-signing key - // The QueryKeys response should contain the master key hopefully. - // First we need to marshal the blob back into JSON so we can verify - // it. - j, err := json.Marshal(sig) - if err != nil { - return fmt.Errorf("json.Marshal: %w", err) - } - for originUserID, forOriginUserID := range sig.Signatures { for originKeyID, originSig := range forOriginUserID { - originSelfSigningKeys, ok := queryRes.SelfSigningKeys[originUserID] - if !ok { - return fmt.Errorf("missing self-signing key for user %q", originUserID) - } - - var originSelfSigningKeyID gomatrixserverlib.KeyID - var originSelfSigningKey gomatrixserverlib.Base64Bytes - for keyID, key := range originSelfSigningKeys.Keys { - originSelfSigningKeyID, originSelfSigningKey = keyID, key - break - } - - originSelfSigningKeyPublic := ed25519.PublicKey(originSelfSigningKey) - - if err := gomatrixserverlib.VerifyJSON(originUserID, originSelfSigningKeyID, originSelfSigningKeyPublic, j); err != nil { - return fmt.Errorf("gomatrixserverlib.VerifyJSON: %w", err) - } - if err := a.DB.StoreCrossSigningSigsForTarget( ctx, originUserID, originKeyID, targetUserID, targetKeyID, originSig, ); err != nil { diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index fcfe24de..60306755 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -65,7 +65,7 @@ func NewInternalAPI( } }() - keyconsumer := consumers.NewOutputSigningKeyUpdateConsumer( + keyconsumer := consumers.NewOutputCrossSigningKeyUpdateConsumer( base.ProcessContext, base.Cfg, consumer, db, ap, ) if err := keyconsumer.Start(); err != nil { diff --git a/keyserver/producers/keychange.go b/keyserver/producers/keychange.go index 0fe21d8b..782675c2 100644 --- a/keyserver/producers/keychange.go +++ b/keyserver/producers/keychange.go @@ -19,6 +19,7 @@ import ( "encoding/json" "github.com/Shopify/sarama" + eduapi "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/keyserver/storage" "github.com/sirupsen/logrus" @@ -73,3 +74,36 @@ func (p *KeyChange) ProduceKeyChanges(keys []api.DeviceMessage) error { } return nil } + +func (p *KeyChange) ProduceSigningKeyUpdate(key eduapi.CrossSigningKeyUpdate) error { + var m sarama.ProducerMessage + output := &api.DeviceMessage{ + Type: api.TypeCrossSigningUpdate, + OutputCrossSigningKeyUpdate: &eduapi.OutputCrossSigningKeyUpdate{ + CrossSigningKeyUpdate: key, + }, + } + + value, err := json.Marshal(output) + if err != nil { + return err + } + + m.Topic = string(p.Topic) + m.Key = sarama.StringEncoder(key.UserID) + m.Value = sarama.ByteEncoder(value) + + partition, offset, err := p.Producer.SendMessage(&m) + if err != nil { + return err + } + err = p.DB.StoreKeyChange(context.Background(), partition, offset, key.UserID) + if err != nil { + return err + } + + logrus.WithFields(logrus.Fields{ + "user_id": key.UserID, + }).Infof("Produced to cross-signing update topic '%s'", p.Topic) + return nil +} diff --git a/setup/config/config_kafka.go b/setup/config/config_kafka.go index 15b3ad71..36191428 100644 --- a/setup/config/config_kafka.go +++ b/setup/config/config_kafka.go @@ -10,7 +10,6 @@ const ( TopicOutputRoomEvent = "OutputRoomEvent" TopicOutputClientData = "OutputClientData" TopicOutputReceiptEvent = "OutputReceiptEvent" - TopicOutputSigningKeyUpdate = "OutputSigningKeyUpdate" ) type Kafka struct { diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go index 0d2ecd44..05fcf37d 100644 --- a/syncapi/consumers/keychange.go +++ b/syncapi/consumers/keychange.go @@ -29,6 +29,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" ) @@ -104,13 +105,23 @@ func (s *OutputKeyChangeEventConsumer) updateOffset(msg *sarama.ConsumerMessage) func (s *OutputKeyChangeEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { defer s.updateOffset(msg) - var output api.DeviceMessage - if err := json.Unmarshal(msg.Value, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Error("syncapi: failed to unmarshal key change event from key server") - sentry.CaptureException(err) - return err + var m api.DeviceMessage + if err := json.Unmarshal(msg.Value, &m); err != nil { + logrus.WithError(err).Errorf("failed to read device message from key change topic") + return nil } + switch m.Type { + case api.TypeCrossSigningUpdate: + return s.onCrossSigningMessage(m, msg.Offset, msg.Partition) + case api.TypeDeviceKeyUpdate: + fallthrough + default: + return s.onDeviceKeyMessage(m, msg.Offset, msg.Partition) + } +} + +func (s *OutputKeyChangeEventConsumer) onDeviceKeyMessage(m api.DeviceMessage, offset int64, partition int32) error { + output := m.DeviceKeys // work out who we need to notify about the new key var queryRes roomserverAPI.QuerySharedUsersResponse err := s.rsAPI.QuerySharedUsers(context.Background(), &roomserverAPI.QuerySharedUsersRequest{ @@ -124,8 +135,35 @@ func (s *OutputKeyChangeEventConsumer) onMessage(msg *sarama.ConsumerMessage) er // make sure we get our own key updates too! queryRes.UserIDsToCount[output.UserID] = 1 posUpdate := types.LogPosition{ - Offset: msg.Offset, - Partition: msg.Partition, + Offset: offset, + Partition: partition, + } + + s.stream.Advance(posUpdate) + for userID := range queryRes.UserIDsToCount { + s.notifier.OnNewKeyChange(types.StreamingToken{DeviceListPosition: posUpdate}, userID, output.UserID) + } + + return nil +} + +func (s *OutputKeyChangeEventConsumer) onCrossSigningMessage(m api.DeviceMessage, offset int64, partition int32) error { + output := m.CrossSigningKeyUpdate + // work out who we need to notify about the new key + var queryRes roomserverAPI.QuerySharedUsersResponse + err := s.rsAPI.QuerySharedUsers(context.Background(), &roomserverAPI.QuerySharedUsersRequest{ + UserID: output.UserID, + }, &queryRes) + if err != nil { + log.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server") + sentry.CaptureException(err) + return err + } + // make sure we get our own key updates too! + queryRes.UserIDsToCount[output.UserID] = 1 + posUpdate := types.LogPosition{ + Offset: offset, + Partition: partition, } s.stream.Advance(posUpdate) diff --git a/sytest-whitelist b/sytest-whitelist index d2f2a1c7..9f3eb893 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -554,3 +554,5 @@ Can upload self-signing keys Fails to upload self-signing keys with no auth Fails to upload self-signing key without master key can fetch self-signing keys over federation +Changing master key notifies local users +Changing user-signing key notifies local users From 1bee1ae2048051b52bc5b5084808a0418c0def04 Mon Sep 17 00:00:00 2001 From: Sambhav Saggi <17993169+sambhavsaggi@users.noreply.github.com> Date: Tue, 17 Aug 2021 11:18:51 -0400 Subject: [PATCH 27/64] Syslog integration (#1952) * Syslog integration, part 1 * Add protocol, make sure syslog actually logs * Make golangci-lint happy about shadow variables * Add syslog tag, wrap syslog in logLevelHook --- internal/log.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/internal/log.go b/internal/log.go index d2b233c5..f0656d7d 100644 --- a/internal/log.go +++ b/internal/log.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "io" + "log/syslog" "net/http" "os" "path" @@ -30,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dugong" "github.com/sirupsen/logrus" + lSyslog "github.com/sirupsen/logrus/hooks/syslog" ) type utcFormatter struct { @@ -128,6 +130,9 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) { case "file": checkFileHookParams(hook.Params) setupFileHook(hook, level, componentName) + case "syslog": + checkSyslogHookParams(hook.Params) + setupSyslogHook(hook, level, componentName) default: logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) } @@ -173,6 +178,34 @@ func setupFileHook(hook config.LogrusHook, level logrus.Level, componentName str }) } +func checkSyslogHookParams(params map[string]interface{}) { + addr, ok := params["address"] + if !ok { + logrus.Fatalf("Expecting a parameter \"address\" for logging hook of type \"syslog\"") + } + + if _, ok := addr.(string); !ok { + logrus.Fatalf("Parameter \"address\" for logging hook of type \"syslog\" should be a string") + } + + proto, ok2 := params["protocol"] + if !ok2 { + logrus.Fatalf("Expecting a parameter \"protocol\" for logging hook of type \"syslog\"") + } + + if _, ok2 := proto.(string); !ok2 { + logrus.Fatalf("Parameter \"protocol\" for logging hook of type \"syslog\" should be a string") + } + +} + +func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) { + syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName) + if err == nil { + logrus.AddHook(&logLevelHook{level, syslogHook}) + } +} + //CloseAndLogIfError Closes io.Closer and logs the error if any func CloseAndLogIfError(ctx context.Context, closer io.Closer, message string) { if closer == nil { From 614e67280defda4a9156f620f2751e3ef136da81 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 18 Aug 2021 12:07:09 +0100 Subject: [PATCH 28/64] Delete device keys/signatures from key server when deleting devices (#1979) * Delete device keys/signatures from key server when deleting device from user API * Move loop to within database transaction * Don't fall over deleting no rows --- keyserver/api/api.go | 13 +++++++++++++ keyserver/internal/internal.go | 8 ++++++++ keyserver/inthttp/client.go | 18 ++++++++++++++++++ keyserver/inthttp/server.go | 11 +++++++++++ keyserver/storage/interface.go | 4 ++++ .../postgres/cross_signing_sigs_table.go | 15 +++++++++++++++ .../storage/postgres/device_keys_table.go | 12 ++++++++++++ keyserver/storage/shared/storage.go | 16 ++++++++++++++++ .../sqlite3/cross_signing_sigs_table.go | 15 +++++++++++++++ keyserver/storage/sqlite3/device_keys_table.go | 12 ++++++++++++ keyserver/storage/tables/interface.go | 2 ++ syncapi/internal/keychange_test.go | 2 ++ userapi/internal/api.go | 12 ++++++++++++ 13 files changed, 140 insertions(+) diff --git a/keyserver/api/api.go b/keyserver/api/api.go index 40120236..5a109cc6 100644 --- a/keyserver/api/api.go +++ b/keyserver/api/api.go @@ -34,6 +34,7 @@ type KeyInternalAPI interface { PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse) // PerformClaimKeys claims one-time keys for use in pre-key messages PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) + PerformDeleteKeys(ctx context.Context, req *PerformDeleteKeysRequest, res *PerformDeleteKeysResponse) PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse) PerformUploadDeviceSignatures(ctx context.Context, req *PerformUploadDeviceSignaturesRequest, res *PerformUploadDeviceSignaturesResponse) QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse) @@ -145,6 +146,18 @@ type PerformUploadKeysResponse struct { OneTimeKeyCounts []OneTimeKeysCount } +// PerformDeleteKeysRequest asks the keyserver to forget about certain +// keys, and signatures related to those keys. +type PerformDeleteKeysRequest struct { + UserID string + KeyIDs []gomatrixserverlib.KeyID +} + +// PerformDeleteKeysResponse is the response to PerformDeleteKeysRequest. +type PerformDeleteKeysResponse struct { + Error *KeyError +} + // KeyError sets a key error field on KeyErrors func (r *PerformUploadKeysResponse) KeyError(userID, deviceID string, err *KeyError) { if r.KeyErrors[userID] == nil { diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index 47eda179..a546e94b 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -182,6 +182,14 @@ func (a *KeyInternalAPI) claimRemoteKeys( util.GetLogger(ctx).WithField("num_keys", keysClaimed).Info("Claimed remote keys") } +func (a *KeyInternalAPI) PerformDeleteKeys(ctx context.Context, req *api.PerformDeleteKeysRequest, res *api.PerformDeleteKeysResponse) { + if err := a.DB.DeleteDeviceKeys(ctx, req.UserID, req.KeyIDs); err != nil { + res.Error = &api.KeyError{ + Err: fmt.Sprintf("Failed to delete device keys: %s", err), + } + } +} + func (a *KeyInternalAPI) QueryOneTimeKeys(ctx context.Context, req *api.QueryOneTimeKeysRequest, res *api.QueryOneTimeKeysResponse) { count, err := a.DB.OneTimeKeysCount(ctx, req.UserID, req.DeviceID) if err != nil { diff --git a/keyserver/inthttp/client.go b/keyserver/inthttp/client.go index 15870571..f50789b8 100644 --- a/keyserver/inthttp/client.go +++ b/keyserver/inthttp/client.go @@ -30,6 +30,7 @@ const ( InputDeviceListUpdatePath = "/keyserver/inputDeviceListUpdate" PerformUploadKeysPath = "/keyserver/performUploadKeys" PerformClaimKeysPath = "/keyserver/performClaimKeys" + PerformDeleteKeysPath = "/keyserver/performDeleteKeys" PerformUploadDeviceKeysPath = "/keyserver/performUploadDeviceKeys" PerformUploadDeviceSignaturesPath = "/keyserver/performUploadDeviceSignatures" QueryKeysPath = "/keyserver/queryKeys" @@ -94,6 +95,23 @@ func (h *httpKeyInternalAPI) PerformClaimKeys( } } +func (h *httpKeyInternalAPI) PerformDeleteKeys( + ctx context.Context, + request *api.PerformDeleteKeysRequest, + response *api.PerformDeleteKeysResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformClaimKeys") + defer span.Finish() + + apiURL := h.apiURL + PerformClaimKeysPath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.Error = &api.KeyError{ + Err: err.Error(), + } + } +} + func (h *httpKeyInternalAPI) PerformUploadKeys( ctx context.Context, request *api.PerformUploadKeysRequest, diff --git a/keyserver/inthttp/server.go b/keyserver/inthttp/server.go index 475544a5..8d557a76 100644 --- a/keyserver/inthttp/server.go +++ b/keyserver/inthttp/server.go @@ -47,6 +47,17 @@ func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(PerformDeleteKeysPath, + httputil.MakeInternalAPI("performDeleteKeys", func(req *http.Request) util.JSONResponse { + request := api.PerformDeleteKeysRequest{} + response := api.PerformDeleteKeysResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + s.PerformDeleteKeys(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle(PerformUploadKeysPath, httputil.MakeInternalAPI("performUploadKeys", func(req *http.Request) util.JSONResponse { request := api.PerformUploadKeysRequest{} diff --git a/keyserver/storage/interface.go b/keyserver/storage/interface.go index b9db81ad..99842bc5 100644 --- a/keyserver/storage/interface.go +++ b/keyserver/storage/interface.go @@ -58,6 +58,10 @@ type Database interface { // If there are some missing keys, they are omitted from the returned slice. There is no ordering on the returned slice. DeviceKeysForUser(ctx context.Context, userID string, deviceIDs []string) ([]api.DeviceMessage, error) + // DeleteDeviceKeys removes the device keys for a given user/device, and any accompanying + // cross-signing signatures relating to that device. + DeleteDeviceKeys(ctx context.Context, userID string, deviceIDs []gomatrixserverlib.KeyID) error + // ClaimKeys based on the 3-uple of user_id, device_id and algorithm name. Returns the keys claimed. Returns no error if a key // cannot be claimed or if none exist for this (user, device, algorithm), instead it is omitted from the returned slice. ClaimKeys(ctx context.Context, userToDeviceToAlgorithm map[string]map[string]string) ([]api.OneTimeKeys, error) diff --git a/keyserver/storage/postgres/cross_signing_sigs_table.go b/keyserver/storage/postgres/cross_signing_sigs_table.go index 677e7a48..e1185395 100644 --- a/keyserver/storage/postgres/cross_signing_sigs_table.go +++ b/keyserver/storage/postgres/cross_signing_sigs_table.go @@ -46,10 +46,14 @@ const upsertCrossSigningSigsForTargetSQL = "" + " VALUES($1, $2, $3, $4, $5)" + " ON CONFLICT (origin_user_id, target_user_id, target_key_id) DO UPDATE SET (origin_key_id, signature) = ($2, $5)" +const deleteCrossSigningSigsForTargetSQL = "" + + "DELETE FROM keyserver_cross_signing_sigs WHERE target_user_id=$1 AND target_key_id=$2" + type crossSigningSigsStatements struct { db *sql.DB selectCrossSigningSigsForTargetStmt *sql.Stmt upsertCrossSigningSigsForTargetStmt *sql.Stmt + deleteCrossSigningSigsForTargetStmt *sql.Stmt } func NewPostgresCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, error) { @@ -63,6 +67,7 @@ func NewPostgresCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, erro return s, sqlutil.StatementList{ {&s.selectCrossSigningSigsForTargetStmt, selectCrossSigningSigsForTargetSQL}, {&s.upsertCrossSigningSigsForTargetStmt, upsertCrossSigningSigsForTargetSQL}, + {&s.deleteCrossSigningSigsForTargetStmt, deleteCrossSigningSigsForTargetSQL}, }.Prepare(db) } @@ -101,3 +106,13 @@ func (s *crossSigningSigsStatements) UpsertCrossSigningSigsForTarget( } return nil } + +func (s *crossSigningSigsStatements) DeleteCrossSigningSigsForTarget( + ctx context.Context, txn *sql.Tx, + targetUserID string, targetKeyID gomatrixserverlib.KeyID, +) error { + if _, err := sqlutil.TxStmt(txn, s.deleteCrossSigningSigsForTargetStmt).ExecContext(ctx, targetUserID, targetKeyID); err != nil { + return fmt.Errorf("s.deleteCrossSigningSigsForTargetStmt: %w", err) + } + return nil +} diff --git a/keyserver/storage/postgres/device_keys_table.go b/keyserver/storage/postgres/device_keys_table.go index e5f68fd0..5ae0da96 100644 --- a/keyserver/storage/postgres/device_keys_table.go +++ b/keyserver/storage/postgres/device_keys_table.go @@ -62,6 +62,9 @@ const selectMaxStreamForUserSQL = "" + const countStreamIDsForUserSQL = "" + "SELECT COUNT(*) FROM keyserver_device_keys WHERE user_id=$1 AND stream_id = ANY($2)" +const deleteDeviceKeysSQL = "" + + "DELETE FROM keyserver_device_keys WHERE user_id=$1 AND device_id=$2" + const deleteAllDeviceKeysSQL = "" + "DELETE FROM keyserver_device_keys WHERE user_id=$1" @@ -72,6 +75,7 @@ type deviceKeysStatements struct { selectBatchDeviceKeysStmt *sql.Stmt selectMaxStreamForUserStmt *sql.Stmt countStreamIDsForUserStmt *sql.Stmt + deleteDeviceKeysStmt *sql.Stmt deleteAllDeviceKeysStmt *sql.Stmt } @@ -98,6 +102,9 @@ func NewPostgresDeviceKeysTable(db *sql.DB) (tables.DeviceKeys, error) { if s.countStreamIDsForUserStmt, err = db.Prepare(countStreamIDsForUserSQL); err != nil { return nil, err } + if s.deleteDeviceKeysStmt, err = db.Prepare(deleteDeviceKeysSQL); err != nil { + return nil, err + } if s.deleteAllDeviceKeysStmt, err = db.Prepare(deleteAllDeviceKeysSQL); err != nil { return nil, err } @@ -163,6 +170,11 @@ func (s *deviceKeysStatements) InsertDeviceKeys(ctx context.Context, txn *sql.Tx return nil } +func (s *deviceKeysStatements) DeleteDeviceKeys(ctx context.Context, txn *sql.Tx, userID, deviceID string) error { + _, err := sqlutil.TxStmt(txn, s.deleteDeviceKeysStmt).ExecContext(ctx, userID, deviceID) + return err +} + func (s *deviceKeysStatements) DeleteAllDeviceKeys(ctx context.Context, txn *sql.Tx, userID string) error { _, err := sqlutil.TxStmt(txn, s.deleteAllDeviceKeysStmt).ExecContext(ctx, userID) return err diff --git a/keyserver/storage/shared/storage.go b/keyserver/storage/shared/storage.go index 64ce53ef..5bd8be36 100644 --- a/keyserver/storage/shared/storage.go +++ b/keyserver/storage/shared/storage.go @@ -158,6 +158,22 @@ func (d *Database) MarkDeviceListStale(ctx context.Context, userID string, isSta }) } +// DeleteDeviceKeys removes the device keys for a given user/device, and any accompanying +// cross-signing signatures relating to that device. +func (d *Database) DeleteDeviceKeys(ctx context.Context, userID string, deviceIDs []gomatrixserverlib.KeyID) error { + return d.Writer.Do(nil, nil, func(txn *sql.Tx) error { + for _, deviceID := range deviceIDs { + if err := d.CrossSigningSigsTable.DeleteCrossSigningSigsForTarget(ctx, txn, userID, deviceID); err != nil && err != sql.ErrNoRows { + return fmt.Errorf("d.CrossSigningSigsTable.DeleteCrossSigningSigsForTarget: %w", err) + } + if err := d.DeviceKeysTable.DeleteDeviceKeys(ctx, txn, userID, string(deviceID)); err != nil && err != sql.ErrNoRows { + return fmt.Errorf("d.DeviceKeysTable.DeleteDeviceKeys: %w", err) + } + } + return nil + }) +} + // CrossSigningKeysForUser returns the latest known cross-signing keys for a user, if any. func (d *Database) CrossSigningKeysForUser(ctx context.Context, userID string) (map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, error) { keyMap, err := d.CrossSigningKeysTable.SelectCrossSigningKeysForUser(ctx, nil, userID) diff --git a/keyserver/storage/sqlite3/cross_signing_sigs_table.go b/keyserver/storage/sqlite3/cross_signing_sigs_table.go index aa702583..9abf5436 100644 --- a/keyserver/storage/sqlite3/cross_signing_sigs_table.go +++ b/keyserver/storage/sqlite3/cross_signing_sigs_table.go @@ -45,10 +45,14 @@ const upsertCrossSigningSigsForTargetSQL = "" + "INSERT OR REPLACE INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" + " VALUES($1, $2, $3, $4, $5)" +const deleteCrossSigningSigsForTargetSQL = "" + + "DELETE FROM keyserver_cross_signing_sigs WHERE target_user_id=$1 AND target_key_id=$2" + type crossSigningSigsStatements struct { db *sql.DB selectCrossSigningSigsForTargetStmt *sql.Stmt upsertCrossSigningSigsForTargetStmt *sql.Stmt + deleteCrossSigningSigsForTargetStmt *sql.Stmt } func NewSqliteCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, error) { @@ -62,6 +66,7 @@ func NewSqliteCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, error) return s, sqlutil.StatementList{ {&s.selectCrossSigningSigsForTargetStmt, selectCrossSigningSigsForTargetSQL}, {&s.upsertCrossSigningSigsForTargetStmt, upsertCrossSigningSigsForTargetSQL}, + {&s.deleteCrossSigningSigsForTargetStmt, deleteCrossSigningSigsForTargetSQL}, }.Prepare(db) } @@ -100,3 +105,13 @@ func (s *crossSigningSigsStatements) UpsertCrossSigningSigsForTarget( } return nil } + +func (s *crossSigningSigsStatements) DeleteCrossSigningSigsForTarget( + ctx context.Context, txn *sql.Tx, + targetUserID string, targetKeyID gomatrixserverlib.KeyID, +) error { + if _, err := sqlutil.TxStmt(txn, s.deleteCrossSigningSigsForTargetStmt).ExecContext(ctx, targetUserID, targetKeyID); err != nil { + return fmt.Errorf("s.deleteCrossSigningSigsForTargetStmt: %w", err) + } + return nil +} diff --git a/keyserver/storage/sqlite3/device_keys_table.go b/keyserver/storage/sqlite3/device_keys_table.go index ca7ed9cf..fa1c930d 100644 --- a/keyserver/storage/sqlite3/device_keys_table.go +++ b/keyserver/storage/sqlite3/device_keys_table.go @@ -58,6 +58,9 @@ const selectMaxStreamForUserSQL = "" + const countStreamIDsForUserSQL = "" + "SELECT COUNT(*) FROM keyserver_device_keys WHERE user_id=$1 AND stream_id IN ($2)" +const deleteDeviceKeysSQL = "" + + "DELETE FROM keyserver_device_keys WHERE user_id=$1 AND device_id=$2" + const deleteAllDeviceKeysSQL = "" + "DELETE FROM keyserver_device_keys WHERE user_id=$1" @@ -67,6 +70,7 @@ type deviceKeysStatements struct { selectDeviceKeysStmt *sql.Stmt selectBatchDeviceKeysStmt *sql.Stmt selectMaxStreamForUserStmt *sql.Stmt + deleteDeviceKeysStmt *sql.Stmt deleteAllDeviceKeysStmt *sql.Stmt } @@ -90,12 +94,20 @@ func NewSqliteDeviceKeysTable(db *sql.DB) (tables.DeviceKeys, error) { if s.selectMaxStreamForUserStmt, err = db.Prepare(selectMaxStreamForUserSQL); err != nil { return nil, err } + if s.deleteDeviceKeysStmt, err = db.Prepare(deleteDeviceKeysSQL); err != nil { + return nil, err + } if s.deleteAllDeviceKeysStmt, err = db.Prepare(deleteAllDeviceKeysSQL); err != nil { return nil, err } return s, nil } +func (s *deviceKeysStatements) DeleteDeviceKeys(ctx context.Context, txn *sql.Tx, userID, deviceID string) error { + _, err := sqlutil.TxStmt(txn, s.deleteDeviceKeysStmt).ExecContext(ctx, userID, deviceID) + return err +} + func (s *deviceKeysStatements) DeleteAllDeviceKeys(ctx context.Context, txn *sql.Tx, userID string) error { _, err := sqlutil.TxStmt(txn, s.deleteAllDeviceKeysStmt).ExecContext(ctx, userID) return err diff --git a/keyserver/storage/tables/interface.go b/keyserver/storage/tables/interface.go index 0649b680..612eeb86 100644 --- a/keyserver/storage/tables/interface.go +++ b/keyserver/storage/tables/interface.go @@ -39,6 +39,7 @@ type DeviceKeys interface { SelectMaxStreamIDForUser(ctx context.Context, txn *sql.Tx, userID string) (streamID int32, err error) CountStreamIDsForUser(ctx context.Context, userID string, streamIDs []int64) (int, error) SelectBatchDeviceKeys(ctx context.Context, userID string, deviceIDs []string) ([]api.DeviceMessage, error) + DeleteDeviceKeys(ctx context.Context, txn *sql.Tx, userID, deviceID string) error DeleteAllDeviceKeys(ctx context.Context, txn *sql.Tx, userID string) error } @@ -62,4 +63,5 @@ type CrossSigningKeys interface { type CrossSigningSigs interface { SelectCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (r types.CrossSigningSigMap, err error) UpsertCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error + DeleteCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID) error } diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index 0c567a96..e52e5556 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -33,6 +33,8 @@ func (k *mockKeyAPI) SetUserAPI(i userapi.UserInternalAPI) {} // PerformClaimKeys claims one-time keys for use in pre-key messages func (k *mockKeyAPI) PerformClaimKeys(ctx context.Context, req *keyapi.PerformClaimKeysRequest, res *keyapi.PerformClaimKeysResponse) { } +func (k *mockKeyAPI) PerformDeleteKeys(ctx context.Context, req *keyapi.PerformDeleteKeysRequest, res *keyapi.PerformDeleteKeysResponse) { +} func (k *mockKeyAPI) PerformUploadDeviceKeys(ctx context.Context, req *keyapi.PerformUploadDeviceKeysRequest, res *keyapi.PerformUploadDeviceKeysResponse) { } func (k *mockKeyAPI) PerformUploadDeviceSignatures(ctx context.Context, req *keyapi.PerformUploadDeviceSignaturesRequest, res *keyapi.PerformUploadDeviceSignaturesResponse) { diff --git a/userapi/internal/api.go b/userapi/internal/api.go index a2bc8ecf..518edef4 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -145,6 +145,18 @@ func (a *UserInternalAPI) PerformDeviceDeletion(ctx context.Context, req *api.Pe if err != nil { return err } + // Ask the keyserver to delete device keys and signatures for those devices + deleteReq := &keyapi.PerformDeleteKeysRequest{ + UserID: req.UserID, + } + for _, keyID := range req.DeviceIDs { + deleteReq.KeyIDs = append(deleteReq.KeyIDs, gomatrixserverlib.KeyID(keyID)) + } + deleteRes := &keyapi.PerformDeleteKeysResponse{} + a.KeyAPI.PerformDeleteKeys(ctx, deleteReq, deleteRes) + if err := deleteRes.Error; err != nil { + return fmt.Errorf("a.KeyAPI.PerformDeleteKeys: %w", err) + } // create empty device keys and upload them to delete what was once there and trigger device list changes return a.deviceListUpdate(req.UserID, deletedDeviceIDs) } From fe5050c0a5b1ef3dddde294d64ab24d8ba26e53f Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 18 Aug 2021 04:41:30 -0700 Subject: [PATCH 29/64] Add and use M_ROOM_IN_USE (#1972) Signed-off-by: nivekuil Co-authored-by: Neil Alexander --- clientapi/jsonerror/jsonerror.go | 6 ++++++ clientapi/routing/createroom.go | 10 ++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go index 7accde5f..caa216e6 100644 --- a/clientapi/jsonerror/jsonerror.go +++ b/clientapi/jsonerror/jsonerror.go @@ -111,6 +111,12 @@ func UserInUse(msg string) *MatrixError { return &MatrixError{"M_USER_IN_USE", msg} } +// RoomInUse is an error returned when the client tries to make a room +// that already exists +func RoomInUse(msg string) *MatrixError { + return &MatrixError{"M_ROOM_IN_USE", msg} +} + // ASExclusive is an error returned when an application service tries to // register an username that is outside of its registered namespace, or if a // user attempts to register a username or room alias within an exclusive diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index b3b996ec..8f96c3d3 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -325,7 +325,10 @@ func createRoom( return jsonerror.InternalServerError() } if aliasResp.RoomID != "" { - return util.MessageResponse(400, "Alias already exists") + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.RoomInUse("Room ID already exists."), + } } aliasEvent = &fledglingEvent{ @@ -484,7 +487,10 @@ func createRoom( } if aliasResp.AliasExists { - return util.MessageResponse(400, "Alias already exists") + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.RoomInUse("Room alias already exists."), + } } } From cf42ae60646343da28e55650383745d09298b0b9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 18 Aug 2021 14:13:38 +0100 Subject: [PATCH 30/64] Build Docker images using Go 1.17 --- build/docker/Dockerfile.monolith | 2 +- build/docker/Dockerfile.polylith | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith index e59fd087..7fd25674 100644 --- a/build/docker/Dockerfile.monolith +++ b/build/docker/Dockerfile.monolith @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.15-alpine AS base +FROM docker.io/golang:1.17-alpine AS base RUN apk --update --no-cache add bash build-base diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith index 5cb53bbd..819926c4 100644 --- a/build/docker/Dockerfile.polylith +++ b/build/docker/Dockerfile.polylith @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.15-alpine AS base +FROM docker.io/golang:1.17-alpine AS base RUN apk --update --no-cache add bash build-base From cd3823308c3d263bc7d4618f430f4bf9fdda43f5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 18 Aug 2021 14:29:45 +0100 Subject: [PATCH 31/64] Version 0.5.0rc1 --- internal/version.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/version.go b/internal/version.go index 55997ffc..6ba1f13a 100644 --- a/internal/version.go +++ b/internal/version.go @@ -16,9 +16,9 @@ var build string const ( VersionMajor = 0 - VersionMinor = 4 - VersionPatch = 1 - VersionTag = "" // example: "rc1" + VersionMinor = 5 + VersionPatch = 0 + VersionTag = "rc1" // example: "rc1" ) func VersionString() string { From 9a2dcd6078a8559ff4d09c0a5a27521e1ea2c6c4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 19 Aug 2021 15:02:50 +0100 Subject: [PATCH 32/64] Update to matrix-org/pinecone@21e3b392289b9412aa0d290f9f03fe9a1f8909e9 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a1832854..69f7bf4e 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 - github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b + github.com/matrix-org/pinecone v0.0.0-20210819133928-21e3b392289b github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.8 github.com/morikuni/aec v1.0.0 // indirect diff --git a/go.sum b/go.sum index 7f3be4c0..44dc73e6 100644 --- a/go.sum +++ b/go.sum @@ -998,8 +998,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 h1:b8 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b h1:5X5vdWQ13xrNkJVqaJHPsrt7rKkMJH5iac0EtfOuxSg= -github.com/matrix-org/pinecone v0.0.0-20210623102758-74f885644c1b/go.mod h1:CVlrvs1R5iz7Omy2GqAjJJKbACn07GZgUq1Gli18FYE= +github.com/matrix-org/pinecone v0.0.0-20210819133928-21e3b392289b h1:l2R+uSQlNeuHG+JRTBrzQOfY1W49sP1mOQCfTkGYc6E= +github.com/matrix-org/pinecone v0.0.0-20210819133928-21e3b392289b/go.mod h1:CVlrvs1R5iz7Omy2GqAjJJKbACn07GZgUq1Gli18FYE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 329c9e671c763ce3f7adc8c734246eb53b47df94 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 19 Aug 2021 16:07:28 +0100 Subject: [PATCH 33/64] Update to matrix-org/pinecone@e692df1 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 69f7bf4e..7f883bea 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/h2non/filetype v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 github.com/juju/testing v0.0.0-20210324180055-18c50b0c2098 // indirect - github.com/lib/pq v1.10.2 + github.com/lib/pq v1.10.1 github.com/libp2p/go-libp2p v0.13.0 github.com/libp2p/go-libp2p-circuit v0.4.0 github.com/libp2p/go-libp2p-core v0.8.3 @@ -33,7 +33,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 - github.com/matrix-org/pinecone v0.0.0-20210819133928-21e3b392289b + github.com/matrix-org/pinecone v0.0.0-20210819150600-e692df1a5c42 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.8 github.com/morikuni/aec v1.0.0 // indirect diff --git a/go.sum b/go.sum index 44dc73e6..65cf1465 100644 --- a/go.sum +++ b/go.sum @@ -746,8 +746,8 @@ github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvf github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= -github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo= +github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2 h1:7cWK5cdA5x72jX0g8iLrQWm5TRJZ6CzGdPEhWj7plWU= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= @@ -998,8 +998,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 h1:b8 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20210819133928-21e3b392289b h1:l2R+uSQlNeuHG+JRTBrzQOfY1W49sP1mOQCfTkGYc6E= -github.com/matrix-org/pinecone v0.0.0-20210819133928-21e3b392289b/go.mod h1:CVlrvs1R5iz7Omy2GqAjJJKbACn07GZgUq1Gli18FYE= +github.com/matrix-org/pinecone v0.0.0-20210819150600-e692df1a5c42 h1:ZO39w5Kbq9Aw3uHHT5QjFR2kpVHxuJxHD3zOhmU5BVI= +github.com/matrix-org/pinecone v0.0.0-20210819150600-e692df1a5c42/go.mod h1:CVlrvs1R5iz7Omy2GqAjJJKbACn07GZgUq1Gli18FYE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 4b0cc32af29a6292ea11b93800effb26370613c1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 24 Aug 2021 11:44:03 +0100 Subject: [PATCH 34/64] Hopefully fix UIA for cross-signing setup (closes #1983) (#1986) --- clientapi/routing/key_crosssigning.go | 52 +++++++++++++++++---------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/clientapi/routing/key_crosssigning.go b/clientapi/routing/key_crosssigning.go index 756598db..7b9d8acd 100644 --- a/clientapi/routing/key_crosssigning.go +++ b/clientapi/routing/key_crosssigning.go @@ -15,11 +15,10 @@ package routing import ( - "encoding/json" - "io/ioutil" "net/http" "github.com/matrix-org/dendrite/clientapi/auth" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/keyserver/api" @@ -29,37 +28,52 @@ import ( "github.com/matrix-org/util" ) +type crossSigningRequest struct { + api.PerformUploadDeviceKeysRequest + Auth newPasswordAuth `json:"auth"` +} + func UploadCrossSigningDeviceKeys( req *http.Request, userInteractiveAuth *auth.UserInteractive, keyserverAPI api.KeyInternalAPI, device *userapi.Device, accountDB accounts.Database, cfg *config.ClientAPI, ) util.JSONResponse { - uploadReq := &api.PerformUploadDeviceKeysRequest{} + uploadReq := &crossSigningRequest{} uploadRes := &api.PerformUploadDeviceKeysResponse{} - ctx := req.Context() - defer req.Body.Close() // nolint:errcheck - bodyBytes, err := ioutil.ReadAll(req.Body) - if err != nil { + resErr := httputil.UnmarshalJSONRequest(req, &uploadReq) + if resErr != nil { + return *resErr + } + sessionID := uploadReq.Auth.Session + if sessionID == "" { + sessionID = util.RandomString(sessionIDLength) + } + if uploadReq.Auth.Type != authtypes.LoginTypePassword { return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()), + Code: http.StatusUnauthorized, + JSON: newUserInteractiveResponse( + sessionID, + []authtypes.Flow{ + { + Stages: []authtypes.LoginType{authtypes.LoginTypePassword}, + }, + }, + nil, + ), } } - - if _, err := userInteractiveAuth.Verify(ctx, bodyBytes, device); err != nil { - return *err + typePassword := auth.LoginTypePassword{ + GetAccountByPassword: accountDB.GetAccountByPassword, + Config: cfg, } - - if err = json.Unmarshal(bodyBytes, &uploadReq); err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("The request body could not be unmarshalled: " + err.Error()), - } + if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil { + return *authErr } + AddCompletedSessionStage(sessionID, authtypes.LoginTypePassword) uploadReq.UserID = device.UserID - keyserverAPI.PerformUploadDeviceKeys(req.Context(), uploadReq, uploadRes) + keyserverAPI.PerformUploadDeviceKeys(req.Context(), &uploadReq.PerformUploadDeviceKeysRequest, uploadRes) if err := uploadRes.Error; err != nil { switch { From 037ff4fb2364cbee7a3a9a856cd2a711f5398682 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 24 Aug 2021 15:03:28 +0100 Subject: [PATCH 35/64] Version 0.5.0 --- CHANGES.md | 16 ++++++++++++++++ internal/version.go | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8207d484..3e0db8c3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,21 @@ # Changelog +## Dendrite 0.5.0 (2021-08-24) + +### Features + +* Support for serverside key backups has been added, allowing your E2EE keys to be backed up and to be restored after logging out or when logging in from a new device +* Experimental support for cross-signing has been added, allowing verifying your own device keys and verifying other user's public keys +* Dendrite can now send logs to a TCP syslog server by using the `syslog` logger type (contributed by [sambhavsaggi](https://github.com/sambhavsaggi)) +* Go 1.15 is now the minimum supported version for Dendrite + +### Fixes + +* Device keys are now cleaned up from the keyserver when the user API removes a device session +* The `M_ROOM_IN_USE` error code is now returned when a room alias is already taken (contributed by [nivekuil](https://github.com/nivekuil)) +* A bug in the state storage migration has been fixed where room create events had incorrect state snapshots +* A bug when deactivating accounts caused by only reading the deprecated username field has been fixed + ## Dendrite 0.4.1 (2021-07-26) ### Features diff --git a/internal/version.go b/internal/version.go index 6ba1f13a..cdda60e2 100644 --- a/internal/version.go +++ b/internal/version.go @@ -18,7 +18,7 @@ const ( VersionMajor = 0 VersionMinor = 5 VersionPatch = 0 - VersionTag = "rc1" // example: "rc1" + VersionTag = "" // example: "rc1" ) func VersionString() string { From 2dd5fd1fd676c570a13c2ed062fa47a026ca826f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 27 Aug 2021 15:48:27 +0100 Subject: [PATCH 36/64] publicRooms should accept POST as well as GET (#1991) --- federationapi/routing/routing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 8f33c766..503e512e 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -449,7 +449,7 @@ func Setup( httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { return GetPostPublicRooms(req, rsAPI) }), - ).Methods(http.MethodGet) + ).Methods(http.MethodGet, http.MethodPost) v1fedmux.Handle("/user/keys/claim", httputil.MakeFedAPI( "federation_keys_claim", cfg.Matrix.ServerName, keys, wakeup, From 51b119107c972007f5a1bc626758a6229ece14db Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 27 Aug 2021 16:50:30 +0100 Subject: [PATCH 37/64] Don't return nonsense canonical room aliases in the public rooms responses (#1992) --- federationapi/routing/publicrooms.go | 4 +++- roomserver/api/wrapper.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/federationapi/routing/publicrooms.go b/federationapi/routing/publicrooms.go index ddd92c5c..5b9be880 100644 --- a/federationapi/routing/publicrooms.go +++ b/federationapi/routing/publicrooms.go @@ -156,7 +156,9 @@ func fillInRooms(ctx context.Context, roomIDs []string, rsAPI roomserverAPI.Room case topicTuple: pub.Topic = contentVal case canonicalTuple: - pub.CanonicalAlias = contentVal + if _, _, err := gomatrixserverlib.SplitID('#', contentVal); err == nil { + pub.CanonicalAlias = contentVal + } case visibilityTuple: pub.WorldReadable = contentVal == "world_readable" // need both of these to determine whether guests can join diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go index 2ebe2f64..de66df80 100644 --- a/roomserver/api/wrapper.go +++ b/roomserver/api/wrapper.go @@ -215,7 +215,9 @@ func PopulatePublicRooms(ctx context.Context, roomIDs []string, rsAPI Roomserver case topicTuple: pub.Topic = contentVal case canonicalTuple: - pub.CanonicalAlias = contentVal + if _, _, err := gomatrixserverlib.SplitID('#', contentVal); err == nil { + pub.CanonicalAlias = contentVal + } case visibilityTuple: pub.WorldReadable = contentVal == "world_readable" // need both of these to determine whether guests can join From 7dc8fb1fe780888d5758bf665949e9e09d21e56a Mon Sep 17 00:00:00 2001 From: kegsay Date: Tue, 7 Sep 2021 15:07:14 +0100 Subject: [PATCH 38/64] Add more logs (#2005) * Add more logs To help debug the migration issue in #1924 along with manual data-loss-inducing fixes. Also log the origin server on processed txns to help debug buggy server origins. * Fix query --- federationapi/routing/send.go | 2 +- .../2021041615092700_state_blocks_refactor.go | 44 ++++++++++++++++--- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 2b282826..aacc8a91 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -345,7 +345,7 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res } if c := len(results); c > 0 { - util.GetLogger(ctx).Infof("Processed %d PDUs from transaction %q", c, t.TransactionID) + util.GetLogger(ctx).Infof("Processed %d PDUs from %v in transaction %q", c, t.Origin, t.TransactionID) } return &gomatrixserverlib.RespSend{PDUs: results}, nil } diff --git a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go index 6b72de51..06740dc8 100644 --- a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go +++ b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go @@ -220,7 +220,6 @@ func UpStateBlocksRefactor(tx *sql.Tx) error { index := stateSnapshotData{snapshot.StateSnapshotNID, snapshot.RoomNID} newsnapshots[index] = append(newsnapshots[index], blocknid) } - for snapshotdata, newblocks := range newsnapshots { var newblocksarray pq.Int64Array for _, b := range newblocks { @@ -229,11 +228,11 @@ func UpStateBlocksRefactor(tx *sql.Tx) error { var newNID types.StateSnapshotNID err = tx.QueryRow(` - INSERT INTO roomserver_state_snapshots (state_snapshot_hash, room_nid, state_block_nids) - VALUES ($1, $2, $3) - ON CONFLICT (state_snapshot_hash) DO UPDATE SET room_nid=$2 - RETURNING state_snapshot_nid - `, newblocks.Hash(), snapshotdata.RoomNID, newblocksarray).Scan(&newNID) + INSERT INTO roomserver_state_snapshots (state_snapshot_hash, room_nid, state_block_nids) + VALUES ($1, $2, $3) + ON CONFLICT (state_snapshot_hash) DO UPDATE SET room_nid=$2 + RETURNING state_snapshot_nid + `, newblocks.Hash(), snapshotdata.RoomNID, newblocksarray).Scan(&newNID) if err != nil { return fmt.Errorf("tx.QueryRow.Scan (insert new snapshot): %w", err) } @@ -252,16 +251,49 @@ func UpStateBlocksRefactor(tx *sql.Tx) error { // If we do, this is a problem if Dendrite tries to load the snapshot as it will not exist // in roomserver_state_snapshots var count int64 + if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_events WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil { return fmt.Errorf("assertion query failed: %s", err) } if count > 0 { + var debugEventID, debugRoomID string + var debugEventTypeNID, debugStateKeyNID, debugSnapNID, debugDepth int64 + err = tx.QueryRow( + `SELECT event_id, event_type_nid, event_state_key_nid, roomserver_events.state_snapshot_nid, depth, room_id FROM roomserver_events + JOIN roomserver_rooms ON roomserver_rooms.room_nid = roomserver_events.room_nid WHERE roomserver_events.state_snapshot_nid < $1 AND roomserver_events.state_snapshot_nid != 0`, maxsnapshotid, + ).Scan(&debugEventID, &debugEventTypeNID, &debugStateKeyNID, &debugSnapNID, &debugDepth, &debugRoomID) + if err != nil { + logrus.Errorf("cannot extract debug info: %v", err) + } else { + logrus.Errorf( + "Affected row: event_id=%v room_id=%v type=%v state_key=%v snapshot=%v depth=%v", + debugEventID, debugRoomID, debugEventTypeNID, debugStateKeyNID, debugSnapNID, debugDepth, + ) + logrus.Errorf("To fix this manually, run this query first then retry the migration: "+ + "UPDATE roomserver_events SET state_snapshot_nid=0 WHERE event_id='%v'", debugEventID) + } return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count) } if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil { return fmt.Errorf("assertion query failed: %s", err) } if count > 0 { + var debugRoomID string + var debugSnapNID, debugLastEventNID int64 + err = tx.QueryRow( + `SELECT room_id, state_snapshot_nid, last_event_sent_nid FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid, + ).Scan(&debugRoomID, &debugSnapNID, &debugLastEventNID) + if err != nil { + logrus.Errorf("cannot extract debug info: %v", err) + } else { + logrus.Errorf( + "Affected row: room_id=%v snapshot=%v last_sent=%v", + debugRoomID, debugSnapNID, debugLastEventNID, + ) + logrus.Errorf("To fix this manually, run this query first then retry the migration: "+ + "UPDATE roomserver_rooms SET state_snapshot_nid=0 WHERE room_id='%v'", debugRoomID) + logrus.Errorf("Running this UPDATE will cause the room in question to become unavailable on this server. Leave and re-join the room afterwards.") + } return fmt.Errorf("%d rooms exist in roomserver_rooms which have not been converted to a new state_snapshot_nid; this is a bug, please report", count) } From a624eab309ee6ee7b3540dfba884a42c9a4f4ad9 Mon Sep 17 00:00:00 2001 From: Ryan W Date: Wed, 8 Sep 2021 17:31:03 +0100 Subject: [PATCH 39/64] - Removed double imports (#1989) - Lower cased error messages Signed-off-by: Ryan Whittington Co-authored-by: kegsay --- build/gobind-pinecone/monolith.go | 3 +- clientapi/httputil/parse.go | 2 +- clientapi/routing/device.go | 15 ++-- clientapi/routing/directory.go | 5 +- clientapi/routing/logout.go | 13 ++-- clientapi/routing/membership.go | 23 +++--- clientapi/routing/password.go | 11 ++- clientapi/routing/redaction.go | 5 +- clientapi/routing/register_secret.go | 8 +- clientapi/threepid/threepid.go | 4 +- clientapi/userutil/userutil.go | 4 +- cmd/dendrite-demo-pinecone/main.go | 3 +- cmd/dendrite-upgrade-tests/main.go | 2 +- federationapi/routing/send.go | 2 +- federationsender/consumers/keychange.go | 9 +-- federationsender/internal/perform.go | 4 +- federationsender/queue/destinationqueue.go | 23 +++--- internal/eventutil/events.go | 2 +- internal/httputil/http.go | 4 +- internal/sqlutil/migrate.go | 10 +-- internal/sqlutil/sql.go | 2 +- mediaapi/fileutils/fileutils.go | 24 +++--- roomserver/internal/perform/perform_invite.go | 2 +- roomserver/internal/perform/perform_join.go | 75 +++++++++---------- roomserver/internal/perform/perform_leave.go | 18 ++--- roomserver/internal/perform/perform_peek.go | 8 +- roomserver/state/state.go | 12 +-- roomserver/storage/shared/storage.go | 2 +- setup/mscs/msc2836/msc2836.go | 2 +- setup/mscs/msc2946/msc2946.go | 2 +- syncapi/consumers/keychange.go | 5 +- syncapi/internal/keychange.go | 9 +-- syncapi/storage/shared/syncserver.go | 7 +- syncapi/types/types.go | 4 +- userapi/internal/api.go | 8 +- userapi/storage/accounts/interface.go | 2 +- userapi/storage/accounts/postgres/storage.go | 3 +- userapi/storage/accounts/sqlite3/storage.go | 2 +- 38 files changed, 162 insertions(+), 177 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 20228404..310e3a04 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -40,7 +40,6 @@ import ( "golang.org/x/net/http2/h2c" pineconeMulticast "github.com/matrix-org/pinecone/multicast" - "github.com/matrix-org/pinecone/router" pineconeRouter "github.com/matrix-org/pinecone/router" pineconeSessions "github.com/matrix-org/pinecone/sessions" "github.com/matrix-org/pinecone/types" @@ -196,7 +195,7 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e func (m *DendriteMonolith) staticPeerConnect() { attempt := func() { - if m.PineconeRouter.PeerCount(router.PeerTypeRemote) == 0 { + if m.PineconeRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 { m.staticPeerMutex.RLock() uri := m.staticPeerURI m.staticPeerMutex.RUnlock() diff --git a/clientapi/httputil/parse.go b/clientapi/httputil/parse.go index ee603341..c8358334 100644 --- a/clientapi/httputil/parse.go +++ b/clientapi/httputil/parse.go @@ -32,7 +32,7 @@ func ParseTSParam(req *http.Request) (time.Time, error) { // The parameter exists, parse into a Time object ts, err := strconv.ParseInt(tsStr, 10, 64) if err != nil { - return time.Time{}, fmt.Errorf("Param 'ts' is no valid int (%s)", err.Error()) + return time.Time{}, fmt.Errorf("param 'ts' is no valid int (%s)", err.Error()) } return time.Unix(ts/1000, 0), nil diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index 6adaa769..9f54a625 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/userapi/api" - userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -50,18 +49,18 @@ type devicesDeleteJSON struct { // GetDeviceByID handles /devices/{deviceID} func GetDeviceByID( - req *http.Request, userAPI userapi.UserInternalAPI, device *api.Device, + req *http.Request, userAPI api.UserInternalAPI, device *api.Device, deviceID string, ) util.JSONResponse { - var queryRes userapi.QueryDevicesResponse - err := userAPI.QueryDevices(req.Context(), &userapi.QueryDevicesRequest{ + var queryRes api.QueryDevicesResponse + err := userAPI.QueryDevices(req.Context(), &api.QueryDevicesRequest{ UserID: device.UserID, }, &queryRes) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("QueryDevices failed") return jsonerror.InternalServerError() } - var targetDevice *userapi.Device + var targetDevice *api.Device for _, device := range queryRes.Devices { if device.ID == deviceID { targetDevice = &device @@ -88,10 +87,10 @@ func GetDeviceByID( // GetDevicesByLocalpart handles /devices func GetDevicesByLocalpart( - req *http.Request, userAPI userapi.UserInternalAPI, device *api.Device, + req *http.Request, userAPI api.UserInternalAPI, device *api.Device, ) util.JSONResponse { - var queryRes userapi.QueryDevicesResponse - err := userAPI.QueryDevices(req.Context(), &userapi.QueryDevicesRequest{ + var queryRes api.QueryDevicesResponse + err := userAPI.QueryDevices(req.Context(), &api.QueryDevicesRequest{ UserID: device.UserID, }, &queryRes) if err != nil { diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index ae466065..96cb0262 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -23,7 +23,6 @@ import ( federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -115,7 +114,7 @@ func DirectoryRoom( // SetLocalAlias implements PUT /directory/room/{roomAlias} func SetLocalAlias( req *http.Request, - device *api.Device, + device *userapi.Device, alias string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, @@ -192,7 +191,7 @@ func SetLocalAlias( // RemoveLocalAlias implements DELETE /directory/room/{roomAlias} func RemoveLocalAlias( req *http.Request, - device *api.Device, + device *userapi.Device, alias string, rsAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { diff --git a/clientapi/routing/logout.go b/clientapi/routing/logout.go index cb300e9f..cfbb6f9f 100644 --- a/clientapi/routing/logout.go +++ b/clientapi/routing/logout.go @@ -19,16 +19,15 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/userapi/api" - userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" ) // Logout handles POST /logout func Logout( - req *http.Request, userAPI userapi.UserInternalAPI, device *api.Device, + req *http.Request, userAPI api.UserInternalAPI, device *api.Device, ) util.JSONResponse { - var performRes userapi.PerformDeviceDeletionResponse - err := userAPI.PerformDeviceDeletion(req.Context(), &userapi.PerformDeviceDeletionRequest{ + var performRes api.PerformDeviceDeletionResponse + err := userAPI.PerformDeviceDeletion(req.Context(), &api.PerformDeviceDeletionRequest{ UserID: device.UserID, DeviceIDs: []string{device.ID}, }, &performRes) @@ -45,10 +44,10 @@ func Logout( // LogoutAll handles POST /logout/all func LogoutAll( - req *http.Request, userAPI userapi.UserInternalAPI, device *api.Device, + req *http.Request, userAPI api.UserInternalAPI, device *api.Device, ) util.JSONResponse { - var performRes userapi.PerformDeviceDeletionResponse - err := userAPI.PerformDeviceDeletion(req.Context(), &userapi.PerformDeviceDeletionRequest{ + var performRes api.PerformDeviceDeletionResponse + err := userAPI.PerformDeviceDeletion(req.Context(), &api.PerformDeviceDeletionRequest{ UserID: device.UserID, DeviceIDs: nil, }, &performRes) diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index b85cfde0..33fb3883 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -26,7 +26,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/internal/eventutil" - "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -107,7 +106,7 @@ func sendMembership(ctx context.Context, accountDB accounts.Database, device *us if err = roomserverAPI.SendEvents( ctx, rsAPI, - api.KindNew, + roomserverAPI.KindNew, []*gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)}, cfg.Matrix.ServerName, nil, @@ -328,11 +327,11 @@ func loadProfile( return profile, err } -func extractRequestData(req *http.Request, roomID string, rsAPI api.RoomserverInternalAPI) ( +func extractRequestData(req *http.Request, roomID string, rsAPI roomserverAPI.RoomserverInternalAPI) ( body *threepid.MembershipRequest, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, resErr *util.JSONResponse, ) { - verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} - verRes := api.QueryRoomVersionForRoomResponse{} + verReq := roomserverAPI.QueryRoomVersionForRoomRequest{RoomID: roomID} + verRes := roomserverAPI.QueryRoomVersionForRoomResponse{} if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { resErr = &util.JSONResponse{ Code: http.StatusBadRequest, @@ -402,13 +401,13 @@ func checkAndProcessThreepid( return } -func checkMemberInRoom(ctx context.Context, rsAPI api.RoomserverInternalAPI, userID, roomID string) *util.JSONResponse { +func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, userID, roomID string) *util.JSONResponse { tuple := gomatrixserverlib.StateKeyTuple{ EventType: gomatrixserverlib.MRoomMember, StateKey: userID, } - var membershipRes api.QueryCurrentStateResponse - err := rsAPI.QueryCurrentState(ctx, &api.QueryCurrentStateRequest{ + var membershipRes roomserverAPI.QueryCurrentStateResponse + err := rsAPI.QueryCurrentState(ctx, &roomserverAPI.QueryCurrentStateRequest{ RoomID: roomID, StateTuples: []gomatrixserverlib.StateKeyTuple{tuple}, }, &membershipRes) @@ -445,8 +444,8 @@ func SendForget( ) util.JSONResponse { ctx := req.Context() logger := util.GetLogger(ctx).WithField("roomID", roomID).WithField("userID", device.UserID) - var membershipRes api.QueryMembershipForUserResponse - membershipReq := api.QueryMembershipForUserRequest{ + var membershipRes roomserverAPI.QueryMembershipForUserResponse + membershipReq := roomserverAPI.QueryMembershipForUserRequest{ RoomID: roomID, UserID: device.UserID, } @@ -468,11 +467,11 @@ func SendForget( } } - request := api.PerformForgetRequest{ + request := roomserverAPI.PerformForgetRequest{ RoomID: roomID, UserID: device.UserID, } - response := api.PerformForgetResponse{} + response := roomserverAPI.PerformForgetResponse{} if err := rsAPI.PerformForget(ctx, &request, &response); err != nil { logger.WithError(err).Error("PerformForget: unable to forget room") return jsonerror.InternalServerError() diff --git a/clientapi/routing/password.go b/clientapi/routing/password.go index 87d5f8ff..b2442443 100644 --- a/clientapi/routing/password.go +++ b/clientapi/routing/password.go @@ -9,7 +9,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/api" - userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -29,7 +28,7 @@ type newPasswordAuth struct { func Password( req *http.Request, - userAPI userapi.UserInternalAPI, + userAPI api.UserInternalAPI, accountDB accounts.Database, device *api.Device, cfg *config.ClientAPI, @@ -90,11 +89,11 @@ func Password( } // Ask the user API to perform the password change. - passwordReq := &userapi.PerformPasswordUpdateRequest{ + passwordReq := &api.PerformPasswordUpdateRequest{ Localpart: localpart, Password: r.NewPassword, } - passwordRes := &userapi.PerformPasswordUpdateResponse{} + passwordRes := &api.PerformPasswordUpdateResponse{} if err := userAPI.PerformPasswordUpdate(req.Context(), passwordReq, passwordRes); err != nil { util.GetLogger(req.Context()).WithError(err).Error("PerformPasswordUpdate failed") return jsonerror.InternalServerError() @@ -107,12 +106,12 @@ func Password( // If the request asks us to log out all other devices then // ask the user API to do that. if r.LogoutDevices { - logoutReq := &userapi.PerformDeviceDeletionRequest{ + logoutReq := &api.PerformDeviceDeletionRequest{ UserID: device.UserID, DeviceIDs: nil, ExceptDeviceID: device.ID, } - logoutRes := &userapi.PerformDeviceDeletionResponse{} + logoutRes := &api.PerformDeviceDeletionResponse{} if err := userAPI.PerformDeviceDeletion(req.Context(), logoutReq, logoutRes); err != nil { util.GetLogger(req.Context()).WithError(err).Error("PerformDeviceDeletion failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index 92375974..c25ca4ef 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -22,7 +22,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/eventutil" - "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -113,7 +112,7 @@ func SendRedaction( return jsonerror.InternalServerError() } - var queryRes api.QueryLatestEventsAndStateResponse + var queryRes roomserverAPI.QueryLatestEventsAndStateResponse e, err := eventutil.QueryAndBuildEvent(req.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes) if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ @@ -121,7 +120,7 @@ func SendRedaction( JSON: jsonerror.NotFound("Room does not exist"), } } - if err = roomserverAPI.SendEvents(context.Background(), rsAPI, api.KindNew, []*gomatrixserverlib.HeaderedEvent{e}, cfg.Matrix.ServerName, nil); err != nil { + if err = roomserverAPI.SendEvents(context.Background(), rsAPI, roomserverAPI.KindNew, []*gomatrixserverlib.HeaderedEvent{e}, cfg.Matrix.ServerName, nil); err != nil { util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/register_secret.go b/clientapi/routing/register_secret.go index f0436e32..1a974b77 100644 --- a/clientapi/routing/register_secret.go +++ b/clientapi/routing/register_secret.go @@ -68,18 +68,18 @@ func (r *SharedSecretRegistration) IsValidMacLogin( ) (bool, error) { // Check that shared secret registration isn't disabled. if r.sharedSecret == "" { - return false, errors.New("Shared secret registration is disabled") + return false, errors.New("shared secret registration is disabled") } if !r.validNonce(nonce) { - return false, fmt.Errorf("Incorrect or expired nonce: %s", nonce) + return false, fmt.Errorf("incorrect or expired nonce: %s", nonce) } // Check that username/password don't contain the HMAC delimiters. if strings.Contains(username, "\x00") { - return false, errors.New("Username contains invalid character") + return false, errors.New("username contains invalid character") } if strings.Contains(password, "\x00") { - return false, errors.New("Password contains invalid character") + return false, errors.New("password contains invalid character") } adminString := "notadmin" diff --git a/clientapi/threepid/threepid.go b/clientapi/threepid/threepid.go index 2f817ef4..1e64e303 100644 --- a/clientapi/threepid/threepid.go +++ b/clientapi/threepid/threepid.go @@ -81,7 +81,7 @@ func CreateSession( // Error if the status isn't OK if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("Could not create a session on the server %s", req.IDServer) + return "", fmt.Errorf("could not create a session on the server %s", req.IDServer) } // Extract the SID from the response and return it @@ -168,7 +168,7 @@ func PublishAssociation(creds Credentials, userID string, cfg *config.ClientAPI) // Error if the status isn't OK if resp.StatusCode != http.StatusOK { - return fmt.Errorf("Could not publish the association on the server %s", creds.IDServer) + return fmt.Errorf("could not publish the association on the server %s", creds.IDServer) } return nil diff --git a/clientapi/userutil/userutil.go b/clientapi/userutil/userutil.go index 4cea3c18..7e909ffa 100644 --- a/clientapi/userutil/userutil.go +++ b/clientapi/userutil/userutil.go @@ -31,11 +31,11 @@ func ParseUsernameParam(usernameParam string, expectedServerName *gomatrixserver lp, domain, err := gomatrixserverlib.SplitID('@', usernameParam) if err != nil { - return "", errors.New("Invalid username") + return "", errors.New("invalid username") } if expectedServerName != nil && domain != *expectedServerName { - return "", errors.New("User ID does not belong to this server") + return "", errors.New("user ID does not belong to this server") } localpart = lp diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 1933c5ad..e919e749 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -49,7 +49,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" pineconeMulticast "github.com/matrix-org/pinecone/multicast" - "github.com/matrix-org/pinecone/router" pineconeRouter "github.com/matrix-org/pinecone/router" pineconeSessions "github.com/matrix-org/pinecone/sessions" @@ -125,7 +124,7 @@ func main() { connectToStaticPeer := func() { attempt := func() { - if pRouter.PeerCount(router.PeerTypeRemote) == 0 { + if pRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 { uri := *instancePeer if uri == "" { return diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index a6cc2d3f..aa8c7fdc 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -148,7 +148,7 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, // add top level Dockerfile err = ioutil.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm) if err != nil { - return "", fmt.Errorf("Custom HEAD: failed to inject /Dockerfile: %w", err) + return "", fmt.Errorf("custom HEAD: failed to inject /Dockerfile: %w", err) } // now tarball it var buffer bytes.Buffer diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index aacc8a91..d31d38dc 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -711,7 +711,7 @@ withNextEvent: } if missing := len(missingAuthEvents); missing > 0 { - return fmt.Errorf("Event refers to %d auth_events which we failed to fetch", missing) + return fmt.Errorf("event refers to %d auth_events which we failed to fetch", missing) } return nil } diff --git a/federationsender/consumers/keychange.go b/federationsender/consumers/keychange.go index 675328df..a9f6d0f8 100644 --- a/federationsender/consumers/keychange.go +++ b/federationsender/consumers/keychange.go @@ -30,7 +30,6 @@ import ( "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" - log "github.com/sirupsen/logrus" ) // KeyChangeConsumer consumes events that originate in key server. @@ -82,7 +81,7 @@ func (t *KeyChangeConsumer) Start() error { func (t *KeyChangeConsumer) onMessage(msg *sarama.ConsumerMessage) error { var m api.DeviceMessage if err := json.Unmarshal(msg.Value, &m); err != nil { - log.WithError(err).Errorf("failed to read device message from key change topic") + logrus.WithError(err).Errorf("failed to read device message from key change topic") return nil } switch m.Type { @@ -96,7 +95,7 @@ func (t *KeyChangeConsumer) onMessage(msg *sarama.ConsumerMessage) error { } func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) error { - logger := log.WithField("user_id", m.UserID) + logger := logrus.WithField("user_id", m.UserID) // only send key change events which originated from us _, originServerName, err := gomatrixserverlib.SplitID('@', m.UserID) @@ -142,7 +141,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) error { return err } - log.Infof("Sending device list update message to %q", destinations) + logrus.Infof("Sending device list update message to %q", destinations) return t.queues.SendEDU(edu, t.serverName, destinations) } @@ -158,7 +157,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) error { // end up parroting information we received from other servers. return nil } - logger := log.WithField("user_id", output.UserID) + logger := logrus.WithField("user_id", output.UserID) var queryRes roomserverAPI.QueryRoomsForUserResponse err = t.rsAPI.QueryRoomsForUser(context.Background(), &roomserverAPI.QueryRoomsForUserRequest{ diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 968df247..2df936e3 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -409,7 +409,7 @@ func (r *FederationSenderInternalAPI) performOutboundPeekUsingServer( return fmt.Errorf("sanityCheckAuthChain: %w", err) } if err = respState.Check(ctx, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName)); err != nil { - return fmt.Errorf("Error checking state returned from peeking: %w", err) + return fmt.Errorf("error checking state returned from peeking: %w", err) } // If we've got this far, the remote server is peeking. @@ -523,7 +523,7 @@ func (r *FederationSenderInternalAPI) PerformLeave( // If we reach here then we didn't complete a leave for some reason. return fmt.Errorf( - "Failed to leave room %q through %d server(s)", + "failed to leave room %q through %d server(s)", request.RoomID, len(request.ServerNames), ) } diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index 33f77a42..0123fa24 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -29,7 +29,6 @@ import ( "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" - log "github.com/sirupsen/logrus" "go.uber.org/atomic" ) @@ -72,7 +71,7 @@ type destinationQueue struct { // start sending events to that destination. func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, receipt *shared.Receipt) { if event == nil { - log.Errorf("attempt to send nil PDU with destination %q", oq.destination) + logrus.Errorf("attempt to send nil PDU with destination %q", oq.destination) return } // Create a database entry that associates the given PDU NID with @@ -84,7 +83,7 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re oq.destination, // the destination server name receipt, // NIDs from federationsender_queue_json table ); err != nil { - log.WithError(err).Errorf("failed to associate PDU %q with destination %q", event.EventID(), oq.destination) + logrus.WithError(err).Errorf("failed to associate PDU %q with destination %q", event.EventID(), oq.destination) return } // Check if the destination is blacklisted. If it isn't then wake @@ -116,7 +115,7 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re // start sending events to that destination. func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *shared.Receipt) { if event == nil { - log.Errorf("attempt to send nil EDU with destination %q", oq.destination) + logrus.Errorf("attempt to send nil EDU with destination %q", oq.destination) return } // Create a database entry that associates the given PDU NID with @@ -127,7 +126,7 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share oq.destination, // the destination server name receipt, // NIDs from federationsender_queue_json table ); err != nil { - log.WithError(err).Errorf("failed to associate EDU with destination %q", oq.destination) + logrus.WithError(err).Errorf("failed to associate EDU with destination %q", oq.destination) return } // Check if the destination is blacklisted. If it isn't then wake @@ -281,7 +280,7 @@ func (oq *destinationQueue) backgroundSend() { // It's been suggested that we should give up because the backoff // has exceeded a maximum allowable value. Clean up the in-memory // buffers at this point. The PDU clean-up is already on a defer. - log.Warnf("Blacklisting %q due to exceeding backoff threshold", oq.destination) + logrus.Warnf("Blacklisting %q due to exceeding backoff threshold", oq.destination) oq.pendingMutex.Lock() for i := range oq.pendingPDUs { oq.pendingPDUs[i] = nil @@ -298,7 +297,7 @@ func (oq *destinationQueue) backgroundSend() { // We haven't backed off yet, so wait for the suggested amount of // time. duration := time.Until(*until) - log.Warnf("Backing off %q for %s", oq.destination, duration) + logrus.Warnf("Backing off %q for %s", oq.destination, duration) oq.backingOff.Store(true) destinationQueueBackingOff.Inc() select { @@ -421,13 +420,13 @@ func (oq *destinationQueue) nextTransaction( if pduReceipts != nil { //logrus.Infof("Cleaning PDUs %q", pduReceipt.String()) if err = oq.db.CleanPDUs(context.Background(), oq.destination, pduReceipts); err != nil { - log.WithError(err).Errorf("Failed to clean PDUs for server %q", t.Destination) + logrus.WithError(err).Errorf("Failed to clean PDUs for server %q", t.Destination) } } if eduReceipts != nil { //logrus.Infof("Cleaning EDUs %q", eduReceipt.String()) if err = oq.db.CleanEDUs(context.Background(), oq.destination, eduReceipts); err != nil { - log.WithError(err).Errorf("Failed to clean EDUs for server %q", t.Destination) + logrus.WithError(err).Errorf("Failed to clean EDUs for server %q", t.Destination) } } // Reset the transaction ID. @@ -440,9 +439,9 @@ func (oq *destinationQueue) nextTransaction( // will retry again, subject to backoff. return false, 0, 0, err default: - log.WithFields(log.Fields{ - "destination": oq.destination, - log.ErrorKey: err, + logrus.WithFields(logrus.Fields{ + "destination": oq.destination, + logrus.ErrorKey: err, }).Debugf("Failed to send transaction %q", t.TransactionID) return false, 0, 0, err } diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go index b8691c50..47c83d51 100644 --- a/internal/eventutil/events.go +++ b/internal/eventutil/events.go @@ -28,7 +28,7 @@ import ( // ErrRoomNoExists is returned when trying to lookup the state of a room that // doesn't exist -var ErrRoomNoExists = errors.New("Room does not exist") +var ErrRoomNoExists = errors.New("room does not exist") // QueryAndBuildEvent builds a Matrix event using the event builder and roomserver query // API client provided. If also fills roomserver query API response (if provided) diff --git a/internal/httputil/http.go b/internal/httputil/http.go index 9197371a..a469c8ac 100644 --- a/internal/httputil/http.go +++ b/internal/httputil/http.go @@ -73,9 +73,9 @@ func PostJSON( Message string `json:"message"` } if msgerr := json.NewDecoder(res.Body).Decode(&errorBody); msgerr == nil { - return fmt.Errorf("Internal API: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message) + return fmt.Errorf("internal API: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message) } - return fmt.Errorf("Internal API: %d from %s", res.StatusCode, apiURL) + return fmt.Errorf("internal API: %d from %s", res.StatusCode, apiURL) } return json.NewDecoder(res.Body).Decode(response) } diff --git a/internal/sqlutil/migrate.go b/internal/sqlutil/migrate.go index 62b1c8fa..7518df3c 100644 --- a/internal/sqlutil/migrate.go +++ b/internal/sqlutil/migrate.go @@ -46,7 +46,7 @@ func (m *Migrations) RunDeltas(db *sql.DB, props *config.DatabaseOptions) error minVer := int64(0) migrations, err := m.collect(minVer, maxVer) if err != nil { - return fmt.Errorf("RunDeltas: Failed to collect migrations: %w", err) + return fmt.Errorf("runDeltas: Failed to collect migrations: %w", err) } if props.ConnectionString.IsPostgres() { if err = goose.SetDialect("postgres"); err != nil { @@ -57,12 +57,12 @@ func (m *Migrations) RunDeltas(db *sql.DB, props *config.DatabaseOptions) error return err } } else { - return fmt.Errorf("Unknown connection string: %s", props.ConnectionString) + return fmt.Errorf("unknown connection string: %s", props.ConnectionString) } for { current, err := goose.EnsureDBVersion(db) if err != nil { - return fmt.Errorf("RunDeltas: Failed to EnsureDBVersion: %w", err) + return fmt.Errorf("runDeltas: Failed to EnsureDBVersion: %w", err) } next, err := migrations.Next(current) @@ -71,11 +71,11 @@ func (m *Migrations) RunDeltas(db *sql.DB, props *config.DatabaseOptions) error return nil } - return fmt.Errorf("RunDeltas: Failed to load next migration to %+v : %w", next, err) + return fmt.Errorf("runDeltas: Failed to load next migration to %+v : %w", next, err) } if err = next.Up(db); err != nil { - return fmt.Errorf("RunDeltas: Failed run migration: %w", err) + return fmt.Errorf("runDeltas: Failed run migration: %w", err) } } } diff --git a/internal/sqlutil/sql.go b/internal/sqlutil/sql.go index 98cc396a..8d0d2dfa 100644 --- a/internal/sqlutil/sql.go +++ b/internal/sqlutil/sql.go @@ -25,7 +25,7 @@ import ( ) // ErrUserExists is returned if a username already exists in the database. -var ErrUserExists = errors.New("Username already exists") +var ErrUserExists = errors.New("username already exists") // A Transaction is something that can be committed or rolledback. type Transaction interface { diff --git a/mediaapi/fileutils/fileutils.go b/mediaapi/fileutils/fileutils.go index 7309cb88..754e4644 100644 --- a/mediaapi/fileutils/fileutils.go +++ b/mediaapi/fileutils/fileutils.go @@ -37,10 +37,10 @@ import ( // For example, if Base64Hash is 'qwerty', the path will be 'q/w/erty/file'. func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath config.Path) (string, error) { if len(base64Hash) < 3 { - return "", fmt.Errorf("Invalid filePath (Base64Hash too short - min 3 characters): %q", base64Hash) + return "", fmt.Errorf("invalid filePath (Base64Hash too short - min 3 characters): %q", base64Hash) } if len(base64Hash) > 255 { - return "", fmt.Errorf("Invalid filePath (Base64Hash too long - max 255 characters): %q", base64Hash) + return "", fmt.Errorf("invalid filePath (Base64Hash too long - max 255 characters): %q", base64Hash) } filePath, err := filepath.Abs(filepath.Join( @@ -51,14 +51,14 @@ func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath config.Path) "file", )) if err != nil { - return "", fmt.Errorf("Unable to construct filePath: %w", err) + return "", fmt.Errorf("unable to construct filePath: %w", err) } // check if the absolute absBasePath is a prefix of the absolute filePath // if so, no directory escape has occurred and the filePath is valid // Note: absBasePath is already absolute if !strings.HasPrefix(filePath, string(absBasePath)) { - return "", fmt.Errorf("Invalid filePath (not within absBasePath %v): %v", absBasePath, filePath) + return "", fmt.Errorf("invalid filePath (not within absBasePath %v): %v", absBasePath, filePath) } return filePath, nil @@ -102,7 +102,7 @@ func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata func RemoveDir(dir types.Path, logger *log.Entry) { dirErr := os.RemoveAll(string(dir)) if dirErr != nil { - logger.WithError(dirErr).WithField("dir", dir).Warn("Failed to remove directory") + logger.WithError(dirErr).WithField("dir", dir).Warn("failed to remove directory") } } @@ -153,11 +153,11 @@ func moveFile(src types.Path, dst types.Path) error { err := os.MkdirAll(dstDir, 0770) if err != nil { - return fmt.Errorf("Failed to make directory: %w", err) + return fmt.Errorf("failed to make directory: %w", err) } err = os.Rename(string(src), string(dst)) if err != nil { - return fmt.Errorf("Failed to move directory: %w", err) + return fmt.Errorf("failed to move directory: %w", err) } return nil } @@ -165,11 +165,11 @@ func moveFile(src types.Path, dst types.Path) error { func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, types.Path, error) { tmpDir, err := createTempDir(absBasePath) if err != nil { - return nil, nil, "", fmt.Errorf("Failed to create temp dir: %w", err) + return nil, nil, "", fmt.Errorf("failed to create temp dir: %w", err) } writer, tmpFile, err := createFileWriter(tmpDir) if err != nil { - return nil, nil, "", fmt.Errorf("Failed to create file writer: %w", err) + return nil, nil, "", fmt.Errorf("failed to create file writer: %w", err) } return writer, tmpFile, tmpDir, nil } @@ -178,11 +178,11 @@ func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, typ func createTempDir(baseDirectory config.Path) (types.Path, error) { baseTmpDir := filepath.Join(string(baseDirectory), "tmp") if err := os.MkdirAll(baseTmpDir, 0770); err != nil { - return "", fmt.Errorf("Failed to create base temp dir: %w", err) + return "", fmt.Errorf("failed to create base temp dir: %w", err) } tmpDir, err := ioutil.TempDir(baseTmpDir, "") if err != nil { - return "", fmt.Errorf("Failed to create temp dir: %w", err) + return "", fmt.Errorf("failed to create temp dir: %w", err) } return types.Path(tmpDir), nil } @@ -194,7 +194,7 @@ func createFileWriter(directory types.Path) (*bufio.Writer, *os.File, error) { filePath := filepath.Join(string(directory), "content") file, err := os.Create(filePath) if err != nil { - return nil, nil, fmt.Errorf("Failed to create file: %w", err) + return nil, nil, fmt.Errorf("failed to create file: %w", err) } return bufio.NewWriter(file), file, nil diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go index c6ad79d9..024c0930 100644 --- a/roomserver/internal/perform/perform_invite.go +++ b/roomserver/internal/perform/perform_invite.go @@ -51,7 +51,7 @@ func (r *Inviter) PerformInvite( targetUserID := *event.StateKey() info, err := r.DB.RoomInfo(ctx, roomID) if err != nil { - return nil, fmt.Errorf("Failed to load RoomInfo: %w", err) + return nil, fmt.Errorf("failed to load RoomInfo: %w", err) } log.WithFields(log.Fields{ diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index 876888e2..772c9d7d 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -24,7 +24,6 @@ import ( "github.com/getsentry/sentry-go" fsAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/eventutil" - "github.com/matrix-org/dendrite/roomserver/api" rsAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal/helpers" "github.com/matrix-org/dendrite/roomserver/internal/input" @@ -49,17 +48,17 @@ type Joiner struct { // PerformJoin handles joining matrix rooms, including over federation by talking to the federationsender. func (r *Joiner) PerformJoin( ctx context.Context, - req *api.PerformJoinRequest, - res *api.PerformJoinResponse, + req *rsAPI.PerformJoinRequest, + res *rsAPI.PerformJoinResponse, ) { roomID, joinedVia, err := r.performJoin(ctx, req) if err != nil { sentry.CaptureException(err) - perr, ok := err.(*api.PerformError) + perr, ok := err.(*rsAPI.PerformError) if ok { res.Error = perr } else { - res.Error = &api.PerformError{ + res.Error = &rsAPI.PerformError{ Msg: err.Error(), } } @@ -70,18 +69,18 @@ func (r *Joiner) PerformJoin( func (r *Joiner) performJoin( ctx context.Context, - req *api.PerformJoinRequest, + req *rsAPI.PerformJoinRequest, ) (string, gomatrixserverlib.ServerName, error) { _, domain, err := gomatrixserverlib.SplitID('@', req.UserID) if err != nil { - return "", "", &api.PerformError{ - Code: api.PerformErrorBadRequest, + return "", "", &rsAPI.PerformError{ + Code: rsAPI.PerformErrorBadRequest, Msg: fmt.Sprintf("Supplied user ID %q in incorrect format", req.UserID), } } if domain != r.Cfg.Matrix.ServerName { - return "", "", &api.PerformError{ - Code: api.PerformErrorBadRequest, + return "", "", &rsAPI.PerformError{ + Code: rsAPI.PerformErrorBadRequest, Msg: fmt.Sprintf("User %q does not belong to this homeserver", req.UserID), } } @@ -91,20 +90,20 @@ func (r *Joiner) performJoin( if strings.HasPrefix(req.RoomIDOrAlias, "#") { return r.performJoinRoomByAlias(ctx, req) } - return "", "", &api.PerformError{ - Code: api.PerformErrorBadRequest, + return "", "", &rsAPI.PerformError{ + Code: rsAPI.PerformErrorBadRequest, Msg: fmt.Sprintf("Room ID or alias %q is invalid", req.RoomIDOrAlias), } } func (r *Joiner) performJoinRoomByAlias( ctx context.Context, - req *api.PerformJoinRequest, + req *rsAPI.PerformJoinRequest, ) (string, gomatrixserverlib.ServerName, error) { // Get the domain part of the room alias. _, domain, err := gomatrixserverlib.SplitID('#', req.RoomIDOrAlias) if err != nil { - return "", "", fmt.Errorf("Alias %q is not in the correct format", req.RoomIDOrAlias) + return "", "", fmt.Errorf("alias %q is not in the correct format", req.RoomIDOrAlias) } req.ServerNames = append(req.ServerNames, domain) @@ -122,7 +121,7 @@ func (r *Joiner) performJoinRoomByAlias( err = r.FSAPI.PerformDirectoryLookup(ctx, &dirReq, &dirRes) if err != nil { logrus.WithError(err).Errorf("error looking up alias %q", req.RoomIDOrAlias) - return "", "", fmt.Errorf("Looking up alias %q over federation failed: %w", req.RoomIDOrAlias, err) + return "", "", fmt.Errorf("looking up alias %q over federation failed: %w", req.RoomIDOrAlias, err) } roomID = dirRes.RoomID req.ServerNames = append(req.ServerNames, dirRes.ServerNames...) @@ -135,14 +134,14 @@ func (r *Joiner) performJoinRoomByAlias( // Otherwise, look up if we know this room alias locally. err = r.RSAPI.GetRoomIDForAlias(ctx, &getRoomReq, &getRoomRes) if err != nil { - return "", "", fmt.Errorf("Lookup room alias %q failed: %w", req.RoomIDOrAlias, err) + return "", "", fmt.Errorf("lookup room alias %q failed: %w", req.RoomIDOrAlias, err) } roomID = getRoomRes.RoomID } // If the room ID is empty then we failed to look up the alias. if roomID == "" { - return "", "", fmt.Errorf("Alias %q not found", req.RoomIDOrAlias) + return "", "", fmt.Errorf("alias %q not found", req.RoomIDOrAlias) } // If we do, then pluck out the room ID and continue the join. @@ -153,7 +152,7 @@ func (r *Joiner) performJoinRoomByAlias( // TODO: Break this function up a bit func (r *Joiner) performJoinRoomByID( ctx context.Context, - req *api.PerformJoinRequest, + req *rsAPI.PerformJoinRequest, ) (string, gomatrixserverlib.ServerName, error) { // The original client request ?server_name=... may include this HS so filter that out so we // don't attempt to make_join with ourselves @@ -168,8 +167,8 @@ func (r *Joiner) performJoinRoomByID( // Get the domain part of the room ID. _, domain, err := gomatrixserverlib.SplitID('!', req.RoomIDOrAlias) if err != nil { - return "", "", &api.PerformError{ - Code: api.PerformErrorBadRequest, + return "", "", &rsAPI.PerformError{ + Code: rsAPI.PerformErrorBadRequest, Msg: fmt.Sprintf("Room ID %q is invalid: %s", req.RoomIDOrAlias, err), } } @@ -207,10 +206,10 @@ func (r *Joiner) performJoinRoomByID( // Force a federated join if we aren't in the room and we've been // given some server names to try joining by. - inRoomReq := &api.QueryServerJoinedToRoomRequest{ + inRoomReq := &rsAPI.QueryServerJoinedToRoomRequest{ RoomID: req.RoomIDOrAlias, } - inRoomRes := &api.QueryServerJoinedToRoomResponse{} + inRoomRes := &rsAPI.QueryServerJoinedToRoomResponse{} if err = r.Queryer.QueryServerJoinedToRoom(ctx, inRoomReq, inRoomRes); err != nil { return "", "", fmt.Errorf("r.Queryer.QueryServerJoinedToRoom: %w", err) } @@ -267,21 +266,21 @@ func (r *Joiner) performJoinRoomByID( // If we haven't already joined the room then send an event // into the room changing our membership status. if !alreadyJoined { - inputReq := api.InputRoomEventsRequest{ - InputRoomEvents: []api.InputRoomEvent{ + inputReq := rsAPI.InputRoomEventsRequest{ + InputRoomEvents: []rsAPI.InputRoomEvent{ { - Kind: api.KindNew, + Kind: rsAPI.KindNew, Event: event.Headered(buildRes.RoomVersion), AuthEventIDs: event.AuthEventIDs(), SendAsServer: string(r.Cfg.Matrix.ServerName), }, }, } - inputRes := api.InputRoomEventsResponse{} + inputRes := rsAPI.InputRoomEventsResponse{} r.Inputer.InputRoomEvents(ctx, &inputReq, &inputRes) if err = inputRes.Err(); err != nil { - return "", "", &api.PerformError{ - Code: api.PerformErrorNotAllowed, + return "", "", &rsAPI.PerformError{ + Code: rsAPI.PerformErrorNotAllowed, Msg: fmt.Sprintf("InputRoomEvents auth failed: %s", err), } } @@ -296,9 +295,9 @@ func (r *Joiner) performJoinRoomByID( // Otherwise we'll try a federated join as normal, since it's quite // possible that the room still exists on other servers. if len(req.ServerNames) == 0 { - return "", "", &api.PerformError{ - Code: api.PerformErrorNoRoom, - Msg: fmt.Sprintf("Room ID %q does not exist", req.RoomIDOrAlias), + return "", "", &rsAPI.PerformError{ + Code: rsAPI.PerformErrorNoRoom, + Msg: fmt.Sprintf("room ID %q does not exist", req.RoomIDOrAlias), } } } @@ -309,7 +308,7 @@ func (r *Joiner) performJoinRoomByID( default: // Something else went wrong. - return "", "", fmt.Errorf("Error joining local room: %q", err) + return "", "", fmt.Errorf("error joining local room: %q", err) } // By this point, if req.RoomIDOrAlias contained an alias, then @@ -321,7 +320,7 @@ func (r *Joiner) performJoinRoomByID( func (r *Joiner) performFederatedJoinRoomByID( ctx context.Context, - req *api.PerformJoinRequest, + req *rsAPI.PerformJoinRequest, ) (gomatrixserverlib.ServerName, error) { // Try joining by all of the supplied server names. fedReq := fsAPI.PerformJoinRequest{ @@ -333,8 +332,8 @@ func (r *Joiner) performFederatedJoinRoomByID( fedRes := fsAPI.PerformJoinResponse{} r.FSAPI.PerformJoin(ctx, &fedReq, &fedRes) if fedRes.LastError != nil { - return "", &api.PerformError{ - Code: api.PerformErrRemote, + return "", &rsAPI.PerformError{ + Code: rsAPI.PerformErrRemote, Msg: fedRes.LastError.Message, RemoteCode: fedRes.LastError.Code, } @@ -344,7 +343,7 @@ func (r *Joiner) performFederatedJoinRoomByID( func buildEvent( ctx context.Context, db storage.Database, cfg *config.Global, builder *gomatrixserverlib.EventBuilder, -) (*gomatrixserverlib.HeaderedEvent, *api.QueryLatestEventsAndStateResponse, error) { +) (*gomatrixserverlib.HeaderedEvent, *rsAPI.QueryLatestEventsAndStateResponse, error) { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) if err != nil { return nil, nil, fmt.Errorf("gomatrixserverlib.StateNeededForEventBuilder: %w", err) @@ -354,8 +353,8 @@ func buildEvent( return nil, nil, errors.New("expecting state tuples for event builder, got none") } - var queryRes api.QueryLatestEventsAndStateResponse - err = helpers.QueryLatestEventsAndState(ctx, db, &api.QueryLatestEventsAndStateRequest{ + var queryRes rsAPI.QueryLatestEventsAndStateResponse + err = helpers.QueryLatestEventsAndState(ctx, db, &rsAPI.QueryLatestEventsAndStateRequest{ RoomID: builder.RoomID, StateToFetch: eventsNeeded.Tuples(), }, &queryRes) diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index 88eb7e1e..a51de546 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -45,15 +45,15 @@ func (r *Leaver) PerformLeave( ) ([]api.OutputEvent, error) { _, domain, err := gomatrixserverlib.SplitID('@', req.UserID) if err != nil { - return nil, fmt.Errorf("Supplied user ID %q in incorrect format", req.UserID) + return nil, fmt.Errorf("supplied user ID %q in incorrect format", req.UserID) } if domain != r.Cfg.Matrix.ServerName { - return nil, fmt.Errorf("User %q does not belong to this homeserver", req.UserID) + return nil, fmt.Errorf("user %q does not belong to this homeserver", req.UserID) } if strings.HasPrefix(req.RoomID, "!") { return r.performLeaveRoomByID(ctx, req, res) } - return nil, fmt.Errorf("Room ID %q is invalid", req.RoomID) + return nil, fmt.Errorf("room ID %q is invalid", req.RoomID) } func (r *Leaver) performLeaveRoomByID( @@ -68,7 +68,7 @@ func (r *Leaver) performLeaveRoomByID( var host gomatrixserverlib.ServerName _, host, err = gomatrixserverlib.SplitID('@', senderUser) if err != nil { - return nil, fmt.Errorf("Sender %q is invalid", senderUser) + return nil, fmt.Errorf("sender %q is invalid", senderUser) } if host != r.Cfg.Matrix.ServerName { return r.performFederatedRejectInvite(ctx, req, res, senderUser, eventID) @@ -91,19 +91,19 @@ func (r *Leaver) performLeaveRoomByID( return nil, err } if !latestRes.RoomExists { - return nil, fmt.Errorf("Room %q does not exist", req.RoomID) + return nil, fmt.Errorf("room %q does not exist", req.RoomID) } // Now let's see if the user is in the room. if len(latestRes.StateEvents) == 0 { - return nil, fmt.Errorf("User %q is not a member of room %q", req.UserID, req.RoomID) + return nil, fmt.Errorf("user %q is not a member of room %q", req.UserID, req.RoomID) } membership, err := latestRes.StateEvents[0].Membership() if err != nil { - return nil, fmt.Errorf("Error getting membership: %w", err) + return nil, fmt.Errorf("error getting membership: %w", err) } if membership != gomatrixserverlib.Join && membership != gomatrixserverlib.Invite { - return nil, fmt.Errorf("User %q is not joined to the room (membership is %q)", req.UserID, membership) + return nil, fmt.Errorf("user %q is not joined to the room (membership is %q)", req.UserID, membership) } // Prepare the template for the leave event. @@ -161,7 +161,7 @@ func (r *Leaver) performFederatedRejectInvite( ) ([]api.OutputEvent, error) { _, domain, err := gomatrixserverlib.SplitID('@', senderUser) if err != nil { - return nil, fmt.Errorf("User ID %q invalid: %w", senderUser, err) + return nil, fmt.Errorf("user ID %q invalid: %w", senderUser, err) } // Ask the federation sender to perform a federated leave for us. diff --git a/roomserver/internal/perform/perform_peek.go b/roomserver/internal/perform/perform_peek.go index 443276cd..bd799667 100644 --- a/roomserver/internal/perform/perform_peek.go +++ b/roomserver/internal/perform/perform_peek.go @@ -96,7 +96,7 @@ func (r *Peeker) performPeekRoomByAlias( // Get the domain part of the room alias. _, domain, err := gomatrixserverlib.SplitID('#', req.RoomIDOrAlias) if err != nil { - return "", fmt.Errorf("Alias %q is not in the correct format", req.RoomIDOrAlias) + return "", fmt.Errorf("alias %q is not in the correct format", req.RoomIDOrAlias) } req.ServerNames = append(req.ServerNames, domain) @@ -114,7 +114,7 @@ func (r *Peeker) performPeekRoomByAlias( err = r.FSAPI.PerformDirectoryLookup(ctx, &dirReq, &dirRes) if err != nil { logrus.WithError(err).Errorf("error looking up alias %q", req.RoomIDOrAlias) - return "", fmt.Errorf("Looking up alias %q over federation failed: %w", req.RoomIDOrAlias, err) + return "", fmt.Errorf("looking up alias %q over federation failed: %w", req.RoomIDOrAlias, err) } roomID = dirRes.RoomID req.ServerNames = append(req.ServerNames, dirRes.ServerNames...) @@ -122,13 +122,13 @@ func (r *Peeker) performPeekRoomByAlias( // Otherwise, look up if we know this room alias locally. roomID, err = r.DB.GetRoomIDForAlias(ctx, req.RoomIDOrAlias) if err != nil { - return "", fmt.Errorf("Lookup room alias %q failed: %w", req.RoomIDOrAlias, err) + return "", fmt.Errorf("lookup room alias %q failed: %w", req.RoomIDOrAlias, err) } } // If the room ID is empty then we failed to look up the alias. if roomID == "" { - return "", fmt.Errorf("Alias %q not found", req.RoomIDOrAlias) + return "", fmt.Errorf("alias %q not found", req.RoomIDOrAlias) } // If we do, then pluck out the room ID and continue the peek. diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 3d71dbb6..bae8b24c 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -71,7 +71,7 @@ func (v *StateResolution) LoadStateAtSnapshot( if !ok { // This should only get hit if the database is corrupt. // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) + panic(fmt.Errorf("corrupt DB: Missing state block numeric ID %d", stateBlockNID)) } fullState = append(fullState, entries...) } @@ -146,7 +146,7 @@ func (v *StateResolution) LoadCombinedStateAfterEvents( if !ok { // This should only get hit if the database is corrupt. // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state snapshot numeric ID %d", prevState.BeforeStateSnapshotNID)) + panic(fmt.Errorf("corrupt DB: Missing state snapshot numeric ID %d", prevState.BeforeStateSnapshotNID)) } // Combine all the state entries for this snapshot. @@ -157,7 +157,7 @@ func (v *StateResolution) LoadCombinedStateAfterEvents( if !ok { // This should only get hit if the database is corrupt. // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) + panic(fmt.Errorf("corrupt DB: Missing state block numeric ID %d", stateBlockNID)) } fullState = append(fullState, entries...) } @@ -757,7 +757,7 @@ func (v *StateResolution) resolveConflictsV1( for _, resolvedEvent := range resolvedEvents { entry, ok := eventIDMap[resolvedEvent.EventID()] if !ok { - panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) + panic(fmt.Errorf("missing state entry for event ID %q", resolvedEvent.EventID())) } notConflicted = append(notConflicted, entry) } @@ -880,7 +880,7 @@ func (v *StateResolution) resolveConflictsV2( for _, resolvedEvent := range resolvedEvents { entry, ok := eventIDMap[resolvedEvent.EventID()] if !ok { - panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) + panic(fmt.Errorf("missing state entry for event ID %q", resolvedEvent.EventID())) } notConflicted = append(notConflicted, entry) } @@ -958,7 +958,7 @@ func (v *StateResolution) loadStateEvents( for _, entry := range eventEntries { event, ok := eventMap(events).lookup(entry.EventNID) if !ok { - panic(fmt.Errorf("Corrupt DB: Missing event numeric ID %d", entry.EventNID)) + panic(fmt.Errorf("corrupt DB: Missing event numeric ID %d", entry.EventNID)) } result = append(result, event.Event) eventIDMap[event.Event.EventID()] = entry diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index e1851679..b4fc15f1 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -1136,7 +1136,7 @@ func (d *Database) loadStateAtSnapshot( if !ok { // This should only get hit if the database is corrupt. // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) + panic(fmt.Errorf("corrupt DB: Missing state block numeric ID %d", stateBlockNID)) } fullState = append(fullState, entries...) } diff --git a/setup/mscs/msc2836/msc2836.go b/setup/mscs/msc2836/msc2836.go index f47a42d9..a538299d 100644 --- a/setup/mscs/msc2836/msc2836.go +++ b/setup/mscs/msc2836/msc2836.go @@ -98,7 +98,7 @@ func Enable( ) error { db, err := NewDatabase(&base.Cfg.MSCs.Database) if err != nil { - return fmt.Errorf("Cannot enable MSC2836: %w", err) + return fmt.Errorf("cannot enable MSC2836: %w", err) } hooks.Enable() hooks.Attach(hooks.KindNewEventPersisted, func(headeredEvent interface{}) { diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index 08a54f75..121a73fd 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -57,7 +57,7 @@ func Enable( ) error { db, err := NewDatabase(&base.Cfg.MSCs.Database) if err != nil { - return fmt.Errorf("Cannot enable MSC2946: %w", err) + return fmt.Errorf("cannot enable MSC2946: %w", err) } hooks.Enable() hooks.Attach(hooks.KindNewEventPersisted, func(headeredEvent interface{}) { diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go index 05fcf37d..1938ff9b 100644 --- a/syncapi/consumers/keychange.go +++ b/syncapi/consumers/keychange.go @@ -30,7 +30,6 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" - log "github.com/sirupsen/logrus" ) // OutputKeyChangeEventConsumer consumes events that originated in the key server. @@ -128,7 +127,7 @@ func (s *OutputKeyChangeEventConsumer) onDeviceKeyMessage(m api.DeviceMessage, o UserID: output.UserID, }, &queryRes) if err != nil { - log.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server") + logrus.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server") sentry.CaptureException(err) return err } @@ -155,7 +154,7 @@ func (s *OutputKeyChangeEventConsumer) onCrossSigningMessage(m api.DeviceMessage UserID: output.UserID, }, &queryRes) if err != nil { - log.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server") + logrus.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server") sentry.CaptureException(err) return err } diff --git a/syncapi/internal/keychange.go b/syncapi/internal/keychange.go index 0bbaf31e..56a438fb 100644 --- a/syncapi/internal/keychange.go +++ b/syncapi/internal/keychange.go @@ -19,7 +19,6 @@ import ( "strings" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/keyserver/api" keyapi "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" @@ -31,8 +30,8 @@ const DeviceListLogName = "dl" // DeviceOTKCounts adds one-time key counts to the /sync response func DeviceOTKCounts(ctx context.Context, keyAPI keyapi.KeyInternalAPI, userID, deviceID string, res *types.Response) error { - var queryRes api.QueryOneTimeKeysResponse - keyAPI.QueryOneTimeKeys(ctx, &api.QueryOneTimeKeysRequest{ + var queryRes keyapi.QueryOneTimeKeysResponse + keyAPI.QueryOneTimeKeys(ctx, &keyapi.QueryOneTimeKeysRequest{ UserID: userID, DeviceID: deviceID, }, &queryRes) @@ -81,8 +80,8 @@ func DeviceListCatchup( if toLog := to; toLog.Partition == partition && toLog.Offset > 0 { toOffset = toLog.Offset } - var queryRes api.QueryKeyChangesResponse - keyAPI.QueryKeyChanges(ctx, &api.QueryKeyChangesRequest{ + var queryRes keyapi.QueryKeyChangesResponse + keyAPI.QueryKeyChanges(ctx, &keyapi.QueryKeyChangesRequest{ Partition: partition, Offset: offset, ToOffset: toOffset, diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index b8271877..e6c68183 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -30,7 +30,6 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" - log "github.com/sirupsen/logrus" ) // Database is a temporary struct until we have made syncserver.go the same for both pq/sqlite @@ -309,7 +308,7 @@ func (d *Database) StreamEventsToEvents(device *userapi.Device, in []types.Strea "transaction_id", in[i].TransactionID.TransactionID, ) if err != nil { - log.WithFields(log.Fields{ + logrus.WithFields(logrus.Fields{ "event_id": out[i].EventID(), }).WithError(err).Warnf("Failed to add transaction ID to event") } @@ -529,7 +528,7 @@ func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, reda return err } if len(redactedEvents) == 0 { - log.WithField("event_id", redactedEventID).WithField("redaction_event", redactedBecause.EventID()).Warnf("missing redacted event for redaction") + logrus.WithField("event_id", redactedEventID).WithField("redaction_event", redactedBecause.EventID()).Warnf("missing redacted event for redaction") return nil } eventToRedact := redactedEvents[0].Unwrap() @@ -645,7 +644,7 @@ func (d *Database) fetchMissingStateEvents( return nil, err } if len(stateEvents) != len(missing) { - log.WithContext(ctx).Warnf("Failed to map all event IDs to events (got %d, wanted %d)", len(stateEvents), len(missing)) + logrus.WithContext(ctx).Warnf("Failed to map all event IDs to events (got %d, wanted %d)", len(stateEvents), len(missing)) // TODO: Why is this happening? It's probably the roomserver. Uncomment // this error again when we work out what it is and fix it, otherwise we diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 49fa1a16..44e718b3 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -29,10 +29,10 @@ var ( // ErrInvalidSyncTokenType is returned when an attempt at creating a // new instance of SyncToken with an invalid type (i.e. neither "s" // nor "t"). - ErrInvalidSyncTokenType = fmt.Errorf("Sync token has an unknown prefix (should be either s or t)") + ErrInvalidSyncTokenType = fmt.Errorf("sync token has an unknown prefix (should be either s or t)") // ErrInvalidSyncTokenLen is returned when the pagination token is an // invalid length - ErrInvalidSyncTokenLen = fmt.Errorf("Sync token has an invalid length") + ErrInvalidSyncTokenLen = fmt.Errorf("sync token has an invalid length") ) type StateDelta struct { diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 518edef4..4ff8f51d 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -177,10 +177,10 @@ func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string) er DeviceKeys: deviceKeys, }, &uploadRes) if uploadRes.Error != nil { - return fmt.Errorf("Failed to delete device keys: %v", uploadRes.Error) + return fmt.Errorf("failed to delete device keys: %v", uploadRes.Error) } if len(uploadRes.KeyErrors) > 0 { - return fmt.Errorf("Failed to delete device keys, key errors: %+v", uploadRes.KeyErrors) + return fmt.Errorf("failed to delete device keys, key errors: %+v", uploadRes.KeyErrors) } return nil } @@ -242,10 +242,10 @@ func (a *UserInternalAPI) PerformDeviceUpdate(ctx context.Context, req *api.Perf OnlyDisplayNameUpdates: true, }, &uploadRes) if uploadRes.Error != nil { - return fmt.Errorf("Failed to update device key display name: %v", uploadRes.Error) + return fmt.Errorf("failed to update device key display name: %v", uploadRes.Error) } if len(uploadRes.KeyErrors) > 0 { - return fmt.Errorf("Failed to update device key display name, key errors: %+v", uploadRes.KeyErrors) + return fmt.Errorf("failed to update device key display name, key errors: %+v", uploadRes.KeyErrors) } } return nil diff --git a/userapi/storage/accounts/interface.go b/userapi/storage/accounts/interface.go index 887f7193..7af2f15f 100644 --- a/userapi/storage/accounts/interface.go +++ b/userapi/storage/accounts/interface.go @@ -67,4 +67,4 @@ type Database interface { // Err3PIDInUse is the error returned when trying to save an association involving // a third-party identifier which is already associated to a local user. -var Err3PIDInUse = errors.New("This third-party identifier is already in use") +var Err3PIDInUse = errors.New("this third-party identifier is already in use") diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index 6bddbfc3..2f829062 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -28,7 +28,6 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas" - _ "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -271,7 +270,7 @@ func (d *Database) hashPassword(plaintext string) (hash string, err error) { // Err3PIDInUse is the error returned when trying to save an association involving // a third-party identifier which is already associated to a local user. -var Err3PIDInUse = errors.New("This third-party identifier is already in use") +var Err3PIDInUse = errors.New("this third-party identifier is already in use") // SaveThreePIDAssociation saves the association between a third party identifier // and a local Matrix user (identified by the user's ID's local part). diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index d752e3db..2b731b75 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -304,7 +304,7 @@ func (d *Database) hashPassword(plaintext string) (hash string, err error) { // Err3PIDInUse is the error returned when trying to save an association involving // a third-party identifier which is already associated to a local user. -var Err3PIDInUse = errors.New("This third-party identifier is already in use") +var Err3PIDInUse = errors.New("this third-party identifier is already in use") // SaveThreePIDAssociation saves the association between a third party identifier // and a local Matrix user (identified by the user's ID's local part). From 1cd4d50181585d664ca7337e3f48ea00a1b1f5c6 Mon Sep 17 00:00:00 2001 From: Ryan W Date: Fri, 10 Sep 2021 10:05:31 +0100 Subject: [PATCH 40/64] Added .well-known/matrix/server endpoint (#1988) * Added .well-known/matrix/server endpoint Signed-off-by: Ryan Whittington * Replaced tabs with spaces Signed-off-by: Ryan Whittington --- build/docker/config/dendrite-config.yaml | 4 ++++ build/gobind-pinecone/monolith.go | 1 + build/gobind-yggdrasil/monolith.go | 1 + cmd/dendrite-demo-libp2p/main.go | 1 + cmd/dendrite-demo-pinecone/main.go | 1 + cmd/dendrite-demo-yggdrasil/main.go | 1 + cmd/dendrite-monolith-server/main.go | 1 + .../personalities/federationapi.go | 2 +- cmd/dendritejs-pinecone/main.go | 1 + dendrite-config.yaml | 4 ++++ federationapi/federationapi.go | 4 ++-- federationapi/federationapi_test.go | 2 +- federationapi/routing/routing.go | 18 +++++++++++++++++- internal/httputil/paths.go | 1 + setup/base.go | 3 +++ setup/config/config_global.go | 3 +++ setup/config/config_test.go | 1 + setup/monolith.go | 4 ++-- 18 files changed, 46 insertions(+), 7 deletions(-) diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index ffcf6a45..bc5d6699 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -52,6 +52,10 @@ global: # considered valid by other homeservers. key_validity_period: 168h0m0s + # The server name to delegate server-server communications to, with optional port + # e.g. localhost:443 + well_known_server_name: "" + # Lists of domains that the server will trust as identity servers to verify third # party identifiers such as phone numbers and email addresses. trusted_third_party_id_servers: diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 310e3a04..3372b6de 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -332,6 +332,7 @@ func (m *DendriteMonolith) Start() { base.PublicClientAPIMux, base.PublicFederationAPIMux, base.PublicKeyAPIMux, + base.PublicWellKnownAPIMux, base.PublicMediaAPIMux, base.SynapseAdminMux, ) diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 6fc5f244..693e9236 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -156,6 +156,7 @@ func (m *DendriteMonolith) Start() { base.PublicClientAPIMux, base.PublicFederationAPIMux, base.PublicKeyAPIMux, + base.PublicWellKnownAPIMux, base.PublicMediaAPIMux, base.SynapseAdminMux, ) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 4caa617f..7606e418 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -198,6 +198,7 @@ func main() { base.Base.PublicClientAPIMux, base.Base.PublicFederationAPIMux, base.Base.PublicKeyAPIMux, + base.Base.PublicWellKnownAPIMux, base.Base.PublicMediaAPIMux, base.Base.SynapseAdminMux, ) diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index e919e749..5f914680 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -210,6 +210,7 @@ func main() { base.PublicClientAPIMux, base.PublicFederationAPIMux, base.PublicKeyAPIMux, + base.PublicWellKnownAPIMux, base.PublicMediaAPIMux, base.SynapseAdminMux, ) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index dbdb9a76..1b5e1a51 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -145,6 +145,7 @@ func main() { base.PublicClientAPIMux, base.PublicFederationAPIMux, base.PublicKeyAPIMux, + base.PublicWellKnownAPIMux, base.PublicMediaAPIMux, base.SynapseAdminMux, ) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 4d8e2ee1..ec8751df 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -161,6 +161,7 @@ func main() { base.PublicClientAPIMux, base.PublicFederationAPIMux, base.PublicKeyAPIMux, + base.PublicWellKnownAPIMux, base.PublicMediaAPIMux, base.SynapseAdminMux, ) diff --git a/cmd/dendrite-polylith-multi/personalities/federationapi.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go index 5ff08528..5488fbf3 100644 --- a/cmd/dendrite-polylith-multi/personalities/federationapi.go +++ b/cmd/dendrite-polylith-multi/personalities/federationapi.go @@ -30,7 +30,7 @@ func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { keyAPI := base.KeyServerHTTPClient() federationapi.AddPublicRoutes( - base.PublicFederationAPIMux, base.PublicKeyAPIMux, + base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, &base.Cfg.FederationAPI, userAPI, federation, keyRing, rsAPI, fsAPI, base.EDUServerClient(), keyAPI, &base.Cfg.MSCs, nil, diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index b44c609c..2b40be43 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -221,6 +221,7 @@ func startup() { base.PublicClientAPIMux, base.PublicFederationAPIMux, base.PublicKeyAPIMux, + base.PublicWellKnownAPIMux, base.PublicMediaAPIMux, base.SynapseAdminMux, ) diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 31b83066..5f2a9de2 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -54,6 +54,10 @@ global: # considered valid by other homeservers. key_validity_period: 168h0m0s + # The server name to delegate server-server communications to, with optional port + # e.g. localhost:443 + well_known_server_name: "" + # Lists of domains that the server will trust as identity servers to verify third # party identifiers such as phone numbers and email addresses. trusted_third_party_id_servers: diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index b3297434..c40d77a6 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -30,7 +30,7 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component. func AddPublicRoutes( - fedRouter, keyRouter *mux.Router, + fedRouter, keyRouter, wellKnownRouter *mux.Router, cfg *config.FederationAPI, userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient, @@ -43,7 +43,7 @@ func AddPublicRoutes( servers federationAPI.ServersInRoomProvider, ) { routing.Setup( - fedRouter, keyRouter, cfg, rsAPI, + fedRouter, keyRouter, wellKnownRouter, cfg, rsAPI, eduAPI, federationSenderAPI, keyRing, federation, userAPI, keyAPI, mscCfg, servers, diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index 505a11da..cb4d8103 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -31,7 +31,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) { fsAPI := base.FederationSenderHTTPClient() // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. - federationapi.AddPublicRoutes(base.PublicFederationAPIMux, base.PublicKeyAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, &cfg.MSCs, nil) + federationapi.AddPublicRoutes(base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, &cfg.MSCs, nil) baseURL, cancel := test.ListenAndServe(t, base.PublicFederationAPIMux, true) defer cancel() serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://")) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 503e512e..7446f1fb 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -30,6 +30,7 @@ import ( userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" ) // Setup registers HTTP handlers with the given ServeMux. @@ -41,7 +42,7 @@ import ( // applied: // nolint: gocyclo func Setup( - fedMux, keyMux *mux.Router, + fedMux, keyMux, wkMux *mux.Router, cfg *config.FederationAPI, rsAPI roomserverAPI.RoomserverInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI, @@ -85,6 +86,21 @@ func Setup( return NotaryKeys(req, cfg, fsAPI, pkReq) }) + if cfg.Matrix.WellKnownServerName != "" { + logrus.Infof("Setting m.server as %s at /.well-known/matrix/server", cfg.Matrix.WellKnownServerName) + wkMux.Handle("/server", httputil.MakeExternalAPI("wellknown", func(req *http.Request) util.JSONResponse { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct { + ServerName string `json:"m.server"` + }{ + ServerName: cfg.Matrix.WellKnownServerName, + }, + } + }), + ).Methods(http.MethodGet, http.MethodOptions) + } + // Ignore the {keyID} argument as we only have a single server key so we always // return that key. // Even if we had more than one server key, we would probably still ignore the diff --git a/internal/httputil/paths.go b/internal/httputil/paths.go index b0f4b8cb..a1009fc2 100644 --- a/internal/httputil/paths.go +++ b/internal/httputil/paths.go @@ -19,5 +19,6 @@ const ( PublicFederationPathPrefix = "/_matrix/federation/" PublicKeyPathPrefix = "/_matrix/key/" PublicMediaPathPrefix = "/_matrix/media/" + PublicWellKnownPrefix = "/.well-known/matrix/" InternalPathPrefix = "/api/" ) diff --git a/setup/base.go b/setup/base.go index 1a52d1c2..d4acdbfb 100644 --- a/setup/base.go +++ b/setup/base.go @@ -76,6 +76,7 @@ type BaseDendrite struct { PublicFederationAPIMux *mux.Router PublicKeyAPIMux *mux.Router PublicMediaAPIMux *mux.Router + PublicWellKnownAPIMux *mux.Router InternalAPIMux *mux.Router SynapseAdminMux *mux.Router UseHTTPAPIs bool @@ -198,6 +199,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo PublicFederationAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath(), PublicKeyAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath(), PublicMediaAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicMediaPathPrefix).Subrouter().UseEncodedPath(), + PublicWellKnownAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicWellKnownPrefix).Subrouter().UseEncodedPath(), InternalAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(), SynapseAdminMux: mux.NewRouter().SkipClean(true).PathPrefix("/_synapse/").Subrouter().UseEncodedPath(), apiHttpClient: &apiClient, @@ -394,6 +396,7 @@ func (b *BaseDendrite) SetupAndServeHTTP( } externalRouter.PathPrefix("/_synapse/").Handler(b.SynapseAdminMux) externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(b.PublicMediaAPIMux) + externalRouter.PathPrefix(httputil.PublicWellKnownPrefix).Handler(b.PublicWellKnownAPIMux) if internalAddr != NoListener && internalAddr != externalAddr { go func() { diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 90a92f2b..d5d9f7f5 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -34,6 +34,9 @@ type Global struct { // Defaults to 24 hours. KeyValidityPeriod time.Duration `yaml:"key_validity_period"` + // The server name to delegate server-server communications to, with optional port + WellKnownServerName string `yaml:"well_known_server_name"` + // Disables federation. Dendrite will not be able to make any outbound HTTP requests // to other servers and the federation API will not be exposed. DisableFederation bool `yaml:"disable_federation"` diff --git a/setup/config/config_test.go b/setup/config/config_test.go index 4107b684..5c51a363 100644 --- a/setup/config/config_test.go +++ b/setup/config/config_test.go @@ -39,6 +39,7 @@ global: private_key: matrix_key.pem key_id: ed25519:auto key_validity_period: 168h0m0s + well_known_server_name: "localhost:443" trusted_third_party_id_servers: - matrix.org - vector.im diff --git a/setup/monolith.go b/setup/monolith.go index 5ceb4ed3..a77cdd56 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -57,7 +57,7 @@ type Monolith struct { } // AddAllPublicRoutes attaches all public paths to the given router -func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, mediaMux, synapseMux *mux.Router) { +func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, wkMux, mediaMux, synapseMux *mux.Router) { clientapi.AddPublicRoutes( csMux, synapseMux, &m.Config.ClientAPI, m.AccountDB, m.FedClient, m.RoomserverAPI, @@ -66,7 +66,7 @@ func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ss &m.Config.MSCs, ) federationapi.AddPublicRoutes( - ssMux, keyMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient, + ssMux, keyMux, wkMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI, m.EDUInternalAPI, m.KeyAPI, &m.Config.MSCs, nil, ) From 57fb44ac4bf81aa9b9835baac795cf9f42f83412 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 10 Sep 2021 14:47:59 +0100 Subject: [PATCH 41/64] Update pinecone to matrix-org/pinecone#12 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7f883bea..cb30a237 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 - github.com/matrix-org/pinecone v0.0.0-20210819150600-e692df1a5c42 + github.com/matrix-org/pinecone v0.0.0-20210910134625-4ec11c22f2c8 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.8 github.com/morikuni/aec v1.0.0 // indirect diff --git a/go.sum b/go.sum index 65cf1465..35ad1380 100644 --- a/go.sum +++ b/go.sum @@ -998,8 +998,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 h1:b8 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20210819150600-e692df1a5c42 h1:ZO39w5Kbq9Aw3uHHT5QjFR2kpVHxuJxHD3zOhmU5BVI= -github.com/matrix-org/pinecone v0.0.0-20210819150600-e692df1a5c42/go.mod h1:CVlrvs1R5iz7Omy2GqAjJJKbACn07GZgUq1Gli18FYE= +github.com/matrix-org/pinecone v0.0.0-20210910134625-4ec11c22f2c8 h1:oE7rDEoz3J3PVkugf10XxjPnGO9PGZOSMwNazfI/0dM= +github.com/matrix-org/pinecone v0.0.0-20210910134625-4ec11c22f2c8/go.mod h1:CVlrvs1R5iz7Omy2GqAjJJKbACn07GZgUq1Gli18FYE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From d08399d1a41063561b7c497ca77ffd88b52dbc1e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 20 Sep 2021 16:22:10 +0100 Subject: [PATCH 42/64] Update to matrix-org/pinecone@4f07afa --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cb30a237..ab133cfd 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 - github.com/matrix-org/pinecone v0.0.0-20210910134625-4ec11c22f2c8 + github.com/matrix-org/pinecone v0.0.0-20210920152116-4f07afaed998 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.8 github.com/morikuni/aec v1.0.0 // indirect diff --git a/go.sum b/go.sum index 35ad1380..ae5abc06 100644 --- a/go.sum +++ b/go.sum @@ -998,8 +998,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 h1:b8 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20210910134625-4ec11c22f2c8 h1:oE7rDEoz3J3PVkugf10XxjPnGO9PGZOSMwNazfI/0dM= -github.com/matrix-org/pinecone v0.0.0-20210910134625-4ec11c22f2c8/go.mod h1:CVlrvs1R5iz7Omy2GqAjJJKbACn07GZgUq1Gli18FYE= +github.com/matrix-org/pinecone v0.0.0-20210920152116-4f07afaed998 h1:Jdxog8Gh1/8IOhkeEub4LeFhaYWNJmOpG16F+eas6r4= +github.com/matrix-org/pinecone v0.0.0-20210920152116-4f07afaed998/go.mod h1:CVlrvs1R5iz7Omy2GqAjJJKbACn07GZgUq1Gli18FYE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From be9e1b536745e7e821f954239f832a309a88e67b Mon Sep 17 00:00:00 2001 From: Aaron Bieber Date: Tue, 5 Oct 2021 09:03:55 -0600 Subject: [PATCH 43/64] Bump github.com/matrix-org/dugong (#2011) The latest version of dugong has a fix for removing log files after they are compressed. Signed-off-by: Aaron Bieber --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ab133cfd..c2d3c276 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.4.1 github.com/libp2p/go-libp2p-record v0.1.3 github.com/lucas-clemente/quic-go v0.22.0 - github.com/matrix-org/dugong v0.0.0-20210603171012-8379174dca81 + github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 diff --git a/go.sum b/go.sum index ae5abc06..590efa3e 100644 --- a/go.sum +++ b/go.sum @@ -985,8 +985,8 @@ github.com/masterzen/azure-sdk-for-go v3.2.0-beta.0.20161014135628-ee4f0065d00c+ github.com/masterzen/simplexml v0.0.0-20160608183007-4572e39b1ab9/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc= github.com/masterzen/winrm v0.0.0-20161014151040-7a535cd943fc/go.mod h1:CfZSN7zwz5gJiFhZJz49Uzk7mEBHIceWmbFmYx7Hf7E= github.com/masterzen/xmlpath v0.0.0-20140218185901-13f4951698ad/go.mod h1:A0zPC53iKKKcXYxr4ROjpQRQ5FgJXtelNdSmHHuq/tY= -github.com/matrix-org/dugong v0.0.0-20210603171012-8379174dca81 h1:lHNIqMZ77Y++RbHuR4eSehyv6KUKb2sJ8Gdfro4P5f4= -github.com/matrix-org/dugong v0.0.0-20210603171012-8379174dca81/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= +github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM= +github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 h1:eqE5OnGx9ZMWmrRbD3KF/3KtTunw0iQulI7YxOIdxo4= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4/go.mod h1:3WluEZ9QXSwU30tWYqktnpC1x9mwZKx1r8uAv8Iq+a4= github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d h1:mGhPVaTht5NViFN/UpdrIlRApmH2FWcVaKUH5MdBKiY= From a5cb8c00bded571b98ca0827f3f9451557d53420 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 8 Oct 2021 15:29:30 +0100 Subject: [PATCH 44/64] Update Pinecone demo --- build/gobind-pinecone/monolith.go | 10 +++++----- cmd/dendrite-demo-pinecone/main.go | 2 +- cmd/dendrite-demo-pinecone/rooms/rooms.go | 6 ++---- go.mod | 7 +++---- go.sum | 19 +++++++++---------- 5 files changed, 20 insertions(+), 24 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 3372b6de..bd1b9998 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -105,7 +105,7 @@ func (m *DendriteMonolith) SetStaticPeer(uri string) { func (m *DendriteMonolith) DisconnectType(peertype int) { for _, p := range m.PineconeRouter.Peers() { if peertype == p.PeerType { - _ = m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil) + m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil) } } } @@ -113,13 +113,13 @@ func (m *DendriteMonolith) DisconnectType(peertype int) { func (m *DendriteMonolith) DisconnectZone(zone string) { for _, p := range m.PineconeRouter.Peers() { if zone == p.Zone { - _ = m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil) + m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil) } } } -func (m *DendriteMonolith) DisconnectPort(port int) error { - return m.PineconeRouter.Disconnect(types.SwitchPortID(port), nil) +func (m *DendriteMonolith) DisconnectPort(port int) { + m.PineconeRouter.Disconnect(types.SwitchPortID(port), nil) } func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error) { @@ -253,7 +253,7 @@ func (m *DendriteMonolith) Start() { logrus.SetOutput(BindLogger{}) logger := log.New(os.Stdout, "PINECONE: ", 0) - m.PineconeRouter = pineconeRouter.NewRouter(logger, "dendrite", sk, pk, nil) + m.PineconeRouter = pineconeRouter.NewRouter(logger, sk, "dendrite", nil) m.PineconeQUIC = pineconeSessions.NewSessions(logger, m.PineconeRouter) m.PineconeMulticast = pineconeMulticast.NewMulticast(logger, m.PineconeRouter) diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 5f914680..7aa72632 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -91,7 +91,7 @@ func main() { } logger := log.New(os.Stdout, "", 0) - pRouter := pineconeRouter.NewRouter(logger, "dendrite", sk, pk, nil) + pRouter := pineconeRouter.NewRouter(logger, sk, "dendrite", nil) go func() { listener, err := net.Listen("tcp", *instanceListen) diff --git a/cmd/dendrite-demo-pinecone/rooms/rooms.go b/cmd/dendrite-demo-pinecone/rooms/rooms.go index ed6ab78c..fc387a17 100644 --- a/cmd/dendrite-demo-pinecone/rooms/rooms.go +++ b/cmd/dendrite-demo-pinecone/rooms/rooms.go @@ -50,11 +50,9 @@ func NewPineconeRoomProvider( } func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom { - known := p.r.KnownNodes() - //known = append(known, p.s.Sessions()...) list := []gomatrixserverlib.ServerName{} - for _, k := range known { - list = append(list, gomatrixserverlib.ServerName(k.String())) + for _, k := range p.r.Peers() { + list = append(list, gomatrixserverlib.ServerName(k.PublicKey)) } return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, list) } diff --git a/go.mod b/go.mod index c2d3c276..f74b07f7 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 - github.com/matrix-org/pinecone v0.0.0-20210920152116-4f07afaed998 + github.com/matrix-org/pinecone v0.0.0-20211008142213-06dcf8511285 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.8 github.com/morikuni/aec v1.0.0 // indirect @@ -53,10 +53,9 @@ require ( github.com/uber/jaeger-lib v2.4.1+incompatible github.com/yggdrasil-network/yggdrasil-go v0.4.1-0.20210715083903-52309d094c00 go.uber.org/atomic v1.9.0 - golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554 - golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + golang.org/x/net v0.0.0-20210927181540-4e4d966f7476 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b gopkg.in/h2non/bimg.v1 v1.1.5 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 590efa3e..af7a3b59 100644 --- a/go.sum +++ b/go.sum @@ -116,9 +116,7 @@ github.com/anacrolix/missinggo v1.1.2-0.20190815015349-b888af804467/go.mod h1:MB github.com/anacrolix/missinggo v1.2.1 h1:0IE3TqX5y5D0IxeMwTyIgqdDew4QrzcXaaEnJQyjHvw= github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y= github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ= -github.com/anacrolix/sync v0.2.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DCOj/I0g= github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= -github.com/anacrolix/utp v0.1.0/go.mod h1:MDwc+vsGEq7RMw6lr2GKOEqjWny5hO5OZXRVNaBJ2Dk= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -998,8 +996,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 h1:b8 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20210920152116-4f07afaed998 h1:Jdxog8Gh1/8IOhkeEub4LeFhaYWNJmOpG16F+eas6r4= -github.com/matrix-org/pinecone v0.0.0-20210920152116-4f07afaed998/go.mod h1:CVlrvs1R5iz7Omy2GqAjJJKbACn07GZgUq1Gli18FYE= +github.com/matrix-org/pinecone v0.0.0-20211008142213-06dcf8511285 h1:W5OSspxTxZlb03wUEywga5DickEmUdQzzWFSJfmy6Oc= +github.com/matrix-org/pinecone v0.0.0-20211008142213-06dcf8511285/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= @@ -1517,8 +1515,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1606,8 +1604,8 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 h1:4CSI6oo7cOjJKajidEljs9h+uP0rRZBPPPhcCbj5mw8= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210927181540-4e4d966f7476 h1:s5hu7bTnLKswvidgtqc4GwsW83m9LZu8UAqzmWOZtI4= +golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1713,8 +1711,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210611083646-a4fc73990273/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= @@ -1791,6 +1789,7 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1N golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.zx2c4.com/wireguard v0.0.0-20210510202332-9844c74f67ec/go.mod h1:a057zjmoc00UN7gVkaJt2sXVK523kMJcogDTEvPIasg= golang.zx2c4.com/wireguard v0.0.0-20210604143328-f9b48a961cd2/go.mod h1:laHzsbfMhGSobUmruXWAyMKKHSqvIcrqZJMyHD+/3O8= +golang.zx2c4.com/wireguard v0.0.0-20210927201915-bb745b2ea326/go.mod h1:SDoazCvdy7RDjBPNEMBwrXhomlmtG7svs8mgwWEqtVI= golang.zx2c4.com/wireguard/windows v0.3.14/go.mod h1:3P4IEAsb+BjlKZmpUXgy74c0iX9AVwwr3WcVJ8nPgME= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= From 50222273d1c29712998ad61150bd183d511df0ab Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 8 Oct 2021 15:33:03 +0100 Subject: [PATCH 45/64] Fix dendritejs-pinecone --- cmd/dendritejs-pinecone/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index 2b40be43..abfcfbd9 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main @@ -155,7 +156,7 @@ func startup() { pk := sk.Public().(ed25519.PublicKey) logger := log.New(os.Stdout, "", 0) - pRouter := pineconeRouter.NewRouter(logger, "dendrite", sk, pk, nil) + pRouter := pineconeRouter.NewRouter(logger, sk, "dendrite", nil) pSessions := pineconeSessions.NewSessions(logger, pRouter) cfg := &config.Dendrite{} From 2f68ab7c2f2dc0b67e28bcdf91e3f8d0473650ee Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 12 Oct 2021 14:48:51 +0100 Subject: [PATCH 46/64] Update to matrix-org/pinecone@d568c316384c42e249b87dcbf35dd6f72aa9257f --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f74b07f7..a9e52f1e 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 - github.com/matrix-org/pinecone v0.0.0-20211008142213-06dcf8511285 + github.com/matrix-org/pinecone v0.0.0-20211012134609-d568c316384c github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.8 github.com/morikuni/aec v1.0.0 // indirect diff --git a/go.sum b/go.sum index af7a3b59..142bca6b 100644 --- a/go.sum +++ b/go.sum @@ -996,8 +996,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 h1:b8 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20211008142213-06dcf8511285 h1:W5OSspxTxZlb03wUEywga5DickEmUdQzzWFSJfmy6Oc= -github.com/matrix-org/pinecone v0.0.0-20211008142213-06dcf8511285/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= +github.com/matrix-org/pinecone v0.0.0-20211012134609-d568c316384c h1:YgcBXZE0RQ2HlttDWCb1nY4qGFdsRmuZxDHrrQUpKG4= +github.com/matrix-org/pinecone v0.0.0-20211012134609-d568c316384c/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From d751e97e03bf96514a77e37789077c8ae3edba8c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 13 Oct 2021 09:19:00 +0100 Subject: [PATCH 47/64] Update sytest blacklist/whitelist for known flaky tests (closes matrix-org/sytest#1155) --- sytest-blacklist | 2 ++ sytest-whitelist | 1 + 2 files changed, 3 insertions(+) diff --git a/sytest-blacklist b/sytest-blacklist index a0aba69c..aa458997 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -73,6 +73,8 @@ Local device key changes get to remote servers with correct prev_id # Flakey Local device key changes appear in /keys/changes +Device list doesn't change if remote server is down +If a device list update goes missing, the server resyncs on the next one # we don't support groups Remove group category diff --git a/sytest-whitelist b/sytest-whitelist index 9f3eb893..0067e2d2 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -556,3 +556,4 @@ Fails to upload self-signing key without master key can fetch self-signing keys over federation Changing master key notifies local users Changing user-signing key notifies local users +Inbound federation correctly handles soft failed events as extremities From 09511c60862b451a47a68f370631836399121e50 Mon Sep 17 00:00:00 2001 From: kegsay Date: Thu, 21 Oct 2021 12:13:01 +0100 Subject: [PATCH 48/64] Upgrade containerd dep (#2024) --- go.mod | 2 +- go.sum | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a9e52f1e..de6a0f5d 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/Shopify/sarama v1.29.1 github.com/codeclysm/extract v2.2.0+incompatible - github.com/containerd/containerd v1.5.5 // indirect + github.com/containerd/containerd v1.5.7 // indirect github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 github.com/getsentry/sentry-go v0.11.0 diff --git a/go.sum b/go.sum index 142bca6b..b33ce961 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,7 @@ github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2 github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= github.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -219,6 +220,8 @@ github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoT github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.5 h1:q1gxsZsGZ8ddVe98yO6pR21b5xQSMiR61lD0W96pgQo= github.com/containerd/containerd v1.5.5/go.mod h1:oSTh0QpT1w6jYcGmbiSbxv9OSQYaa88mPyWIuU79zyo= +github.com/containerd/containerd v1.5.7 h1:rQyoYtj4KddB3bxG6SAqd4+08gePNyJjRqvOIfV3rkM= +github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -575,6 +578,7 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -1191,6 +1195,7 @@ github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= github.com/opencontainers/runc v1.0.1/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= From 3b7a4ad640f260b3918a35684b05ced077a09f0c Mon Sep 17 00:00:00 2001 From: kegsay Date: Thu, 21 Oct 2021 15:35:36 +0100 Subject: [PATCH 49/64] sytest: update blacklist and whitelist (#2025) Should make CI happy again --- sytest-blacklist | 4 ++++ sytest-whitelist | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/sytest-blacklist b/sytest-blacklist index aa458997..6ab9b352 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -79,3 +79,7 @@ If a device list update goes missing, the server resyncs on the next one # we don't support groups Remove group category Remove group role + +# See https://github.com/matrix-org/sytest/pull/1142 +Device list doesn't change if remote server is down +If a device list update goes missing, the server resyncs on the next one diff --git a/sytest-whitelist b/sytest-whitelist index 0067e2d2..c9740049 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -557,3 +557,11 @@ can fetch self-signing keys over federation Changing master key notifies local users Changing user-signing key notifies local users Inbound federation correctly handles soft failed events as extremities +User can create and send/receive messages in a room with version 7 +local user can join room with version 7 +User can invite local user to room with version 7 +remote user can join room with version 7 +User can invite remote user to room with version 7 +Remote user can backfill in a room with version 7 +Can reject invites over federation for rooms with version 7 +Can receive redactions from regular users over federation in room version 7 From 4c129e3e3d6e8d952a87f79d9fe6cb343e2de341 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 Oct 2021 11:38:04 +0100 Subject: [PATCH 50/64] Update Pinecone demo --- build/gobind-pinecone/monolith.go | 4 ++-- cmd/dendrite-demo-pinecone/conn/client.go | 2 +- cmd/dendrite-demo-pinecone/main.go | 6 +++--- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index bd1b9998..21dc128c 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -132,7 +132,7 @@ func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error) for i := 1; i <= 10; i++ { logrus.Errorf("Attempting authenticated connect (attempt %d)", i) var err error - conduit.port, err = m.PineconeRouter.AuthenticatedConnect(l, zone, peertype) + conduit.port, err = m.PineconeRouter.AuthenticatedConnect(l, zone, peertype, true) switch err { case io.ErrClosedPipe: logrus.Errorf("Authenticated connect failed due to closed pipe (attempt %d)", i) @@ -253,7 +253,7 @@ func (m *DendriteMonolith) Start() { logrus.SetOutput(BindLogger{}) logger := log.New(os.Stdout, "PINECONE: ", 0) - m.PineconeRouter = pineconeRouter.NewRouter(logger, sk, "dendrite", nil) + m.PineconeRouter = pineconeRouter.NewRouter(logger, sk, false) m.PineconeQUIC = pineconeSessions.NewSessions(logger, m.PineconeRouter) m.PineconeMulticast = pineconeMulticast.NewMulticast(logger, m.PineconeRouter) diff --git a/cmd/dendrite-demo-pinecone/conn/client.go b/cmd/dendrite-demo-pinecone/conn/client.go index 5ba8e8ec..29436fda 100644 --- a/cmd/dendrite-demo-pinecone/conn/client.go +++ b/cmd/dendrite-demo-pinecone/conn/client.go @@ -34,7 +34,7 @@ func ConnectToPeer(pRouter *pineconeRouter.Router, peer string) error { if parent == nil { return fmt.Errorf("failed to wrap connection") } - _, err := pRouter.AuthenticatedConnect(parent, "static", pineconeRouter.PeerTypeRemote) + _, err := pRouter.AuthenticatedConnect(parent, "static", pineconeRouter.PeerTypeRemote, true) return err } diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 7aa72632..60b83dc7 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -91,7 +91,7 @@ func main() { } logger := log.New(os.Stdout, "", 0) - pRouter := pineconeRouter.NewRouter(logger, sk, "dendrite", nil) + pRouter := pineconeRouter.NewRouter(logger, sk, false) go func() { listener, err := net.Listen("tcp", *instanceListen) @@ -108,7 +108,7 @@ func main() { continue } - port, err := pRouter.AuthenticatedConnect(conn, "", pineconeRouter.PeerTypeRemote) + port, err := pRouter.AuthenticatedConnect(conn, "", pineconeRouter.PeerTypeRemote, true) if err != nil { logrus.WithError(err).Error("pSwitch.AuthenticatedConnect failed") continue @@ -231,7 +231,7 @@ func main() { return } conn := conn.WrapWebSocketConn(c) - if _, err = pRouter.AuthenticatedConnect(conn, "websocket", pineconeRouter.PeerTypeRemote); err != nil { + if _, err = pRouter.AuthenticatedConnect(conn, "websocket", pineconeRouter.PeerTypeRemote, true); err != nil { logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch") } }) diff --git a/go.mod b/go.mod index de6a0f5d..72a3a11b 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 - github.com/matrix-org/pinecone v0.0.0-20211012134609-d568c316384c + github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.8 github.com/morikuni/aec v1.0.0 // indirect diff --git a/go.sum b/go.sum index b33ce961..268dc671 100644 --- a/go.sum +++ b/go.sum @@ -1000,8 +1000,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 h1:b8 github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20211012134609-d568c316384c h1:YgcBXZE0RQ2HlttDWCb1nY4qGFdsRmuZxDHrrQUpKG4= -github.com/matrix-org/pinecone v0.0.0-20211012134609-d568c316384c/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= +github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 h1:6JkIymZ1vxfI0shSpg6gNPTJaF4/95Evy34slPVZGKM= +github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From aa26c2ed5076a5c93680edb79236dfee16c7ae9f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 Oct 2021 11:40:24 +0100 Subject: [PATCH 51/64] Fix dendritejs build --- cmd/dendritejs-pinecone/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index abfcfbd9..46e65783 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -156,7 +156,7 @@ func startup() { pk := sk.Public().(ed25519.PublicKey) logger := log.New(os.Stdout, "", 0) - pRouter := pineconeRouter.NewRouter(logger, sk, "dendrite", nil) + pRouter := pineconeRouter.NewRouter(logger, sk, false) pSessions := pineconeSessions.NewSessions(logger, pRouter) cfg := &config.Dendrite{} From b8d9d792d7b8f16ea2a03e5023a9de01452e81fc Mon Sep 17 00:00:00 2001 From: S7evinK Date: Mon, 25 Oct 2021 11:08:02 +0200 Subject: [PATCH 52/64] Update are-we-synapse-yet (#2018) * Update are-we-synapse-yet Signed-off-by: Till Faelligen * More missing presence tests * Move tests to presence only --- are-we-synapse-yet.list | 280 ++++++++++++++++++++-------------------- 1 file changed, 140 insertions(+), 140 deletions(-) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index d057e8e3..c2c726e5 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -152,219 +152,219 @@ jon Joining room twice is idempotent syn New room members see their own join event v1s New room members see existing users' presence in room initialSync syn Existing members see new members' join events -syn Existing members see new members' presence +pre Existing members see new members' presence v1s All room members see all room members' presence in global initialSync f,jon Remote users can join room by alias syn New room members see their own join event v1s New room members see existing members' presence in room initialSync syn Existing members see new members' join events -syn Existing members see new member's presence -v1s New room members see first user's profile information in global initialSync -v1s New room members see first user's profile information in per-room initialSync -f,jon Remote users may not join unfederated rooms +pre Existing members see new member's presence +v1s New room members see first user's profile information in global initialSync +v1s New room members see first user's profile information in per-room initialSync +f,jon Remote users may not join unfederated rooms syn Local room members see posted message events v1s Fetching eventstream a second time doesn't yield the message again syn Local non-members don't see posted message events -get Local room members can get room messages +get Local room members can get room messages f,syn Remote room members also see posted message events -f,get Remote room members can get room messages +f,get Remote room members can get room messages get Message history can be paginated f,get Message history can be paginated over federation -eph Ephemeral messages received from clients are correctly expired +eph Ephemeral messages received from clients are correctly expired ali Room aliases can contain Unicode f,ali Remote room alias queries can handle Unicode -ali Canonical alias can be set -ali Canonical alias can include alt_aliases +ali Canonical alias can be set +ali Canonical alias can include alt_aliases ali Regular users can add and delete aliases in the default room configuration ali Regular users can add and delete aliases when m.room.aliases is restricted ali Deleting a non-existent alias should return a 404 ali Users can't delete other's aliases -ali Users with sufficient power-level can delete other's aliases -ali Can delete canonical alias -ali Alias creators can delete alias with no ops -ali Alias creators can delete canonical alias with no ops -msc Only room members can list aliases of a room -inv Can invite users to invite-only rooms -inv Uninvited users cannot join the room -inv Invited user can reject invite -f,inv Invited user can reject invite over federation -f,inv Invited user can reject invite over federation several times -inv Invited user can reject invite for empty room -f,inv Invited user can reject invite over federation for empty room -inv Invited user can reject local invite after originator leaves -inv Invited user can see room metadata -f,inv Remote invited user can see room metadata -inv Users cannot invite themselves to a room -inv Users cannot invite a user that is already in the room -ban Banned user is kicked and may not rejoin until unbanned -f,ban Remote banned user is kicked and may not rejoin until unbanned -ban 'ban' event respects room powerlevel -plv setting 'm.room.name' respects room powerlevel +ali Users with sufficient power-level can delete other's aliases +ali Can delete canonical alias +ali Alias creators can delete alias with no ops +ali Alias creators can delete canonical alias with no ops +msc Only room members can list aliases of a room +inv Can invite users to invite-only rooms +inv Uninvited users cannot join the room +inv Invited user can reject invite +f,inv Invited user can reject invite over federation +f,inv Invited user can reject invite over federation several times +inv Invited user can reject invite for empty room +f,inv Invited user can reject invite over federation for empty room +inv Invited user can reject local invite after originator leaves +inv Invited user can see room metadata +f,inv Remote invited user can see room metadata +inv Users cannot invite themselves to a room +inv Users cannot invite a user that is already in the room +ban Banned user is kicked and may not rejoin until unbanned +f,ban Remote banned user is kicked and may not rejoin until unbanned +ban 'ban' event respects room powerlevel +plv setting 'm.room.name' respects room powerlevel plv setting 'm.room.power_levels' respects room powerlevel (2 subtests) plv Unprivileged users can set m.room.topic if it only needs level 0 plv Users cannot set ban powerlevel higher than their own (2 subtests) plv Users cannot set kick powerlevel higher than their own (2 subtests) plv Users cannot set redact powerlevel higher than their own (2 subtests) -v1s Check that event streams started after a client joined a room work (SYT-1) -v1s Event stream catches up fully after many messages -xxx POST /rooms/:room_id/redact/:event_id as power user redacts message -xxx POST /rooms/:room_id/redact/:event_id as original message sender redacts message -xxx POST /rooms/:room_id/redact/:event_id as random user does not redact message -xxx POST /redact disallows redaction of event in different room -xxx Redaction of a redaction redacts the redaction reason -v1s A departed room is still included in /initialSync (SPEC-216) -v1s Can get rooms/{roomId}/initialSync for a departed room (SPEC-216) -rst Can get rooms/{roomId}/state for a departed room (SPEC-216) +v1s Check that event streams started after a client joined a room work (SYT-1) +v1s Event stream catches up fully after many messages +xxx POST /rooms/:room_id/redact/:event_id as power user redacts message +xxx POST /rooms/:room_id/redact/:event_id as original message sender redacts message +xxx POST /rooms/:room_id/redact/:event_id as random user does not redact message +xxx POST /redact disallows redaction of event in different room +xxx Redaction of a redaction redacts the redaction reason +v1s A departed room is still included in /initialSync (SPEC-216) +v1s Can get rooms/{roomId}/initialSync for a departed room (SPEC-216) +rst Can get rooms/{roomId}/state for a departed room (SPEC-216) mem Can get rooms/{roomId}/members for a departed room (SPEC-216) -get Can get rooms/{roomId}/messages for a departed room (SPEC-216) -rst Can get 'm.room.name' state for a departed room (SPEC-216) +get Can get rooms/{roomId}/messages for a departed room (SPEC-216) +rst Can get 'm.room.name' state for a departed room (SPEC-216) syn Getting messages going forward is limited for a departed room (SPEC-216) -3pd Can invite existing 3pid -3pd Can invite existing 3pid with no ops into a private room -3pd Can invite existing 3pid in createRoom -3pd Can invite unbound 3pid -f,3pd Can invite unbound 3pid over federation -3pd Can invite unbound 3pid with no ops into a private room -f,3pd Can invite unbound 3pid over federation with no ops into a private room -f,3pd Can invite unbound 3pid over federation with users from both servers -3pd Can accept unbound 3pid invite after inviter leaves -3pd Can accept third party invite with /join +3pd Can invite existing 3pid +3pd Can invite existing 3pid with no ops into a private room +3pd Can invite existing 3pid in createRoom +3pd Can invite unbound 3pid +f,3pd Can invite unbound 3pid over federation +3pd Can invite unbound 3pid with no ops into a private room +f,3pd Can invite unbound 3pid over federation with no ops into a private room +f,3pd Can invite unbound 3pid over federation with users from both servers +3pd Can accept unbound 3pid invite after inviter leaves +3pd Can accept third party invite with /join 3pd 3pid invite join with wrong but valid signature are rejected 3pd 3pid invite join valid signature but revoked keys are rejected 3pd 3pid invite join valid signature but unreachable ID server are rejected gst Guest user cannot call /events globally gst Guest users can join guest_access rooms -gst Guest users can send messages to guest_access rooms if joined -gst Guest user calling /events doesn't tightloop -gst Guest users are kicked from guest_access rooms on revocation of guest_access +gst Guest users can send messages to guest_access rooms if joined +gst Guest user calling /events doesn't tightloop +gst Guest users are kicked from guest_access rooms on revocation of guest_access gst Guest user can set display names -gst Guest users are kicked from guest_access rooms on revocation of guest_access over federation -gst Guest user can upgrade to fully featured user +gst Guest users are kicked from guest_access rooms on revocation of guest_access over federation +gst Guest user can upgrade to fully featured user gst Guest user cannot upgrade other users -pub GET /publicRooms lists rooms -pub GET /publicRooms includes avatar URLs -gst Guest users can accept invites to private rooms over federation -gst Guest users denied access over federation if guest access prohibited -mem Room members can override their displayname on a room-specific basis +pub GET /publicRooms lists rooms +pub GET /publicRooms includes avatar URLs +gst Guest users can accept invites to private rooms over federation +gst Guest users denied access over federation if guest access prohibited +mem Room members can override their displayname on a room-specific basis mem Room members can join a room with an overridden displayname -mem Users cannot kick users from a room they are not in -mem Users cannot kick users who have already left a room -typ Typing notification sent to local room members -f,typ Typing notifications also sent to remote room members -typ Typing can be explicitly stopped -rct Read receipts are visible to /initialSync -rct Read receipts are sent as events -rct Receipts must be m.read -pro displayname updates affect room member events -pro avatar_url updates affect room member events +mem Users cannot kick users from a room they are not in +mem Users cannot kick users who have already left a room +typ Typing notification sent to local room members +f,typ Typing notifications also sent to remote room members +typ Typing can be explicitly stopped +rct Read receipts are visible to /initialSync +rct Read receipts are sent as events +rct Receipts must be m.read +pro displayname updates affect room member events +pro avatar_url updates affect room member events gst m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users -gst m.room.history_visibility == "shared" allows/forbids appropriately for Guest users -gst m.room.history_visibility == "invited" allows/forbids appropriately for Guest users -gst m.room.history_visibility == "joined" allows/forbids appropriately for Guest users -gst m.room.history_visibility == "default" allows/forbids appropriately for Guest users +gst m.room.history_visibility == "shared" allows/forbids appropriately for Guest users +gst m.room.history_visibility == "invited" allows/forbids appropriately for Guest users +gst m.room.history_visibility == "joined" allows/forbids appropriately for Guest users +gst m.room.history_visibility == "default" allows/forbids appropriately for Guest users gst Guest non-joined user cannot call /events on shared room gst Guest non-joined user cannot call /events on invited room gst Guest non-joined user cannot call /events on joined room gst Guest non-joined user cannot call /events on default room -gst Guest non-joined user can call /events on world_readable room +gst Guest non-joined user can call /events on world_readable room gst Guest non-joined users can get state for world_readable rooms gst Guest non-joined users can get individual state for world_readable rooms gst Guest non-joined users cannot room initalSync for non-world_readable rooms -gst Guest non-joined users can room initialSync for world_readable rooms +gst Guest non-joined users can room initialSync for world_readable rooms gst Guest non-joined users can get individual state for world_readable rooms after leaving gst Guest non-joined users cannot send messages to guest_access rooms if not joined gst Guest users can sync from world_readable guest_access rooms if joined -gst Guest users can sync from shared guest_access rooms if joined -gst Guest users can sync from invited guest_access rooms if joined -gst Guest users can sync from joined guest_access rooms if joined +gst Guest users can sync from shared guest_access rooms if joined +gst Guest users can sync from invited guest_access rooms if joined +gst Guest users can sync from joined guest_access rooms if joined gst Guest users can sync from default guest_access rooms if joined ath m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users -ath m.room.history_visibility == "shared" allows/forbids appropriately for Real users -ath m.room.history_visibility == "invited" allows/forbids appropriately for Real users -ath m.room.history_visibility == "joined" allows/forbids appropriately for Real users -ath m.room.history_visibility == "default" allows/forbids appropriately for Real users +ath m.room.history_visibility == "shared" allows/forbids appropriately for Real users +ath m.room.history_visibility == "invited" allows/forbids appropriately for Real users +ath m.room.history_visibility == "joined" allows/forbids appropriately for Real users +ath m.room.history_visibility == "default" allows/forbids appropriately for Real users ath Real non-joined user cannot call /events on shared room ath Real non-joined user cannot call /events on invited room ath Real non-joined user cannot call /events on joined room ath Real non-joined user cannot call /events on default room -ath Real non-joined user can call /events on world_readable room +ath Real non-joined user can call /events on world_readable room ath Real non-joined users can get state for world_readable rooms ath Real non-joined users can get individual state for world_readable rooms ath Real non-joined users cannot room initalSync for non-world_readable rooms -ath Real non-joined users can room initialSync for world_readable rooms -ath Real non-joined users can get individual state for world_readable rooms after leaving +ath Real non-joined users can room initialSync for world_readable rooms +ath Real non-joined users can get individual state for world_readable rooms after leaving ath Real non-joined users cannot send messages to guest_access rooms if not joined ath Real users can sync from world_readable guest_access rooms if joined -ath Real users can sync from shared guest_access rooms if joined -ath Real users can sync from invited guest_access rooms if joined -ath Real users can sync from joined guest_access rooms if joined +ath Real users can sync from shared guest_access rooms if joined +ath Real users can sync from invited guest_access rooms if joined +ath Real users can sync from joined guest_access rooms if joined ath Real users can sync from default guest_access rooms if joined -ath Only see history_visibility changes on boundaries +ath Only see history_visibility changes on boundaries f,ath Backfill works correctly with history visibility set to joined -fgt Forgotten room messages cannot be paginated -fgt Forgetting room does not show up in v2 /sync -fgt Can forget room you've been kicked from +fgt Forgotten room messages cannot be paginated +fgt Forgetting room does not show up in v2 /sync +fgt Can forget room you've been kicked from fgt Can't forget room you're still in fgt Can re-join room if re-invited -ath Only original members of the room can see messages from erased users +ath Only original members of the room can see messages from erased users mem /joined_rooms returns only joined rooms -mem /joined_members return joined members -ctx /context/ on joined room works -ctx /context/ on non world readable room does not work -ctx /context/ returns correct number of events -ctx /context/ with lazy_load_members filter works +mem /joined_members return joined members +ctx /context/ on joined room works +ctx /context/ on non world readable room does not work +ctx /context/ returns correct number of events +ctx /context/ with lazy_load_members filter works get /event/ on joined room works get /event/ on non world readable room does not work get /event/ does not allow access to events before the user joined mem Can get rooms/{roomId}/members -mem Can get rooms/{roomId}/members at a given point -mem Can filter rooms/{roomId}/members -upg /upgrade creates a new room -upg /upgrade should preserve room visibility for public rooms -upg /upgrade should preserve room visibility for private rooms -upg /upgrade copies >100 power levels to the new room -upg /upgrade copies the power levels to the new room -upg /upgrade preserves the power level of the upgrading user in old and new rooms -upg /upgrade copies important state to the new room -upg /upgrade copies ban events to the new room -upg local user has push rules copied to upgraded room -f,upg remote user has push rules copied to upgraded room -upg /upgrade moves aliases to the new room -upg /upgrade moves remote aliases to the new room -upg /upgrade preserves direct room state -upg /upgrade preserves room federation ability -upg /upgrade restricts power levels in the old room -upg /upgrade restricts power levels in the old room when the old PLs are unusual -upg /upgrade to an unknown version is rejected -upg /upgrade is rejected if the user can't send state events -upg /upgrade of a bogus room fails gracefully -upg Cannot send tombstone event that points to the same room -f,upg Local and remote users' homeservers remove a room from their public directory on upgrade -rst Name/topic keys are correct +mem Can get rooms/{roomId}/members at a given point +mem Can filter rooms/{roomId}/members +upg /upgrade creates a new room +upg /upgrade should preserve room visibility for public rooms +upg /upgrade should preserve room visibility for private rooms +upg /upgrade copies >100 power levels to the new room +upg /upgrade copies the power levels to the new room +upg /upgrade preserves the power level of the upgrading user in old and new rooms +upg /upgrade copies important state to the new room +upg /upgrade copies ban events to the new room +upg local user has push rules copied to upgraded room +f,upg remote user has push rules copied to upgraded room +upg /upgrade moves aliases to the new room +upg /upgrade moves remote aliases to the new room +upg /upgrade preserves direct room state +upg /upgrade preserves room federation ability +upg /upgrade restricts power levels in the old room +upg /upgrade restricts power levels in the old room when the old PLs are unusual +upg /upgrade to an unknown version is rejected +upg /upgrade is rejected if the user can't send state events +upg /upgrade of a bogus room fails gracefully +upg Cannot send tombstone event that points to the same room +f,upg Local and remote users' homeservers remove a room from their public directory on upgrade +rst Name/topic keys are correct f,pub Can get remote public room list pub Can paginate public room list -pub Can search public room list +pub Can search public room list syn Can create filter syn Can download filter syn Can sync syn Can sync a joined room syn Full state sync includes joined rooms syn Newly joined room is included in an incremental sync -syn Newly joined room has correct timeline in incremental sync -syn Newly joined room includes presence in incremental sync -syn Get presence for newly joined members in incremental sync -syn Can sync a room with a single message -syn Can sync a room with a message with a transaction id +syn Newly joined room has correct timeline in incremental sync +pre Newly joined room includes presence in incremental sync +pre Get presence for newly joined members in incremental sync +syn Can sync a room with a single message +syn Can sync a room with a message with a transaction id syn A message sent after an initial sync appears in the timeline of an incremental sync. -syn A filtered timeline reaches its limit -syn Syncing a new room with a large timeline limit isn't limited -syn A full_state incremental update returns only recent timeline -syn A prev_batch token can be used in the v1 messages API -syn A next_batch token can be used in the v1 messages API +syn A filtered timeline reaches its limit +syn Syncing a new room with a large timeline limit isn't limited +syn A full_state incremental update returns only recent timeline +syn A prev_batch token can be used in the v1 messages API +syn A next_batch token can be used in the v1 messages API syn A prev_batch token from incremental sync can be used in the v1 messages API -syn User sees their own presence in a sync -syn User is offline if they set_presence=offline in their sync -syn User sees updates to presence from other users in the incremental sync. +pre User sees their own presence in a sync +pre User is offline if they set_presence=offline in their sync +pre User sees updates to presence from other users in the incremental sync. syn State is included in the timeline in the initial sync f,syn State from remote users is included in the state in the initial sync syn Changes to state are included in an incremental sync From ba2f456e86b0cb06f95c28da09d84d5a997b3d7e Mon Sep 17 00:00:00 2001 From: cofob Date: Mon, 25 Oct 2021 16:08:47 +0700 Subject: [PATCH 53/64] Update README.md (#2012) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d8d6b2b6..30bf19f6 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ We are prioritising features that will benefit single-user homeservers first (e. than features that massive deployments may be interested in (User Directory, OpenID, Guests, Admin APIs, AS API). This means Dendrite supports amongst others: - Core room functionality (creating rooms, invites, auth rules) - - Federation in rooms v1-v6 + - Federation in rooms v1-v7 - Backfilling locally and via federation - Accounts, Profiles and Devices - Published room lists From 23c20c40c12dea6b5ae99e8d601baad37c46d0a4 Mon Sep 17 00:00:00 2001 From: kegsay Date: Fri, 29 Oct 2021 12:06:10 +0100 Subject: [PATCH 54/64] generate-config: add flags for db and server name (#2030) --- cmd/generate-config/main.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index bd70cbc5..e300a1ca 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -5,16 +5,35 @@ import ( "fmt" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" "gopkg.in/yaml.v2" ) func main() { defaultsForCI := flag.Bool("ci", false, "sane defaults for CI testing") + serverName := flag.String("server", "", "The domain name of the server if not 'localhost'") + dbURI := flag.String("db", "", "The DB URI to use for all components if not SQLite files") flag.Parse() cfg := &config.Dendrite{} cfg.Defaults() + if *serverName != "" { + cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName) + } + if *dbURI != "" { + cfg.Global.Kafka.Database.ConnectionString = config.DataSource(*dbURI) + cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(*dbURI) + cfg.FederationSender.Database.ConnectionString = config.DataSource(*dbURI) + cfg.KeyServer.Database.ConnectionString = config.DataSource(*dbURI) + cfg.MSCs.Database.ConnectionString = config.DataSource(*dbURI) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(*dbURI) + cfg.RoomServer.Database.ConnectionString = config.DataSource(*dbURI) + cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(*dbURI) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(*dbURI) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(*dbURI) + cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(*dbURI) + } cfg.Global.TrustedIDServers = []string{ "matrix.org", "vector.im", From a22a6d360fd7d3b3897da9528f2fe6333689def8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 29 Oct 2021 12:06:38 +0100 Subject: [PATCH 55/64] Add 'Federation publicRoom Name/topic keys are correct' to the sytest whitelist (#2029) Sytest had two tests with the name `Name/topic keys are correct`. https://github.com/matrix-org/sytest/pull/1098 edited one so that the names were unique. It seems that this test name was in the dendrite Sytest whitelist. This PR adds the additional, new name to the whitelist as well. Time to see if CI passes! --- sytest-whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/sytest-whitelist b/sytest-whitelist index c9740049..019caac4 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -565,3 +565,4 @@ User can invite remote user to room with version 7 Remote user can backfill in a room with version 7 Can reject invites over federation for rooms with version 7 Can receive redactions from regular users over federation in room version 7 +Federation publicRoom Name/topic keys are correct From 92026a73ceb41ef2e7e65eac7edb79fa729342ef Mon Sep 17 00:00:00 2001 From: kegsay Date: Fri, 29 Oct 2021 15:01:22 +0100 Subject: [PATCH 56/64] Remove tests from the blacklist (#2031) * Remove tests from the blacklist They seem to be more reliable now * Update whitelist --- sytest-blacklist | 49 ------------------------------------------------ sytest-whitelist | 5 ++++- 2 files changed, 4 insertions(+), 50 deletions(-) diff --git a/sytest-blacklist b/sytest-blacklist index 6ab9b352..5e562845 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -1,52 +1,9 @@ -# Blacklisted due to flakiness -Remote users can join room by alias - -# Blacklisted due to flakiness -POST /login can log in as a user with just the local part of the id - -# Blacklisted due to flakiness -avatar_url updates affect room member events - -# Blacklisted due to flakiness -displayname updates affect room member events - -# Blacklisted due to flakiness -Room members can override their displayname on a room-specific basis - -# Blacklisted due to flakiness -Alias creators can delete alias with no ops - -# Blacklisted because matrix-org/dendrite#847 might have broken it but we're not -# really sure and we need it pretty badly anyway -Real non-joined users can get individual state for world_readable rooms after leaving - # Blacklisted until matrix-org/dendrite#862 is reverted due to Riot bug Latest account data appears in v2 /sync -# Blacklisted due to flakiness -Outbound federation can backfill events - -# Blacklisted due to alias work on Synapse -Alias creators can delete canonical alias with no ops - -# Blacklisted because we need to implement v2 invite endpoints for room versions -# to be supported (currently fails with M_UNSUPPORTED_ROOM_VERSION) -Inbound federation rejects invites which are not signed by the sender - # Blacklisted because we don't support ignores yet Ignore invite in incremental sync -# Blacklisted because this test calls /r0/events which we don't implement -New room members see their own join event -Existing members see new members' join events - -# See https://github.com/matrix-org/sytest/pull/901 -Remote invited user can see room metadata - -# We don't implement soft-failed events yet, but because the /send response is vague, -# this test thinks it's all fine... -Inbound federation accepts a second soft-failed event - # Relies on a rejected PL event which will never be accepted into the DAG # Caused by https://github.com/matrix-org/sytest/pull/911 Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state @@ -59,15 +16,9 @@ Invited user can reject local invite after originator leaves Invited user can reject invite for empty room If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes -# Blacklisted due to flakiness -A prev_batch token from incremental sync can be used in the v1 messages API - # Blacklisted due to flakiness Forgotten room messages cannot be paginated -# Blacklisted due to flakiness -Can re-join room if re-invited - # Blacklisted due to flakiness after #1774 Local device key changes get to remote servers with correct prev_id diff --git a/sytest-whitelist b/sytest-whitelist index 019caac4..16c7e544 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -136,7 +136,6 @@ query for user with no keys returns empty key dict Can claim one time key using POST Can claim remote one time key using POST Local device key changes appear in v2 /sync -Local device key changes appear in /keys/changes New users appear in /keys/changes Local delete device changes appear in v2 /sync Local new device changes appear in v2 /sync @@ -566,3 +565,7 @@ Remote user can backfill in a room with version 7 Can reject invites over federation for rooms with version 7 Can receive redactions from regular users over federation in room version 7 Federation publicRoom Name/topic keys are correct +Remote invited user can see room metadata +Can re-join room if re-invited +A prev_batch token from incremental sync can be used in the v1 messages API +Inbound federation rejects invites which are not signed by the sender From 42973e7d65c218eb6771de4707e0aeaa707867fc Mon Sep 17 00:00:00 2001 From: kegsay Date: Mon, 1 Nov 2021 15:05:20 +0000 Subject: [PATCH 57/64] Add part of #1322 tests (#2034) * Add part of #1322 tests * Fix #1322 --- sytest-whitelist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sytest-whitelist b/sytest-whitelist index 16c7e544..afe8a7e0 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -569,3 +569,5 @@ Remote invited user can see room metadata Can re-join room if re-invited A prev_batch token from incremental sync can be used in the v1 messages API Inbound federation rejects invites which are not signed by the sender +Invited user can reject invite over federation several times +Test that we can be reinvited to a room we created From fbd1a0ab13b330026b3893ee40582fcacbfae376 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 2 Nov 2021 10:13:38 +0000 Subject: [PATCH 58/64] Update to matrix-org/gomatrixserverlib@5e02b64 --- federationapi/routing/send.go | 4 ++-- federationsender/internal/perform.go | 8 +------- go.mod | 5 ++--- go.sum | 23 ++++++++++------------- 4 files changed, 15 insertions(+), 25 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index d31d38dc..3cae837c 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -306,7 +306,7 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res } continue } - if err = gomatrixserverlib.VerifyAllEventSignatures(ctx, []*gomatrixserverlib.Event{event}, t.keys); err != nil { + if err = event.VerifyEventSignatures(ctx, t.keys); err != nil { util.GetLogger(ctx).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID()) results[event.EventID()] = gomatrixserverlib.PDUResult{ Error: err.Error(), @@ -1358,7 +1358,7 @@ func (t *txnReq) lookupEvent(ctx context.Context, roomVersion gomatrixserverlib. util.GetLogger(ctx).WithField("event_id", missingEventID).Warnf("Failed to get missing /event for event ID from %d server(s)", len(servers)) return nil, fmt.Errorf("wasn't able to find event via %d server(s)", len(servers)) } - if err := gomatrixserverlib.VerifyAllEventSignatures(ctx, []*gomatrixserverlib.Event{event}, t.keys); err != nil { + if err := event.VerifyEventSignatures(ctx, t.keys); err != nil { util.GetLogger(ctx).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID()) return nil, verifySigError{event.EventID(), err} } diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 2df936e3..53fa974b 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -713,14 +713,8 @@ func federatedAuthProvider( } // Check the signatures of the event. - if res, err := gomatrixserverlib.VerifyEventSignatures(ctx, []*gomatrixserverlib.Event{ev}, keyRing); err != nil { + if err := ev.VerifyEventSignatures(ctx, keyRing); err != nil { return nil, fmt.Errorf("missingAuth VerifyEventSignatures: %w", err) - } else { - for _, err := range res { - if err != nil { - return nil, fmt.Errorf("missingAuth VerifyEventSignatures: %w", err) - } - } } // If the event is OK then add it to the results and the retry map. diff --git a/go.mod b/go.mod index 72a3a11b..82cda435 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 + github.com/matrix-org/gomatrixserverlib v0.0.0-20211102101113-5e02b64e5312 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 @@ -46,8 +46,7 @@ require ( github.com/pressly/goose v2.7.0+incompatible github.com/prometheus/client_golang v1.11.0 github.com/sirupsen/logrus v1.8.1 - github.com/tidwall/gjson v1.8.1 - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/gjson v1.9.3 github.com/tidwall/sjson v1.1.7 github.com/uber/jaeger-client-go v2.29.1+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible diff --git a/go.sum b/go.sum index 268dc671..1d0dacac 100644 --- a/go.sum +++ b/go.sum @@ -77,7 +77,6 @@ github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg3 github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= -github.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= @@ -218,8 +217,6 @@ github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= -github.com/containerd/containerd v1.5.5 h1:q1gxsZsGZ8ddVe98yO6pR21b5xQSMiR61lD0W96pgQo= -github.com/containerd/containerd v1.5.5/go.mod h1:oSTh0QpT1w6jYcGmbiSbxv9OSQYaa88mPyWIuU79zyo= github.com/containerd/containerd v1.5.7 h1:rQyoYtj4KddB3bxG6SAqd4+08gePNyJjRqvOIfV3rkM= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -996,8 +993,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 h1:b8cyR4aYv9Lmf1lKgASJ+PFSp/GBv8ZFgb/O42ZXLGA= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211102101113-5e02b64e5312 h1:of3kbkOrjE8793Vsni8ofxMSy/vpO1SB+ZkHSfv4wrs= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211102101113-5e02b64e5312/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 h1:6JkIymZ1vxfI0shSpg6gNPTJaF4/95Evy34slPVZGKM= @@ -1036,6 +1033,7 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -1194,7 +1192,6 @@ github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59P github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= -github.com/opencontainers/runc v1.0.1/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1369,15 +1366,12 @@ github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= -github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/gjson v1.8.0/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= -github.com/tidwall/gjson v1.8.1 h1:8j5EE9Hrh3l9Od1OIEDAb7IpezNA20UdRngNAj5N0WU= -github.com/tidwall/gjson v1.8.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= -github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= -github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= +github.com/tidwall/gjson v1.9.3 h1:hqzS9wAHMO+KVBBkLxYdkEeeFHuqr95GfClRLKlgK0E= +github.com/tidwall/gjson v1.9.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -1504,6 +1498,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1665,6 +1660,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1760,6 +1756,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From 1ec5b0709b65541c8acee1a34e7f419f8d2b60f1 Mon Sep 17 00:00:00 2001 From: hacktivista Date: Tue, 2 Nov 2021 10:15:13 +0000 Subject: [PATCH 59/64] Support WebP based thumbnails (#2033) Co-authored-by: Felix Freeman --- go.mod | 1 + go.sum | 3 +++ mediaapi/thumbnailer/thumbnailer_nfnt.go | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/go.mod b/go.mod index 82cda435..6e358997 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.4.1-0.20210715083903-52309d094c00 go.uber.org/atomic v1.9.0 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 + golang.org/x/image v0.0.0-20211028202545-6944b10bf410 golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554 golang.org/x/net v0.0.0-20210927181540-4e4d966f7476 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b diff --git a/go.sum b/go.sum index 1d0dacac..a36ee34d 100644 --- a/go.sum +++ b/go.sum @@ -1529,7 +1529,10 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= diff --git a/mediaapi/thumbnailer/thumbnailer_nfnt.go b/mediaapi/thumbnailer/thumbnailer_nfnt.go index aa9faf4c..d7aa294a 100644 --- a/mediaapi/thumbnailer/thumbnailer_nfnt.go +++ b/mediaapi/thumbnailer/thumbnailer_nfnt.go @@ -27,6 +27,10 @@ import ( // Imported for png codec _ "image/png" + + // Imported for webp codec + _ "golang.org/x/image/webp" + "os" "time" From 5b969d172b6dc900e89b77a630a8b1b72dff44a1 Mon Sep 17 00:00:00 2001 From: database64128 Date: Wed, 3 Nov 2021 00:35:15 +0800 Subject: [PATCH 60/64] =?UTF-8?q?0=EF=B8=8F=E2=83=A3=20Media=20API:=20Fix?= =?UTF-8?q?=20maxFileSizeBytes=3D0=20handling=20(#1993)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mediaapi/routing/download.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 39b86b44..2358915e 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -737,7 +737,7 @@ func (r *downloadRequest) fetchRemoteFile( return "", false, parseErr } - if contentLength > int64(maxFileSizeBytes) { + if maxFileSizeBytes > 0 && contentLength > int64(maxFileSizeBytes) { // TODO: Bubble up this as a 413 return "", false, fmt.Errorf("remote file is too large (%v > %v bytes)", contentLength, maxFileSizeBytes) } From b99f594a93dda7645fabda4b81498ea4ad40bc97 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 2 Nov 2021 16:47:39 +0000 Subject: [PATCH 61/64] Fix #2028 (#2036) --- roomserver/storage/shared/storage.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index b4fc15f1..3685332f 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -165,14 +165,21 @@ func (d *Database) AddState( if berr != nil { return 0, fmt.Errorf("d.StateBlockTable.BulkSelectStateBlockEntries: %w", berr) } + var found bool for i := len(state) - 1; i >= 0; i-- { + found = false for _, events := range blocks { for _, event := range events { if state[i].EventNID == event { - state = append(state[:i], state[i+1:]...) + found = true + break } } } + if found { + state = append(state[:i], state[i+1:]...) + i-- + } } } err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { From dec05c334707346506a979e1cdb3b295dbad5c4f Mon Sep 17 00:00:00 2001 From: PiotrKozimor <37144818+PiotrKozimor@users.noreply.github.com> Date: Tue, 2 Nov 2021 17:48:48 +0100 Subject: [PATCH 62/64] Run gofmt on dendrite - apply go 1.17 preferred build tags (#2021) --- appservice/storage/storage.go | 1 + build/gobind-pinecone/platform_ios.go | 1 + build/gobind-pinecone/platform_other.go | 1 + build/gobind-yggdrasil/platform_ios.go | 1 + build/gobind-yggdrasil/platform_other.go | 1 + cmd/dendrite-demo-pinecone/embed/embed_elementweb.go | 1 + cmd/dendrite-demo-pinecone/embed/embed_other.go | 1 + cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go | 1 + cmd/dendrite-demo-yggdrasil/embed/embed_other.go | 1 + cmd/dendritejs-pinecone/jsServer.go | 1 + cmd/dendritejs-pinecone/main_noop.go | 1 + cmd/dendritejs-pinecone/main_test.go | 1 + cmd/dendritejs/jsServer.go | 1 + cmd/dendritejs/keyfetcher.go | 1 + cmd/dendritejs/main.go | 1 + cmd/dendritejs/main_noop.go | 1 + cmd/dendritejs/publicrooms.go | 1 + federationsender/storage/storage.go | 1 + internal/sqlutil/postgres.go | 1 + internal/sqlutil/postgres_wasm.go | 1 + internal/sqlutil/trace_driver.go | 1 + internal/sqlutil/trace_driver_wasm.go | 1 + keyserver/storage/storage.go | 1 + mediaapi/storage/storage.go | 1 + mediaapi/thumbnailer/thumbnailer_bimg.go | 1 + mediaapi/thumbnailer/thumbnailer_nfnt.go | 1 + roomserver/storage/storage.go | 1 + signingkeyserver/storage/keydb.go | 1 + signingkeyserver/storage/keydb_wasm.go | 1 + syncapi/storage/storage.go | 1 + userapi/storage/accounts/sqlite3/constraint_wasm.go | 1 + userapi/storage/accounts/storage.go | 1 + userapi/storage/devices/storage.go | 1 + 33 files changed, 33 insertions(+) diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index b0df2b7d..97b8501e 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/build/gobind-pinecone/platform_ios.go b/build/gobind-pinecone/platform_ios.go index 01f8a6a0..802d7fac 100644 --- a/build/gobind-pinecone/platform_ios.go +++ b/build/gobind-pinecone/platform_ios.go @@ -1,3 +1,4 @@ +//go:build ios // +build ios package gobind diff --git a/build/gobind-pinecone/platform_other.go b/build/gobind-pinecone/platform_other.go index fdfb13bc..2e81e2f4 100644 --- a/build/gobind-pinecone/platform_other.go +++ b/build/gobind-pinecone/platform_other.go @@ -1,3 +1,4 @@ +//go:build !ios // +build !ios package gobind diff --git a/build/gobind-yggdrasil/platform_ios.go b/build/gobind-yggdrasil/platform_ios.go index 01f8a6a0..802d7fac 100644 --- a/build/gobind-yggdrasil/platform_ios.go +++ b/build/gobind-yggdrasil/platform_ios.go @@ -1,3 +1,4 @@ +//go:build ios // +build ios package gobind diff --git a/build/gobind-yggdrasil/platform_other.go b/build/gobind-yggdrasil/platform_other.go index fdfb13bc..2e81e2f4 100644 --- a/build/gobind-yggdrasil/platform_other.go +++ b/build/gobind-yggdrasil/platform_other.go @@ -1,3 +1,4 @@ +//go:build !ios // +build !ios package gobind diff --git a/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go b/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go index 4d2da55c..8b3be72c 100644 --- a/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go +++ b/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go @@ -1,3 +1,4 @@ +//go:build elementweb // +build elementweb package embed diff --git a/cmd/dendrite-demo-pinecone/embed/embed_other.go b/cmd/dendrite-demo-pinecone/embed/embed_other.go index 04c2188c..a4b22345 100644 --- a/cmd/dendrite-demo-pinecone/embed/embed_other.go +++ b/cmd/dendrite-demo-pinecone/embed/embed_other.go @@ -1,3 +1,4 @@ +//go:build !elementweb // +build !elementweb package embed diff --git a/cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go b/cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go index 8d49c553..e7725ec8 100644 --- a/cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go +++ b/cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go @@ -1,3 +1,4 @@ +//go:build elementweb // +build elementweb package embed diff --git a/cmd/dendrite-demo-yggdrasil/embed/embed_other.go b/cmd/dendrite-demo-yggdrasil/embed/embed_other.go index 04c2188c..a4b22345 100644 --- a/cmd/dendrite-demo-yggdrasil/embed/embed_other.go +++ b/cmd/dendrite-demo-yggdrasil/embed/embed_other.go @@ -1,3 +1,4 @@ +//go:build !elementweb // +build !elementweb package embed diff --git a/cmd/dendritejs-pinecone/jsServer.go b/cmd/dendritejs-pinecone/jsServer.go index 074d20cb..4298c2ae 100644 --- a/cmd/dendritejs-pinecone/jsServer.go +++ b/cmd/dendritejs-pinecone/jsServer.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs-pinecone/main_noop.go b/cmd/dendritejs-pinecone/main_noop.go index dcea032f..0cc7e47e 100644 --- a/cmd/dendritejs-pinecone/main_noop.go +++ b/cmd/dendritejs-pinecone/main_noop.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package main diff --git a/cmd/dendritejs-pinecone/main_test.go b/cmd/dendritejs-pinecone/main_test.go index 751700cb..17fea6cc 100644 --- a/cmd/dendritejs-pinecone/main_test.go +++ b/cmd/dendritejs-pinecone/main_test.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs/jsServer.go b/cmd/dendritejs/jsServer.go index 074d20cb..4298c2ae 100644 --- a/cmd/dendritejs/jsServer.go +++ b/cmd/dendritejs/jsServer.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go index cef04537..cdf93764 100644 --- a/cmd/dendritejs/keyfetcher.go +++ b/cmd/dendritejs/keyfetcher.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index d8fc8b83..10aadb6e 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs/main_noop.go b/cmd/dendritejs/main_noop.go index dcea032f..0cc7e47e 100644 --- a/cmd/dendritejs/main_noop.go +++ b/cmd/dendritejs/main_noop.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package main diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go index a4623ba3..19afc5bc 100644 --- a/cmd/dendritejs/publicrooms.go +++ b/cmd/dendritejs/publicrooms.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index 5462c352..46e01f25 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/internal/sqlutil/postgres.go b/internal/sqlutil/postgres.go index 41a5508a..5e656b1d 100644 --- a/internal/sqlutil/postgres.go +++ b/internal/sqlutil/postgres.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package sqlutil diff --git a/internal/sqlutil/postgres_wasm.go b/internal/sqlutil/postgres_wasm.go index c45842f0..34086f45 100644 --- a/internal/sqlutil/postgres_wasm.go +++ b/internal/sqlutil/postgres_wasm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package sqlutil diff --git a/internal/sqlutil/trace_driver.go b/internal/sqlutil/trace_driver.go index f123b1e4..b7bb3676 100644 --- a/internal/sqlutil/trace_driver.go +++ b/internal/sqlutil/trace_driver.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package sqlutil diff --git a/internal/sqlutil/trace_driver_wasm.go b/internal/sqlutil/trace_driver_wasm.go index a3c163f5..51b60c3c 100644 --- a/internal/sqlutil/trace_driver_wasm.go +++ b/internal/sqlutil/trace_driver_wasm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package sqlutil diff --git a/keyserver/storage/storage.go b/keyserver/storage/storage.go index 8f05d003..742e8463 100644 --- a/keyserver/storage/storage.go +++ b/keyserver/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index a976f795..56059f1c 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/mediaapi/thumbnailer/thumbnailer_bimg.go b/mediaapi/thumbnailer/thumbnailer_bimg.go index 087385a7..6ca53317 100644 --- a/mediaapi/thumbnailer/thumbnailer_bimg.go +++ b/mediaapi/thumbnailer/thumbnailer_bimg.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build bimg // +build bimg package thumbnailer diff --git a/mediaapi/thumbnailer/thumbnailer_nfnt.go b/mediaapi/thumbnailer/thumbnailer_nfnt.go index d7aa294a..beae88c5 100644 --- a/mediaapi/thumbnailer/thumbnailer_nfnt.go +++ b/mediaapi/thumbnailer/thumbnailer_nfnt.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !bimg // +build !bimg package thumbnailer diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 9359312d..9f98ea3e 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/signingkeyserver/storage/keydb.go b/signingkeyserver/storage/keydb.go index aa247f1d..82b6d0ad 100644 --- a/signingkeyserver/storage/keydb.go +++ b/signingkeyserver/storage/keydb.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/signingkeyserver/storage/keydb_wasm.go b/signingkeyserver/storage/keydb_wasm.go index 187d9669..b112993a 100644 --- a/signingkeyserver/storage/keydb_wasm.go +++ b/signingkeyserver/storage/keydb_wasm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package storage diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index 15386c33..7f9c28e9 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/userapi/storage/accounts/sqlite3/constraint_wasm.go b/userapi/storage/accounts/sqlite3/constraint_wasm.go index 0dd5b1fe..6c4ee762 100644 --- a/userapi/storage/accounts/sqlite3/constraint_wasm.go +++ b/userapi/storage/accounts/sqlite3/constraint_wasm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package sqlite3 diff --git a/userapi/storage/accounts/storage.go b/userapi/storage/accounts/storage.go index 3489c9d0..a21f7d94 100644 --- a/userapi/storage/accounts/storage.go +++ b/userapi/storage/accounts/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package accounts diff --git a/userapi/storage/devices/storage.go b/userapi/storage/devices/storage.go index bfce924d..3c203430 100644 --- a/userapi/storage/devices/storage.go +++ b/userapi/storage/devices/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package devices From 0c2872828b7b8aa06a9c6be2c7a5ac0ce119fb6b Mon Sep 17 00:00:00 2001 From: S7evinK Date: Tue, 2 Nov 2021 17:50:59 +0100 Subject: [PATCH 63/64] Fix building on Windows (#2008) Signed-off-by: Till Faelligen --- cmd/create-account/main.go | 5 +-- internal/log.go | 61 --------------------------- internal/log_unix.go | 84 ++++++++++++++++++++++++++++++++++++++ internal/log_windows.go | 48 ++++++++++++++++++++++ 4 files changed, 134 insertions(+), 64 deletions(-) create mode 100644 internal/log_unix.go create mode 100644 internal/log_windows.go diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index a1e254f8..3ac07770 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -22,7 +22,6 @@ import ( "io/ioutil" "os" "strings" - "syscall" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" @@ -121,13 +120,13 @@ func getPassword(password, pwdFile *string, pwdStdin, askPass *bool, r io.Reader // ask the user to provide the password if *askPass { fmt.Print("Enter Password: ") - bytePassword, err := term.ReadPassword(syscall.Stdin) + bytePassword, err := term.ReadPassword(int(os.Stdin.Fd())) if err != nil { logrus.Fatalln("Unable to read password:", err) } fmt.Println() fmt.Print("Confirm Password: ") - bytePassword2, err := term.ReadPassword(syscall.Stdin) + bytePassword2, err := term.ReadPassword(int(os.Stdin.Fd())) if err != nil { logrus.Fatalln("Unable to read password:", err) } diff --git a/internal/log.go b/internal/log.go index f0656d7d..bba0ac6e 100644 --- a/internal/log.go +++ b/internal/log.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io" - "log/syslog" "net/http" "os" "path" @@ -31,7 +30,6 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dugong" "github.com/sirupsen/logrus" - lSyslog "github.com/sirupsen/logrus/hooks/syslog" ) type utcFormatter struct { @@ -108,37 +106,6 @@ func SetupStdLogging() { }) } -// SetupHookLogging configures the logging hooks defined in the configuration. -// If something fails here it means that the logging was improperly configured, -// so we just exit with the error -func SetupHookLogging(hooks []config.LogrusHook, componentName string) { - logrus.SetReportCaller(true) - for _, hook := range hooks { - // Check we received a proper logging level - level, err := logrus.ParseLevel(hook.Level) - if err != nil { - logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err) - } - - // Perform a first filter on the logs according to the lowest level of all - // (Eg: If we have hook for info and above, prevent logrus from processing debug logs) - if logrus.GetLevel() < level { - logrus.SetLevel(level) - } - - switch hook.Type { - case "file": - checkFileHookParams(hook.Params) - setupFileHook(hook, level, componentName) - case "syslog": - checkSyslogHookParams(hook.Params) - setupSyslogHook(hook, level, componentName) - default: - logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) - } - } -} - // File type hooks should be provided a path to a directory to store log files func checkFileHookParams(params map[string]interface{}) { path, ok := params["path"] @@ -178,34 +145,6 @@ func setupFileHook(hook config.LogrusHook, level logrus.Level, componentName str }) } -func checkSyslogHookParams(params map[string]interface{}) { - addr, ok := params["address"] - if !ok { - logrus.Fatalf("Expecting a parameter \"address\" for logging hook of type \"syslog\"") - } - - if _, ok := addr.(string); !ok { - logrus.Fatalf("Parameter \"address\" for logging hook of type \"syslog\" should be a string") - } - - proto, ok2 := params["protocol"] - if !ok2 { - logrus.Fatalf("Expecting a parameter \"protocol\" for logging hook of type \"syslog\"") - } - - if _, ok2 := proto.(string); !ok2 { - logrus.Fatalf("Parameter \"protocol\" for logging hook of type \"syslog\" should be a string") - } - -} - -func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) { - syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName) - if err == nil { - logrus.AddHook(&logLevelHook{level, syslogHook}) - } -} - //CloseAndLogIfError Closes io.Closer and logs the error if any func CloseAndLogIfError(ctx context.Context, closer io.Closer, message string) { if closer == nil { diff --git a/internal/log_unix.go b/internal/log_unix.go new file mode 100644 index 00000000..25ad0420 --- /dev/null +++ b/internal/log_unix.go @@ -0,0 +1,84 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package internal + +import ( + "log/syslog" + + "github.com/matrix-org/dendrite/setup/config" + "github.com/sirupsen/logrus" + lSyslog "github.com/sirupsen/logrus/hooks/syslog" +) + +// SetupHookLogging configures the logging hooks defined in the configuration. +// If something fails here it means that the logging was improperly configured, +// so we just exit with the error +func SetupHookLogging(hooks []config.LogrusHook, componentName string) { + logrus.SetReportCaller(true) + for _, hook := range hooks { + // Check we received a proper logging level + level, err := logrus.ParseLevel(hook.Level) + if err != nil { + logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err) + } + + // Perform a first filter on the logs according to the lowest level of all + // (Eg: If we have hook for info and above, prevent logrus from processing debug logs) + if logrus.GetLevel() < level { + logrus.SetLevel(level) + } + + switch hook.Type { + case "file": + checkFileHookParams(hook.Params) + setupFileHook(hook, level, componentName) + case "syslog": + checkSyslogHookParams(hook.Params) + setupSyslogHook(hook, level, componentName) + default: + logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) + } + } +} + +func checkSyslogHookParams(params map[string]interface{}) { + addr, ok := params["address"] + if !ok { + logrus.Fatalf("Expecting a parameter \"address\" for logging hook of type \"syslog\"") + } + + if _, ok := addr.(string); !ok { + logrus.Fatalf("Parameter \"address\" for logging hook of type \"syslog\" should be a string") + } + + proto, ok2 := params["protocol"] + if !ok2 { + logrus.Fatalf("Expecting a parameter \"protocol\" for logging hook of type \"syslog\"") + } + + if _, ok2 := proto.(string); !ok2 { + logrus.Fatalf("Parameter \"protocol\" for logging hook of type \"syslog\" should be a string") + } + +} + +func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) { + syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName) + if err == nil { + logrus.AddHook(&logLevelHook{level, syslogHook}) + } +} diff --git a/internal/log_windows.go b/internal/log_windows.go new file mode 100644 index 00000000..39562328 --- /dev/null +++ b/internal/log_windows.go @@ -0,0 +1,48 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "github.com/matrix-org/dendrite/setup/config" + "github.com/sirupsen/logrus" +) + +// SetupHookLogging configures the logging hooks defined in the configuration. +// If something fails here it means that the logging was improperly configured, +// so we just exit with the error +func SetupHookLogging(hooks []config.LogrusHook, componentName string) { + logrus.SetReportCaller(true) + for _, hook := range hooks { + // Check we received a proper logging level + level, err := logrus.ParseLevel(hook.Level) + if err != nil { + logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err) + } + + // Perform a first filter on the logs according to the lowest level of all + // (Eg: If we have hook for info and above, prevent logrus from processing debug logs) + if logrus.GetLevel() < level { + logrus.SetLevel(level) + } + + switch hook.Type { + case "file": + checkFileHookParams(hook.Params) + setupFileHook(hook, level, componentName) + default: + logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) + } + } +} From b4a007ecceafd2b93fee2e775b0a61283d4a3844 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 2 Nov 2021 17:15:06 +0000 Subject: [PATCH 64/64] Support joining version 8 and 9 rooms (update to matrix-org/gomatrixserverlib@13366e7) --- go.mod | 2 +- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 6e358997..0a223f5a 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20211102101113-5e02b64e5312 + github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index a36ee34d..21f4807b 100644 --- a/go.sum +++ b/go.sum @@ -993,8 +993,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211102101113-5e02b64e5312 h1:of3kbkOrjE8793Vsni8ofxMSy/vpO1SB+ZkHSfv4wrs= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211102101113-5e02b64e5312/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1 h1:Pv7+98sreiHltpamJ4em6RCX/WPVN1wl53Gli9Cz744= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 h1:6JkIymZ1vxfI0shSpg6gNPTJaF4/95Evy34slPVZGKM= @@ -1529,7 +1529,6 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ= golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=