From 98dfaf38561839650d2c456ce4f38a4f5e73a7e5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 25 Jan 2021 15:04:01 +0000 Subject: [PATCH] Try jsoniter in gmsl --- federationapi/routing/backfill.go | 4 ++-- federationapi/routing/events.go | 4 ++-- federationapi/routing/send.go | 3 ++- federationapi/routing/send_test.go | 16 ++++++++-------- federationsender/consumers/eduserver.go | 3 ++- federationsender/queue/destinationqueue.go | 6 +++--- go.mod | 3 ++- go.sum | 5 +++++ 8 files changed, 26 insertions(+), 18 deletions(-) diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 31005209..462b8ccf 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -15,12 +15,12 @@ package routing import ( - "encoding/json" "fmt" "net/http" "strconv" "time" + jsoniter "github.com/json-iterator/go" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -102,7 +102,7 @@ func Backfill( } } - eventJSONs := []json.RawMessage{} + eventJSONs := []jsoniter.RawMessage{} for _, e := range gomatrixserverlib.ReverseTopologicalOrdering( evs, gomatrixserverlib.TopologicalOrderByPrevEvents, diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index 312ef9f8..c40bc216 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -16,10 +16,10 @@ package routing import ( "context" - "encoding/json" "net/http" "time" + jsoniter "github.com/json-iterator/go" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -45,7 +45,7 @@ func GetEvent( return util.JSONResponse{Code: http.StatusOK, JSON: gomatrixserverlib.Transaction{ Origin: origin, OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), - PDUs: []json.RawMessage{ + PDUs: []jsoniter.RawMessage{ event.JSON(), }, }} diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 96b5355e..c4f2a650 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -23,6 +23,7 @@ import ( "sync" "time" + jsoniter "github.com/json-iterator/go" "github.com/matrix-org/dendrite/clientapi/jsonerror" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" keyapi "github.com/matrix-org/dendrite/keyserver/api" @@ -56,7 +57,7 @@ func Send( } var txnEvents struct { - PDUs []json.RawMessage `json:"pdus"` + PDUs []jsoniter.RawMessage `json:"pdus"` EDUs []gomatrixserverlib.EDU `json:"edus"` } diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 8bdf54c4..020ccd96 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -2,12 +2,12 @@ package routing import ( "context" - "encoding/json" "fmt" "reflect" "testing" "time" + jsoniter "github.com/json-iterator/go" eduAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/api" @@ -21,7 +21,7 @@ const ( var ( testRoomVersion = gomatrixserverlib.RoomVersionV1 - testData = []json.RawMessage{ + testData = []jsoniter.RawMessage{ []byte(`{"auth_events":[],"content":{"creator":"@userid:kaer.morhen"},"depth":0,"event_id":"$0ok8ynDp7kjc95e3:kaer.morhen","hashes":{"sha256":"17kPoH+h0Dk4Omn7Sus0qMb6+oGcf+CZFEgDhv7UKWs"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"jP4a04f5/F10Pw95FPpdCyKAO44JOwUQ/MZOOeA/RTU1Dn+AHPMzGSaZnuGjRr/xQuADt+I3ctb5ZQfLKNzHDw"}},"state_key":"","type":"m.room.create"}`), []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"content":{"membership":"join"},"depth":1,"event_id":"$LEwEu0kxrtu5fOiS:kaer.morhen","hashes":{"sha256":"B7M88PhXf3vd1LaFtjQutFu4x/w7fHD28XKZ4sAsJTo"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"p2vqmuJn7ZBRImctSaKbXCAxCcBlIjPH9JHte1ouIUGy84gpu4eLipOvSBCLL26hXfC0Zrm4WUto6Hr+ohdrCg"}},"state_key":"@userid:kaer.morhen","type":"m.room.member"}`), []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"join_rule":"public"},"depth":2,"event_id":"$SMHlqUrNhhBBRLeN:kaer.morhen","hashes":{"sha256":"vIuJQvmMjrGxshAkj1SXe0C4RqvMbv4ZADDw9pFCWqQ"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"hBMsb3Qppo3RaqqAl4JyTgaiWEbW5hlckATky6PrHun+F3YM203TzG7w9clwuQU5F5pZoB1a6nw+to0hN90FAw"}},"state_key":"","type":"m.room.join_rules"}`), @@ -362,7 +362,7 @@ func (c *txnFedClient) LookupMissingEvents(ctx context.Context, s gomatrixserver return c.getMissingEvents(missing) } -func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederationClient, pdus []json.RawMessage) *txnReq { +func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederationClient, pdus []jsoniter.RawMessage) *txnReq { t := &txnReq{ rsAPI: rsAPI, eduAPI: &testEDUProducer{}, @@ -445,7 +445,7 @@ func TestBasicTransaction(t *testing.T) { } }, } - pdus := []json.RawMessage{ + pdus := []jsoniter.RawMessage{ testData[len(testData)-1], // a message event } txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus) @@ -465,7 +465,7 @@ func TestTransactionFailAuthChecks(t *testing.T) { } }, } - pdus := []json.RawMessage{ + pdus := []jsoniter.RawMessage{ testData[len(testData)-1], // a message event } txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus) @@ -550,7 +550,7 @@ func TestTransactionFetchMissingPrevEvents(t *testing.T) { }, } - pdus := []json.RawMessage{ + pdus := []jsoniter.RawMessage{ inputEvent.JSON(), } txn := mustCreateTransaction(rsAPI, cli, pdus) @@ -713,7 +713,7 @@ func TestTransactionFetchMissingStateByStateIDs(t *testing.T) { // /event for event B returns it getEvent: map[string]gomatrixserverlib.Transaction{ eventB.EventID(): { - PDUs: []json.RawMessage{ + PDUs: []jsoniter.RawMessage{ eventB.JSON(), }, }, @@ -735,7 +735,7 @@ func TestTransactionFetchMissingStateByStateIDs(t *testing.T) { }, } - pdus := []json.RawMessage{ + pdus := []jsoniter.RawMessage{ eventD.JSON(), } txn := mustCreateTransaction(rsAPI, cli, pdus) diff --git a/federationsender/consumers/eduserver.go b/federationsender/consumers/eduserver.go index 6d11eb88..a9b6e9bf 100644 --- a/federationsender/consumers/eduserver.go +++ b/federationsender/consumers/eduserver.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/Shopify/sarama" + jsoniter "github.com/json-iterator/go" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" @@ -131,7 +132,7 @@ func (t *OutputEDUConsumer) onSendToDeviceEvent(msg *sarama.ConsumerMessage) err Sender: ote.Sender, Type: ote.Type, MessageID: util.RandomString(32), - Messages: map[string]map[string]json.RawMessage{ + Messages: map[string]map[string]jsoniter.RawMessage{ ote.UserID: { ote.DeviceID: ote.Content, }, diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index c8b0bf65..19272794 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -16,11 +16,11 @@ package queue import ( "context" - "encoding/json" "fmt" "sync" "time" + jsoniter "github.com/json-iterator/go" "github.com/matrix-org/dendrite/federationsender/statistics" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/storage/shared" @@ -367,7 +367,7 @@ func (oq *destinationQueue) nextTransaction( // Create the transaction. t := gomatrixserverlib.Transaction{ - PDUs: []json.RawMessage{}, + PDUs: []jsoniter.RawMessage{}, EDUs: []gomatrixserverlib.EDU{}, } t.Origin = oq.origin @@ -390,7 +390,7 @@ func (oq *destinationQueue) nextTransaction( if pdu == nil || pdu.pdu == nil { continue } - // Append the JSON of the event, since this is a json.RawMessage type in the + // Append the JSON of the event, since this is a jsoniter.RawMessage type in the // gomatrixserverlib.Transaction struct t.PDUs = append(t.PDUs, pdu.pdu.JSON()) pduReceipts = append(pduReceipts, pdu.receipt) diff --git a/go.mod b/go.mod index 891d0f57..6e8dffc3 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/gologme/log v1.2.0 github.com/gorilla/mux v1.8.0 github.com/hashicorp/golang-lru v0.5.4 + github.com/json-iterator/go v1.1.10 github.com/lib/pq v1.8.0 github.com/libp2p/go-libp2p v0.11.0 github.com/libp2p/go-libp2p-circuit v0.3.1 @@ -22,7 +23,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20210122154608-a38974bd8a37 + github.com/matrix-org/gomatrixserverlib v0.0.0-20210125150012-01e52c47a2a5 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index acdfaf2c..644b13b0 100644 --- a/go.sum +++ b/go.sum @@ -282,6 +282,7 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -569,6 +570,8 @@ github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHRE github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20210122154608-a38974bd8a37 h1:si2CZZpwOLWZfDXfgHPkaTlaAkdJvpJzr1zVqyKXd0I= github.com/matrix-org/gomatrixserverlib v0.0.0-20210122154608-a38974bd8a37/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210125150012-01e52c47a2a5 h1:CB4qoqmsNCozYHZi0051R5pg8F8QJWdwMyDfmdRfk7s= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210125150012-01e52c47a2a5/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= @@ -607,8 +610,10 @@ github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=