Merge branch 'master' into neilalexander/restrictedjoin

appservice/storage/storage.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package storage

build/gobind-pinecone/platform_ios.go

@@ -1,3 +1,4 @@
+//go:build ios
 // +build ios
 
 package gobind

build/gobind-pinecone/platform_other.go

@@ -1,3 +1,4 @@
+//go:build !ios
 // +build !ios
 
 package gobind

build/gobind-yggdrasil/platform_ios.go

@@ -1,3 +1,4 @@
+//go:build ios
 // +build ios
 
 package gobind

build/gobind-yggdrasil/platform_other.go

@@ -1,3 +1,4 @@
+//go:build !ios
 // +build !ios
 
 package gobind

cmd/create-account/main.go

@@ -22,7 +22,6 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
-	"syscall"
 
 	"github.com/matrix-org/dendrite/setup"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -121,13 +120,13 @@ func getPassword(password, pwdFile *string, pwdStdin, askPass *bool, r io.Reader
 	// ask the user to provide the password
 	if *askPass {
 		fmt.Print("Enter Password: ")
-		bytePassword, err := term.ReadPassword(syscall.Stdin)
+		bytePassword, err := term.ReadPassword(int(os.Stdin.Fd()))
 		if err != nil {
 			logrus.Fatalln("Unable to read password:", err)
 		}
 		fmt.Println()
 		fmt.Print("Confirm Password: ")
-		bytePassword2, err := term.ReadPassword(syscall.Stdin)
+		bytePassword2, err := term.ReadPassword(int(os.Stdin.Fd()))
 		if err != nil {
 			logrus.Fatalln("Unable to read password:", err)
 		}

cmd/dendrite-demo-pinecone/embed/embed_elementweb.go

@@ -1,3 +1,4 @@
+//go:build elementweb
 // +build elementweb
 
 package embed

cmd/dendrite-demo-pinecone/embed/embed_other.go

@@ -1,3 +1,4 @@
+//go:build !elementweb
 // +build !elementweb
 
 package embed

cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go

@@ -1,3 +1,4 @@
+//go:build elementweb
 // +build elementweb
 
 package embed

cmd/dendrite-demo-yggdrasil/embed/embed_other.go

@@ -1,3 +1,4 @@
+//go:build !elementweb
 // +build !elementweb
 
 package embed

cmd/dendritejs-pinecone/jsServer.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main

cmd/dendritejs-pinecone/main_noop.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package main

cmd/dendritejs-pinecone/main_test.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main

cmd/dendritejs/jsServer.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main

cmd/dendritejs/keyfetcher.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main

cmd/dendritejs/main.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main

cmd/dendritejs/main_noop.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package main

cmd/dendritejs/publicrooms.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package main

federationapi/routing/send.go

@@ -148,7 +148,8 @@ type inputWorker struct {
 	input   *sendFIFOQueue
 }
 
-var inputWorkers sync.Map // room ID -> *inputWorker
+var inFlightTxnsPerOrigin sync.Map // transaction ID -> chan util.JSONResponse
+var inputWorkers sync.Map          // room ID -> *inputWorker
 
 // Send implements /_matrix/federation/v1/send/{txnID}
 func Send(
@@ -164,6 +165,37 @@ func Send(
 	mu *internal.MutexByRoom,
 	servers federationAPI.ServersInRoomProvider,
 ) util.JSONResponse {
+	// First we should check if this origin has already submitted this
+	// txn ID to us. If they have and the txnIDs map contains an entry,
+	// the transaction is still being worked on. The new client can wait
+	// for it to complete rather than creating more work.
+	index := string(request.Origin()) + "\000" + string(txnID)
+	v, ok := inFlightTxnsPerOrigin.LoadOrStore(index, make(chan util.JSONResponse, 1))
+	ch := v.(chan util.JSONResponse)
+	if ok {
+		// This origin already submitted this txn ID to us, and the work
+		// is still taking place, so we'll just wait for it to finish.
+		ctx, cancel := context.WithTimeout(httpReq.Context(), time.Minute*5)
+		defer cancel()
+		select {
+		case <-ctx.Done():
+			// If the caller gives up then return straight away. We don't
+			// want to attempt to process what they sent us any further.
+			return util.JSONResponse{Code: http.StatusRequestTimeout}
+		case res := <-ch:
+			// The original task just finished processing so let's return
+			// the result of it.
+			if res.Code == 0 {
+				return util.JSONResponse{Code: http.StatusAccepted}
+			}
+			return res
+		}
+	}
+	// Otherwise, store that we're currently working on this txn from
+	// this origin. When we're done processing, close the channel.
+	defer close(ch)
+	defer inFlightTxnsPerOrigin.Delete(index)
+
 	t := txnReq{
 		rsAPI:      rsAPI,
 		eduAPI:     eduAPI,
@@ -205,7 +237,7 @@ func Send(
 
 	util.GetLogger(httpReq.Context()).Infof("Received transaction %q from %q containing %d PDUs, %d EDUs", txnID, request.Origin(), len(t.PDUs), len(t.EDUs))
 
-	resp, jsonErr := t.processTransaction(httpReq.Context())
+	resp, jsonErr := t.processTransaction(context.Background())
 	if jsonErr != nil {
 		util.GetLogger(httpReq.Context()).WithField("jsonErr", jsonErr).Error("t.processTransaction failed")
 		return *jsonErr
@@ -215,10 +247,12 @@ func Send(
 	// Status code 200:
 	// The result of processing the transaction. The server is to use this response
 	// even in the event of one or more PDUs failing to be processed.
-	return util.JSONResponse{
+	res := util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: resp,
 	}
+	ch <- res
+	return res
 }
 
 type txnReq struct {

federationsender/storage/storage.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package storage

go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20211108101507-7a3ac5aa8c82
 	github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0
 	github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4

go.sum

@@ -993,8 +993,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1 h1:Pv7+98sreiHltpamJ4em6RCX/WPVN1wl53Gli9Cz744=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20211108101507-7a3ac5aa8c82 h1:hULlw0rjPgoH53BGSNQZNASJIzYgoW3LH4yJ4Cx8Bak=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20211108101507-7a3ac5aa8c82/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc=
 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU=
 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE=
 github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 h1:6JkIymZ1vxfI0shSpg6gNPTJaF4/95Evy34slPVZGKM=
@@ -1884,7 +1884,6 @@ gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

internal/log.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"log/syslog"
 	"net/http"
 	"os"
 	"path"
@@ -31,7 +30,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dugong"
 	"github.com/sirupsen/logrus"
-	lSyslog "github.com/sirupsen/logrus/hooks/syslog"
 )
 
 type utcFormatter struct {
@@ -108,37 +106,6 @@ func SetupStdLogging() {
 	})
 }
 
-// SetupHookLogging configures the logging hooks defined in the configuration.
-// If something fails here it means that the logging was improperly configured,
-// so we just exit with the error
-func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
-	logrus.SetReportCaller(true)
-	for _, hook := range hooks {
-		// Check we received a proper logging level
-		level, err := logrus.ParseLevel(hook.Level)
-		if err != nil {
-			logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err)
-		}
-
-		// Perform a first filter on the logs according to the lowest level of all
-		// (Eg: If we have hook for info and above, prevent logrus from processing debug logs)
-		if logrus.GetLevel() < level {
-			logrus.SetLevel(level)
-		}
-
-		switch hook.Type {
-		case "file":
-			checkFileHookParams(hook.Params)
-			setupFileHook(hook, level, componentName)
-		case "syslog":
-			checkSyslogHookParams(hook.Params)
-			setupSyslogHook(hook, level, componentName)
-		default:
-			logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type)
-		}
-	}
-}
-
 // File type hooks should be provided a path to a directory to store log files
 func checkFileHookParams(params map[string]interface{}) {
 	path, ok := params["path"]
@@ -178,34 +145,6 @@ func setupFileHook(hook config.LogrusHook, level logrus.Level, componentName str
 	})
 }
 
-func checkSyslogHookParams(params map[string]interface{}) {
-	addr, ok := params["address"]
-	if !ok {
-		logrus.Fatalf("Expecting a parameter \"address\" for logging hook of type \"syslog\"")
-	}
-
-	if _, ok := addr.(string); !ok {
-		logrus.Fatalf("Parameter \"address\" for logging hook of type \"syslog\" should be a string")
-	}
-
-	proto, ok2 := params["protocol"]
-	if !ok2 {
-		logrus.Fatalf("Expecting a parameter \"protocol\" for logging hook of type \"syslog\"")
-	}
-
-	if _, ok2 := proto.(string); !ok2 {
-		logrus.Fatalf("Parameter \"protocol\" for logging hook of type \"syslog\" should be a string")
-	}
-
-}
-
-func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) {
-	syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName)
-	if err == nil {
-		logrus.AddHook(&logLevelHook{level, syslogHook})
-	}
-}
-
 //CloseAndLogIfError Closes io.Closer and logs the error if any
 func CloseAndLogIfError(ctx context.Context, closer io.Closer, message string) {
 	if closer == nil {

internal/log_unix.go

@@ -0,0 +1,84 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package internal
+
+import (
+	"log/syslog"
+
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/sirupsen/logrus"
+	lSyslog "github.com/sirupsen/logrus/hooks/syslog"
+)
+
+// SetupHookLogging configures the logging hooks defined in the configuration.
+// If something fails here it means that the logging was improperly configured,
+// so we just exit with the error
+func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
+	logrus.SetReportCaller(true)
+	for _, hook := range hooks {
+		// Check we received a proper logging level
+		level, err := logrus.ParseLevel(hook.Level)
+		if err != nil {
+			logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err)
+		}
+
+		// Perform a first filter on the logs according to the lowest level of all
+		// (Eg: If we have hook for info and above, prevent logrus from processing debug logs)
+		if logrus.GetLevel() < level {
+			logrus.SetLevel(level)
+		}
+
+		switch hook.Type {
+		case "file":
+			checkFileHookParams(hook.Params)
+			setupFileHook(hook, level, componentName)
+		case "syslog":
+			checkSyslogHookParams(hook.Params)
+			setupSyslogHook(hook, level, componentName)
+		default:
+			logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type)
+		}
+	}
+}
+
+func checkSyslogHookParams(params map[string]interface{}) {
+	addr, ok := params["address"]
+	if !ok {
+		logrus.Fatalf("Expecting a parameter \"address\" for logging hook of type \"syslog\"")
+	}
+
+	if _, ok := addr.(string); !ok {
+		logrus.Fatalf("Parameter \"address\" for logging hook of type \"syslog\" should be a string")
+	}
+
+	proto, ok2 := params["protocol"]
+	if !ok2 {
+		logrus.Fatalf("Expecting a parameter \"protocol\" for logging hook of type \"syslog\"")
+	}
+
+	if _, ok2 := proto.(string); !ok2 {
+		logrus.Fatalf("Parameter \"protocol\" for logging hook of type \"syslog\" should be a string")
+	}
+
+}
+
+func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) {
+	syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName)
+	if err == nil {
+		logrus.AddHook(&logLevelHook{level, syslogHook})
+	}
+}

internal/log_windows.go

@@ -0,0 +1,48 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/sirupsen/logrus"
+)
+
+// SetupHookLogging configures the logging hooks defined in the configuration.
+// If something fails here it means that the logging was improperly configured,
+// so we just exit with the error
+func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
+	logrus.SetReportCaller(true)
+	for _, hook := range hooks {
+		// Check we received a proper logging level
+		level, err := logrus.ParseLevel(hook.Level)
+		if err != nil {
+			logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err)
+		}
+
+		// Perform a first filter on the logs according to the lowest level of all
+		// (Eg: If we have hook for info and above, prevent logrus from processing debug logs)
+		if logrus.GetLevel() < level {
+			logrus.SetLevel(level)
+		}
+
+		switch hook.Type {
+		case "file":
+			checkFileHookParams(hook.Params)
+			setupFileHook(hook, level, componentName)
+		default:
+			logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type)
+		}
+	}
+}

internal/sqlutil/postgres.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package sqlutil

internal/sqlutil/postgres_wasm.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package sqlutil

internal/sqlutil/trace_driver.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package sqlutil

internal/sqlutil/trace_driver_wasm.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package sqlutil

keyserver/storage/storage.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package storage

mediaapi/routing/download.go

@@ -737,7 +737,7 @@ func (r *downloadRequest) fetchRemoteFile(
 		return "", false, parseErr
 	}
 
-	if contentLength > int64(maxFileSizeBytes) {
+	if maxFileSizeBytes > 0 && contentLength > int64(maxFileSizeBytes) {
 		// TODO: Bubble up this as a 413
 		return "", false, fmt.Errorf("remote file is too large (%v > %v bytes)", contentLength, maxFileSizeBytes)
 	}

mediaapi/storage/storage.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package storage

mediaapi/thumbnailer/thumbnailer_bimg.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build bimg
 // +build bimg
 
 package thumbnailer

mediaapi/thumbnailer/thumbnailer_nfnt.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !bimg
 // +build !bimg
 
 package thumbnailer

roomserver/state/state.go

@@ -778,7 +778,8 @@ func (v *StateResolution) resolveConflictsV2(
 	ctx context.Context,
 	notConflicted, conflicted []types.StateEntry,
 ) ([]types.StateEntry, error) {
-	eventIDMap := make(map[string]types.StateEntry)
+	estimate := len(conflicted) + len(notConflicted)
+	eventIDMap := make(map[string]types.StateEntry, estimate)
 
 	// Load the conflicted events
 	conflictedEvents, conflictedEventMap, err := v.loadStateEvents(ctx, conflicted)
@@ -800,18 +801,20 @@ func (v *StateResolution) resolveConflictsV2(
 
 	// For each conflicted event, we will add a new set of auth events. Auth
 	// events may be duplicated across these sets but that's OK.
-	authSets := make(map[string][]*gomatrixserverlib.Event)
-	var authEvents []*gomatrixserverlib.Event
-	var authDifference []*gomatrixserverlib.Event
+	authSets := make(map[string][]*gomatrixserverlib.Event, len(conflicted))
+	authEvents := make([]*gomatrixserverlib.Event, 0, estimate*3)
+	authDifference := make([]*gomatrixserverlib.Event, 0, estimate)
 
 	// For each conflicted event, let's try and get the needed auth events.
+	neededStateKeys := make([]string, 16)
+	authEntries := make([]types.StateEntry, 16)
 	for _, conflictedEvent := range conflictedEvents {
 		// Work out which auth events we need to load.
 		key := conflictedEvent.EventID()
 		needed := gomatrixserverlib.StateNeededForAuth([]*gomatrixserverlib.Event{conflictedEvent})
 
 		// Find the numeric IDs for the necessary state keys.
-		var neededStateKeys []string
+		neededStateKeys = neededStateKeys[:0]
 		neededStateKeys = append(neededStateKeys, needed.Member...)
 		neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...)
 		stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys)
@@ -821,7 +824,7 @@ func (v *StateResolution) resolveConflictsV2(
 
 		// Load the necessary auth events.
 		tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed)
-		var authEntries []types.StateEntry
+		authEntries = authEntries[:0]
 		for _, tuple := range tuplesNeeded {
 			if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok {
 				authEntries = append(authEntries, types.StateEntry{

roomserver/storage/shared/storage.go

@@ -165,14 +165,21 @@ func (d *Database) AddState(
 		if berr != nil {
 			return 0, fmt.Errorf("d.StateBlockTable.BulkSelectStateBlockEntries: %w", berr)
 		}
+		var found bool
 		for i := len(state) - 1; i >= 0; i-- {
+			found = false
 			for _, events := range blocks {
 				for _, event := range events {
 					if state[i].EventNID == event {
-						state = append(state[:i], state[i+1:]...)
+						found = true
+						break
 					}
 				}
 			}
+			if found {
+				state = append(state[:i], state[i+1:]...)
+				i--
+			}
 		}
 	}
 	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {

roomserver/storage/storage.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package storage

signingkeyserver/storage/keydb.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package storage

signingkeyserver/storage/keydb_wasm.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package storage

syncapi/storage/postgres/output_room_events_table.go

@@ -116,7 +116,7 @@ const updateEventJSONSQL = "" +
 
 // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id).
 const selectStateInRangeSQL = "" +
-	"SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" +
+	"SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" +
 	" FROM syncapi_output_room_events" +
 	" WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" +
 	" AND ( $3::text[] IS NULL OR     sender  = ANY($3)  )" +
@@ -221,13 +221,14 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 
 	for rows.Next() {
 		var (
+			eventID         string
 			streamPos       types.StreamPosition
 			eventBytes      []byte
 			excludeFromSync bool
 			addIDs          pq.StringArray
 			delIDs          pq.StringArray
 		)
-		if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil {
+		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil {
 			return nil, nil, err
 		}
 		// Sanity check for deleted state and whine if we see it. We don't need to do anything
@@ -243,7 +244,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 
 		// TODO: Handle redacted events
 		var ev gomatrixserverlib.HeaderedEvent
-		if err := json.Unmarshal(eventBytes, &ev); err != nil {
+		if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil {
 			return nil, nil, err
 		}
 		needSet := stateNeeded[ev.RoomID()]
@@ -258,7 +259,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 		}
 		stateNeeded[ev.RoomID()] = needSet
 
-		eventIDToEvent[ev.EventID()] = types.StreamEvent{
+		eventIDToEvent[eventID] = types.StreamEvent{
 			HeaderedEvent:   &ev,
 			StreamPosition:  streamPos,
 			ExcludeFromSync: excludeFromSync,

syncapi/storage/sqlite3/output_room_events_table.go

@@ -81,7 +81,7 @@ const updateEventJSONSQL = "" +
 	"UPDATE syncapi_output_room_events SET headered_event_json=$1 WHERE event_id=$2"
 
 const selectStateInRangeSQL = "" +
-	"SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" +
+	"SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" +
 	" FROM syncapi_output_room_events" +
 	" WHERE (id > $1 AND id <= $2)" +
 	" AND ((add_state_ids IS NOT NULL AND add_state_ids != '') OR (remove_state_ids IS NOT NULL AND remove_state_ids != ''))"
@@ -173,13 +173,14 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 
 	for rows.Next() {
 		var (
+			eventID         string
 			streamPos       types.StreamPosition
 			eventBytes      []byte
 			excludeFromSync bool
 			addIDsJSON      string
 			delIDsJSON      string
 		)
-		if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON); err != nil {
+		if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON); err != nil {
 			return nil, nil, err
 		}
 
@@ -201,7 +202,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 
 		// TODO: Handle redacted events
 		var ev gomatrixserverlib.HeaderedEvent
-		if err := json.Unmarshal(eventBytes, &ev); err != nil {
+		if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil {
 			return nil, nil, err
 		}
 		needSet := stateNeeded[ev.RoomID()]
@@ -216,7 +217,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
 		}
 		stateNeeded[ev.RoomID()] = needSet
 
-		eventIDToEvent[ev.EventID()] = types.StreamEvent{
+		eventIDToEvent[eventID] = types.StreamEvent{
 			HeaderedEvent:   &ev,
 			StreamPosition:  streamPos,
 			ExcludeFromSync: excludeFromSync,

syncapi/storage/storage.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package storage

userapi/storage/accounts/sqlite3/constraint_wasm.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build wasm
 // +build wasm
 
 package sqlite3

userapi/storage/accounts/storage.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package accounts

userapi/storage/devices/storage.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !wasm
 // +build !wasm
 
 package devices