hoernschen/dendrite
1
0
Fork 0
mirror of https://github.com/hoernschen/dendrite.git synced 2025-07-31 13:22:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

bugfix: E2EE device keys could sometimes not be sent to remote servers (#2466)

Browse source 
* Fix flakey sytest 'Local device key changes get to remote servers'

* Debug logs

* Remove internal/test and use /test only

Remove a lot of ancient code too.

* Use FederationRoomserverAPI in more places

* Use more interfaces in federationapi; begin adding regression test

* Linting

* Add regression test

* Unbreak tests

* ALL THE LOGS

* Fix a race condition which could cause events to not be sent to servers

If a new room event which rewrites state arrives, we remove all joined hosts
then re-calculate them. This wasn't done in a transaction so for a brief period
we would have no joined hosts. During this interim, key change events which arrive
would not be sent to destination servers. This would sporadically fail on sytest.

* Unbreak new tests

* Linting
This commit is contained in:
kegsay 2022-05-17 13:23:35 +01:00 • committed by GitHub
parent cd82460513
commit 6de29c1cd2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
48 changed files with 566 additions and 618 deletions
Download patch file Download diff file Expand all files Collapse all files

2
internal/caching/cache_typing_test.go
View file

@ -20,7 +20,7 @@ import (
"testing"
"time"
"github.com/matrix-org/dendrite/internal/test"
"github.com/matrix-org/dendrite/test"
)
func TestEDUCache(t *testing.T) {

158
internal/test/client.go
View file

@ -1,158 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"net/http"
"sync"
"time"
"github.com/matrix-org/gomatrixserverlib"
)
// Request contains the information necessary to issue a request and test its result
type Request struct {
Req *http.Request
WantedBody string
WantedStatusCode int
LastErr *LastRequestErr
}
// LastRequestErr is a synchronised error wrapper
// Useful for obtaining the last error from a set of requests
type LastRequestErr struct {
sync.Mutex
Err error
}
// Set sets the error
func (r *LastRequestErr) Set(err error) {
r.Lock()
defer r.Unlock()
r.Err = err
}
// Get gets the error
func (r *LastRequestErr) Get() error {
r.Lock()
defer r.Unlock()
return r.Err
}
// CanonicalJSONInput canonicalises a slice of JSON strings
// Useful for test input
func CanonicalJSONInput(jsonData []string) []string {
for i := range jsonData {
jsonBytes, err := gomatrixserverlib.CanonicalJSON([]byte(jsonData[i]))
if err != nil && err != io.EOF {
panic(err)
}
jsonData[i] = string(jsonBytes)
}
return jsonData
}
// Do issues a request and checks the status code and body of the response
func (r *Request) Do() (err error) {
client := &http.Client{
Timeout: 5 * time.Second,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
}
res, err := client.Do(r.Req)
if err != nil {
return err
}
defer (func() { err = res.Body.Close() })()
if res.StatusCode != r.WantedStatusCode {
return fmt.Errorf("incorrect status code. Expected: %d Got: %d", r.WantedStatusCode, res.StatusCode)
}
if r.WantedBody != "" {
resBytes, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
jsonBytes, err := gomatrixserverlib.CanonicalJSON(resBytes)
if err != nil {
return err
}
if string(jsonBytes) != r.WantedBody {
return fmt.Errorf("returned wrong bytes. Expected:\n%s\n\nGot:\n%s", r.WantedBody, string(jsonBytes))
}
}
return nil
}
// DoUntilSuccess blocks and repeats the same request until the response returns the desired status code and body.
// It then closes the given channel and returns.
func (r *Request) DoUntilSuccess(done chan error) {
r.LastErr = &LastRequestErr{}
for {
if err := r.Do(); err != nil {
r.LastErr.Set(err)
time.Sleep(1 * time.Second) // don't tightloop
continue
}
close(done)
return
}
}
// Run repeatedly issues a request until success, error or a timeout is reached
func (r *Request) Run(label string, timeout time.Duration, serverCmdChan chan error) {
fmt.Printf("==TESTING== %v (timeout: %v)\n", label, timeout)
done := make(chan error, 1)
// We need to wait for the server to:
// - have connected to the database
// - have created the tables
// - be listening on the given port
go r.DoUntilSuccess(done)
// wait for one of:
// - the test to pass (done channel is closed)
// - the server to exit with an error (error sent on serverCmdChan)
// - our test timeout to expire
// We don't need to clean up since the main() function handles that in the event we panic
select {
case <-time.After(timeout):
fmt.Printf("==TESTING== %v TIMEOUT\n", label)
if reqErr := r.LastErr.Get(); reqErr != nil {
fmt.Println("Last /sync request error:")
fmt.Println(reqErr)
}
panic(fmt.Sprintf("%v server timed out", label))
case err := <-serverCmdChan:
if err != nil {
fmt.Println("=============================================================================================")
fmt.Printf("%v server failed to run. If failing with 'pq: password authentication failed for user' try:", label)
fmt.Println(" export PGHOST=/var/run/postgresql")
fmt.Println("=============================================================================================")
panic(err)
}
case <-done:
fmt.Printf("==TESTING== %v PASSED\n", label)
}
}

273
internal/test/config.go
View file

@ -1,273 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"errors"
"fmt"
"io/ioutil"
"math/big"
"os"
"path/filepath"
"strings"
"time"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"gopkg.in/yaml.v2"
)
const (
// ConfigFile is the name of the config file for a server.
ConfigFile = "dendrite.yaml"
// ServerKeyFile is the name of the file holding the matrix server private key.
ServerKeyFile = "server_key.pem"
// TLSCertFile is the name of the file holding the TLS certificate used for federation.
TLSCertFile = "tls_cert.pem"
// TLSKeyFile is the name of the file holding the TLS key used for federation.
TLSKeyFile = "tls_key.pem"
// MediaDir is the name of the directory used to store media.
MediaDir = "media"
)
// MakeConfig makes a config suitable for running integration tests.
// Generates new matrix and TLS keys for the server.
func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) {
var cfg config.Dendrite
cfg.Defaults(true)
port := startPort
assignAddress := func() config.HTTPAddress {
result := config.HTTPAddress(fmt.Sprintf("http://%s:%d", host, port))
port++
return result
}
serverKeyPath := filepath.Join(configDir, ServerKeyFile)
tlsCertPath := filepath.Join(configDir, TLSKeyFile)
tlsKeyPath := filepath.Join(configDir, TLSCertFile)
mediaBasePath := filepath.Join(configDir, MediaDir)
if err := NewMatrixKey(serverKeyPath); err != nil {
return nil, 0, err
}
if err := NewTLSKey(tlsKeyPath, tlsCertPath); err != nil {
return nil, 0, err
}
cfg.Version = config.Version
cfg.Global.ServerName = gomatrixserverlib.ServerName(assignAddress())
cfg.Global.PrivateKeyPath = config.Path(serverKeyPath)
cfg.MediaAPI.BasePath = config.Path(mediaBasePath)
cfg.Global.JetStream.Addresses = []string{kafkaURI}
// TODO: Use different databases for the different schemas.
// Using the same database for every schema currently works because
// the table names are globally unique. But we might not want to
// rely on that in the future.
cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(database)
cfg.FederationAPI.Database.ConnectionString = config.DataSource(database)
cfg.KeyServer.Database.ConnectionString = config.DataSource(database)
cfg.MediaAPI.Database.ConnectionString = config.DataSource(database)
cfg.RoomServer.Database.ConnectionString = config.DataSource(database)
cfg.SyncAPI.Database.ConnectionString = config.DataSource(database)
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database)
cfg.AppServiceAPI.InternalAPI.Listen = assignAddress()
cfg.FederationAPI.InternalAPI.Listen = assignAddress()
cfg.KeyServer.InternalAPI.Listen = assignAddress()
cfg.MediaAPI.InternalAPI.Listen = assignAddress()
cfg.RoomServer.InternalAPI.Listen = assignAddress()
cfg.SyncAPI.InternalAPI.Listen = assignAddress()
cfg.UserAPI.InternalAPI.Listen = assignAddress()
cfg.AppServiceAPI.InternalAPI.Connect = cfg.AppServiceAPI.InternalAPI.Listen
cfg.FederationAPI.InternalAPI.Connect = cfg.FederationAPI.InternalAPI.Listen
cfg.KeyServer.InternalAPI.Connect = cfg.KeyServer.InternalAPI.Listen
cfg.MediaAPI.InternalAPI.Connect = cfg.MediaAPI.InternalAPI.Listen
cfg.RoomServer.InternalAPI.Connect = cfg.RoomServer.InternalAPI.Listen
cfg.SyncAPI.InternalAPI.Connect = cfg.SyncAPI.InternalAPI.Listen
cfg.UserAPI.InternalAPI.Connect = cfg.UserAPI.InternalAPI.Listen
return &cfg, port, nil
}
// WriteConfig writes the config file to the directory.
func WriteConfig(cfg *config.Dendrite, configDir string) error {
data, err := yaml.Marshal(cfg)
if err != nil {
return err
}
return ioutil.WriteFile(filepath.Join(configDir, ConfigFile), data, 0666)
}
// NewMatrixKey generates a new ed25519 matrix server key and writes it to a file.
func NewMatrixKey(matrixKeyPath string) (err error) {
var data [35]byte
_, err = rand.Read(data[:])
if err != nil {
return err
}
keyOut, err := os.OpenFile(matrixKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer (func() {
err = keyOut.Close()
})()
keyID := base64.RawURLEncoding.EncodeToString(data[:])
keyID = strings.ReplaceAll(keyID, "-", "")
keyID = strings.ReplaceAll(keyID, "_", "")
err = pem.Encode(keyOut, &pem.Block{
Type: "MATRIX PRIVATE KEY",
Headers: map[string]string{
"Key-ID": fmt.Sprintf("ed25519:%s", keyID[:6]),
},
Bytes: data[3:],
})
return err
}
const certificateDuration = time.Hour * 24 * 365 * 10
func generateTLSTemplate(dnsNames []string) (*rsa.PrivateKey, *x509.Certificate, error) {
priv, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
return nil, nil, err
}
notBefore := time.Now()
notAfter := notBefore.Add(certificateDuration)
serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
if err != nil {
return nil, nil, err
}
template := x509.Certificate{
SerialNumber: serialNumber,
NotBefore: notBefore,
NotAfter: notAfter,
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
DNSNames: dnsNames,
}
return priv, &template, nil
}
func writeCertificate(tlsCertPath string, derBytes []byte) error {
certOut, err := os.Create(tlsCertPath)
if err != nil {
return err
}
defer certOut.Close() // nolint: errcheck
return pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
}
func writePrivateKey(tlsKeyPath string, priv *rsa.PrivateKey) error {
keyOut, err := os.OpenFile(tlsKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer keyOut.Close() // nolint: errcheck
err = pem.Encode(keyOut, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(priv),
})
return err
}
// NewTLSKey generates a new RSA TLS key and certificate and writes it to a file.
func NewTLSKey(tlsKeyPath, tlsCertPath string) error {
priv, template, err := generateTLSTemplate(nil)
if err != nil {
return err
}
// Self-signed certificate: template == parent
derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
if err != nil {
return err
}
if err = writeCertificate(tlsCertPath, derBytes); err != nil {
return err
}
return writePrivateKey(tlsKeyPath, priv)
}
func NewTLSKeyWithAuthority(serverName, tlsKeyPath, tlsCertPath, authorityKeyPath, authorityCertPath string) error {
priv, template, err := generateTLSTemplate([]string{serverName})
if err != nil {
return err
}
// load the authority key
dat, err := ioutil.ReadFile(authorityKeyPath)
if err != nil {
return err
}
block, _ := pem.Decode([]byte(dat))
if block == nil || block.Type != "RSA PRIVATE KEY" {
return errors.New("authority .key is not a valid pem encoded rsa private key")
}
authorityPriv, err := x509.ParsePKCS1PrivateKey(block.Bytes)
if err != nil {
return err
}
// load the authority certificate
dat, err = ioutil.ReadFile(authorityCertPath)
if err != nil {
return err
}
block, _ = pem.Decode([]byte(dat))
if block == nil || block.Type != "CERTIFICATE" {
return errors.New("authority .crt is not a valid pem encoded x509 cert")
}
var caCerts []*x509.Certificate
caCerts, err = x509.ParseCertificates(block.Bytes)
if err != nil {
return err
}
if len(caCerts) != 1 {
return errors.New("authority .crt contains none or more than one cert")
}
authorityCert := caCerts[0]
// Sign the new certificate using the authority's key/cert
derBytes, err := x509.CreateCertificate(rand.Reader, template, authorityCert, &priv.PublicKey, authorityPriv)
if err != nil {
return err
}
if err = writeCertificate(tlsCertPath, derBytes); err != nil {
return err
}
return writePrivateKey(tlsKeyPath, priv)
}

76
internal/test/kafka.go
View file

@ -1,76 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"io"
"os/exec"
"path/filepath"
"strings"
)
// KafkaExecutor executes kafka scripts.
type KafkaExecutor struct {
// The location of Zookeeper. Typically this is `localhost:2181`.
ZookeeperURI string
// The directory where Kafka is installed to. Used to locate kafka scripts.
KafkaDirectory string
// The location of the Kafka logs. Typically this is `localhost:9092`.
KafkaURI string
// Where stdout and stderr should be written to. Typically this is `os.Stderr`.
OutputWriter io.Writer
}
// CreateTopic creates a new kafka topic. This is created with a single partition.
func (e *KafkaExecutor) CreateTopic(topic string) error {
cmd := exec.Command(
filepath.Join(e.KafkaDirectory, "bin", "kafka-topics.sh"),
"--create",
"--zookeeper", e.ZookeeperURI,
"--replication-factor", "1",
"--partitions", "1",
"--topic", topic,
)
cmd.Stdout = e.OutputWriter
cmd.Stderr = e.OutputWriter
return cmd.Run()
}
// WriteToTopic writes data to a kafka topic.
func (e *KafkaExecutor) WriteToTopic(topic string, data []string) error {
cmd := exec.Command(
filepath.Join(e.KafkaDirectory, "bin", "kafka-console-producer.sh"),
"--broker-list", e.KafkaURI,
"--topic", topic,
)
cmd.Stdout = e.OutputWriter
cmd.Stderr = e.OutputWriter
cmd.Stdin = strings.NewReader(strings.Join(data, "\n"))
return cmd.Run()
}
// DeleteTopic deletes a given kafka topic if it exists.
func (e *KafkaExecutor) DeleteTopic(topic string) error {
cmd := exec.Command(
filepath.Join(e.KafkaDirectory, "bin", "kafka-topics.sh"),
"--delete",
"--if-exists",
"--zookeeper", e.ZookeeperURI,
"--topic", topic,
)
cmd.Stderr = e.OutputWriter
cmd.Stdout = e.OutputWriter
return cmd.Run()
}

31
internal/test/keyring.go
View file

@ -1,31 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"context"
"github.com/matrix-org/gomatrixserverlib"
)
// NopJSONVerifier is a JSONVerifier that verifies nothing and returns no errors.
type NopJSONVerifier struct {
// this verifier verifies nothing
}
func (t *NopJSONVerifier) VerifyJSONs(ctx context.Context, requests []gomatrixserverlib.VerifyJSONRequest) ([]gomatrixserverlib.VerifyJSONResult, error) {
result := make([]gomatrixserverlib.VerifyJSONResult, len(requests))
return result, nil
}

152
internal/test/server.go
View file

@ -1,152 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"context"
"fmt"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"github.com/matrix-org/dendrite/setup/config"
)
// Defaulting allows assignment of string variables with a fallback default value
// Useful for use with os.Getenv() for example
func Defaulting(value, defaultValue string) string {
if value == "" {
value = defaultValue
}
return value
}
// CreateDatabase creates a new database, dropping it first if it exists
func CreateDatabase(command string, args []string, database string) error {
cmd := exec.Command(command, args...)
cmd.Stdin = strings.NewReader(
fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s;", database, database),
)
// Send stdout and stderr to our stderr so that we see error messages from
// the psql process
cmd.Stdout = os.Stderr
cmd.Stderr = os.Stderr
return cmd.Run()
}
// CreateBackgroundCommand creates an executable command
// The Cmd being executed is returned. A channel is also returned,
// which will have any termination errors sent down it, followed immediately by the channel being closed.
func CreateBackgroundCommand(command string, args []string) (*exec.Cmd, chan error) {
cmd := exec.Command(command, args...)
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stderr
if err := cmd.Start(); err != nil {
panic("failed to start server: " + err.Error())
}
cmdChan := make(chan error, 1)
go func() {
cmdChan <- cmd.Wait()
close(cmdChan)
}()
return cmd, cmdChan
}
// InitDatabase creates the database and config file needed for the server to run
func InitDatabase(postgresDatabase, postgresContainerName string, databases []string) {
if len(databases) > 0 {
var dbCmd string
var dbArgs []string
if postgresContainerName == "" {
dbCmd = "psql"
dbArgs = []string{postgresDatabase}
} else {
dbCmd = "docker"
dbArgs = []string{
"exec", "-i", postgresContainerName, "psql", "-U", "postgres", postgresDatabase,
}
}
for _, database := range databases {
if err := CreateDatabase(dbCmd, dbArgs, database); err != nil {
panic(err)
}
}
}
}
// StartProxy creates a reverse proxy
func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) {
proxyArgs := []string{
"--bind-address", bindAddr,
"--sync-api-server-url", "http://" + string(cfg.SyncAPI.InternalAPI.Connect),
"--client-api-server-url", "http://" + string(cfg.ClientAPI.InternalAPI.Connect),
"--media-api-server-url", "http://" + string(cfg.MediaAPI.InternalAPI.Connect),
"--tls-cert", "server.crt",
"--tls-key", "server.key",
}
return CreateBackgroundCommand(
filepath.Join(filepath.Dir(os.Args[0]), "client-api-proxy"),
proxyArgs,
)
}
// ListenAndServe will listen on a random high-numbered port and attach the given router.
// Returns the base URL to send requests to. Call `cancel` to shutdown the server, which will block until it has closed.
func ListenAndServe(t *testing.T, router http.Handler, useTLS bool) (apiURL string, cancel func()) {
listener, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatalf("failed to listen: %s", err)
}
port := listener.Addr().(*net.TCPAddr).Port
srv := http.Server{}
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
srv.Handler = router
var err error
if useTLS {
certFile := filepath.Join(os.TempDir(), "dendrite.cert")
keyFile := filepath.Join(os.TempDir(), "dendrite.key")
err = NewTLSKey(keyFile, certFile)
if err != nil {
t.Logf("failed to generate tls key/cert: %s", err)
return
}
err = srv.ServeTLS(listener, certFile, keyFile)
} else {
err = srv.Serve(listener)
}
if err != nil && err != http.ErrServerClosed {
t.Logf("Listen failed: %s", err)
}
}()
secure := ""
if useTLS {
secure = "s"
}
return fmt.Sprintf("http%s://localhost:%d", secure, port), func() {
_ = srv.Shutdown(context.Background())
wg.Wait()
}
}

34
internal/test/slice.go
View file

@ -1,34 +0,0 @@
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import "sort"
// UnsortedStringSliceEqual returns true if the slices have same length & elements.
// Does not modify the given slice.
func UnsortedStringSliceEqual(first, second []string) bool {
if len(first) != len(second) {
return false
}
a, b := first[:], second[:]
sort.Strings(a)
sort.Strings(b)
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}