Add test infrastructure code for dendrite unit/integ tests (#2331)

* Add test infrastructure code for dendrite unit/integ tests

Start re-enabling some syncapi storage tests in the process.

* Linting

* Add postgres service to unit tests

* dendrite not syncv3

* Skip test which doesn't work

* Linting

* Add `jetstream.PrepareForTests`

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>

test/db.go

@@ -0,0 +1,127 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/user"
+	"testing"
+)
+
+type DBType int
+
+var DBTypeSQLite DBType = 1
+var DBTypePostgres DBType = 2
+
+var Quiet = false
+
+func createLocalDB(dbName string) string {
+	if !Quiet {
+		fmt.Println("Note: tests require a postgres install accessible to the current user")
+	}
+	createDB := exec.Command("createdb", dbName)
+	if !Quiet {
+		createDB.Stdout = os.Stdout
+		createDB.Stderr = os.Stderr
+	}
+	err := createDB.Run()
+	if err != nil && !Quiet {
+		fmt.Println("createLocalDB returned error:", err)
+	}
+	return dbName
+}
+
+func currentUser() string {
+	user, err := user.Current()
+	if err != nil {
+		if !Quiet {
+			fmt.Println("cannot get current user: ", err)
+		}
+		os.Exit(2)
+	}
+	return user.Username
+}
+
+// Prepare a sqlite or postgres connection string for testing.
+// Returns the connection string to use and a close function which must be called when the test finishes.
+// Calling this function twice will return the same database, which will have data from previous tests
+// unless close() is called.
+// TODO: namespace for concurrent package tests
+func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, close func()) {
+	if dbType == DBTypeSQLite {
+		dbname := "dendrite_test.db"
+		return fmt.Sprintf("file:%s", dbname), func() {
+			err := os.Remove(dbname)
+			if err != nil {
+				t.Fatalf("failed to cleanup sqlite db '%s': %s", dbname, err)
+			}
+		}
+	}
+
+	// Required vars: user and db
+	// We'll try to infer from the local env if they are missing
+	user := os.Getenv("POSTGRES_USER")
+	if user == "" {
+		user = currentUser()
+	}
+	dbName := os.Getenv("POSTGRES_DB")
+	if dbName == "" {
+		dbName = createLocalDB("dendrite_test")
+	}
+	connStr = fmt.Sprintf(
+		"user=%s dbname=%s sslmode=disable",
+		user, dbName,
+	)
+	// optional vars, used in CI
+	password := os.Getenv("POSTGRES_PASSWORD")
+	if password != "" {
+		connStr += fmt.Sprintf(" password=%s", password)
+	}
+	host := os.Getenv("POSTGRES_HOST")
+	if host != "" {
+		connStr += fmt.Sprintf(" host=%s", host)
+	}
+
+	return connStr, func() {
+		// Drop all tables on the database to get a fresh instance
+		db, err := sql.Open("postgres", connStr)
+		if err != nil {
+			t.Fatalf("failed to connect to postgres db '%s': %s", connStr, err)
+		}
+		_, err = db.Exec(`DROP SCHEMA public CASCADE;
+		CREATE SCHEMA public;`)
+		if err != nil {
+			t.Fatalf("failed to cleanup postgres db '%s': %s", connStr, err)
+		}
+		_ = db.Close()
+	}
+}
+
+// Creates subtests with each known DBType
+func WithAllDatabases(t *testing.T, testFn func(t *testing.T, db DBType)) {
+	dbs := map[string]DBType{
+		"postgres": DBTypePostgres,
+		"sqlite":   DBTypeSQLite,
+	}
+	for dbName, dbType := range dbs {
+		dbt := dbType
+		t.Run(dbName, func(tt *testing.T) {
+			testFn(tt, dbt)
+		})
+	}
+}

test/event.go

@@ -0,0 +1,51 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"crypto/ed25519"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type eventMods struct {
+	originServerTS time.Time
+	origin         gomatrixserverlib.ServerName
+	stateKey       *string
+	unsigned       interface{}
+	keyID          gomatrixserverlib.KeyID
+	privKey        ed25519.PrivateKey
+}
+
+type eventModifier func(e *eventMods)
+
+func WithTimestamp(ts time.Time) eventModifier {
+	return func(e *eventMods) {
+		e.originServerTS = ts
+	}
+}
+
+func WithStateKey(skey string) eventModifier {
+	return func(e *eventMods) {
+		e.stateKey = &skey
+	}
+}
+
+func WithUnsigned(unsigned interface{}) eventModifier {
+	return func(e *eventMods) {
+		e.unsigned = unsigned
+	}
+}

test/room.go

@@ -0,0 +1,223 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"crypto/ed25519"
+	"encoding/json"
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type Preset int
+
+var (
+	PresetNone               Preset = 0
+	PresetPrivateChat        Preset = 1
+	PresetPublicChat         Preset = 2
+	PresetTrustedPrivateChat Preset = 3
+
+	roomIDCounter = int64(0)
+
+	testKeyID      = gomatrixserverlib.KeyID("ed25519:test")
+	testPrivateKey = ed25519.NewKeyFromSeed([]byte{
+		1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+		17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
+	})
+)
+
+type Room struct {
+	ID      string
+	Version gomatrixserverlib.RoomVersion
+	preset  Preset
+	creator *User
+
+	authEvents gomatrixserverlib.AuthEvents
+	events     []*gomatrixserverlib.HeaderedEvent
+}
+
+// Create a new test room. Automatically creates the initial create events.
+func NewRoom(t *testing.T, creator *User, modifiers ...roomModifier) *Room {
+	t.Helper()
+	counter := atomic.AddInt64(&roomIDCounter, 1)
+
+	// set defaults then let roomModifiers override
+	r := &Room{
+		ID:         fmt.Sprintf("!%d:localhost", counter),
+		creator:    creator,
+		authEvents: gomatrixserverlib.NewAuthEvents(nil),
+		preset:     PresetPublicChat,
+		Version:    gomatrixserverlib.RoomVersionV9,
+	}
+	for _, m := range modifiers {
+		m(t, r)
+	}
+	r.insertCreateEvents(t)
+	return r
+}
+
+func (r *Room) insertCreateEvents(t *testing.T) {
+	t.Helper()
+	var joinRule gomatrixserverlib.JoinRuleContent
+	var hisVis gomatrixserverlib.HistoryVisibilityContent
+	plContent := eventutil.InitialPowerLevelsContent(r.creator.ID)
+	switch r.preset {
+	case PresetTrustedPrivateChat:
+		fallthrough
+	case PresetPrivateChat:
+		joinRule.JoinRule = "invite"
+		hisVis.HistoryVisibility = "shared"
+	case PresetPublicChat:
+		joinRule.JoinRule = "public"
+		hisVis.HistoryVisibility = "shared"
+	}
+	r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomCreate, map[string]interface{}{
+		"creator":      r.creator.ID,
+		"room_version": r.Version,
+	}, WithStateKey(""))
+	r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomMember, map[string]interface{}{
+		"membership": "join",
+	}, WithStateKey(r.creator.ID))
+	r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomPowerLevels, plContent, WithStateKey(""))
+	r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomJoinRules, joinRule, WithStateKey(""))
+	r.CreateAndInsert(t, r.creator, gomatrixserverlib.MRoomHistoryVisibility, hisVis, WithStateKey(""))
+}
+
+// Create an event in this room but do not insert it. Does not modify the room in any way (depth, fwd extremities, etc) so is thread-safe.
+func (r *Room) CreateEvent(t *testing.T, creator *User, eventType string, content interface{}, mods ...eventModifier) *gomatrixserverlib.HeaderedEvent {
+	t.Helper()
+	depth := 1 + len(r.events) // depth starts at 1
+
+	// possible event modifiers (optional fields)
+	mod := &eventMods{}
+	for _, m := range mods {
+		m(mod)
+	}
+
+	if mod.privKey == nil {
+		mod.privKey = testPrivateKey
+	}
+	if mod.keyID == "" {
+		mod.keyID = testKeyID
+	}
+	if mod.originServerTS.IsZero() {
+		mod.originServerTS = time.Now()
+	}
+	if mod.origin == "" {
+		mod.origin = gomatrixserverlib.ServerName("localhost")
+	}
+
+	var unsigned gomatrixserverlib.RawJSON
+	var err error
+	if mod.unsigned != nil {
+		unsigned, err = json.Marshal(mod.unsigned)
+		if err != nil {
+			t.Fatalf("CreateEvent[%s]: failed to marshal unsigned field: %s", eventType, err)
+		}
+	}
+
+	builder := &gomatrixserverlib.EventBuilder{
+		Sender:   creator.ID,
+		RoomID:   r.ID,
+		Type:     eventType,
+		StateKey: mod.stateKey,
+		Depth:    int64(depth),
+		Unsigned: unsigned,
+	}
+	err = builder.SetContent(content)
+	if err != nil {
+		t.Fatalf("CreateEvent[%s]: failed to SetContent: %s", eventType, err)
+	}
+	if depth > 1 {
+		builder.PrevEvents = []gomatrixserverlib.EventReference{r.events[len(r.events)-1].EventReference()}
+	}
+
+	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
+	if err != nil {
+		t.Fatalf("CreateEvent[%s]: failed to StateNeededForEventBuilder: %s", eventType, err)
+	}
+	refs, err := eventsNeeded.AuthEventReferences(&r.authEvents)
+	if err != nil {
+		t.Fatalf("CreateEvent[%s]: failed to AuthEventReferences: %s", eventType, err)
+	}
+	builder.AuthEvents = refs
+	ev, err := builder.Build(
+		mod.originServerTS, mod.origin, mod.keyID,
+		mod.privKey, r.Version,
+	)
+	if err != nil {
+		t.Fatalf("CreateEvent[%s]: failed to build event: %s", eventType, err)
+	}
+	if err = gomatrixserverlib.Allowed(ev, &r.authEvents); err != nil {
+		t.Fatalf("CreateEvent[%s]: failed to verify event was allowed: %s", eventType, err)
+	}
+	return ev.Headered(r.Version)
+}
+
+// Add a new event to this room DAG. Not thread-safe.
+func (r *Room) InsertEvent(t *testing.T, he *gomatrixserverlib.HeaderedEvent) {
+	t.Helper()
+	// Add the event to the list of auth events
+	r.events = append(r.events, he)
+	if he.StateKey() != nil {
+		err := r.authEvents.AddEvent(he.Unwrap())
+		if err != nil {
+			t.Fatalf("InsertEvent: failed to add event to auth events: %s", err)
+		}
+	}
+}
+
+func (r *Room) Events() []*gomatrixserverlib.HeaderedEvent {
+	return r.events
+}
+
+func (r *Room) CreateAndInsert(t *testing.T, creator *User, eventType string, content interface{}, mods ...eventModifier) *gomatrixserverlib.HeaderedEvent {
+	t.Helper()
+	he := r.CreateEvent(t, creator, eventType, content, mods...)
+	r.InsertEvent(t, he)
+	return he
+}
+
+// All room modifiers are below
+
+type roomModifier func(t *testing.T, r *Room)
+
+func RoomPreset(p Preset) roomModifier {
+	return func(t *testing.T, r *Room) {
+		switch p {
+		case PresetPrivateChat:
+			fallthrough
+		case PresetPublicChat:
+			fallthrough
+		case PresetTrustedPrivateChat:
+			fallthrough
+		case PresetNone:
+			r.preset = p
+		default:
+			t.Errorf("invalid RoomPreset: %v", p)
+		}
+	}
+}
+
+func RoomVersion(ver gomatrixserverlib.RoomVersion) roomModifier {
+	return func(t *testing.T, r *Room) {
+		r.Version = ver
+	}
+}

test/user.go

@@ -0,0 +1,36 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"fmt"
+	"sync/atomic"
+)
+
+var (
+	userIDCounter = int64(0)
+)
+
+type User struct {
+	ID string
+}
+
+func NewUser() *User {
+	counter := atomic.AddInt64(&userIDCounter, 1)
+	u := &User{
+		ID: fmt.Sprintf("@%d:localhost", counter),
+	}
+	return u
+}