Automatically upgrade databases on startup (#1529)

* Support auto-upgrading accounts DB

* Auto-upgrade device DB deltas

* Support up/downgrading from cmd/goose

* Linting

* Create tables then do migrations then prepare statements

To avoid failing due to some things not existing

* Linting

userapi/storage/accounts/sqlite3/accounts_table.go

@@ -74,13 +74,13 @@ type accountsStatements struct {
 	serverName                    gomatrixserverlib.ServerName
 }
 
+func (s *accountsStatements) execSchema(db *sql.DB) error {
+	_, err := db.Exec(accountsSchema)
+	return err
+}
+
 func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) {
 	s.db = db
-
-	_, err = db.Exec(accountsSchema)
-	if err != nil {
-		return
-	}
 	if s.insertAccountStmt, err = db.Prepare(insertAccountSQL); err != nil {
 		return
 	}

userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go

@@ -0,0 +1,64 @@
+package deltas
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/pressly/goose"
+)
+
+func LoadFromGoose() {
+	goose.AddMigration(UpIsActive, DownIsActive)
+}
+
+func LoadIsActive(m *sqlutil.Migrations) {
+	m.AddMigration(UpIsActive, DownIsActive)
+}
+
+func UpIsActive(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+	ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
+CREATE TABLE account_accounts (
+    localpart TEXT NOT NULL PRIMARY KEY,
+    created_ts BIGINT NOT NULL,
+    password_hash TEXT,
+    appservice_id TEXT,
+    is_deactivated BOOLEAN DEFAULT 0
+);
+INSERT
+    INTO account_accounts (
+      localpart, created_ts, password_hash, appservice_id
+    ) SELECT
+        localpart, created_ts, password_hash, appservice_id
+    FROM account_accounts_tmp
+;
+DROP TABLE account_accounts_tmp;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownIsActive(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+	ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
+CREATE TABLE account_accounts (
+    localpart TEXT NOT NULL PRIMARY KEY,
+    created_ts BIGINT NOT NULL,
+    password_hash TEXT,
+    appservice_id TEXT
+);
+INSERT
+    INTO account_accounts (
+      localpart, created_ts, password_hash, appservice_id
+    ) SELECT
+        localpart, created_ts, password_hash, appservice_id
+    FROM account_accounts_tmp
+;
+DROP TABLE account_accounts_tmp;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}

userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql

@@ -1,38 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
-CREATE TABLE account_accounts (
-    localpart TEXT NOT NULL PRIMARY KEY,
-    created_ts BIGINT NOT NULL,
-    password_hash TEXT,
-    appservice_id TEXT,
-    is_deactivated BOOLEAN DEFAULT 0
-);
-INSERT
-    INTO account_accounts (
-      localpart, created_ts, password_hash, appservice_id
-    ) SELECT
-        localpart, created_ts, password_hash, appservice_id
-    FROM account_accounts_tmp
-;
-DROP TABLE account_accounts_tmp;
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
-ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
-CREATE TABLE account_accounts (
-    localpart TEXT NOT NULL PRIMARY KEY,
-    created_ts BIGINT NOT NULL,
-    password_hash TEXT,
-    appservice_id TEXT
-);
-INSERT
-    INTO account_accounts (
-      localpart, created_ts, password_hash, appservice_id
-    ) SELECT
-        localpart, created_ts, password_hash, appservice_id
-    FROM account_accounts_tmp
-;
-DROP TABLE account_accounts_tmp;
--- +goose StatementEnd

userapi/storage/accounts/sqlite3/storage.go

@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3/deltas"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
 	// Import the sqlite3 database driver.
@@ -60,6 +61,18 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
 		db:         db,
 		writer:     sqlutil.NewExclusiveWriter(),
 	}
+
+	// Create tables before executing migrations so we don't fail if the table is missing,
+	// and THEN prepare statements so we don't fail due to referencing new columns
+	if err = d.accounts.execSchema(db); err != nil {
+		return nil, err
+	}
+	m := sqlutil.NewMigrations()
+	deltas.LoadIsActive(m)
+	if err = m.RunDeltas(db, dbProperties); err != nil {
+		return nil, err
+	}
+
 	partitions := sqlutil.PartitionOffsetStatements{}
 	if err = partitions.Prepare(db, d.writer, "account"); err != nil {
 		return nil, err
@@ -76,6 +89,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
 	if err = d.threepids.prepare(db); err != nil {
 		return nil, err
 	}
+
 	return d, nil
 }