Merge both user API databases into one (#2186)

* Merge user API databases into one

* Remove DeviceDatabase from config

* Fix tests

* Try that again

* Clean up keyserver device keys when the devices no longer exist in the user API

* Tweak ordering

* Fix UserExists flag, device check

* Allow including empty entries so we can clean them up

* Remove logging

userapi/storage/sqlite3/deltas/20200929203058_is_active.go

@@ -0,0 +1,66 @@
+package deltas
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/pressly/goose"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+func LoadFromGoose() {
+	goose.AddMigration(UpIsActive, DownIsActive)
+	goose.AddMigration(UpAddAccountType, DownAddAccountType)
+}
+
+func LoadIsActive(m *sqlutil.Migrations) {
+	m.AddMigration(UpIsActive, DownIsActive)
+}
+
+func UpIsActive(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+	ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
+CREATE TABLE account_accounts (
+    localpart TEXT NOT NULL PRIMARY KEY,
+    created_ts BIGINT NOT NULL,
+    password_hash TEXT,
+    appservice_id TEXT,
+    is_deactivated BOOLEAN DEFAULT 0
+);
+INSERT
+    INTO account_accounts (
+      localpart, created_ts, password_hash, appservice_id
+    ) SELECT
+        localpart, created_ts, password_hash, appservice_id
+    FROM account_accounts_tmp
+;
+DROP TABLE account_accounts_tmp;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownIsActive(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+	ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
+CREATE TABLE account_accounts (
+    localpart TEXT NOT NULL PRIMARY KEY,
+    created_ts BIGINT NOT NULL,
+    password_hash TEXT,
+    appservice_id TEXT
+);
+INSERT
+    INTO account_accounts (
+      localpart, created_ts, password_hash, appservice_id
+    ) SELECT
+        localpart, created_ts, password_hash, appservice_id
+    FROM account_accounts_tmp
+;
+DROP TABLE account_accounts_tmp;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}

userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go

@@ -0,0 +1,65 @@
+package deltas
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+func LoadLastSeenTSIP(m *sqlutil.Migrations) {
+	m.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP)
+}
+
+func UpLastSeenTSIP(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+    ALTER TABLE device_devices RENAME TO device_devices_tmp;
+    CREATE TABLE device_devices (
+        access_token TEXT PRIMARY KEY,
+        session_id INTEGER,
+        device_id TEXT ,
+        localpart TEXT ,
+        created_ts BIGINT,
+        display_name TEXT,
+        last_seen_ts BIGINT,
+        ip TEXT,
+        user_agent TEXT,
+        UNIQUE (localpart, device_id)
+    );
+    INSERT
+    INTO device_devices (
+        access_token, session_id, device_id, localpart, created_ts, display_name, last_seen_ts, ip, user_agent
+    )  SELECT
+           access_token, session_id, device_id, localpart, created_ts, display_name, created_ts, '', ''
+    FROM device_devices_tmp;
+    DROP TABLE device_devices_tmp;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownLastSeenTSIP(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+ALTER TABLE device_devices RENAME TO device_devices_tmp;
+CREATE TABLE IF NOT EXISTS device_devices (
+    access_token TEXT PRIMARY KEY,
+    session_id INTEGER,
+    device_id TEXT ,
+    localpart TEXT ,
+    created_ts BIGINT,
+    display_name TEXT,
+    UNIQUE (localpart, device_id)
+);
+INSERT
+INTO device_devices (
+    access_token, session_id, device_id, localpart, created_ts, display_name
+) SELECT
+       access_token, session_id, device_id, localpart, created_ts, display_name
+FROM device_devices_tmp;
+DROP TABLE device_devices_tmp;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}

userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go

@@ -0,0 +1,54 @@
+package deltas
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/pressly/goose"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+func init() {
+	goose.AddMigration(UpAddAccountType, DownAddAccountType)
+}
+
+func LoadAddAccountType(m *sqlutil.Migrations) {
+	m.AddMigration(UpAddAccountType, DownAddAccountType)
+}
+
+func UpAddAccountType(tx *sql.Tx) error {
+	// initially set every account to useraccount, change appservice and guest accounts afterwards
+	// (user = 1, guest = 2, admin = 3, appservice = 4)
+	_, err := tx.Exec(`ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
+CREATE TABLE account_accounts (
+    localpart TEXT NOT NULL PRIMARY KEY,
+    created_ts BIGINT NOT NULL,
+    password_hash TEXT,
+    appservice_id TEXT,
+    is_deactivated BOOLEAN DEFAULT 0,
+    account_type INTEGER NOT NULL
+);
+INSERT
+    INTO account_accounts (
+      localpart, created_ts, password_hash, appservice_id, account_type
+    ) SELECT
+        localpart, created_ts, password_hash, appservice_id, 1
+    FROM account_accounts_tmp
+;
+UPDATE account_accounts SET account_type = 4 WHERE appservice_id <> '';
+UPDATE account_accounts SET account_type = 2 WHERE localpart GLOB '[0-9]*';
+DROP TABLE account_accounts_tmp;`)
+	if err != nil {
+		return fmt.Errorf("failed to add column: %w", err)
+	}
+	return nil
+}
+
+func DownAddAccountType(tx *sql.Tx) error {
+	_, err := tx.Exec(`ALTER TABLE account_accounts DROP COLUMN account_type;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}