From 73ae1bd58cada3aecb15c8778129dc45ae6ea03a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Nov 2017 16:31:46 +0000 Subject: [PATCH] Migrate clientapi storage to use migrate library --- .../accounts/1511790253_initial.down.sql | 6 ++ db-schema/accounts/1511790253_initial.up.sql | 73 +++++++++++++++++++ db-schema/devices/1511791787_initial.down.sql | 1 + db-schema/devices/1511791787_initial.up.sql | 21 ++++++ .../storage/accounts/account_data_table.go | 20 ----- .../auth/storage/accounts/accounts_table.go | 18 ----- .../auth/storage/accounts/filter_table.go | 20 ----- .../auth/storage/accounts/membership_table.go | 22 ------ .../auth/storage/accounts/profile_table.go | 16 ---- .../auth/storage/accounts/storage.go | 8 +- .../auth/storage/accounts/threepid_table.go | 20 ----- .../auth/storage/devices/devices_table.go | 28 ------- .../clientapi/auth/storage/devices/storage.go | 6 ++ 13 files changed, 113 insertions(+), 146 deletions(-) create mode 100644 db-schema/accounts/1511790253_initial.down.sql create mode 100644 db-schema/accounts/1511790253_initial.up.sql create mode 100644 db-schema/devices/1511791787_initial.down.sql create mode 100644 db-schema/devices/1511791787_initial.up.sql diff --git a/db-schema/accounts/1511790253_initial.down.sql b/db-schema/accounts/1511790253_initial.down.sql new file mode 100644 index 00000000..093a816c --- /dev/null +++ b/db-schema/accounts/1511790253_initial.down.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS account_data; +DROP TABLE IF EXISTS account_accounts; +DROP TABLE IF EXISTS account_filter; +DROP TABLE IF EXISTS account_memberships; +DROP TABLE IF EXISTS account_profiles; +DROP TABLE IF EXISTS account_threepid; diff --git a/db-schema/accounts/1511790253_initial.up.sql b/db-schema/accounts/1511790253_initial.up.sql new file mode 100644 index 00000000..3ade999d --- /dev/null +++ b/db-schema/accounts/1511790253_initial.up.sql @@ -0,0 +1,73 @@ +CREATE TABLE IF NOT EXISTS account_data ( + -- The Matrix user ID localpart for this account + localpart TEXT NOT NULL, + -- The room ID for this data (empty string if not specific to a room) + room_id TEXT, + -- The account data type + type TEXT NOT NULL, + -- The account data content + content TEXT NOT NULL, + + PRIMARY KEY(localpart, room_id, type) +); + +CREATE TABLE IF NOT EXISTS account_accounts ( + -- The Matrix user ID localpart for this account + localpart TEXT NOT NULL PRIMARY KEY, + -- When this account was first created, as a unix timestamp (ms resolution). + created_ts BIGINT NOT NULL, + -- The password hash for this account. Can be NULL if this is a passwordless account. + password_hash TEXT + -- TODO: + -- is_guest, is_admin, appservice_id, upgraded_ts, devices, any email reset stuff? +); + +CREATE TABLE IF NOT EXISTS account_filter ( + -- The filter + filter TEXT NOT NULL, + -- The ID + id SERIAL UNIQUE, + -- The localpart of the Matrix user ID associated to this filter + localpart TEXT NOT NULL, + + PRIMARY KEY(id, localpart) +); + +CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart); + +CREATE TABLE IF NOT EXISTS account_memberships ( + -- The Matrix user ID localpart for the member + localpart TEXT NOT NULL, + -- The room this user is a member of + room_id TEXT NOT NULL, + -- The ID of the join membership event + event_id TEXT NOT NULL, + + -- A user can only be member of a room once + PRIMARY KEY (localpart, room_id) +); + +-- Use index to process deletion by ID more efficiently +CREATE UNIQUE INDEX IF NOT EXISTS account_membership_event_id ON account_memberships(event_id); + +CREATE TABLE IF NOT EXISTS account_profiles ( + -- The Matrix user ID localpart for this account + localpart TEXT NOT NULL PRIMARY KEY, + -- The display name for this account + display_name TEXT, + -- The URL of the avatar for this account + avatar_url TEXT +); + +CREATE TABLE IF NOT EXISTS account_threepid ( + -- The third party identifier + threepid TEXT NOT NULL, + -- The 3PID medium + medium TEXT NOT NULL DEFAULT 'email', + -- The localpart of the Matrix user ID associated to this 3PID + localpart TEXT NOT NULL, + + PRIMARY KEY(threepid, medium) +); + +CREATE INDEX IF NOT EXISTS account_threepid_localpart ON account_threepid(localpart); diff --git a/db-schema/devices/1511791787_initial.down.sql b/db-schema/devices/1511791787_initial.down.sql new file mode 100644 index 00000000..e8bba2f1 --- /dev/null +++ b/db-schema/devices/1511791787_initial.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS device_devices; diff --git a/db-schema/devices/1511791787_initial.up.sql b/db-schema/devices/1511791787_initial.up.sql new file mode 100644 index 00000000..6b7eab20 --- /dev/null +++ b/db-schema/devices/1511791787_initial.up.sql @@ -0,0 +1,21 @@ +-- Stores data about devices. +CREATE TABLE IF NOT EXISTS device_devices ( + -- The access token granted to this device. This has to be the primary key + -- so we can distinguish which device is making a given request. + access_token TEXT NOT NULL PRIMARY KEY, + -- The device identifier. This only needs to uniquely identify a device for a given user, not globally. + -- access_tokens will be clobbered based on the device ID for a user. + device_id TEXT NOT NULL, + -- The Matrix user ID localpart for this device. This is preferable to storing the full user_id + -- as it is smaller, makes it clearer that we only manage devices for our own users, and may make + -- migration to different domain names easier. + localpart TEXT NOT NULL, + -- When this devices was first recognised on the network, as a unix timestamp (ms resolution). + created_ts BIGINT NOT NULL, + -- The display name, human friendlier than device_id and updatable + display_name TEXT + -- TODO: device keys, device display names, last used ts and IP address?, token restrictions (if 3rd-party OAuth app) +); + +-- Device IDs must be unique for a given user. +CREATE UNIQUE INDEX IF NOT EXISTS device_localpart_id_idx ON device_devices(localpart, device_id); diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/account_data_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/account_data_table.go index 0d73cb31..4d7bd98f 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/account_data_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/account_data_table.go @@ -21,22 +21,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -const accountDataSchema = ` --- Stores data about accounts data. -CREATE TABLE IF NOT EXISTS account_data ( - -- The Matrix user ID localpart for this account - localpart TEXT NOT NULL, - -- The room ID for this data (empty string if not specific to a room) - room_id TEXT, - -- The account data type - type TEXT NOT NULL, - -- The account data content - content TEXT NOT NULL, - - PRIMARY KEY(localpart, room_id, type) -); -` - const insertAccountDataSQL = ` INSERT INTO account_data(localpart, room_id, type, content) VALUES($1, $2, $3, $4) ON CONFLICT (localpart, room_id, type) DO UPDATE SET content = EXCLUDED.content @@ -55,10 +39,6 @@ type accountDataStatements struct { } func (s *accountDataStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(accountDataSchema) - if err != nil { - return - } if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil { return } diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/accounts_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/accounts_table.go index 68a80917..74bd18f7 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/accounts_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/accounts_table.go @@ -24,20 +24,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -const accountsSchema = ` --- Stores data about accounts. -CREATE TABLE IF NOT EXISTS account_accounts ( - -- The Matrix user ID localpart for this account - localpart TEXT NOT NULL PRIMARY KEY, - -- When this account was first created, as a unix timestamp (ms resolution). - created_ts BIGINT NOT NULL, - -- The password hash for this account. Can be NULL if this is a passwordless account. - password_hash TEXT - -- TODO: - -- is_guest, is_admin, appservice_id, upgraded_ts, devices, any email reset stuff? -); -` - const insertAccountSQL = "" + "INSERT INTO account_accounts(localpart, created_ts, password_hash) VALUES ($1, $2, $3)" @@ -57,10 +43,6 @@ type accountsStatements struct { } func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { - _, err = db.Exec(accountsSchema) - if err != nil { - return - } if s.insertAccountStmt, err = db.Prepare(insertAccountSQL); err != nil { return } diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go index 81bae454..46e77181 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go @@ -21,22 +21,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -const filterSchema = ` --- Stores data about filters -CREATE TABLE IF NOT EXISTS account_filter ( - -- The filter - filter TEXT NOT NULL, - -- The ID - id SERIAL UNIQUE, - -- The localpart of the Matrix user ID associated to this filter - localpart TEXT NOT NULL, - - PRIMARY KEY(id, localpart) -); - -CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart); -` - const selectFilterSQL = "" + "SELECT filter FROM account_filter WHERE localpart = $1 AND id = $2" @@ -53,10 +37,6 @@ type filterStatements struct { } func (s *filterStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(filterSchema) - if err != nil { - return - } if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil { return } diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/membership_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/membership_table.go index 1a0d0fed..374758c2 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/membership_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/membership_table.go @@ -22,24 +22,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) -const membershipSchema = ` --- Stores data about users memberships to rooms. -CREATE TABLE IF NOT EXISTS account_memberships ( - -- The Matrix user ID localpart for the member - localpart TEXT NOT NULL, - -- The room this user is a member of - room_id TEXT NOT NULL, - -- The ID of the join membership event - event_id TEXT NOT NULL, - - -- A user can only be member of a room once - PRIMARY KEY (localpart, room_id) -); - --- Use index to process deletion by ID more efficiently -CREATE UNIQUE INDEX IF NOT EXISTS account_membership_event_id ON account_memberships(event_id); -` - const insertMembershipSQL = ` INSERT INTO account_memberships(localpart, room_id, event_id) VALUES ($1, $2, $3) ON CONFLICT (localpart, room_id) DO UPDATE SET event_id = EXCLUDED.event_id @@ -58,10 +40,6 @@ type membershipStatements struct { } func (s *membershipStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(membershipSchema) - if err != nil { - return - } if s.deleteMembershipsByEventIDsStmt, err = db.Prepare(deleteMembershipsByEventIDsSQL); err != nil { return } diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/profile_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/profile_table.go index 157bb99b..715a46ff 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/profile_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/profile_table.go @@ -21,18 +21,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) -const profilesSchema = ` --- Stores data about accounts profiles. -CREATE TABLE IF NOT EXISTS account_profiles ( - -- The Matrix user ID localpart for this account - localpart TEXT NOT NULL PRIMARY KEY, - -- The display name for this account - display_name TEXT, - -- The URL of the avatar for this account - avatar_url TEXT -); -` - const insertProfileSQL = "" + "INSERT INTO account_profiles(localpart, display_name, avatar_url) VALUES ($1, $2, $3)" @@ -53,10 +41,6 @@ type profilesStatements struct { } func (s *profilesStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(profilesSchema) - if err != nil { - return - } if s.insertProfileStmt, err = db.Prepare(insertProfileSQL); err != nil { return } diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/storage.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/storage.go index d5712eb5..255d80ae 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/storage.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/storage.go @@ -23,8 +23,6 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" - // Import the postgres database driver. - _ "github.com/lib/pq" ) // Database represents an account database @@ -47,6 +45,12 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) if db, err = sql.Open("postgres", dataSourceName); err != nil { return nil, err } + + err = common.DoMigrations(db, "accounts") + if err != nil { + return nil, err + } + partitions := common.PartitionOffsetStatements{} if err = partitions.Prepare(db, "account"); err != nil { return nil, err diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/threepid_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/threepid_table.go index 5900260a..cdd4745b 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/threepid_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/threepid_table.go @@ -23,22 +23,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) -const threepidSchema = ` --- Stores data about third party identifiers -CREATE TABLE IF NOT EXISTS account_threepid ( - -- The third party identifier - threepid TEXT NOT NULL, - -- The 3PID medium - medium TEXT NOT NULL DEFAULT 'email', - -- The localpart of the Matrix user ID associated to this 3PID - localpart TEXT NOT NULL, - - PRIMARY KEY(threepid, medium) -); - -CREATE INDEX IF NOT EXISTS account_threepid_localpart ON account_threepid(localpart); -` - const selectLocalpartForThreePIDSQL = "" + "SELECT localpart FROM account_threepid WHERE threepid = $1 AND medium = $2" @@ -59,10 +43,6 @@ type threepidStatements struct { } func (s *threepidStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(threepidSchema) - if err != nil { - return - } if s.selectLocalpartForThreePIDStmt, err = db.Prepare(selectLocalpartForThreePIDSQL); err != nil { return } diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go index c8ece6b9..f251c83b 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/devices_table.go @@ -26,30 +26,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -const devicesSchema = ` --- Stores data about devices. -CREATE TABLE IF NOT EXISTS device_devices ( - -- The access token granted to this device. This has to be the primary key - -- so we can distinguish which device is making a given request. - access_token TEXT NOT NULL PRIMARY KEY, - -- The device identifier. This only needs to uniquely identify a device for a given user, not globally. - -- access_tokens will be clobbered based on the device ID for a user. - device_id TEXT NOT NULL, - -- The Matrix user ID localpart for this device. This is preferable to storing the full user_id - -- as it is smaller, makes it clearer that we only manage devices for our own users, and may make - -- migration to different domain names easier. - localpart TEXT NOT NULL, - -- When this devices was first recognised on the network, as a unix timestamp (ms resolution). - created_ts BIGINT NOT NULL, - -- The display name, human friendlier than device_id and updatable - display_name TEXT - -- TODO: device keys, device display names, last used ts and IP address?, token restrictions (if 3rd-party OAuth app) -); - --- Device IDs must be unique for a given user. -CREATE UNIQUE INDEX IF NOT EXISTS device_localpart_id_idx ON device_devices(localpart, device_id); -` - const insertDeviceSQL = "" + "INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5)" @@ -83,10 +59,6 @@ type devicesStatements struct { } func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { - _, err = db.Exec(devicesSchema) - if err != nil { - return - } if s.insertDeviceStmt, err = db.Prepare(insertDeviceSQL); err != nil { return } diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go index 6ac475a6..6b554a14 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/devices/storage.go @@ -37,6 +37,12 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) if db, err = sql.Open("postgres", dataSourceName); err != nil { return nil, err } + + err = common.DoMigrations(db, "devices") + if err != nil { + return nil, err + } + d := devicesStatements{} if err = d.prepare(db, serverName); err != nil { return nil, err