Support for m.login.token (#2014)

* Add GOPATH to PATH in find-lint.sh.

The user doesn't necessarily have it in PATH.

* Refactor LoginTypePassword and Type to support m.login.token and m.login.sso.

For login token:

* m.login.token will require deleting the token after completeAuth has
  generated an access token, so a cleanup function is returned by
  Type.Login.
* Allowing different login types will require parsing the /login body
  twice: first to extract the "type" and then the type-specific parsing.
  Thus, we will have to buffer the request JSON in /login, like
  UserInteractive already does.

For SSO:

* NewUserInteractive will have to also use GetAccountByLocalpart. It
  makes more sense to just pass a (narrowed-down) accountDB interface
  to it than adding more function pointers.

Code quality:

* Passing around (and down-casting) interface{} for login request types
  has drawbacks in terms of type-safety, and no inherent benefits. We
  always decode JSON anyway. Hence renaming to Type.LoginFromJSON. Code
  that directly uses LoginTypePassword with parsed data can still use
  Login.
* Removed a TODO for SSO. This is already tracked in #1297.
* httputil.UnmarshalJSON is useful because it returns a JSONResponse.

This change is intended to have no functional changes.

* Support login tokens in User API.

This adds full lifecycle functions for login tokens: create, query, delete.

* Support m.login.token in /login.

* Fixes for PR review.

* Set @matrix-org/dendrite-core as repository code owner

* Return event NID from `StoreEvent`, match PSQL vs SQLite behaviour, tweak backfill persistence (#2071)

Co-authored-by: kegsay <kegan@matrix.org>
Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>

userapi/storage/devices/interface.go

@@ -38,4 +38,15 @@ type Database interface {
 	RemoveDevices(ctx context.Context, localpart string, devices []string) error
 	// RemoveAllDevices deleted all devices for this user. Returns the devices deleted.
 	RemoveAllDevices(ctx context.Context, localpart, exceptDeviceID string) (devices []api.Device, err error)
+
+	// CreateLoginToken generates a token, stores and returns it. The lifetime is
+	// determined by the loginTokenLifetime given to the Database constructor.
+	CreateLoginToken(ctx context.Context, data *api.LoginTokenData) (*api.LoginTokenMetadata, error)
+
+	// RemoveLoginToken removes the named token (and may clean up other expired tokens).
+	RemoveLoginToken(ctx context.Context, token string) error
+
+	// GetLoginTokenDataByToken returns the data associated with the given token.
+	// May return sql.ErrNoRows.
+	GetLoginTokenDataByToken(ctx context.Context, token string) (*api.LoginTokenData, error)
 }

userapi/storage/devices/postgres/logintoken_table.go

@@ -0,0 +1,93 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+)
+
+type loginTokenStatements struct {
+	insertStmt        *sql.Stmt
+	deleteStmt        *sql.Stmt
+	selectByTokenStmt *sql.Stmt
+}
+
+// execSchema ensures tables and indices exist.
+func (s *loginTokenStatements) execSchema(db *sql.DB) error {
+	_, err := db.Exec(`
+CREATE TABLE IF NOT EXISTS login_tokens (
+	-- The random value of the token issued to a user
+	token TEXT NOT NULL PRIMARY KEY,
+	-- When the token expires
+	token_expires_at TIMESTAMP NOT NULL,
+
+    -- The mxid for this account
+	user_id TEXT NOT NULL
+);
+
+-- This index allows efficient garbage collection of expired tokens.
+CREATE INDEX IF NOT EXISTS login_tokens_expiration_idx ON login_tokens(token_expires_at);
+`)
+	return err
+}
+
+// prepare runs statement preparation.
+func (s *loginTokenStatements) prepare(db *sql.DB) error {
+	return sqlutil.StatementList{
+		{&s.insertStmt, "INSERT INTO login_tokens(token, token_expires_at, user_id) VALUES ($1, $2, $3)"},
+		{&s.deleteStmt, "DELETE FROM login_tokens WHERE token = $1 OR token_expires_at <= $2"},
+		{&s.selectByTokenStmt, "SELECT user_id FROM login_tokens WHERE token = $1 AND token_expires_at > $2"},
+	}.Prepare(db)
+}
+
+// insert adds an already generated token to the database.
+func (s *loginTokenStatements) insert(ctx context.Context, txn *sql.Tx, metadata *api.LoginTokenMetadata, data *api.LoginTokenData) error {
+	stmt := sqlutil.TxStmt(txn, s.insertStmt)
+	_, err := stmt.ExecContext(ctx, metadata.Token, metadata.Expiration.UTC(), data.UserID)
+	return err
+}
+
+// deleteByToken removes the named token.
+//
+// As a simple way to garbage-collect stale tokens, we also remove all expired tokens.
+// The login_tokens_expiration_idx index should make that efficient.
+func (s *loginTokenStatements) deleteByToken(ctx context.Context, txn *sql.Tx, token string) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteStmt)
+	res, err := stmt.ExecContext(ctx, token, time.Now().UTC())
+	if err != nil {
+		return err
+	}
+	if n, err := res.RowsAffected(); err == nil && n > 1 {
+		util.GetLogger(ctx).WithField("num_deleted", n).Infof("Deleted %d login tokens (%d likely additional expired token)", n, n-1)
+	}
+	return nil
+}
+
+// selectByToken returns the data associated with the given token. May return sql.ErrNoRows.
+func (s *loginTokenStatements) selectByToken(ctx context.Context, token string) (*api.LoginTokenData, error) {
+	var data api.LoginTokenData
+	err := s.selectByTokenStmt.QueryRowContext(ctx, token, time.Now().UTC()).Scan(&data.UserID)
+	if err != nil {
+		return nil, err
+	}
+
+	return &data, nil
+}

userapi/storage/devices/postgres/storage.go

@@ -19,6 +19,7 @@ import (
 	"crypto/rand"
 	"database/sql"
 	"encoding/base64"
+	"time"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -27,28 +28,38 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-// The length of generated device IDs
-var deviceIDByteLength = 6
+const (
+	// The length of generated device IDs
+	deviceIDByteLength   = 6
+	loginTokenByteLength = 32
+)
 
 // Database represents a device database.
 type Database struct {
-	db      *sql.DB
-	devices devicesStatements
+	db                 *sql.DB
+	devices            devicesStatements
+	loginTokens        loginTokenStatements
+	loginTokenLifetime time.Duration
 }
 
 // NewDatabase creates a new device database
-func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (*Database, error) {
+func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, loginTokenLifetime time.Duration) (*Database, error) {
 	db, err := sqlutil.Open(dbProperties)
 	if err != nil {
 		return nil, err
 	}
-	d := devicesStatements{}
+	var d devicesStatements
+	var lt loginTokenStatements
 
 	// Create tables before executing migrations so we don't fail if the table is missing,
 	// and THEN prepare statements so we don't fail due to referencing new columns
 	if err = d.execSchema(db); err != nil {
 		return nil, err
 	}
+	if err = lt.execSchema(db); err != nil {
+		return nil, err
+	}
+
 	m := sqlutil.NewMigrations()
 	deltas.LoadLastSeenTSIP(m)
 	if err = m.RunDeltas(db, dbProperties); err != nil {
@@ -58,8 +69,11 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
 	if err = d.prepare(db, serverName); err != nil {
 		return nil, err
 	}
+	if err = lt.prepare(db); err != nil {
+		return nil, err
+	}
 
-	return &Database{db, d}, nil
+	return &Database{db, d, lt, loginTokenLifetime}, nil
 }
 
 // GetDeviceByAccessToken returns the device matching the given access token.
@@ -210,3 +224,47 @@ func (d *Database) UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID
 		return d.devices.updateDeviceLastSeen(ctx, txn, localpart, deviceID, ipAddr)
 	})
 }
+
+// CreateLoginToken generates a token, stores and returns it. The lifetime is
+// determined by the loginTokenLifetime given to the Database constructor.
+func (d *Database) CreateLoginToken(ctx context.Context, data *api.LoginTokenData) (*api.LoginTokenMetadata, error) {
+	tok, err := generateLoginToken()
+	if err != nil {
+		return nil, err
+	}
+	meta := &api.LoginTokenMetadata{
+		Token:      tok,
+		Expiration: time.Now().Add(d.loginTokenLifetime),
+	}
+
+	err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
+		return d.loginTokens.insert(ctx, txn, meta, data)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return meta, nil
+}
+
+func generateLoginToken() (string, error) {
+	b := make([]byte, loginTokenByteLength)
+	_, err := rand.Read(b)
+	if err != nil {
+		return "", err
+	}
+	return base64.RawURLEncoding.EncodeToString(b), nil
+}
+
+// RemoveLoginToken removes the named token (and may clean up other expired tokens).
+func (d *Database) RemoveLoginToken(ctx context.Context, token string) error {
+	return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
+		return d.loginTokens.deleteByToken(ctx, txn, token)
+	})
+}
+
+// GetLoginTokenDataByToken returns the data associated with the given token.
+// May return sql.ErrNoRows.
+func (d *Database) GetLoginTokenDataByToken(ctx context.Context, token string) (*api.LoginTokenData, error) {
+	return d.loginTokens.selectByToken(ctx, token)
+}

userapi/storage/devices/sqlite3/logintoken_table.go

@@ -0,0 +1,93 @@
+// Copyright 2021 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite3
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/util"
+)
+
+type loginTokenStatements struct {
+	insertStmt        *sql.Stmt
+	deleteStmt        *sql.Stmt
+	selectByTokenStmt *sql.Stmt
+}
+
+// execSchema ensures tables and indices exist.
+func (s *loginTokenStatements) execSchema(db *sql.DB) error {
+	_, err := db.Exec(`
+CREATE TABLE IF NOT EXISTS login_tokens (
+	-- The random value of the token issued to a user
+	token TEXT NOT NULL PRIMARY KEY,
+	-- When the token expires
+	token_expires_at TIMESTAMP NOT NULL,
+
+    -- The mxid for this account
+	user_id TEXT NOT NULL
+);
+
+-- This index allows efficient garbage collection of expired tokens.
+CREATE INDEX IF NOT EXISTS login_tokens_expiration_idx ON login_tokens(token_expires_at);
+`)
+	return err
+}
+
+// prepare runs statement preparation.
+func (s *loginTokenStatements) prepare(db *sql.DB) error {
+	return sqlutil.StatementList{
+		{&s.insertStmt, "INSERT INTO login_tokens(token, token_expires_at, user_id) VALUES ($1, $2, $3)"},
+		{&s.deleteStmt, "DELETE FROM login_tokens WHERE token = $1 OR token_expires_at <= $2"},
+		{&s.selectByTokenStmt, "SELECT user_id FROM login_tokens WHERE token = $1 AND token_expires_at > $2"},
+	}.Prepare(db)
+}
+
+// insert adds an already generated token to the database.
+func (s *loginTokenStatements) insert(ctx context.Context, txn *sql.Tx, metadata *api.LoginTokenMetadata, data *api.LoginTokenData) error {
+	stmt := sqlutil.TxStmt(txn, s.insertStmt)
+	_, err := stmt.ExecContext(ctx, metadata.Token, metadata.Expiration.UTC(), data.UserID)
+	return err
+}
+
+// deleteByToken removes the named token.
+//
+// As a simple way to garbage-collect stale tokens, we also remove all expired tokens.
+// The login_tokens_expiration_idx index should make that efficient.
+func (s *loginTokenStatements) deleteByToken(ctx context.Context, txn *sql.Tx, token string) error {
+	stmt := sqlutil.TxStmt(txn, s.deleteStmt)
+	res, err := stmt.ExecContext(ctx, token, time.Now().UTC())
+	if err != nil {
+		return err
+	}
+	if n, err := res.RowsAffected(); err == nil && n > 1 {
+		util.GetLogger(ctx).WithField("num_deleted", n).Infof("Deleted %d login tokens (%d likely additional expired token)", n, n-1)
+	}
+	return nil
+}
+
+// selectByToken returns the data associated with the given token. May return sql.ErrNoRows.
+func (s *loginTokenStatements) selectByToken(ctx context.Context, token string) (*api.LoginTokenData, error) {
+	var data api.LoginTokenData
+	err := s.selectByTokenStmt.QueryRowContext(ctx, token, time.Now().UTC()).Scan(&data.UserID)
+	if err != nil {
+		return nil, err
+	}
+
+	return &data, nil
+}

userapi/storage/devices/sqlite3/storage.go

@@ -19,6 +19,7 @@ import (
 	"crypto/rand"
 	"database/sql"
 	"encoding/base64"
+	"time"
 
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/setup/config"
@@ -27,30 +28,41 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-// The length of generated device IDs
-var deviceIDByteLength = 6
+const (
+	// The length of generated device IDs
+	deviceIDByteLength = 6
+
+	loginTokenByteLength = 32
+)
 
 // Database represents a device database.
 type Database struct {
-	db      *sql.DB
-	writer  sqlutil.Writer
-	devices devicesStatements
+	db                 *sql.DB
+	writer             sqlutil.Writer
+	devices            devicesStatements
+	loginTokens        loginTokenStatements
+	loginTokenLifetime time.Duration
 }
 
 // NewDatabase creates a new device database
-func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (*Database, error) {
+func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, loginTokenLifetime time.Duration) (*Database, error) {
 	db, err := sqlutil.Open(dbProperties)
 	if err != nil {
 		return nil, err
 	}
 	writer := sqlutil.NewExclusiveWriter()
-	d := devicesStatements{}
+	var d devicesStatements
+	var lt loginTokenStatements
 
 	// Create tables before executing migrations so we don't fail if the table is missing,
 	// and THEN prepare statements so we don't fail due to referencing new columns
 	if err = d.execSchema(db); err != nil {
 		return nil, err
 	}
+	if err = lt.execSchema(db); err != nil {
+		return nil, err
+	}
+
 	m := sqlutil.NewMigrations()
 	deltas.LoadLastSeenTSIP(m)
 	if err = m.RunDeltas(db, dbProperties); err != nil {
@@ -59,7 +71,10 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
 	if err = d.prepare(db, writer, serverName); err != nil {
 		return nil, err
 	}
-	return &Database{db, writer, d}, nil
+	if err = lt.prepare(db); err != nil {
+		return nil, err
+	}
+	return &Database{db, writer, d, lt, loginTokenLifetime}, nil
 }
 
 // GetDeviceByAccessToken returns the device matching the given access token.
@@ -210,3 +225,47 @@ func (d *Database) UpdateDeviceLastSeen(ctx context.Context, localpart, deviceID
 		return d.devices.updateDeviceLastSeen(ctx, txn, localpart, deviceID, ipAddr)
 	})
 }
+
+// CreateLoginToken generates a token, stores and returns it. The lifetime is
+// determined by the loginTokenLifetime given to the Database constructor.
+func (d *Database) CreateLoginToken(ctx context.Context, data *api.LoginTokenData) (*api.LoginTokenMetadata, error) {
+	tok, err := generateLoginToken()
+	if err != nil {
+		return nil, err
+	}
+	meta := &api.LoginTokenMetadata{
+		Token:      tok,
+		Expiration: time.Now().Add(d.loginTokenLifetime),
+	}
+
+	err = d.writer.Do(d.db, nil, func(txn *sql.Tx) error {
+		return d.loginTokens.insert(ctx, txn, meta, data)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return meta, nil
+}
+
+func generateLoginToken() (string, error) {
+	b := make([]byte, loginTokenByteLength)
+	_, err := rand.Read(b)
+	if err != nil {
+		return "", err
+	}
+	return base64.RawURLEncoding.EncodeToString(b), nil
+}
+
+// RemoveLoginToken removes the named token (and may clean up other expired tokens).
+func (d *Database) RemoveLoginToken(ctx context.Context, token string) error {
+	return d.writer.Do(d.db, nil, func(txn *sql.Tx) error {
+		return d.loginTokens.deleteByToken(ctx, txn, token)
+	})
+}
+
+// GetLoginTokenDataByToken returns the data associated with the given token.
+// May return sql.ErrNoRows.
+func (d *Database) GetLoginTokenDataByToken(ctx context.Context, token string) (*api.LoginTokenData, error) {
+	return d.loginTokens.selectByToken(ctx, token)
+}

userapi/storage/devices/storage.go

@@ -19,6 +19,7 @@ package devices
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/storage/devices/postgres"
@@ -27,13 +28,14 @@ import (
 )
 
 // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
-// and sets postgres connection parameters
-func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName) (Database, error) {
+// and sets postgres connection parameters. loginTokenLifetime determines how long a
+// login token from CreateLoginToken is valid.
+func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, loginTokenLifetime time.Duration) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties, serverName)
+		return sqlite3.NewDatabase(dbProperties, serverName, loginTokenLifetime)
 	case dbProperties.ConnectionString.IsPostgres():
-		return postgres.NewDatabase(dbProperties, serverName)
+		return postgres.NewDatabase(dbProperties, serverName, loginTokenLifetime)
 	default:
 		return nil, fmt.Errorf("unexpected database type")
 	}

userapi/storage/devices/storage_wasm.go

@@ -16,6 +16,7 @@ package devices
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3"
@@ -25,10 +26,11 @@ import (
 func NewDatabase(
 	dbProperties *config.DatabaseOptions,
 	serverName gomatrixserverlib.ServerName,
+	loginTokenLifetime time.Duration,
 ) (Database, error) {
 	switch {
 	case dbProperties.ConnectionString.IsSQLite():
-		return sqlite3.NewDatabase(dbProperties, serverName)
+		return sqlite3.NewDatabase(dbProperties, serverName, loginTokenLifetime)
 	case dbProperties.ConnectionString.IsPostgres():
 		return nil, fmt.Errorf("can't use Postgres implementation")
 	default: