use go module for dependencies (#594)

clientapi/README.md

@@ -0,0 +1,11 @@
+This component roughly corresponds to "Client Room Send" and "Client Sync" on [the WIRING diagram](https://github.com/matrix-org/dendrite/blob/master/WIRING.md).
+This component produces multiple binaries.
+
+## Internals
+
+- HTTP routing is done using `gorilla/mux` and the routing paths are in the `routing` package.
+
+### Writers
+- Each HTTP "write operation" (`/createRoom`, `/rooms/$room_id/send/$type`, etc) is contained entirely to a single file in the `writers` package.
+- This file contains the request and response `struct` definitions, as well as a `Validate() bool` function to validate incoming requests.
+- The entry point for each write operation is a stand-alone function as this makes testing easier. All dependencies should be injected into this function, including server keys/name, etc.

clientapi/auth/auth.go

@@ -0,0 +1,210 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package auth implements authentication checks and storage.
+package auth
+
+import (
+	"context"
+	"crypto/rand"
+	"database/sql"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/matrix-org/dendrite/appservice/types"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/util"
+)
+
+// OWASP recommends at least 128 bits of entropy for tokens: https://www.owasp.org/index.php/Insufficient_Session-ID_Length
+// 32 bytes => 256 bits
+var tokenByteLength = 32
+
+// DeviceDatabase represents a device database.
+type DeviceDatabase interface {
+	// Look up the device matching the given access token.
+	GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error)
+}
+
+// AccountDatabase represents an account database.
+type AccountDatabase interface {
+	// Look up the account matching the given localpart.
+	GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error)
+}
+
+// Data contains information required to authenticate a request.
+type Data struct {
+	AccountDB AccountDatabase
+	DeviceDB  DeviceDatabase
+	// AppServices is the list of all registered AS
+	AppServices []config.ApplicationService
+}
+
+// VerifyUserFromRequest authenticates the HTTP request,
+// on success returns Device of the requester.
+// Finds local user or an application service user.
+// Note: For an AS user, AS dummy device is returned.
+// On failure returns an JSON error response which can be sent to the client.
+func VerifyUserFromRequest(
+	req *http.Request, data Data,
+) (*authtypes.Device, *util.JSONResponse) {
+	// Try to find the Application Service user
+	token, err := ExtractAccessToken(req)
+	if err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: jsonerror.MissingToken(err.Error()),
+		}
+	}
+
+	// Search for app service with given access_token
+	var appService *config.ApplicationService
+	for _, as := range data.AppServices {
+		if as.ASToken == token {
+			appService = &as
+			break
+		}
+	}
+
+	if appService != nil {
+		// Create a dummy device for AS user
+		dev := authtypes.Device{
+			// Use AS dummy device ID
+			ID: types.AppServiceDeviceID,
+			// AS dummy device has AS's token.
+			AccessToken: token,
+		}
+
+		userID := req.URL.Query().Get("user_id")
+		localpart, err := userutil.ParseUsernameParam(userID, nil)
+		if err != nil {
+			return nil, &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.InvalidUsername(err.Error()),
+			}
+		}
+
+		if localpart != "" { // AS is masquerading as another user
+			// Verify that the user is registered
+			account, err := data.AccountDB.GetAccountByLocalpart(req.Context(), localpart)
+			// Verify that account exists & appServiceID matches
+			if err == nil && account.AppServiceID == appService.ID {
+				// Set the userID of dummy device
+				dev.UserID = userID
+				return &dev, nil
+			}
+
+			return nil, &util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden("Application service has not registered this user"),
+			}
+		}
+
+		// AS is not masquerading as any user, so use AS's sender_localpart
+		dev.UserID = appService.SenderLocalpart
+		return &dev, nil
+	}
+
+	// Try to find local user from device database
+	dev, devErr := verifyAccessToken(req, data.DeviceDB)
+	if devErr == nil {
+		return dev, verifyUserParameters(req)
+	}
+
+	return nil, &util.JSONResponse{
+		Code: http.StatusUnauthorized,
+		JSON: jsonerror.UnknownToken("Unrecognized access token"),
+	}
+}
+
+// verifyUserParameters ensures that a request coming from a regular user is not
+// using any query parameters reserved for an application service
+func verifyUserParameters(req *http.Request) *util.JSONResponse {
+	if req.URL.Query().Get("ts") != "" {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown("parameter 'ts' not allowed without valid parameter 'access_token'"),
+		}
+	}
+	return nil
+}
+
+// verifyAccessToken verifies that an access token was supplied in the given HTTP request
+// and returns the device it corresponds to. Returns resErr (an error response which can be
+// sent to the client) if the token is invalid or there was a problem querying the database.
+func verifyAccessToken(req *http.Request, deviceDB DeviceDatabase) (device *authtypes.Device, resErr *util.JSONResponse) {
+	token, err := ExtractAccessToken(req)
+	if err != nil {
+		resErr = &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: jsonerror.MissingToken(err.Error()),
+		}
+		return
+	}
+	device, err = deviceDB.GetDeviceByAccessToken(req.Context(), token)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			resErr = &util.JSONResponse{
+				Code: http.StatusUnauthorized,
+				JSON: jsonerror.UnknownToken("Unknown token"),
+			}
+		} else {
+			jsonErr := httputil.LogThenError(req, err)
+			resErr = &jsonErr
+		}
+	}
+	return
+}
+
+// GenerateAccessToken creates a new access token. Returns an error if failed to generate
+// random bytes.
+func GenerateAccessToken() (string, error) {
+	b := make([]byte, tokenByteLength)
+	if _, err := rand.Read(b); err != nil {
+		return "", err
+	}
+	// url-safe no padding
+	return base64.RawURLEncoding.EncodeToString(b), nil
+}
+
+// ExtractAccessToken from a request, or return an error detailing what went wrong. The
+// error message MUST be human-readable and comprehensible to the client.
+func ExtractAccessToken(req *http.Request) (string, error) {
+	// cf https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/api/auth.py#L631
+	authBearer := req.Header.Get("Authorization")
+	queryToken := req.URL.Query().Get("access_token")
+	if authBearer != "" && queryToken != "" {
+		return "", fmt.Errorf("mixing Authorization headers and access_token query parameters")
+	}
+
+	if queryToken != "" {
+		return queryToken, nil
+	}
+
+	if authBearer != "" {
+		parts := strings.SplitN(authBearer, " ", 2)
+		if len(parts) != 2 || parts[0] != "Bearer" {
+			return "", fmt.Errorf("invalid Authorization header")
+		}
+		return parts[1], nil
+	}
+
+	return "", fmt.Errorf("missing access token")
+}

clientapi/auth/authtypes/account.go

@@ -0,0 +1,31 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package authtypes
+
+import (
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// Account represents a Matrix account on this home server.
+type Account struct {
+	UserID       string
+	Localpart    string
+	ServerName   gomatrixserverlib.ServerName
+	Profile      *Profile
+	AppServiceID string
+	// TODO: Other flags like IsAdmin, IsGuest
+	// TODO: Devices
+	// TODO: Associations (e.g. with application services)
+}

clientapi/auth/authtypes/device.go

@@ -0,0 +1,25 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package authtypes
+
+// Device represents a client's device (mobile, web, etc)
+type Device struct {
+	ID     string
+	UserID string
+	// The access_token granted to this device.
+	// This uniquely identifies the device from all other devices and clients.
+	AccessToken string
+	// TODO: display name, last used timestamp, keys, etc
+}

clientapi/auth/authtypes/flow.go

@@ -0,0 +1,21 @@
+// Copyright Andrew Morgan <andrew@amorgan.xyz>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package authtypes
+
+// Flow represents one possible way that the client can authenticate a request.
+// https://matrix.org/docs/spec/client_server/r0.3.0.html#user-interactive-authentication-api
+type Flow struct {
+	Stages []LoginType `json:"stages"`
+}

clientapi/auth/authtypes/logintypes.go

@@ -0,0 +1,12 @@
+package authtypes
+
+// LoginType are specified by http://matrix.org/docs/spec/client_server/r0.2.0.html#login-types
+type LoginType string
+
+// The relevant login types implemented in Dendrite
+const (
+	LoginTypeDummy              = "m.login.dummy"
+	LoginTypeSharedSecret       = "org.matrix.login.shared_secret"
+	LoginTypeRecaptcha          = "m.login.recaptcha"
+	LoginTypeApplicationService = "m.login.application_service"
+)

clientapi/auth/authtypes/membership.go

@@ -0,0 +1,23 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package authtypes
+
+// Membership represents the relationship between a user and a room they're a
+// member of
+type Membership struct {
+	Localpart string
+	RoomID    string
+	EventID   string
+}

clientapi/auth/authtypes/profile.go

@@ -0,0 +1,22 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package authtypes
+
+// Profile represents the profile for a Matrix account on this home server.
+type Profile struct {
+	Localpart   string
+	DisplayName string
+	AvatarURL   string
+}

clientapi/auth/authtypes/threepid.go

@@ -0,0 +1,21 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package authtypes
+
+// ThreePID represents a third-party identifier
+type ThreePID struct {
+	Address string `json:"address"`
+	Medium  string `json:"medium"`
+}

clientapi/auth/storage/accounts/account_data_table.go

@@ -0,0 +1,148 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package accounts
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const accountDataSchema = `
+-- Stores data about accounts data.
+CREATE TABLE IF NOT EXISTS account_data (
+    -- The Matrix user ID localpart for this account
+    localpart TEXT NOT NULL,
+    -- The room ID for this data (empty string if not specific to a room)
+    room_id TEXT,
+    -- The account data type
+    type TEXT NOT NULL,
+    -- The account data content
+    content TEXT NOT NULL,
+
+    PRIMARY KEY(localpart, room_id, type)
+);
+`
+
+const insertAccountDataSQL = `
+	INSERT INTO account_data(localpart, room_id, type, content) VALUES($1, $2, $3, $4)
+	ON CONFLICT (localpart, room_id, type) DO UPDATE SET content = EXCLUDED.content
+`
+
+const selectAccountDataSQL = "" +
+	"SELECT room_id, type, content FROM account_data WHERE localpart = $1"
+
+const selectAccountDataByTypeSQL = "" +
+	"SELECT content FROM account_data WHERE localpart = $1 AND room_id = $2 AND type = $3"
+
+type accountDataStatements struct {
+	insertAccountDataStmt       *sql.Stmt
+	selectAccountDataStmt       *sql.Stmt
+	selectAccountDataByTypeStmt *sql.Stmt
+}
+
+func (s *accountDataStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(accountDataSchema)
+	if err != nil {
+		return
+	}
+	if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil {
+		return
+	}
+	if s.selectAccountDataStmt, err = db.Prepare(selectAccountDataSQL); err != nil {
+		return
+	}
+	if s.selectAccountDataByTypeStmt, err = db.Prepare(selectAccountDataByTypeSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *accountDataStatements) insertAccountData(
+	ctx context.Context, localpart, roomID, dataType, content string,
+) (err error) {
+	stmt := s.insertAccountDataStmt
+	_, err = stmt.ExecContext(ctx, localpart, roomID, dataType, content)
+	return
+}
+
+func (s *accountDataStatements) selectAccountData(
+	ctx context.Context, localpart string,
+) (
+	global []gomatrixserverlib.ClientEvent,
+	rooms map[string][]gomatrixserverlib.ClientEvent,
+	err error,
+) {
+	rows, err := s.selectAccountDataStmt.QueryContext(ctx, localpart)
+	if err != nil {
+		return
+	}
+
+	global = []gomatrixserverlib.ClientEvent{}
+	rooms = make(map[string][]gomatrixserverlib.ClientEvent)
+
+	for rows.Next() {
+		var roomID string
+		var dataType string
+		var content []byte
+
+		if err = rows.Scan(&roomID, &dataType, &content); err != nil {
+			return
+		}
+
+		ac := gomatrixserverlib.ClientEvent{
+			Type:    dataType,
+			Content: content,
+		}
+
+		if len(roomID) > 0 {
+			rooms[roomID] = append(rooms[roomID], ac)
+		} else {
+			global = append(global, ac)
+		}
+	}
+
+	return
+}
+
+func (s *accountDataStatements) selectAccountDataByType(
+	ctx context.Context, localpart, roomID, dataType string,
+) (data []gomatrixserverlib.ClientEvent, err error) {
+	data = []gomatrixserverlib.ClientEvent{}
+
+	stmt := s.selectAccountDataByTypeStmt
+	rows, err := stmt.QueryContext(ctx, localpart, roomID, dataType)
+	if err != nil {
+		return
+	}
+
+	for rows.Next() {
+		var content []byte
+
+		if err = rows.Scan(&content); err != nil {
+			return
+		}
+
+		ac := gomatrixserverlib.ClientEvent{
+			Type:    dataType,
+			Content: content,
+		}
+
+		data = append(data, ac)
+	}
+
+	return
+}

clientapi/auth/storage/accounts/accounts_table.go

@@ -0,0 +1,153 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package accounts
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const accountsSchema = `
+-- Stores data about accounts.
+CREATE TABLE IF NOT EXISTS account_accounts (
+    -- The Matrix user ID localpart for this account
+    localpart TEXT NOT NULL PRIMARY KEY,
+    -- When this account was first created, as a unix timestamp (ms resolution).
+    created_ts BIGINT NOT NULL,
+    -- The password hash for this account. Can be NULL if this is a passwordless account.
+    password_hash TEXT,
+    -- Identifies which application service this account belongs to, if any.
+    appservice_id TEXT
+    -- TODO:
+    -- is_guest, is_admin, upgraded_ts, devices, any email reset stuff?
+);
+-- Create sequence for autogenerated numeric usernames
+CREATE SEQUENCE IF NOT EXISTS numeric_username_seq START 1;
+`
+
+const insertAccountSQL = "" +
+	"INSERT INTO account_accounts(localpart, created_ts, password_hash, appservice_id) VALUES ($1, $2, $3, $4)"
+
+const selectAccountByLocalpartSQL = "" +
+	"SELECT localpart, appservice_id FROM account_accounts WHERE localpart = $1"
+
+const selectPasswordHashSQL = "" +
+	"SELECT password_hash FROM account_accounts WHERE localpart = $1"
+
+const selectNewNumericLocalpartSQL = "" +
+	"SELECT nextval('numeric_username_seq')"
+
+// TODO: Update password
+
+type accountsStatements struct {
+	insertAccountStmt             *sql.Stmt
+	selectAccountByLocalpartStmt  *sql.Stmt
+	selectPasswordHashStmt        *sql.Stmt
+	selectNewNumericLocalpartStmt *sql.Stmt
+	serverName                    gomatrixserverlib.ServerName
+}
+
+func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) {
+	_, err = db.Exec(accountsSchema)
+	if err != nil {
+		return
+	}
+	if s.insertAccountStmt, err = db.Prepare(insertAccountSQL); err != nil {
+		return
+	}
+	if s.selectAccountByLocalpartStmt, err = db.Prepare(selectAccountByLocalpartSQL); err != nil {
+		return
+	}
+	if s.selectPasswordHashStmt, err = db.Prepare(selectPasswordHashSQL); err != nil {
+		return
+	}
+	if s.selectNewNumericLocalpartStmt, err = db.Prepare(selectNewNumericLocalpartSQL); err != nil {
+		return
+	}
+	s.serverName = server
+	return
+}
+
+// insertAccount creates a new account. 'hash' should be the password hash for this account. If it is missing,
+// this account will be passwordless. Returns an error if this account already exists. Returns the account
+// on success.
+func (s *accountsStatements) insertAccount(
+	ctx context.Context, localpart, hash, appserviceID string,
+) (*authtypes.Account, error) {
+	createdTimeMS := time.Now().UnixNano() / 1000000
+	stmt := s.insertAccountStmt
+
+	var err error
+	if appserviceID == "" {
+		_, err = stmt.ExecContext(ctx, localpart, createdTimeMS, hash, nil)
+	} else {
+		_, err = stmt.ExecContext(ctx, localpart, createdTimeMS, hash, appserviceID)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return &authtypes.Account{
+		Localpart:    localpart,
+		UserID:       userutil.MakeUserID(localpart, s.serverName),
+		ServerName:   s.serverName,
+		AppServiceID: appserviceID,
+	}, nil
+}
+
+func (s *accountsStatements) selectPasswordHash(
+	ctx context.Context, localpart string,
+) (hash string, err error) {
+	err = s.selectPasswordHashStmt.QueryRowContext(ctx, localpart).Scan(&hash)
+	return
+}
+
+func (s *accountsStatements) selectAccountByLocalpart(
+	ctx context.Context, localpart string,
+) (*authtypes.Account, error) {
+	var appserviceIDPtr sql.NullString
+	var acc authtypes.Account
+
+	stmt := s.selectAccountByLocalpartStmt
+	err := stmt.QueryRowContext(ctx, localpart).Scan(&acc.Localpart, &appserviceIDPtr)
+	if err != nil {
+		if err != sql.ErrNoRows {
+			log.WithError(err).Error("Unable to retrieve user from the db")
+		}
+		return nil, err
+	}
+	if appserviceIDPtr.Valid {
+		acc.AppServiceID = appserviceIDPtr.String
+	}
+
+	acc.UserID = userutil.MakeUserID(localpart, s.serverName)
+	acc.ServerName = s.serverName
+
+	return &acc, nil
+}
+
+func (s *accountsStatements) selectNewNumericLocalpart(
+	ctx context.Context,
+) (id int64, err error) {
+	err = s.selectNewNumericLocalpartStmt.QueryRowContext(ctx).Scan(&id)
+	return
+}

clientapi/auth/storage/accounts/filter_table.go

@@ -0,0 +1,107 @@
+// Copyright 2017 Jan Christian Grünhage
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package accounts
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const filterSchema = `
+-- Stores data about filters
+CREATE TABLE IF NOT EXISTS account_filter (
+	-- The filter
+	filter TEXT NOT NULL,
+	-- The ID
+	id SERIAL UNIQUE,
+	-- The localpart of the Matrix user ID associated to this filter
+	localpart TEXT NOT NULL,
+
+	PRIMARY KEY(id, localpart)
+);
+
+CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart);
+`
+
+const selectFilterSQL = "" +
+	"SELECT filter FROM account_filter WHERE localpart = $1 AND id = $2"
+
+const selectFilterIDByContentSQL = "" +
+	"SELECT id FROM account_filter WHERE localpart = $1 AND filter = $2"
+
+const insertFilterSQL = "" +
+	"INSERT INTO account_filter (filter, id, localpart) VALUES ($1, DEFAULT, $2) RETURNING id"
+
+type filterStatements struct {
+	selectFilterStmt            *sql.Stmt
+	selectFilterIDByContentStmt *sql.Stmt
+	insertFilterStmt            *sql.Stmt
+}
+
+func (s *filterStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(filterSchema)
+	if err != nil {
+		return
+	}
+	if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil {
+		return
+	}
+	if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil {
+		return
+	}
+	if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *filterStatements) selectFilter(
+	ctx context.Context, localpart string, filterID string,
+) (filter []byte, err error) {
+	err = s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filter)
+	return
+}
+
+func (s *filterStatements) insertFilter(
+	ctx context.Context, filter []byte, localpart string,
+) (filterID string, err error) {
+	var existingFilterID string
+
+	// This can result in a race condition when two clients try to insert the
+	// same filter and localpart at the same time, however this is not a
+	// problem as both calls will result in the same filterID
+	filterJSON, err := gomatrixserverlib.CanonicalJSON(filter)
+	if err != nil {
+		return "", err
+	}
+
+	// Check if filter already exists in the database
+	err = s.selectFilterIDByContentStmt.QueryRowContext(ctx,
+		localpart, filterJSON).Scan(&existingFilterID)
+	if err != nil && err != sql.ErrNoRows {
+		return "", err
+	}
+	// If it does, return the existing ID
+	if existingFilterID != "" {
+		return existingFilterID, err
+	}
+
+	// Otherwise insert the filter and return the new ID
+	err = s.insertFilterStmt.QueryRowContext(ctx, filterJSON, localpart).
+		Scan(&filterID)
+	return
+}

clientapi/auth/storage/accounts/membership_table.go

@@ -0,0 +1,132 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package accounts
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/lib/pq"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+)
+
+const membershipSchema = `
+-- Stores data about users memberships to rooms.
+CREATE TABLE IF NOT EXISTS account_memberships (
+    -- The Matrix user ID localpart for the member
+    localpart TEXT NOT NULL,
+    -- The room this user is a member of
+    room_id TEXT NOT NULL,
+    -- The ID of the join membership event
+    event_id TEXT NOT NULL,
+
+    -- A user can only be member of a room once
+    PRIMARY KEY (localpart, room_id)
+);
+
+-- Use index to process deletion by ID more efficiently
+CREATE UNIQUE INDEX IF NOT EXISTS account_membership_event_id ON account_memberships(event_id);
+`
+
+const insertMembershipSQL = `
+	INSERT INTO account_memberships(localpart, room_id, event_id) VALUES ($1, $2, $3)
+	ON CONFLICT (localpart, room_id) DO UPDATE SET event_id = EXCLUDED.event_id
+`
+
+const selectMembershipsByLocalpartSQL = "" +
+	"SELECT room_id, event_id FROM account_memberships WHERE localpart = $1"
+
+const selectMembershipInRoomByLocalpartSQL = "" +
+	"SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2"
+
+const deleteMembershipsByEventIDsSQL = "" +
+	"DELETE FROM account_memberships WHERE event_id = ANY($1)"
+
+type membershipStatements struct {
+	deleteMembershipsByEventIDsStmt       *sql.Stmt
+	insertMembershipStmt                  *sql.Stmt
+	selectMembershipInRoomByLocalpartStmt *sql.Stmt
+	selectMembershipsByLocalpartStmt      *sql.Stmt
+}
+
+func (s *membershipStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(membershipSchema)
+	if err != nil {
+		return
+	}
+	if s.deleteMembershipsByEventIDsStmt, err = db.Prepare(deleteMembershipsByEventIDsSQL); err != nil {
+		return
+	}
+	if s.insertMembershipStmt, err = db.Prepare(insertMembershipSQL); err != nil {
+		return
+	}
+	if s.selectMembershipInRoomByLocalpartStmt, err = db.Prepare(selectMembershipInRoomByLocalpartSQL); err != nil {
+		return
+	}
+	if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *membershipStatements) insertMembership(
+	ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string,
+) (err error) {
+	stmt := txn.Stmt(s.insertMembershipStmt)
+	_, err = stmt.ExecContext(ctx, localpart, roomID, eventID)
+	return
+}
+
+func (s *membershipStatements) deleteMembershipsByEventIDs(
+	ctx context.Context, txn *sql.Tx, eventIDs []string,
+) (err error) {
+	stmt := txn.Stmt(s.deleteMembershipsByEventIDsStmt)
+	_, err = stmt.ExecContext(ctx, pq.StringArray(eventIDs))
+	return
+}
+
+func (s *membershipStatements) selectMembershipInRoomByLocalpart(
+	ctx context.Context, localpart, roomID string,
+) (authtypes.Membership, error) {
+	membership := authtypes.Membership{Localpart: localpart, RoomID: roomID}
+	stmt := s.selectMembershipInRoomByLocalpartStmt
+	err := stmt.QueryRowContext(ctx, localpart, roomID).Scan(&membership.EventID)
+
+	return membership, err
+}
+
+func (s *membershipStatements) selectMembershipsByLocalpart(
+	ctx context.Context, localpart string,
+) (memberships []authtypes.Membership, err error) {
+	stmt := s.selectMembershipsByLocalpartStmt
+	rows, err := stmt.QueryContext(ctx, localpart)
+	if err != nil {
+		return
+	}
+
+	memberships = []authtypes.Membership{}
+
+	defer rows.Close() // nolint: errcheck
+	for rows.Next() {
+		var m authtypes.Membership
+		m.Localpart = localpart
+		if err := rows.Scan(&m.RoomID, &m.EventID); err != nil {
+			return nil, err
+		}
+		memberships = append(memberships, m)
+	}
+
+	return
+}

clientapi/auth/storage/accounts/profile_table.go

@@ -0,0 +1,107 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package accounts
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+)
+
+const profilesSchema = `
+-- Stores data about accounts profiles.
+CREATE TABLE IF NOT EXISTS account_profiles (
+    -- The Matrix user ID localpart for this account
+    localpart TEXT NOT NULL PRIMARY KEY,
+    -- The display name for this account
+    display_name TEXT,
+    -- The URL of the avatar for this account
+    avatar_url TEXT
+);
+`
+
+const insertProfileSQL = "" +
+	"INSERT INTO account_profiles(localpart, display_name, avatar_url) VALUES ($1, $2, $3)"
+
+const selectProfileByLocalpartSQL = "" +
+	"SELECT localpart, display_name, avatar_url FROM account_profiles WHERE localpart = $1"
+
+const setAvatarURLSQL = "" +
+	"UPDATE account_profiles SET avatar_url = $1 WHERE localpart = $2"
+
+const setDisplayNameSQL = "" +
+	"UPDATE account_profiles SET display_name = $1 WHERE localpart = $2"
+
+type profilesStatements struct {
+	insertProfileStmt            *sql.Stmt
+	selectProfileByLocalpartStmt *sql.Stmt
+	setAvatarURLStmt             *sql.Stmt
+	setDisplayNameStmt           *sql.Stmt
+}
+
+func (s *profilesStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(profilesSchema)
+	if err != nil {
+		return
+	}
+	if s.insertProfileStmt, err = db.Prepare(insertProfileSQL); err != nil {
+		return
+	}
+	if s.selectProfileByLocalpartStmt, err = db.Prepare(selectProfileByLocalpartSQL); err != nil {
+		return
+	}
+	if s.setAvatarURLStmt, err = db.Prepare(setAvatarURLSQL); err != nil {
+		return
+	}
+	if s.setDisplayNameStmt, err = db.Prepare(setDisplayNameSQL); err != nil {
+		return
+	}
+	return
+}
+
+func (s *profilesStatements) insertProfile(
+	ctx context.Context, localpart string,
+) (err error) {
+	_, err = s.insertProfileStmt.ExecContext(ctx, localpart, "", "")
+	return
+}
+
+func (s *profilesStatements) selectProfileByLocalpart(
+	ctx context.Context, localpart string,
+) (*authtypes.Profile, error) {
+	var profile authtypes.Profile
+	err := s.selectProfileByLocalpartStmt.QueryRowContext(ctx, localpart).Scan(
+		&profile.Localpart, &profile.DisplayName, &profile.AvatarURL,
+	)
+	if err != nil {
+		return nil, err
+	}
+	return &profile, nil
+}
+
+func (s *profilesStatements) setAvatarURL(
+	ctx context.Context, localpart string, avatarURL string,
+) (err error) {
+	_, err = s.setAvatarURLStmt.ExecContext(ctx, avatarURL, localpart)
+	return
+}
+
+func (s *profilesStatements) setDisplayName(
+	ctx context.Context, localpart string, displayName string,
+) (err error) {
+	_, err = s.setDisplayNameStmt.ExecContext(ctx, displayName, localpart)
+	return
+}

clientapi/auth/storage/accounts/storage.go

@@ -0,0 +1,380 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package accounts
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/gomatrixserverlib"
+	"golang.org/x/crypto/bcrypt"
+	// Import the postgres database driver.
+	_ "github.com/lib/pq"
+)
+
+// Database represents an account database
+type Database struct {
+	db *sql.DB
+	common.PartitionOffsetStatements
+	accounts     accountsStatements
+	profiles     profilesStatements
+	memberships  membershipStatements
+	accountDatas accountDataStatements
+	threepids    threepidStatements
+	filter       filterStatements
+	serverName   gomatrixserverlib.ServerName
+}
+
+// NewDatabase creates a new accounts and profiles database
+func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
+	var db *sql.DB
+	var err error
+	if db, err = sql.Open("postgres", dataSourceName); err != nil {
+		return nil, err
+	}
+	partitions := common.PartitionOffsetStatements{}
+	if err = partitions.Prepare(db, "account"); err != nil {
+		return nil, err
+	}
+	a := accountsStatements{}
+	if err = a.prepare(db, serverName); err != nil {
+		return nil, err
+	}
+	p := profilesStatements{}
+	if err = p.prepare(db); err != nil {
+		return nil, err
+	}
+	m := membershipStatements{}
+	if err = m.prepare(db); err != nil {
+		return nil, err
+	}
+	ac := accountDataStatements{}
+	if err = ac.prepare(db); err != nil {
+		return nil, err
+	}
+	t := threepidStatements{}
+	if err = t.prepare(db); err != nil {
+		return nil, err
+	}
+	f := filterStatements{}
+	if err = f.prepare(db); err != nil {
+		return nil, err
+	}
+	return &Database{db, partitions, a, p, m, ac, t, f, serverName}, nil
+}
+
+// GetAccountByPassword returns the account associated with the given localpart and password.
+// Returns sql.ErrNoRows if no account exists which matches the given localpart.
+func (d *Database) GetAccountByPassword(
+	ctx context.Context, localpart, plaintextPassword string,
+) (*authtypes.Account, error) {
+	hash, err := d.accounts.selectPasswordHash(ctx, localpart)
+	if err != nil {
+		return nil, err
+	}
+	if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil {
+		return nil, err
+	}
+	return d.accounts.selectAccountByLocalpart(ctx, localpart)
+}
+
+// GetProfileByLocalpart returns the profile associated with the given localpart.
+// Returns sql.ErrNoRows if no profile exists which matches the given localpart.
+func (d *Database) GetProfileByLocalpart(
+	ctx context.Context, localpart string,
+) (*authtypes.Profile, error) {
+	return d.profiles.selectProfileByLocalpart(ctx, localpart)
+}
+
+// SetAvatarURL updates the avatar URL of the profile associated with the given
+// localpart. Returns an error if something went wrong with the SQL query
+func (d *Database) SetAvatarURL(
+	ctx context.Context, localpart string, avatarURL string,
+) error {
+	return d.profiles.setAvatarURL(ctx, localpart, avatarURL)
+}
+
+// SetDisplayName updates the display name of the profile associated with the given
+// localpart. Returns an error if something went wrong with the SQL query
+func (d *Database) SetDisplayName(
+	ctx context.Context, localpart string, displayName string,
+) error {
+	return d.profiles.setDisplayName(ctx, localpart, displayName)
+}
+
+// CreateAccount makes a new account with the given login name and password, and creates an empty profile
+// for this account. If no password is supplied, the account will be a passwordless account. If the
+// account already exists, it will return nil, nil.
+func (d *Database) CreateAccount(
+	ctx context.Context, localpart, plaintextPassword, appserviceID string,
+) (*authtypes.Account, error) {
+	var err error
+
+	// Generate a password hash if this is not a password-less user
+	hash := ""
+	if plaintextPassword != "" {
+		hash, err = hashPassword(plaintextPassword)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if err := d.profiles.insertProfile(ctx, localpart); err != nil {
+		if common.IsUniqueConstraintViolationErr(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	return d.accounts.insertAccount(ctx, localpart, hash, appserviceID)
+}
+
+// SaveMembership saves the user matching a given localpart as a member of a given
+// room. It also stores the ID of the membership event.
+// If a membership already exists between the user and the room, or if the
+// insert fails, returns the SQL error
+func (d *Database) saveMembership(
+	ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string,
+) error {
+	return d.memberships.insertMembership(ctx, txn, localpart, roomID, eventID)
+}
+
+// removeMembershipsByEventIDs removes the memberships corresponding to the
+// `join` membership events IDs in the eventIDs slice.
+// If the removal fails, or if there is no membership to remove, returns an error
+func (d *Database) removeMembershipsByEventIDs(
+	ctx context.Context, txn *sql.Tx, eventIDs []string,
+) error {
+	return d.memberships.deleteMembershipsByEventIDs(ctx, txn, eventIDs)
+}
+
+// UpdateMemberships adds the "join" membership events included in a given state
+// events array, and removes those which ID is included in a given array of events
+// IDs. All of the process is run in a transaction, which commits only once/if every
+// insertion and deletion has been successfully processed.
+// Returns a SQL error if there was an issue with any part of the process
+func (d *Database) UpdateMemberships(
+	ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string,
+) error {
+	return common.WithTransaction(d.db, func(txn *sql.Tx) error {
+		if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil {
+			return err
+		}
+
+		for _, event := range eventsToAdd {
+			if err := d.newMembership(ctx, txn, event); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+// GetMembershipInRoomByLocalpart returns the membership for an user
+// matching the given localpart if he is a member of the room matching roomID,
+// if not sql.ErrNoRows is returned.
+// If there was an issue during the retrieval, returns the SQL error
+func (d *Database) GetMembershipInRoomByLocalpart(
+	ctx context.Context, localpart, roomID string,
+) (authtypes.Membership, error) {
+	return d.memberships.selectMembershipInRoomByLocalpart(ctx, localpart, roomID)
+}
+
+// GetMembershipsByLocalpart returns an array containing the memberships for all
+// the rooms a user matching a given localpart is a member of
+// If no membership match the given localpart, returns an empty array
+// If there was an issue during the retrieval, returns the SQL error
+func (d *Database) GetMembershipsByLocalpart(
+	ctx context.Context, localpart string,
+) (memberships []authtypes.Membership, err error) {
+	return d.memberships.selectMembershipsByLocalpart(ctx, localpart)
+}
+
+// newMembership saves a new membership in the database.
+// If the event isn't a valid m.room.member event with type `join`, does nothing.
+// If an error occurred, returns the SQL error
+func (d *Database) newMembership(
+	ctx context.Context, txn *sql.Tx, ev gomatrixserverlib.Event,
+) error {
+	if ev.Type() == "m.room.member" && ev.StateKey() != nil {
+		localpart, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey())
+		if err != nil {
+			return err
+		}
+
+		// We only want state events from local users
+		if string(serverName) != string(d.serverName) {
+			return nil
+		}
+
+		eventID := ev.EventID()
+		roomID := ev.RoomID()
+		membership, err := ev.Membership()
+		if err != nil {
+			return err
+		}
+
+		// Only "join" membership events can be considered as new memberships
+		if membership == "join" {
+			if err := d.saveMembership(ctx, txn, localpart, roomID, eventID); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// SaveAccountData saves new account data for a given user and a given room.
+// If the account data is not specific to a room, the room ID should be an empty string
+// If an account data already exists for a given set (user, room, data type), it will
+// update the corresponding row with the new content
+// Returns a SQL error if there was an issue with the insertion/update
+func (d *Database) SaveAccountData(
+	ctx context.Context, localpart, roomID, dataType, content string,
+) error {
+	return d.accountDatas.insertAccountData(ctx, localpart, roomID, dataType, content)
+}
+
+// GetAccountData returns account data related to a given localpart
+// If no account data could be found, returns an empty arrays
+// Returns an error if there was an issue with the retrieval
+func (d *Database) GetAccountData(ctx context.Context, localpart string) (
+	global []gomatrixserverlib.ClientEvent,
+	rooms map[string][]gomatrixserverlib.ClientEvent,
+	err error,
+) {
+	return d.accountDatas.selectAccountData(ctx, localpart)
+}
+
+// GetAccountDataByType returns account data matching a given
+// localpart, room ID and type.
+// If no account data could be found, returns an empty array
+// Returns an error if there was an issue with the retrieval
+func (d *Database) GetAccountDataByType(
+	ctx context.Context, localpart, roomID, dataType string,
+) (data []gomatrixserverlib.ClientEvent, err error) {
+	return d.accountDatas.selectAccountDataByType(
+		ctx, localpart, roomID, dataType,
+	)
+}
+
+// GetNewNumericLocalpart generates and returns a new unused numeric localpart
+func (d *Database) GetNewNumericLocalpart(
+	ctx context.Context,
+) (int64, error) {
+	return d.accounts.selectNewNumericLocalpart(ctx)
+}
+
+func hashPassword(plaintext string) (hash string, err error) {
+	hashBytes, err := bcrypt.GenerateFromPassword([]byte(plaintext), bcrypt.DefaultCost)
+	return string(hashBytes), err
+}
+
+// Err3PIDInUse is the error returned when trying to save an association involving
+// a third-party identifier which is already associated to a local user.
+var Err3PIDInUse = errors.New("This third-party identifier is already in use")
+
+// SaveThreePIDAssociation saves the association between a third party identifier
+// and a local Matrix user (identified by the user's ID's local part).
+// If the third-party identifier is already part of an association, returns Err3PIDInUse.
+// Returns an error if there was a problem talking to the database.
+func (d *Database) SaveThreePIDAssociation(
+	ctx context.Context, threepid, localpart, medium string,
+) (err error) {
+	return common.WithTransaction(d.db, func(txn *sql.Tx) error {
+		user, err := d.threepids.selectLocalpartForThreePID(
+			ctx, txn, threepid, medium,
+		)
+		if err != nil {
+			return err
+		}
+
+		if len(user) > 0 {
+			return Err3PIDInUse
+		}
+
+		return d.threepids.insertThreePID(ctx, txn, threepid, medium, localpart)
+	})
+}
+
+// RemoveThreePIDAssociation removes the association involving a given third-party
+// identifier.
+// If no association exists involving this third-party identifier, returns nothing.
+// If there was a problem talking to the database, returns an error.
+func (d *Database) RemoveThreePIDAssociation(
+	ctx context.Context, threepid string, medium string,
+) (err error) {
+	return d.threepids.deleteThreePID(ctx, threepid, medium)
+}
+
+// GetLocalpartForThreePID looks up the localpart associated with a given third-party
+// identifier.
+// If no association involves the given third-party idenfitier, returns an empty
+// string.
+// Returns an error if there was a problem talking to the database.
+func (d *Database) GetLocalpartForThreePID(
+	ctx context.Context, threepid string, medium string,
+) (localpart string, err error) {
+	return d.threepids.selectLocalpartForThreePID(ctx, nil, threepid, medium)
+}
+
+// GetThreePIDsForLocalpart looks up the third-party identifiers associated with
+// a given local user.
+// If no association is known for this user, returns an empty slice.
+// Returns an error if there was an issue talking to the database.
+func (d *Database) GetThreePIDsForLocalpart(
+	ctx context.Context, localpart string,
+) (threepids []authtypes.ThreePID, err error) {
+	return d.threepids.selectThreePIDsForLocalpart(ctx, localpart)
+}
+
+// GetFilter looks up the filter associated with a given local user and filter ID.
+// Returns a filter represented as a byte slice. Otherwise returns an error if
+// no such filter exists or if there was an error talking to the database.
+func (d *Database) GetFilter(
+	ctx context.Context, localpart string, filterID string,
+) ([]byte, error) {
+	return d.filter.selectFilter(ctx, localpart, filterID)
+}
+
+// PutFilter puts the passed filter into the database.
+// Returns the filterID as a string. Otherwise returns an error if something
+// goes wrong.
+func (d *Database) PutFilter(
+	ctx context.Context, localpart string, filter []byte,
+) (string, error) {
+	return d.filter.insertFilter(ctx, filter, localpart)
+}
+
+// CheckAccountAvailability checks if the username/localpart is already present
+// in the database.
+// If the DB returns sql.ErrNoRows the Localpart isn't taken.
+func (d *Database) CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) {
+	_, err := d.accounts.selectAccountByLocalpart(ctx, localpart)
+	if err == sql.ErrNoRows {
+		return true, nil
+	}
+	return false, err
+}
+
+// GetAccountByLocalpart returns the account associated with the given localpart.
+// This function assumes the request is authenticated or the account data is used only internally.
+// Returns sql.ErrNoRows if no account exists which matches the given localpart.
+func (d *Database) GetAccountByLocalpart(ctx context.Context, localpart string,
+) (*authtypes.Account, error) {
+	return d.accounts.selectAccountByLocalpart(ctx, localpart)
+}

clientapi/auth/storage/accounts/threepid_table.go

@@ -0,0 +1,129 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package accounts
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/common"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+)
+
+const threepidSchema = `
+-- Stores data about third party identifiers
+CREATE TABLE IF NOT EXISTS account_threepid (
+	-- The third party identifier
+	threepid TEXT NOT NULL,
+	-- The 3PID medium
+	medium TEXT NOT NULL DEFAULT 'email',
+	-- The localpart of the Matrix user ID associated to this 3PID
+	localpart TEXT NOT NULL,
+
+	PRIMARY KEY(threepid, medium)
+);
+
+CREATE INDEX IF NOT EXISTS account_threepid_localpart ON account_threepid(localpart);
+`
+
+const selectLocalpartForThreePIDSQL = "" +
+	"SELECT localpart FROM account_threepid WHERE threepid = $1 AND medium = $2"
+
+const selectThreePIDsForLocalpartSQL = "" +
+	"SELECT threepid, medium FROM account_threepid WHERE localpart = $1"
+
+const insertThreePIDSQL = "" +
+	"INSERT INTO account_threepid (threepid, medium, localpart) VALUES ($1, $2, $3)"
+
+const deleteThreePIDSQL = "" +
+	"DELETE FROM account_threepid WHERE threepid = $1 AND medium = $2"
+
+type threepidStatements struct {
+	selectLocalpartForThreePIDStmt  *sql.Stmt
+	selectThreePIDsForLocalpartStmt *sql.Stmt
+	insertThreePIDStmt              *sql.Stmt
+	deleteThreePIDStmt              *sql.Stmt
+}
+
+func (s *threepidStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(threepidSchema)
+	if err != nil {
+		return
+	}
+	if s.selectLocalpartForThreePIDStmt, err = db.Prepare(selectLocalpartForThreePIDSQL); err != nil {
+		return
+	}
+	if s.selectThreePIDsForLocalpartStmt, err = db.Prepare(selectThreePIDsForLocalpartSQL); err != nil {
+		return
+	}
+	if s.insertThreePIDStmt, err = db.Prepare(insertThreePIDSQL); err != nil {
+		return
+	}
+	if s.deleteThreePIDStmt, err = db.Prepare(deleteThreePIDSQL); err != nil {
+		return
+	}
+
+	return
+}
+
+func (s *threepidStatements) selectLocalpartForThreePID(
+	ctx context.Context, txn *sql.Tx, threepid string, medium string,
+) (localpart string, err error) {
+	stmt := common.TxStmt(txn, s.selectLocalpartForThreePIDStmt)
+	err = stmt.QueryRowContext(ctx, threepid, medium).Scan(&localpart)
+	if err == sql.ErrNoRows {
+		return "", nil
+	}
+	return
+}
+
+func (s *threepidStatements) selectThreePIDsForLocalpart(
+	ctx context.Context, localpart string,
+) (threepids []authtypes.ThreePID, err error) {
+	rows, err := s.selectThreePIDsForLocalpartStmt.QueryContext(ctx, localpart)
+	if err != nil {
+		return
+	}
+
+	threepids = []authtypes.ThreePID{}
+	for rows.Next() {
+		var threepid string
+		var medium string
+		if err = rows.Scan(&threepid, &medium); err != nil {
+			return
+		}
+		threepids = append(threepids, authtypes.ThreePID{
+			Address: threepid,
+			Medium:  medium,
+		})
+	}
+
+	return
+}
+
+func (s *threepidStatements) insertThreePID(
+	ctx context.Context, txn *sql.Tx, threepid, medium, localpart string,
+) (err error) {
+	stmt := common.TxStmt(txn, s.insertThreePIDStmt)
+	_, err = stmt.ExecContext(ctx, threepid, medium, localpart)
+	return
+}
+
+func (s *threepidStatements) deleteThreePID(
+	ctx context.Context, threepid string, medium string) (err error) {
+	_, err = s.deleteThreePIDStmt.ExecContext(ctx, threepid, medium)
+	return
+}

clientapi/auth/storage/devices/devices_table.go

@@ -0,0 +1,208 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devices
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/matrix-org/dendrite/common"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+const devicesSchema = `
+-- Stores data about devices.
+CREATE TABLE IF NOT EXISTS device_devices (
+    -- The access token granted to this device. This has to be the primary key
+    -- so we can distinguish which device is making a given request.
+    access_token TEXT NOT NULL PRIMARY KEY,
+    -- The device identifier. This only needs to uniquely identify a device for a given user, not globally.
+    -- access_tokens will be clobbered based on the device ID for a user.
+    device_id TEXT NOT NULL,
+    -- The Matrix user ID localpart for this device. This is preferable to storing the full user_id
+    -- as it is smaller, makes it clearer that we only manage devices for our own users, and may make
+    -- migration to different domain names easier.
+    localpart TEXT NOT NULL,
+    -- When this devices was first recognised on the network, as a unix timestamp (ms resolution).
+    created_ts BIGINT NOT NULL,
+    -- The display name, human friendlier than device_id and updatable
+    display_name TEXT
+    -- TODO: device keys, device display names, last used ts and IP address?, token restrictions (if 3rd-party OAuth app)
+);
+
+-- Device IDs must be unique for a given user.
+CREATE UNIQUE INDEX IF NOT EXISTS device_localpart_id_idx ON device_devices(localpart, device_id);
+`
+
+const insertDeviceSQL = "" +
+	"INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5)"
+
+const selectDeviceByTokenSQL = "" +
+	"SELECT device_id, localpart FROM device_devices WHERE access_token = $1"
+
+const selectDeviceByIDSQL = "" +
+	"SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2"
+
+const selectDevicesByLocalpartSQL = "" +
+	"SELECT device_id, display_name FROM device_devices WHERE localpart = $1"
+
+const updateDeviceNameSQL = "" +
+	"UPDATE device_devices SET display_name = $1 WHERE localpart = $2 AND device_id = $3"
+
+const deleteDeviceSQL = "" +
+	"DELETE FROM device_devices WHERE device_id = $1 AND localpart = $2"
+
+const deleteDevicesByLocalpartSQL = "" +
+	"DELETE FROM device_devices WHERE localpart = $1"
+
+type devicesStatements struct {
+	insertDeviceStmt             *sql.Stmt
+	selectDeviceByTokenStmt      *sql.Stmt
+	selectDeviceByIDStmt         *sql.Stmt
+	selectDevicesByLocalpartStmt *sql.Stmt
+	updateDeviceNameStmt         *sql.Stmt
+	deleteDeviceStmt             *sql.Stmt
+	deleteDevicesByLocalpartStmt *sql.Stmt
+	serverName                   gomatrixserverlib.ServerName
+}
+
+func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) {
+	_, err = db.Exec(devicesSchema)
+	if err != nil {
+		return
+	}
+	if s.insertDeviceStmt, err = db.Prepare(insertDeviceSQL); err != nil {
+		return
+	}
+	if s.selectDeviceByTokenStmt, err = db.Prepare(selectDeviceByTokenSQL); err != nil {
+		return
+	}
+	if s.selectDeviceByIDStmt, err = db.Prepare(selectDeviceByIDSQL); err != nil {
+		return
+	}
+	if s.selectDevicesByLocalpartStmt, err = db.Prepare(selectDevicesByLocalpartSQL); err != nil {
+		return
+	}
+	if s.updateDeviceNameStmt, err = db.Prepare(updateDeviceNameSQL); err != nil {
+		return
+	}
+	if s.deleteDeviceStmt, err = db.Prepare(deleteDeviceSQL); err != nil {
+		return
+	}
+	if s.deleteDevicesByLocalpartStmt, err = db.Prepare(deleteDevicesByLocalpartSQL); err != nil {
+		return
+	}
+	s.serverName = server
+	return
+}
+
+// insertDevice creates a new device. Returns an error if any device with the same access token already exists.
+// Returns an error if the user already has a device with the given device ID.
+// Returns the device on success.
+func (s *devicesStatements) insertDevice(
+	ctx context.Context, txn *sql.Tx, id, localpart, accessToken string,
+	displayName *string,
+) (*authtypes.Device, error) {
+	createdTimeMS := time.Now().UnixNano() / 1000000
+	stmt := common.TxStmt(txn, s.insertDeviceStmt)
+	if _, err := stmt.ExecContext(ctx, id, localpart, accessToken, createdTimeMS, displayName); err != nil {
+		return nil, err
+	}
+	return &authtypes.Device{
+		ID:          id,
+		UserID:      userutil.MakeUserID(localpart, s.serverName),
+		AccessToken: accessToken,
+	}, nil
+}
+
+func (s *devicesStatements) deleteDevice(
+	ctx context.Context, txn *sql.Tx, id, localpart string,
+) error {
+	stmt := common.TxStmt(txn, s.deleteDeviceStmt)
+	_, err := stmt.ExecContext(ctx, id, localpart)
+	return err
+}
+
+func (s *devicesStatements) deleteDevicesByLocalpart(
+	ctx context.Context, txn *sql.Tx, localpart string,
+) error {
+	stmt := common.TxStmt(txn, s.deleteDevicesByLocalpartStmt)
+	_, err := stmt.ExecContext(ctx, localpart)
+	return err
+}
+
+func (s *devicesStatements) updateDeviceName(
+	ctx context.Context, txn *sql.Tx, localpart, deviceID string, displayName *string,
+) error {
+	stmt := common.TxStmt(txn, s.updateDeviceNameStmt)
+	_, err := stmt.ExecContext(ctx, displayName, localpart, deviceID)
+	return err
+}
+
+func (s *devicesStatements) selectDeviceByToken(
+	ctx context.Context, accessToken string,
+) (*authtypes.Device, error) {
+	var dev authtypes.Device
+	var localpart string
+	stmt := s.selectDeviceByTokenStmt
+	err := stmt.QueryRowContext(ctx, accessToken).Scan(&dev.ID, &localpart)
+	if err == nil {
+		dev.UserID = userutil.MakeUserID(localpart, s.serverName)
+		dev.AccessToken = accessToken
+	}
+	return &dev, err
+}
+
+func (s *devicesStatements) selectDeviceByID(
+	ctx context.Context, localpart, deviceID string,
+) (*authtypes.Device, error) {
+	var dev authtypes.Device
+	var created sql.NullInt64
+	stmt := s.selectDeviceByIDStmt
+	err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&created)
+	if err == nil {
+		dev.ID = deviceID
+		dev.UserID = userutil.MakeUserID(localpart, s.serverName)
+	}
+	return &dev, err
+}
+
+func (s *devicesStatements) selectDevicesByLocalpart(
+	ctx context.Context, localpart string,
+) ([]authtypes.Device, error) {
+	devices := []authtypes.Device{}
+
+	rows, err := s.selectDevicesByLocalpartStmt.QueryContext(ctx, localpart)
+
+	if err != nil {
+		return devices, err
+	}
+
+	for rows.Next() {
+		var dev authtypes.Device
+		err = rows.Scan(&dev.ID)
+		if err != nil {
+			return devices, err
+		}
+		dev.UserID = userutil.MakeUserID(localpart, s.serverName)
+		devices = append(devices, dev)
+	}
+
+	return devices, nil
+}

clientapi/auth/storage/devices/storage.go

@@ -0,0 +1,167 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devices
+
+import (
+	"context"
+	"crypto/rand"
+	"database/sql"
+	"encoding/base64"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// The length of generated device IDs
+var deviceIDByteLength = 6
+
+// Database represents a device database.
+type Database struct {
+	db      *sql.DB
+	devices devicesStatements
+}
+
+// NewDatabase creates a new device database
+func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
+	var db *sql.DB
+	var err error
+	if db, err = sql.Open("postgres", dataSourceName); err != nil {
+		return nil, err
+	}
+	d := devicesStatements{}
+	if err = d.prepare(db, serverName); err != nil {
+		return nil, err
+	}
+	return &Database{db, d}, nil
+}
+
+// GetDeviceByAccessToken returns the device matching the given access token.
+// Returns sql.ErrNoRows if no matching device was found.
+func (d *Database) GetDeviceByAccessToken(
+	ctx context.Context, token string,
+) (*authtypes.Device, error) {
+	return d.devices.selectDeviceByToken(ctx, token)
+}
+
+// GetDeviceByID returns the device matching the given ID.
+// Returns sql.ErrNoRows if no matching device was found.
+func (d *Database) GetDeviceByID(
+	ctx context.Context, localpart, deviceID string,
+) (*authtypes.Device, error) {
+	return d.devices.selectDeviceByID(ctx, localpart, deviceID)
+}
+
+// GetDevicesByLocalpart returns the devices matching the given localpart.
+func (d *Database) GetDevicesByLocalpart(
+	ctx context.Context, localpart string,
+) ([]authtypes.Device, error) {
+	return d.devices.selectDevicesByLocalpart(ctx, localpart)
+}
+
+// CreateDevice makes a new device associated with the given user ID localpart.
+// If there is already a device with the same device ID for this user, that access token will be revoked
+// and replaced with the given accessToken. If the given accessToken is already in use for another device,
+// an error will be returned.
+// If no device ID is given one is generated.
+// Returns the device on success.
+func (d *Database) CreateDevice(
+	ctx context.Context, localpart string, deviceID *string, accessToken string,
+	displayName *string,
+) (dev *authtypes.Device, returnErr error) {
+	if deviceID != nil {
+		returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error {
+			var err error
+			// Revoke existing token for this device
+			if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil {
+				return err
+			}
+
+			dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName)
+			return err
+		})
+	} else {
+		// We generate device IDs in a loop in case its already taken.
+		// We cap this at going round 5 times to ensure we don't spin forever
+		var newDeviceID string
+		for i := 1; i <= 5; i++ {
+			newDeviceID, returnErr = generateDeviceID()
+			if returnErr != nil {
+				return
+			}
+
+			returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error {
+				var err error
+				dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName)
+				return err
+			})
+			if returnErr == nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// generateDeviceID creates a new device id. Returns an error if failed to generate
+// random bytes.
+func generateDeviceID() (string, error) {
+	b := make([]byte, deviceIDByteLength)
+	_, err := rand.Read(b)
+	if err != nil {
+		return "", err
+	}
+	// url-safe no padding
+	return base64.RawURLEncoding.EncodeToString(b), nil
+}
+
+// UpdateDevice updates the given device with the display name.
+// Returns SQL error if there are problems and nil on success.
+func (d *Database) UpdateDevice(
+	ctx context.Context, localpart, deviceID string, displayName *string,
+) error {
+	return common.WithTransaction(d.db, func(txn *sql.Tx) error {
+		return d.devices.updateDeviceName(ctx, txn, localpart, deviceID, displayName)
+	})
+}
+
+// RemoveDevice revokes a device by deleting the entry in the database
+// matching with the given device ID and user ID localpart.
+// If the device doesn't exist, it will not return an error
+// If something went wrong during the deletion, it will return the SQL error.
+func (d *Database) RemoveDevice(
+	ctx context.Context, deviceID, localpart string,
+) error {
+	return common.WithTransaction(d.db, func(txn *sql.Tx) error {
+		if err := d.devices.deleteDevice(ctx, txn, deviceID, localpart); err != sql.ErrNoRows {
+			return err
+		}
+		return nil
+	})
+}
+
+// RemoveAllDevices revokes devices by deleting the entry in the
+// database matching the given user ID localpart.
+// If something went wrong during the deletion, it will return the SQL error.
+func (d *Database) RemoveAllDevices(
+	ctx context.Context, localpart string,
+) error {
+	return common.WithTransaction(d.db, func(txn *sql.Tx) error {
+		if err := d.devices.deleteDevicesByLocalpart(ctx, txn, localpart); err != sql.ErrNoRows {
+			return err
+		}
+		return nil
+	})
+}

clientapi/clientapi.go

@@ -0,0 +1,72 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientapi
+
+import (
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/clientapi/consumers"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/clientapi/routing"
+	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/common/transactions"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	typingServerAPI "github.com/matrix-org/dendrite/typingserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+)
+
+// SetupClientAPIComponent sets up and registers HTTP handlers for the ClientAPI
+// component.
+func SetupClientAPIComponent(
+	base *basecomponent.BaseDendrite,
+	deviceDB *devices.Database,
+	accountsDB *accounts.Database,
+	federation *gomatrixserverlib.FederationClient,
+	keyRing *gomatrixserverlib.KeyRing,
+	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	inputAPI roomserverAPI.RoomserverInputAPI,
+	queryAPI roomserverAPI.RoomserverQueryAPI,
+	typingInputAPI typingServerAPI.TypingServerInputAPI,
+	asAPI appserviceAPI.AppServiceQueryAPI,
+	transactionsCache *transactions.Cache,
+) {
+	roomserverProducer := producers.NewRoomserverProducer(inputAPI)
+	typingProducer := producers.NewTypingServerProducer(typingInputAPI)
+
+	userUpdateProducer := &producers.UserUpdateProducer{
+		Producer: base.KafkaProducer,
+		Topic:    string(base.Cfg.Kafka.Topics.UserUpdates),
+	}
+
+	syncProducer := &producers.SyncAPIProducer{
+		Producer: base.KafkaProducer,
+		Topic:    string(base.Cfg.Kafka.Topics.OutputClientData),
+	}
+
+	consumer := consumers.NewOutputRoomEventConsumer(
+		base.Cfg, base.KafkaConsumer, accountsDB, queryAPI,
+	)
+	if err := consumer.Start(); err != nil {
+		logrus.WithError(err).Panicf("failed to start room server consumer")
+	}
+
+	routing.Setup(
+		base.APIMux, *base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI,
+		accountsDB, deviceDB, federation, *keyRing, userUpdateProducer,
+		syncProducer, typingProducer, transactionsCache,
+	)
+}

clientapi/consumers/roomserver.go

@@ -0,0 +1,144 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consumers
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	log "github.com/sirupsen/logrus"
+	sarama "gopkg.in/Shopify/sarama.v1"
+)
+
+// OutputRoomEventConsumer consumes events that originated in the room server.
+type OutputRoomEventConsumer struct {
+	roomServerConsumer *common.ContinualConsumer
+	db                 *accounts.Database
+	query              api.RoomserverQueryAPI
+	serverName         string
+}
+
+// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
+func NewOutputRoomEventConsumer(
+	cfg *config.Dendrite,
+	kafkaConsumer sarama.Consumer,
+	store *accounts.Database,
+	queryAPI api.RoomserverQueryAPI,
+) *OutputRoomEventConsumer {
+
+	consumer := common.ContinualConsumer{
+		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
+		Consumer:       kafkaConsumer,
+		PartitionStore: store,
+	}
+	s := &OutputRoomEventConsumer{
+		roomServerConsumer: &consumer,
+		db:                 store,
+		query:              queryAPI,
+		serverName:         string(cfg.Matrix.ServerName),
+	}
+	consumer.ProcessMessage = s.onMessage
+
+	return s
+}
+
+// Start consuming from room servers
+func (s *OutputRoomEventConsumer) Start() error {
+	return s.roomServerConsumer.Start()
+}
+
+// onMessage is called when the sync server receives a new event from the room server output log.
+// It is not safe for this function to be called from multiple goroutines, or else the
+// sync stream position may race and be incorrectly calculated.
+func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
+	// Parse out the event JSON
+	var output api.OutputEvent
+	if err := json.Unmarshal(msg.Value, &output); err != nil {
+		// If the message was invalid, log it and move on to the next message in the stream
+		log.WithError(err).Errorf("roomserver output log: message parse failure")
+		return nil
+	}
+
+	if output.Type != api.OutputTypeNewRoomEvent {
+		log.WithField("type", output.Type).Debug(
+			"roomserver output log: ignoring unknown output type",
+		)
+		return nil
+	}
+
+	ev := output.NewRoomEvent.Event
+	log.WithFields(log.Fields{
+		"event_id": ev.EventID(),
+		"room_id":  ev.RoomID(),
+		"type":     ev.Type(),
+	}).Info("received event from roomserver")
+
+	events, err := s.lookupStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev)
+	if err != nil {
+		return err
+	}
+
+	return s.db.UpdateMemberships(context.TODO(), events, output.NewRoomEvent.RemovesStateEventIDs)
+}
+
+// lookupStateEvents looks up the state events that are added by a new event.
+func (s *OutputRoomEventConsumer) lookupStateEvents(
+	addsStateEventIDs []string, event gomatrixserverlib.Event,
+) ([]gomatrixserverlib.Event, error) {
+	// Fast path if there aren't any new state events.
+	if len(addsStateEventIDs) == 0 {
+		// If the event is a membership update (e.g. for a profile update), it won't
+		// show up in AddsStateEventIDs, so we need to add it manually
+		if event.Type() == "m.room.member" {
+			return []gomatrixserverlib.Event{event}, nil
+		}
+		return nil, nil
+	}
+
+	// Fast path if the only state event added is the event itself.
+	if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() {
+		return []gomatrixserverlib.Event{event}, nil
+	}
+
+	result := []gomatrixserverlib.Event{}
+	missing := []string{}
+	for _, id := range addsStateEventIDs {
+		// Append the current event in the results if its ID is in the events list
+		if id == event.EventID() {
+			result = append(result, event)
+		} else {
+			// If the event isn't the current one, add it to the list of events
+			// to retrieve from the roomserver
+			missing = append(missing, id)
+		}
+	}
+
+	// Request the missing events from the roomserver
+	eventReq := api.QueryEventsByIDRequest{EventIDs: missing}
+	var eventResp api.QueryEventsByIDResponse
+	if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil {
+		return nil, err
+	}
+
+	result = append(result, eventResp.Events...)
+
+	return result, nil
+}

clientapi/httputil/httputil.go

@@ -0,0 +1,46 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httputil
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/util"
+)
+
+// UnmarshalJSONRequest into the given interface pointer. Returns an error JSON response if
+// there was a problem unmarshalling. Calling this function consumes the request body.
+func UnmarshalJSONRequest(req *http.Request, iface interface{}) *util.JSONResponse {
+	if err := json.NewDecoder(req.Body).Decode(iface); err != nil {
+		// TODO: We may want to suppress the Error() return in production? It's useful when
+		// debugging because an error will be produced for both invalid/malformed JSON AND
+		// valid JSON with incorrect types for values.
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()),
+		}
+	}
+	return nil
+}
+
+// LogThenError logs the given error then returns a matrix-compliant 500 internal server error response.
+// This should be used to log fatal errors which require investigation. It should not be used
+// to log client validation errors, etc.
+func LogThenError(req *http.Request, err error) util.JSONResponse {
+	util.GetLogger(req.Context()).WithError(err).Error("request failed")
+	return jsonerror.InternalServerError()
+}

clientapi/httputil/parse.go

@@ -0,0 +1,39 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httputil
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+// ParseTSParam takes a req (typically from an application service) and parses a Time object
+// from the req if it exists in the query parameters. If it doesn't exist, the
+// current time is returned.
+func ParseTSParam(req *http.Request) (time.Time, error) {
+	// Use the ts parameter's value for event time if present
+	tsStr := req.URL.Query().Get("ts")
+	if tsStr == "" {
+		return time.Now(), nil
+	}
+
+	// The parameter exists, parse into a Time object
+	ts, err := strconv.ParseInt(tsStr, 10, 64)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("Param 'ts' is no valid int (%s)", err.Error())
+	}
+
+	return time.Unix(ts/1000, 0), nil
+}

clientapi/jsonerror/jsonerror.go

@@ -0,0 +1,148 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jsonerror
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/matrix-org/util"
+)
+
+// MatrixError represents the "standard error response" in Matrix.
+// http://matrix.org/docs/spec/client_server/r0.2.0.html#api-standards
+type MatrixError struct {
+	ErrCode string `json:"errcode"`
+	Err     string `json:"error"`
+}
+
+func (e MatrixError) Error() string {
+	return fmt.Sprintf("%s: %s", e.ErrCode, e.Err)
+}
+
+// InternalServerError returns a 500 Internal Server Error in a matrix-compliant
+// format.
+func InternalServerError() util.JSONResponse {
+	return util.JSONResponse{
+		Code: http.StatusInternalServerError,
+		JSON: Unknown("Internal Server Error"),
+	}
+}
+
+// Unknown is an unexpected error
+func Unknown(msg string) *MatrixError {
+	return &MatrixError{"M_UNKNOWN", msg}
+}
+
+// Forbidden is an error when the client tries to access a resource
+// they are not allowed to access.
+func Forbidden(msg string) *MatrixError {
+	return &MatrixError{"M_FORBIDDEN", msg}
+}
+
+// BadJSON is an error when the client supplies malformed JSON.
+func BadJSON(msg string) *MatrixError {
+	return &MatrixError{"M_BAD_JSON", msg}
+}
+
+// NotJSON is an error when the client supplies something that is not JSON
+// to a JSON endpoint.
+func NotJSON(msg string) *MatrixError {
+	return &MatrixError{"M_NOT_JSON", msg}
+}
+
+// NotFound is an error when the client tries to access an unknown resource.
+func NotFound(msg string) *MatrixError {
+	return &MatrixError{"M_NOT_FOUND", msg}
+}
+
+// MissingArgument is an error when the client tries to access a resource
+// without providing an argument that is required.
+func MissingArgument(msg string) *MatrixError {
+	return &MatrixError{"M_MISSING_ARGUMENT", msg}
+}
+
+// InvalidArgumentValue is an error when the client tries to provide an
+// invalid value for a valid argument
+func InvalidArgumentValue(msg string) *MatrixError {
+	return &MatrixError{"M_INVALID_ARGUMENT_VALUE", msg}
+}
+
+// MissingToken is an error when the client tries to access a resource which
+// requires authentication without supplying credentials.
+func MissingToken(msg string) *MatrixError {
+	return &MatrixError{"M_MISSING_TOKEN", msg}
+}
+
+// UnknownToken is an error when the client tries to access a resource which
+// requires authentication and supplies an unrecognized token
+func UnknownToken(msg string) *MatrixError {
+	return &MatrixError{"M_UNKNOWN_TOKEN", msg}
+}
+
+// WeakPassword is an error which is returned when the client tries to register
+// using a weak password. http://matrix.org/docs/spec/client_server/r0.2.0.html#password-based
+func WeakPassword(msg string) *MatrixError {
+	return &MatrixError{"M_WEAK_PASSWORD", msg}
+}
+
+// InvalidUsername is an error returned when the client tries to register an
+// invalid username
+func InvalidUsername(msg string) *MatrixError {
+	return &MatrixError{"M_INVALID_USERNAME", msg}
+}
+
+// UserInUse is an error returned when the client tries to register an
+// username that already exists
+func UserInUse(msg string) *MatrixError {
+	return &MatrixError{"M_USER_IN_USE", msg}
+}
+
+// ASExclusive is an error returned when an application service tries to
+// register an username that is outside of its registered namespace, or if a
+// user attempts to register a username or room alias within an exclusive
+// namespace.
+func ASExclusive(msg string) *MatrixError {
+	return &MatrixError{"M_EXCLUSIVE", msg}
+}
+
+// GuestAccessForbidden is an error which is returned when the client is
+// forbidden from accessing a resource as a guest.
+func GuestAccessForbidden(msg string) *MatrixError {
+	return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg}
+}
+
+// LimitExceededError is a rate-limiting error.
+type LimitExceededError struct {
+	MatrixError
+	RetryAfterMS int64 `json:"retry_after_ms,omitempty"`
+}
+
+// LimitExceeded is an error when the client tries to send events too quickly.
+func LimitExceeded(msg string, retryAfterMS int64) *LimitExceededError {
+	return &LimitExceededError{
+		MatrixError:  MatrixError{"M_LIMIT_EXCEEDED", msg},
+		RetryAfterMS: retryAfterMS,
+	}
+}
+
+// NotTrusted is an error which is returned when the client asks the server to
+// proxy a request (e.g. 3PID association) to a server that isn't trusted
+func NotTrusted(serverName string) *MatrixError {
+	return &MatrixError{
+		ErrCode: "M_SERVER_NOT_TRUSTED",
+		Err:     fmt.Sprintf("Untrusted server '%s'", serverName),
+	}
+}

clientapi/jsonerror/jsonerror_test.go

@@ -0,0 +1,44 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jsonerror
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestLimitExceeded(t *testing.T) {
+	e := LimitExceeded("too fast", 5000)
+	jsonBytes, err := json.Marshal(&e)
+	if err != nil {
+		t.Fatalf("TestLimitExceeded: Failed to marshal LimitExceeded error. %s", err.Error())
+	}
+	want := `{"errcode":"M_LIMIT_EXCEEDED","error":"too fast","retry_after_ms":5000}`
+	if string(jsonBytes) != want {
+		t.Errorf("TestLimitExceeded: want %s, got %s", want, string(jsonBytes))
+	}
+}
+
+func TestForbidden(t *testing.T) {
+	e := Forbidden("you shall not pass")
+	jsonBytes, err := json.Marshal(&e)
+	if err != nil {
+		t.Fatalf("TestForbidden: Failed to marshal Forbidden error. %s", err.Error())
+	}
+	want := `{"errcode":"M_FORBIDDEN","error":"you shall not pass"}`
+	if string(jsonBytes) != want {
+		t.Errorf("TestForbidden: want %s, got %s", want, string(jsonBytes))
+	}
+}

clientapi/producers/roomserver.go

@@ -0,0 +1,112 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package producers
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// RoomserverProducer produces events for the roomserver to consume.
+type RoomserverProducer struct {
+	InputAPI api.RoomserverInputAPI
+}
+
+// NewRoomserverProducer creates a new RoomserverProducer
+func NewRoomserverProducer(inputAPI api.RoomserverInputAPI) *RoomserverProducer {
+	return &RoomserverProducer{
+		InputAPI: inputAPI,
+	}
+}
+
+// SendEvents writes the given events to the roomserver input log. The events are written with KindNew.
+func (c *RoomserverProducer) SendEvents(
+	ctx context.Context, events []gomatrixserverlib.Event, sendAsServer gomatrixserverlib.ServerName,
+	txnID *api.TransactionID,
+) (string, error) {
+	ires := make([]api.InputRoomEvent, len(events))
+	for i, event := range events {
+		ires[i] = api.InputRoomEvent{
+			Kind:          api.KindNew,
+			Event:         event,
+			AuthEventIDs:  event.AuthEventIDs(),
+			SendAsServer:  string(sendAsServer),
+			TransactionID: txnID,
+		}
+	}
+	return c.SendInputRoomEvents(ctx, ires)
+}
+
+// SendEventWithState writes an event with KindNew to the roomserver input log
+// with the state at the event as KindOutlier before it.
+func (c *RoomserverProducer) SendEventWithState(
+	ctx context.Context, state gomatrixserverlib.RespState, event gomatrixserverlib.Event,
+) error {
+	outliers, err := state.Events()
+	if err != nil {
+		return err
+	}
+
+	ires := make([]api.InputRoomEvent, len(outliers)+1)
+	for i, outlier := range outliers {
+		ires[i] = api.InputRoomEvent{
+			Kind:         api.KindOutlier,
+			Event:        outlier,
+			AuthEventIDs: outlier.AuthEventIDs(),
+		}
+	}
+
+	stateEventIDs := make([]string, len(state.StateEvents))
+	for i := range state.StateEvents {
+		stateEventIDs[i] = state.StateEvents[i].EventID()
+	}
+
+	ires[len(outliers)] = api.InputRoomEvent{
+		Kind:          api.KindNew,
+		Event:         event,
+		AuthEventIDs:  event.AuthEventIDs(),
+		HasState:      true,
+		StateEventIDs: stateEventIDs,
+	}
+
+	_, err = c.SendInputRoomEvents(ctx, ires)
+	return err
+}
+
+// SendInputRoomEvents writes the given input room events to the roomserver input API.
+func (c *RoomserverProducer) SendInputRoomEvents(
+	ctx context.Context, ires []api.InputRoomEvent,
+) (eventID string, err error) {
+	request := api.InputRoomEventsRequest{InputRoomEvents: ires}
+	var response api.InputRoomEventsResponse
+	err = c.InputAPI.InputRoomEvents(ctx, &request, &response)
+	eventID = response.EventID
+	return
+}
+
+// SendInvite writes the invite event to the roomserver input API.
+// This should only be needed for invite events that occur outside of a known room.
+// If we are in the room then the event should be sent using the SendEvents method.
+func (c *RoomserverProducer) SendInvite(
+	ctx context.Context, inviteEvent gomatrixserverlib.Event,
+) error {
+	request := api.InputRoomEventsRequest{
+		InputInviteEvents: []api.InputInviteEvent{{Event: inviteEvent}},
+	}
+	var response api.InputRoomEventsResponse
+	return c.InputAPI.InputRoomEvents(ctx, &request, &response)
+}

clientapi/producers/syncapi.go

@@ -0,0 +1,50 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package producers
+
+import (
+	"encoding/json"
+
+	"github.com/matrix-org/dendrite/common"
+
+	sarama "gopkg.in/Shopify/sarama.v1"
+)
+
+// SyncAPIProducer produces events for the sync API server to consume
+type SyncAPIProducer struct {
+	Topic    string
+	Producer sarama.SyncProducer
+}
+
+// SendData sends account data to the sync API server
+func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string) error {
+	var m sarama.ProducerMessage
+
+	data := common.AccountData{
+		RoomID: roomID,
+		Type:   dataType,
+	}
+	value, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+
+	m.Topic = string(p.Topic)
+	m.Key = sarama.StringEncoder(userID)
+	m.Value = sarama.ByteEncoder(value)
+
+	_, _, err = p.Producer.SendMessage(&m)
+	return err
+}

clientapi/producers/typingserver.go

@@ -0,0 +1,54 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package producers
+
+import (
+	"context"
+	"time"
+
+	"github.com/matrix-org/dendrite/typingserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// TypingServerProducer produces events for the typing server to consume
+type TypingServerProducer struct {
+	InputAPI api.TypingServerInputAPI
+}
+
+// NewTypingServerProducer creates a new TypingServerProducer
+func NewTypingServerProducer(inputAPI api.TypingServerInputAPI) *TypingServerProducer {
+	return &TypingServerProducer{
+		InputAPI: inputAPI,
+	}
+}
+
+// Send typing event to typing server
+func (p *TypingServerProducer) Send(
+	ctx context.Context, userID, roomID string,
+	typing bool, timeout int64,
+) error {
+	requestData := api.InputTypingEvent{
+		UserID:         userID,
+		RoomID:         roomID,
+		Typing:         typing,
+		Timeout:        timeout,
+		OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
+	}
+
+	var response api.InputTypingEventResponse
+	err := p.InputAPI.InputTypingEvent(
+		ctx, &api.InputTypingEventRequest{InputTypingEvent: requestData}, &response,
+	)
+
+	return err
+}

clientapi/producers/userupdate.go

@@ -0,0 +1,62 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package producers
+
+import (
+	"encoding/json"
+
+	sarama "gopkg.in/Shopify/sarama.v1"
+)
+
+// UserUpdateProducer produces events related to user updates.
+type UserUpdateProducer struct {
+	Topic    string
+	Producer sarama.SyncProducer
+}
+
+// TODO: Move this struct to `common` so the components that consume the topic
+// can use it when parsing incoming messages
+type profileUpdate struct {
+	Updated  string `json:"updated"`   // Which attribute is updated (can be either `avatar_url` or `displayname`)
+	OldValue string `json:"old_value"` // The attribute's value before the update
+	NewValue string `json:"new_value"` // The attribute's value after the update
+}
+
+// SendUpdate sends an update using kafka to notify the roomserver of the
+// profile update. Returns an error if the update failed to send.
+func (p *UserUpdateProducer) SendUpdate(
+	userID string, updatedAttribute string, oldValue string, newValue string,
+) error {
+	var update profileUpdate
+	var m sarama.ProducerMessage
+
+	m.Topic = string(p.Topic)
+	m.Key = sarama.StringEncoder(userID)
+
+	update = profileUpdate{
+		Updated:  updatedAttribute,
+		OldValue: oldValue,
+		NewValue: newValue,
+	}
+
+	value, err := json.Marshal(update)
+	if err != nil {
+		return err
+	}
+	m.Value = sarama.ByteEncoder(value)
+
+	_, _, err = p.Producer.SendMessage(&m)
+	return err
+}

clientapi/routing/account_data.go

@@ -0,0 +1,76 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"io/ioutil"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/util"
+)
+
+// SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type}
+func SaveAccountData(
+	req *http.Request, accountDB *accounts.Database, device *authtypes.Device,
+	userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer,
+) util.JSONResponse {
+	if req.Method != http.MethodPut {
+		return util.JSONResponse{
+			Code: http.StatusMethodNotAllowed,
+			JSON: jsonerror.NotFound("Bad method"),
+		}
+	}
+
+	if userID != device.UserID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("userID does not match the current user"),
+		}
+	}
+
+	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	defer req.Body.Close() // nolint: errcheck
+
+	body, err := ioutil.ReadAll(req.Body)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if err := accountDB.SaveAccountData(
+		req.Context(), localpart, roomID, dataType, string(body),
+	); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if err := syncProducer.SendData(userID, roomID, dataType); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/createroom.go

@@ -0,0 +1,337 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	log "github.com/sirupsen/logrus"
+)
+
+// https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-createroom
+type createRoomRequest struct {
+	Invite          []string               `json:"invite"`
+	Name            string                 `json:"name"`
+	Visibility      string                 `json:"visibility"`
+	Topic           string                 `json:"topic"`
+	Preset          string                 `json:"preset"`
+	CreationContent map[string]interface{} `json:"creation_content"`
+	InitialState    []fledglingEvent       `json:"initial_state"`
+	RoomAliasName   string                 `json:"room_alias_name"`
+	GuestCanJoin    bool                   `json:"guest_can_join"`
+}
+
+const (
+	presetPrivateChat        = "private_chat"
+	presetTrustedPrivateChat = "trusted_private_chat"
+	presetPublicChat         = "public_chat"
+)
+
+const (
+	joinRulePublic = "public"
+	joinRuleInvite = "invite"
+)
+const (
+	historyVisibilityShared = "shared"
+	// TODO: These should be implemented once history visibility is implemented
+	// historyVisibilityWorldReadable = "world_readable"
+	// historyVisibilityInvited       = "invited"
+)
+
+func (r createRoomRequest) Validate() *util.JSONResponse {
+	whitespace := "\t\n\x0b\x0c\r " // https://docs.python.org/2/library/string.html#string.whitespace
+	// https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/handlers/room.py#L81
+	// Synapse doesn't check for ':' but we will else it will break parsers badly which split things into 2 segments.
+	if strings.ContainsAny(r.RoomAliasName, whitespace+":") {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("room_alias_name cannot contain whitespace or ':'"),
+		}
+	}
+	for _, userID := range r.Invite {
+		// TODO: We should put user ID parsing code into gomatrixserverlib and use that instead
+		//       (see https://github.com/matrix-org/gomatrixserverlib/blob/3394e7c7003312043208aa73727d2256eea3d1f6/eventcontent.go#L347 )
+		//       It should be a struct (with pointers into a single string to avoid copying) and
+		//       we should update all refs to use UserID types rather than strings.
+		// https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/types.py#L92
+		if _, _, err := gomatrixserverlib.SplitID('@', userID); err != nil {
+			return &util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON("user id must be in the form @localpart:domain"),
+			}
+		}
+	}
+	switch r.Preset {
+	case presetPrivateChat, presetTrustedPrivateChat, presetPublicChat, "":
+	default:
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("preset must be any of 'private_chat', 'trusted_private_chat', 'public_chat'"),
+		}
+	}
+
+	return nil
+}
+
+// https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-createroom
+type createRoomResponse struct {
+	RoomID    string `json:"room_id"`
+	RoomAlias string `json:"room_alias,omitempty"` // in synapse not spec
+}
+
+// fledglingEvent is a helper representation of an event used when creating many events in succession.
+type fledglingEvent struct {
+	Type     string      `json:"type"`
+	StateKey string      `json:"state_key"`
+	Content  interface{} `json:"content"`
+}
+
+// CreateRoom implements /createRoom
+func CreateRoom(
+	req *http.Request, device *authtypes.Device,
+	cfg config.Dendrite, producer *producers.RoomserverProducer,
+	accountDB *accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI,
+	asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	// TODO (#267): Check room ID doesn't clash with an existing one, and we
+	//              probably shouldn't be using pseudo-random strings, maybe GUIDs?
+	roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
+	return createRoom(req, device, cfg, roomID, producer, accountDB, aliasAPI, asAPI)
+}
+
+// createRoom implements /createRoom
+// nolint: gocyclo
+func createRoom(
+	req *http.Request, device *authtypes.Device,
+	cfg config.Dendrite, roomID string, producer *producers.RoomserverProducer,
+	accountDB *accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI,
+	asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	logger := util.GetLogger(req.Context())
+	userID := device.UserID
+	var r createRoomRequest
+	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	if resErr != nil {
+		return *resErr
+	}
+	// TODO: apply rate-limit
+
+	if resErr = r.Validate(); resErr != nil {
+		return *resErr
+	}
+
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+	// TODO: visibility/presets/raw initial state/creation content
+	// TODO: Create room alias association
+	// Make sure this doesn't fall into an application service's namespace though!
+
+	logger.WithFields(log.Fields{
+		"userID": userID,
+		"roomID": roomID,
+	}).Info("Creating new room")
+
+	profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	membershipContent := common.MemberContent{
+		Membership:  "join",
+		DisplayName: profile.DisplayName,
+		AvatarURL:   profile.AvatarURL,
+	}
+
+	var joinRules, historyVisibility string
+	switch r.Preset {
+	case presetPrivateChat:
+		joinRules = joinRuleInvite
+		historyVisibility = historyVisibilityShared
+	case presetTrustedPrivateChat:
+		joinRules = joinRuleInvite
+		historyVisibility = historyVisibilityShared
+		// TODO If trusted_private_chat, all invitees are given the same power level as the room creator.
+	case presetPublicChat:
+		joinRules = joinRulePublic
+		historyVisibility = historyVisibilityShared
+	default:
+		// Default room rules, r.Preset was previously checked for valid values so
+		// only a request with no preset should end up here.
+		joinRules = joinRuleInvite
+		historyVisibility = historyVisibilityShared
+	}
+
+	var builtEvents []gomatrixserverlib.Event
+
+	// send events into the room in order of:
+	//  1- m.room.create
+	//  2- room creator join member
+	//  3- m.room.power_levels
+	//  4- m.room.canonical_alias (opt) TODO
+	//  5- m.room.join_rules
+	//  6- m.room.history_visibility
+	//  7- m.room.guest_access (opt)
+	//  8- other initial state items
+	//  9- m.room.name (opt)
+	//  10- m.room.topic (opt)
+	//  11- invite events (opt) - with is_direct flag if applicable TODO
+	//  12- 3pid invite events (opt) TODO
+	//  13- m.room.aliases event for HS (if alias specified) TODO
+	// This differs from Synapse slightly. Synapse would vary the ordering of 3-7
+	// depending on if those events were in "initial_state" or not. This made it
+	// harder to reason about, hence sticking to a strict static ordering.
+	// TODO: Synapse has txn/token ID on each event. Do we need to do this here?
+	eventsToMake := []fledglingEvent{
+		{"m.room.create", "", common.CreateContent{Creator: userID}},
+		{"m.room.member", userID, membershipContent},
+		{"m.room.power_levels", "", common.InitialPowerLevelsContent(userID)},
+		// TODO: m.room.canonical_alias
+		{"m.room.join_rules", "", common.JoinRulesContent{JoinRule: joinRules}},
+		{"m.room.history_visibility", "", common.HistoryVisibilityContent{HistoryVisibility: historyVisibility}},
+	}
+	if r.GuestCanJoin {
+		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", common.GuestAccessContent{GuestAccess: "can_join"}})
+	}
+	eventsToMake = append(eventsToMake, r.InitialState...)
+	if r.Name != "" {
+		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", common.NameContent{Name: r.Name}})
+	}
+	if r.Topic != "" {
+		eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", common.TopicContent{Topic: r.Topic}})
+	}
+	// TODO: invite events
+	// TODO: 3pid invite events
+	// TODO: m.room.aliases
+
+	authEvents := gomatrixserverlib.NewAuthEvents(nil)
+	for i, e := range eventsToMake {
+		depth := i + 1 // depth starts at 1
+
+		builder := gomatrixserverlib.EventBuilder{
+			Sender:   userID,
+			RoomID:   roomID,
+			Type:     e.Type,
+			StateKey: &e.StateKey,
+			Depth:    int64(depth),
+		}
+		err = builder.SetContent(e.Content)
+		if err != nil {
+			return httputil.LogThenError(req, err)
+		}
+		if i > 0 {
+			builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()}
+		}
+		var ev *gomatrixserverlib.Event
+		ev, err = buildEvent(&builder, &authEvents, cfg, evTime)
+		if err != nil {
+			return httputil.LogThenError(req, err)
+		}
+
+		if err = gomatrixserverlib.Allowed(*ev, &authEvents); err != nil {
+			return httputil.LogThenError(req, err)
+		}
+
+		// Add the event to the list of auth events
+		builtEvents = append(builtEvents, *ev)
+		err = authEvents.AddEvent(ev)
+		if err != nil {
+			return httputil.LogThenError(req, err)
+		}
+	}
+
+	// send events to the room server
+	_, err = producer.SendEvents(req.Context(), builtEvents, cfg.Matrix.ServerName, nil)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	// TODO(#269): Reserve room alias while we create the room. This stops us
+	// from creating the room but still failing due to the alias having already
+	// been taken.
+	var roomAlias string
+	if r.RoomAliasName != "" {
+		roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, cfg.Matrix.ServerName)
+
+		aliasReq := roomserverAPI.SetRoomAliasRequest{
+			Alias:  roomAlias,
+			RoomID: roomID,
+			UserID: userID,
+		}
+
+		var aliasResp roomserverAPI.SetRoomAliasResponse
+		err = aliasAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp)
+		if err != nil {
+			return httputil.LogThenError(req, err)
+		}
+
+		if aliasResp.AliasExists {
+			return util.MessageResponse(400, "Alias already exists")
+		}
+	}
+
+	response := createRoomResponse{
+		RoomID:    roomID,
+		RoomAlias: roomAlias,
+	}
+
+	return util.JSONResponse{
+		Code: 200,
+		JSON: response,
+	}
+}
+
+// buildEvent fills out auth_events for the builder then builds the event
+func buildEvent(
+	builder *gomatrixserverlib.EventBuilder,
+	provider gomatrixserverlib.AuthEventProvider,
+	cfg config.Dendrite,
+	evTime time.Time,
+) (*gomatrixserverlib.Event, error) {
+	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
+	if err != nil {
+		return nil, err
+	}
+	refs, err := eventsNeeded.AuthEventReferences(provider)
+	if err != nil {
+		return nil, err
+	}
+	builder.AuthEvents = refs
+	eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
+	event, err := builder.Build(eventID, evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey)
+	if err != nil {
+		return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %s", builder.Type, err)
+	}
+	return &event, nil
+}

clientapi/routing/device.go

@@ -0,0 +1,155 @@
+// Copyright 2017 Paul Tötterman <paul.totterman@iki.fi>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"database/sql"
+	"encoding/json"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+type deviceJSON struct {
+	DeviceID string `json:"device_id"`
+	UserID   string `json:"user_id"`
+}
+
+type devicesJSON struct {
+	Devices []deviceJSON `json:"devices"`
+}
+
+type deviceUpdateJSON struct {
+	DisplayName *string `json:"display_name"`
+}
+
+// GetDeviceByID handles /devices/{deviceID}
+func GetDeviceByID(
+	req *http.Request, deviceDB *devices.Database, device *authtypes.Device,
+	deviceID string,
+) util.JSONResponse {
+	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	ctx := req.Context()
+	dev, err := deviceDB.GetDeviceByID(ctx, localpart, deviceID)
+	if err == sql.ErrNoRows {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Unknown device"),
+		}
+	} else if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: deviceJSON{
+			DeviceID: dev.ID,
+			UserID:   dev.UserID,
+		},
+	}
+}
+
+// GetDevicesByLocalpart handles /devices
+func GetDevicesByLocalpart(
+	req *http.Request, deviceDB *devices.Database, device *authtypes.Device,
+) util.JSONResponse {
+	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	ctx := req.Context()
+	deviceList, err := deviceDB.GetDevicesByLocalpart(ctx, localpart)
+
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	res := devicesJSON{}
+
+	for _, dev := range deviceList {
+		res.Devices = append(res.Devices, deviceJSON{
+			DeviceID: dev.ID,
+			UserID:   dev.UserID,
+		})
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: res,
+	}
+}
+
+// UpdateDeviceByID handles PUT on /devices/{deviceID}
+func UpdateDeviceByID(
+	req *http.Request, deviceDB *devices.Database, device *authtypes.Device,
+	deviceID string,
+) util.JSONResponse {
+	if req.Method != http.MethodPut {
+		return util.JSONResponse{
+			Code: http.StatusMethodNotAllowed,
+			JSON: jsonerror.NotFound("Bad Method"),
+		}
+	}
+
+	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	ctx := req.Context()
+	dev, err := deviceDB.GetDeviceByID(ctx, localpart, deviceID)
+	if err == sql.ErrNoRows {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Unknown device"),
+		}
+	} else if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if dev.UserID != device.UserID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("device not owned by current user"),
+		}
+	}
+
+	defer req.Body.Close() // nolint: errcheck
+
+	payload := deviceUpdateJSON{}
+
+	if err := json.NewDecoder(req.Body).Decode(&payload); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if err := deviceDB.UpdateDevice(ctx, localpart, deviceID, payload.DisplayName); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/directory.go

@@ -0,0 +1,183 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/common/config"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+// DirectoryRoom looks up a room alias
+func DirectoryRoom(
+	req *http.Request,
+	roomAlias string,
+	federation *gomatrixserverlib.FederationClient,
+	cfg *config.Dendrite,
+	rsAPI roomserverAPI.RoomserverAliasAPI,
+) util.JSONResponse {
+	_, domain, err := gomatrixserverlib.SplitID('#', roomAlias)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"),
+		}
+	}
+
+	if domain == cfg.Matrix.ServerName {
+		// Query the roomserver API to check if the alias exists locally
+		queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias}
+		var queryRes roomserverAPI.GetRoomIDForAliasResponse
+		if err = rsAPI.GetRoomIDForAlias(req.Context(), &queryReq, &queryRes); err != nil {
+			return httputil.LogThenError(req, err)
+		}
+
+		// List any roomIDs found associated with this alias
+		if len(queryRes.RoomID) > 0 {
+			return util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: queryRes,
+			}
+		}
+	} else {
+		// Query the federation for this room alias
+		resp, err := federation.LookupRoomAlias(req.Context(), domain, roomAlias)
+		if err != nil {
+			switch err.(type) {
+			case gomatrix.HTTPError:
+			default:
+				// TODO: Return 502 if the remote server errored.
+				// TODO: Return 504 if the remote server timed out.
+				return httputil.LogThenError(req, err)
+			}
+		}
+		if len(resp.RoomID) > 0 {
+			return util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: resp,
+			}
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusNotFound,
+		JSON: jsonerror.NotFound(
+			fmt.Sprintf("Room alias %s not found", roomAlias),
+		),
+	}
+}
+
+// SetLocalAlias implements PUT /directory/room/{roomAlias}
+// TODO: Check if the user has the power level to set an alias
+func SetLocalAlias(
+	req *http.Request,
+	device *authtypes.Device,
+	alias string,
+	cfg *config.Dendrite,
+	aliasAPI roomserverAPI.RoomserverAliasAPI,
+) util.JSONResponse {
+	_, domain, err := gomatrixserverlib.SplitID('#', alias)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"),
+		}
+	}
+
+	if domain != cfg.Matrix.ServerName {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Alias must be on local homeserver"),
+		}
+	}
+
+	// Check that the alias does not fall within an exclusive namespace of an
+	// application service
+	// TODO: This code should eventually be refactored with:
+	// 1. The new method for checking for things matching an AS's namespace
+	// 2. Using an overall Regex object for all AS's just like we did for usernames
+	for _, appservice := range cfg.Derived.ApplicationServices {
+		if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok {
+			for _, namespace := range aliasNamespaces {
+				if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) {
+					return util.JSONResponse{
+						Code: http.StatusBadRequest,
+						JSON: jsonerror.ASExclusive("Alias is reserved by an application service"),
+					}
+				}
+			}
+		}
+	}
+
+	var r struct {
+		RoomID string `json:"room_id"`
+	}
+	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
+		return *resErr
+	}
+
+	queryReq := roomserverAPI.SetRoomAliasRequest{
+		UserID: device.UserID,
+		RoomID: r.RoomID,
+		Alias:  alias,
+	}
+	var queryRes roomserverAPI.SetRoomAliasResponse
+	if err := aliasAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if queryRes.AliasExists {
+		return util.JSONResponse{
+			Code: http.StatusConflict,
+			JSON: jsonerror.Unknown("The alias " + alias + " already exists."),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+// RemoveLocalAlias implements DELETE /directory/room/{roomAlias}
+// TODO: Check if the user has the power level to remove an alias
+func RemoveLocalAlias(
+	req *http.Request,
+	device *authtypes.Device,
+	alias string,
+	aliasAPI roomserverAPI.RoomserverAliasAPI,
+) util.JSONResponse {
+	queryReq := roomserverAPI.RemoveRoomAliasRequest{
+		Alias:  alias,
+		UserID: device.UserID,
+	}
+	var queryRes roomserverAPI.RemoveRoomAliasResponse
+	if err := aliasAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/filter.go

@@ -0,0 +1,123 @@
+// Copyright 2017 Jan Christian Grünhage
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"encoding/json"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+// GetFilter implements GET /_matrix/client/r0/user/{userId}/filter/{filterId}
+func GetFilter(
+	req *http.Request, device *authtypes.Device, accountDB *accounts.Database, userID string, filterID string,
+) util.JSONResponse {
+	if req.Method != http.MethodGet {
+		return util.JSONResponse{
+			Code: http.StatusMethodNotAllowed,
+			JSON: jsonerror.NotFound("Bad method"),
+		}
+	}
+	if userID != device.UserID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Cannot get filters for other users"),
+		}
+	}
+	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	res, err := accountDB.GetFilter(req.Context(), localpart, filterID)
+	if err != nil {
+		//TODO better error handling. This error message is *probably* right,
+		// but if there are obscure db errors, this will also be returned,
+		// even though it is not correct.
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.NotFound("No such filter"),
+		}
+	}
+	filter := gomatrix.Filter{}
+	err = json.Unmarshal(res, &filter)
+	if err != nil {
+		httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: filter,
+	}
+}
+
+type filterResponse struct {
+	FilterID string `json:"filter_id"`
+}
+
+//PutFilter implements POST /_matrix/client/r0/user/{userId}/filter
+func PutFilter(
+	req *http.Request, device *authtypes.Device, accountDB *accounts.Database, userID string,
+) util.JSONResponse {
+	if req.Method != http.MethodPost {
+		return util.JSONResponse{
+			Code: http.StatusMethodNotAllowed,
+			JSON: jsonerror.NotFound("Bad method"),
+		}
+	}
+	if userID != device.UserID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Cannot create filters for other users"),
+		}
+	}
+
+	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	var filter gomatrix.Filter
+
+	if reqErr := httputil.UnmarshalJSONRequest(req, &filter); reqErr != nil {
+		return *reqErr
+	}
+
+	filterArray, err := json.Marshal(filter)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("Filter is malformed"),
+		}
+	}
+
+	filterID, err := accountDB.PutFilter(req.Context(), localpart, filterArray)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: filterResponse{FilterID: filterID},
+	}
+}

clientapi/routing/joinroom.go

@@ -0,0 +1,333 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+// JoinRoomByIDOrAlias implements the "/join/{roomIDOrAlias}" API.
+// https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-join-roomidoralias
+func JoinRoomByIDOrAlias(
+	req *http.Request,
+	device *authtypes.Device,
+	roomIDOrAlias string,
+	cfg config.Dendrite,
+	federation *gomatrixserverlib.FederationClient,
+	producer *producers.RoomserverProducer,
+	queryAPI roomserverAPI.RoomserverQueryAPI,
+	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	keyRing gomatrixserverlib.KeyRing,
+	accountDB *accounts.Database,
+) util.JSONResponse {
+	var content map[string]interface{} // must be a JSON object
+	if resErr := httputil.UnmarshalJSONRequest(req, &content); resErr != nil {
+		return *resErr
+	}
+
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+
+	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	content["membership"] = "join"
+	content["displayname"] = profile.DisplayName
+	content["avatar_url"] = profile.AvatarURL
+
+	r := joinRoomReq{
+		req, evTime, content, device.UserID, cfg, federation, producer, queryAPI, aliasAPI, keyRing,
+	}
+
+	if strings.HasPrefix(roomIDOrAlias, "!") {
+		return r.joinRoomByID(roomIDOrAlias)
+	}
+	if strings.HasPrefix(roomIDOrAlias, "#") {
+		return r.joinRoomByAlias(roomIDOrAlias)
+	}
+	return util.JSONResponse{
+		Code: http.StatusBadRequest,
+		JSON: jsonerror.BadJSON("Invalid first character for room ID or alias"),
+	}
+}
+
+type joinRoomReq struct {
+	req        *http.Request
+	evTime     time.Time
+	content    map[string]interface{}
+	userID     string
+	cfg        config.Dendrite
+	federation *gomatrixserverlib.FederationClient
+	producer   *producers.RoomserverProducer
+	queryAPI   roomserverAPI.RoomserverQueryAPI
+	aliasAPI   roomserverAPI.RoomserverAliasAPI
+	keyRing    gomatrixserverlib.KeyRing
+}
+
+// joinRoomByID joins a room by room ID
+func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse {
+	// A client should only join a room by room ID when it has an invite
+	// to the room. If the server is already in the room then we can
+	// lookup the invite and process the request as a normal state event.
+	// If the server is not in the room the we will need to look up the
+	// remote server the invite came from in order to request a join event
+	// from that server.
+	queryReq := roomserverAPI.QueryInvitesForUserRequest{
+		RoomID: roomID, TargetUserID: r.userID,
+	}
+	var queryRes roomserverAPI.QueryInvitesForUserResponse
+	if err := r.queryAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil {
+		return httputil.LogThenError(r.req, err)
+	}
+
+	servers := []gomatrixserverlib.ServerName{}
+	seenInInviterIDs := map[gomatrixserverlib.ServerName]bool{}
+	for _, userID := range queryRes.InviteSenderUserIDs {
+		_, domain, err := gomatrixserverlib.SplitID('@', userID)
+		if err != nil {
+			return httputil.LogThenError(r.req, err)
+		}
+		if !seenInInviterIDs[domain] {
+			servers = append(servers, domain)
+			seenInInviterIDs[domain] = true
+		}
+	}
+
+	// Also add the domain extracted from the roomID as a last resort to join
+	// in case the client is erroneously trying to join by ID without an invite
+	// or all previous attempts at domains extracted from the inviter IDs fail
+	// Note: It's no guarantee we'll succeed because a room isn't bound to the domain in its ID
+	_, domain, err := gomatrixserverlib.SplitID('!', roomID)
+	if err != nil {
+		return httputil.LogThenError(r.req, err)
+	}
+	if domain != r.cfg.Matrix.ServerName && !seenInInviterIDs[domain] {
+		servers = append(servers, domain)
+	}
+
+	return r.joinRoomUsingServers(roomID, servers)
+
+}
+
+// joinRoomByAlias joins a room using a room alias.
+func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse {
+	_, domain, err := gomatrixserverlib.SplitID('#', roomAlias)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"),
+		}
+	}
+	if domain == r.cfg.Matrix.ServerName {
+		queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias}
+		var queryRes roomserverAPI.GetRoomIDForAliasResponse
+		if err = r.aliasAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil {
+			return httputil.LogThenError(r.req, err)
+		}
+
+		if len(queryRes.RoomID) > 0 {
+			return r.joinRoomUsingServers(queryRes.RoomID, []gomatrixserverlib.ServerName{r.cfg.Matrix.ServerName})
+		}
+		// If the response doesn't contain a non-empty string, return an error
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Room alias " + roomAlias + " not found."),
+		}
+	}
+	// If the room isn't local, use federation to join
+	return r.joinRoomByRemoteAlias(domain, roomAlias)
+}
+
+func (r joinRoomReq) joinRoomByRemoteAlias(
+	domain gomatrixserverlib.ServerName, roomAlias string,
+) util.JSONResponse {
+	resp, err := r.federation.LookupRoomAlias(r.req.Context(), domain, roomAlias)
+	if err != nil {
+		switch x := err.(type) {
+		case gomatrix.HTTPError:
+			if x.Code == http.StatusNotFound {
+				return util.JSONResponse{
+					Code: http.StatusNotFound,
+					JSON: jsonerror.NotFound("Room alias not found"),
+				}
+			}
+		}
+		return httputil.LogThenError(r.req, err)
+	}
+
+	return r.joinRoomUsingServers(resp.RoomID, resp.Servers)
+}
+
+func (r joinRoomReq) writeToBuilder(eb *gomatrixserverlib.EventBuilder, roomID string) error {
+	eb.Type = "m.room.member"
+
+	err := eb.SetContent(r.content)
+	if err != nil {
+		return err
+	}
+
+	err = eb.SetUnsigned(struct{}{})
+	if err != nil {
+		return err
+	}
+
+	eb.Sender = r.userID
+	eb.StateKey = &r.userID
+	eb.RoomID = roomID
+	eb.Redacts = ""
+
+	return nil
+}
+
+func (r joinRoomReq) joinRoomUsingServers(
+	roomID string, servers []gomatrixserverlib.ServerName,
+) util.JSONResponse {
+	var eb gomatrixserverlib.EventBuilder
+	err := r.writeToBuilder(&eb, roomID)
+	if err != nil {
+		return httputil.LogThenError(r.req, err)
+	}
+
+	var queryRes roomserverAPI.QueryLatestEventsAndStateResponse
+	event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.queryAPI, &queryRes)
+	if err == nil {
+		if _, err = r.producer.SendEvents(r.req.Context(), []gomatrixserverlib.Event{*event}, r.cfg.Matrix.ServerName, nil); err != nil {
+			return httputil.LogThenError(r.req, err)
+		}
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: struct {
+				RoomID string `json:"room_id"`
+			}{roomID},
+		}
+	}
+	if err != common.ErrRoomNoExists {
+		return httputil.LogThenError(r.req, err)
+	}
+
+	if len(servers) == 0 {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("No candidate servers found for room"),
+		}
+	}
+
+	var lastErr error
+	for _, server := range servers {
+		var response *util.JSONResponse
+		response, lastErr = r.joinRoomUsingServer(roomID, server)
+		if lastErr != nil {
+			// There was a problem talking to one of the servers.
+			util.GetLogger(r.req.Context()).WithError(lastErr).WithField("server", server).Warn("Failed to join room using server")
+			// Try the next server.
+			continue
+		}
+		return *response
+	}
+
+	// Every server we tried to join through resulted in an error.
+	// We return the error from the last server.
+
+	// TODO: Generate the correct HTTP status code for all different
+	// kinds of errors that could have happened.
+	// The possible errors include:
+	//   1) We can't connect to the remote servers.
+	//   2) None of the servers we could connect to think we are allowed
+	//	    to join the room.
+	//   3) The remote server returned something invalid.
+	//   4) We couldn't fetch the public keys needed to verify the
+	//      signatures on the state events.
+	//   5) ...
+	return httputil.LogThenError(r.req, lastErr)
+}
+
+// joinRoomUsingServer tries to join a remote room using a given matrix server.
+// If there was a failure communicating with the server or the response from the
+// server was invalid this returns an error.
+// Otherwise this returns a JSONResponse.
+func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib.ServerName) (*util.JSONResponse, error) {
+	respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID)
+	if err != nil {
+		// TODO: Check if the user was not allowed to join the room.
+		return nil, err
+	}
+
+	// Set all the fields to be what they should be, this should be a no-op
+	// but it's possible that the remote server returned us something "odd"
+	err = r.writeToBuilder(&respMakeJoin.JoinEvent, roomID)
+	if err != nil {
+		return nil, err
+	}
+
+	eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), r.cfg.Matrix.ServerName)
+	event, err := respMakeJoin.JoinEvent.Build(
+		eventID, r.evTime, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID, r.cfg.Matrix.PrivateKey,
+	)
+	if err != nil {
+		res := httputil.LogThenError(r.req, err)
+		return &res, nil
+	}
+
+	respSendJoin, err := r.federation.SendJoin(r.req.Context(), server, event)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil {
+		return nil, err
+	}
+
+	if err = r.producer.SendEventWithState(
+		r.req.Context(), gomatrixserverlib.RespState(respSendJoin), event,
+	); err != nil {
+		res := httputil.LogThenError(r.req, err)
+		return &res, nil
+	}
+
+	return &util.JSONResponse{
+		Code: http.StatusOK,
+		// TODO: Put the response struct somewhere common.
+		JSON: struct {
+			RoomID string `json:"room_id"`
+		}{roomID},
+	}, nil
+}

clientapi/routing/login.go

@@ -0,0 +1,152 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"context"
+	"database/sql"
+	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+type loginFlows struct {
+	Flows []flow `json:"flows"`
+}
+
+type flow struct {
+	Type   string   `json:"type"`
+	Stages []string `json:"stages"`
+}
+
+type passwordRequest struct {
+	User               string  `json:"user"`
+	Password           string  `json:"password"`
+	InitialDisplayName *string `json:"initial_device_display_name"`
+	DeviceID           string  `json:"device_id"`
+}
+
+type loginResponse struct {
+	UserID      string                       `json:"user_id"`
+	AccessToken string                       `json:"access_token"`
+	HomeServer  gomatrixserverlib.ServerName `json:"home_server"`
+	DeviceID    string                       `json:"device_id"`
+}
+
+func passwordLogin() loginFlows {
+	f := loginFlows{}
+	s := flow{"m.login.password", []string{"m.login.password"}}
+	f.Flows = append(f.Flows, s)
+	return f
+}
+
+// Login implements GET and POST /login
+func Login(
+	req *http.Request, accountDB *accounts.Database, deviceDB *devices.Database,
+	cfg config.Dendrite,
+) util.JSONResponse {
+	if req.Method == http.MethodGet { // TODO: support other forms of login other than password, depending on config options
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: passwordLogin(),
+		}
+	} else if req.Method == http.MethodPost {
+		var r passwordRequest
+		resErr := httputil.UnmarshalJSONRequest(req, &r)
+		if resErr != nil {
+			return *resErr
+		}
+		if r.User == "" {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.BadJSON("'user' must be supplied."),
+			}
+		}
+
+		util.GetLogger(req.Context()).WithField("user", r.User).Info("Processing login request")
+
+		localpart, err := userutil.ParseUsernameParam(r.User, &cfg.Matrix.ServerName)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.InvalidUsername(err.Error()),
+			}
+		}
+
+		acc, err := accountDB.GetAccountByPassword(req.Context(), localpart, r.Password)
+		if err != nil {
+			// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
+			// but that would leak the existence of the user.
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"),
+			}
+		}
+
+		token, err := auth.GenerateAccessToken()
+		if err != nil {
+			httputil.LogThenError(req, err)
+		}
+
+		dev, err := getDevice(req.Context(), r, deviceDB, acc, localpart, token)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: jsonerror.Unknown("failed to create device: " + err.Error()),
+			}
+		}
+
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: loginResponse{
+				UserID:      dev.UserID,
+				AccessToken: dev.AccessToken,
+				HomeServer:  cfg.Matrix.ServerName,
+				DeviceID:    dev.ID,
+			},
+		}
+	}
+	return util.JSONResponse{
+		Code: http.StatusMethodNotAllowed,
+		JSON: jsonerror.NotFound("Bad method"),
+	}
+}
+
+// check if device exists else create one
+func getDevice(
+	ctx context.Context,
+	r passwordRequest,
+	deviceDB *devices.Database,
+	acc *authtypes.Account,
+	localpart, token string,
+) (dev *authtypes.Device, err error) {
+	dev, err = deviceDB.GetDeviceByID(ctx, localpart, r.DeviceID)
+	if err == sql.ErrNoRows {
+		// device doesn't exist, create one
+		dev, err = deviceDB.CreateDevice(
+			ctx, acc.Localpart, nil, token, r.InitialDisplayName,
+		)
+	}
+	return
+}

clientapi/routing/logout.go

@@ -0,0 +1,71 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+// Logout handles POST /logout
+func Logout(
+	req *http.Request, deviceDB *devices.Database, device *authtypes.Device,
+) util.JSONResponse {
+	if req.Method != http.MethodPost {
+		return util.JSONResponse{
+			Code: http.StatusMethodNotAllowed,
+			JSON: jsonerror.NotFound("Bad method"),
+		}
+	}
+
+	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if err := deviceDB.RemoveDevice(req.Context(), device.ID, localpart); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+// LogoutAll handles POST /logout/all
+func LogoutAll(
+	req *http.Request, deviceDB *devices.Database, device *authtypes.Device,
+) util.JSONResponse {
+	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if err := deviceDB.RemoveAllDevices(req.Context(), localpart); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/membership.go

@@ -0,0 +1,217 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"time"
+
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/clientapi/threepid"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/util"
+)
+
+var errMissingUserID = errors.New("'user_id' must be supplied")
+
+// SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite)
+// by building a m.room.member event then sending it to the room server
+func SendMembership(
+	req *http.Request, accountDB *accounts.Database, device *authtypes.Device,
+	roomID string, membership string, cfg config.Dendrite,
+	queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	producer *producers.RoomserverProducer,
+) util.JSONResponse {
+	var body threepid.MembershipRequest
+	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
+		return *reqErr
+	}
+
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+
+	inviteStored, err := threepid.CheckAndProcessInvite(
+		req.Context(), device, &body, cfg, queryAPI, accountDB, producer,
+		membership, roomID, evTime,
+	)
+	if err == threepid.ErrMissingParameter {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(err.Error()),
+		}
+	} else if err == threepid.ErrNotTrusted {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.NotTrusted(body.IDServer),
+		}
+	} else if err == common.ErrRoomNoExists {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound(err.Error()),
+		}
+	} else if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	// If an invite has been stored on an identity server, it means that a
+	// m.room.third_party_invite event has been emitted and that we shouldn't
+	// emit a m.room.member one.
+	if inviteStored {
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: struct{}{},
+		}
+	}
+
+	event, err := buildMembershipEvent(
+		req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, queryAPI, asAPI,
+	)
+	if err == errMissingUserID {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(err.Error()),
+		}
+	} else if err == common.ErrRoomNoExists {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound(err.Error()),
+		}
+	} else if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if _, err := producer.SendEvents(
+		req.Context(), []gomatrixserverlib.Event{*event}, cfg.Matrix.ServerName, nil,
+	); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+func buildMembershipEvent(
+	ctx context.Context,
+	body threepid.MembershipRequest, accountDB *accounts.Database,
+	device *authtypes.Device,
+	membership, roomID string,
+	cfg config.Dendrite, evTime time.Time,
+	queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+) (*gomatrixserverlib.Event, error) {
+	stateKey, reason, err := getMembershipStateKey(body, device, membership)
+	if err != nil {
+		return nil, err
+	}
+
+	profile, err := loadProfile(ctx, stateKey, cfg, accountDB, asAPI)
+	if err != nil {
+		return nil, err
+	}
+
+	builder := gomatrixserverlib.EventBuilder{
+		Sender:   device.UserID,
+		RoomID:   roomID,
+		Type:     "m.room.member",
+		StateKey: &stateKey,
+	}
+
+	// "unban" or "kick" isn't a valid membership value, change it to "leave"
+	if membership == "unban" || membership == "kick" {
+		membership = "leave"
+	}
+
+	content := common.MemberContent{
+		Membership:  membership,
+		DisplayName: profile.DisplayName,
+		AvatarURL:   profile.AvatarURL,
+		Reason:      reason,
+	}
+
+	if err = builder.SetContent(content); err != nil {
+		return nil, err
+	}
+
+	return common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil)
+}
+
+// loadProfile lookups the profile of a given user from the database and returns
+// it if the user is local to this server, or returns an empty profile if not.
+// Returns an error if the retrieval failed or if the first parameter isn't a
+// valid Matrix ID.
+func loadProfile(
+	ctx context.Context,
+	userID string,
+	cfg config.Dendrite,
+	accountDB *accounts.Database,
+	asAPI appserviceAPI.AppServiceQueryAPI,
+) (*authtypes.Profile, error) {
+	_, serverName, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return nil, err
+	}
+
+	var profile *authtypes.Profile
+	if serverName == cfg.Matrix.ServerName {
+		profile, err = appserviceAPI.RetreiveUserProfile(ctx, userID, asAPI, accountDB)
+	} else {
+		profile = &authtypes.Profile{}
+	}
+
+	return profile, err
+}
+
+// getMembershipStateKey extracts the target user ID of a membership change.
+// For "join" and "leave" this will be the ID of the user making the change.
+// For "ban", "unban", "kick" and "invite" the target user ID will be in the JSON request body.
+// In the latter case, if there was an issue retrieving the user ID from the request body,
+// returns a JSONResponse with a corresponding error code and message.
+func getMembershipStateKey(
+	body threepid.MembershipRequest, device *authtypes.Device, membership string,
+) (stateKey string, reason string, err error) {
+	if membership == "ban" || membership == "unban" || membership == "kick" || membership == "invite" {
+		// If we're in this case, the state key is contained in the request body,
+		// possibly along with a reason (for "kick" and "ban") so we need to parse
+		// it
+		if body.UserID == "" {
+			err = errMissingUserID
+			return
+		}
+
+		stateKey = body.UserID
+		reason = body.Reason
+	} else {
+		stateKey = device.UserID
+	}
+
+	return
+}

clientapi/routing/memberships.go

@@ -0,0 +1,60 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+type response struct {
+	Chunk []gomatrixserverlib.ClientEvent `json:"chunk"`
+}
+
+// GetMemberships implements GET /rooms/{roomId}/members
+func GetMemberships(
+	req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool,
+	_ config.Dendrite,
+	queryAPI api.RoomserverQueryAPI,
+) util.JSONResponse {
+	queryReq := api.QueryMembershipsForRoomRequest{
+		JoinedOnly: joinedOnly,
+		RoomID:     roomID,
+		Sender:     device.UserID,
+	}
+	var queryRes api.QueryMembershipsForRoomResponse
+	if err := queryAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if !queryRes.HasBeenInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: response{queryRes.JoinEvents},
+	}
+}

clientapi/routing/profile.go

@@ -0,0 +1,292 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/util"
+)
+
+// GetProfile implements GET /profile/{userID}
+func GetProfile(
+	req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	if req.Method != http.MethodGet {
+		return util.JSONResponse{
+			Code: http.StatusMethodNotAllowed,
+			JSON: jsonerror.NotFound("Bad method"),
+		}
+	}
+	profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	res := common.ProfileResponse{
+		AvatarURL:   profile.AvatarURL,
+		DisplayName: profile.DisplayName,
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: res,
+	}
+}
+
+// GetAvatarURL implements GET /profile/{userID}/avatar_url
+func GetAvatarURL(
+	req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	res := common.AvatarURL{
+		AvatarURL: profile.AvatarURL,
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: res,
+	}
+}
+
+// SetAvatarURL implements PUT /profile/{userID}/avatar_url
+func SetAvatarURL(
+	req *http.Request, accountDB *accounts.Database, device *authtypes.Device,
+	userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite,
+	rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI,
+) util.JSONResponse {
+	if userID != device.UserID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("userID does not match the current user"),
+		}
+	}
+
+	changedKey := "avatar_url"
+
+	var r common.AvatarURL
+	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
+		return *resErr
+	}
+	if r.AvatarURL == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("'avatar_url' must be supplied."),
+		}
+	}
+
+	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+
+	oldProfile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if err = accountDB.SetAvatarURL(req.Context(), localpart, r.AvatarURL); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	memberships, err := accountDB.GetMembershipsByLocalpart(req.Context(), localpart)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	newProfile := authtypes.Profile{
+		Localpart:   localpart,
+		DisplayName: oldProfile.DisplayName,
+		AvatarURL:   r.AvatarURL,
+	}
+
+	events, err := buildMembershipEvents(
+		req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI,
+	)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if _, err := rsProducer.SendEvents(req.Context(), events, cfg.Matrix.ServerName, nil); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if err := producer.SendUpdate(userID, changedKey, oldProfile.AvatarURL, r.AvatarURL); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+// GetDisplayName implements GET /profile/{userID}/displayname
+func GetDisplayName(
+	req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI,
+) util.JSONResponse {
+	profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+	res := common.DisplayName{
+		DisplayName: profile.DisplayName,
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: res,
+	}
+}
+
+// SetDisplayName implements PUT /profile/{userID}/displayname
+func SetDisplayName(
+	req *http.Request, accountDB *accounts.Database, device *authtypes.Device,
+	userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite,
+	rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI,
+) util.JSONResponse {
+	if userID != device.UserID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("userID does not match the current user"),
+		}
+	}
+
+	changedKey := "displayname"
+
+	var r common.DisplayName
+	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
+		return *resErr
+	}
+	if r.DisplayName == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("'displayname' must be supplied."),
+		}
+	}
+
+	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+
+	oldProfile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if err = accountDB.SetDisplayName(req.Context(), localpart, r.DisplayName); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	memberships, err := accountDB.GetMembershipsByLocalpart(req.Context(), localpart)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	newProfile := authtypes.Profile{
+		Localpart:   localpart,
+		DisplayName: r.DisplayName,
+		AvatarURL:   oldProfile.AvatarURL,
+	}
+
+	events, err := buildMembershipEvents(
+		req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI,
+	)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if _, err := rsProducer.SendEvents(req.Context(), events, cfg.Matrix.ServerName, nil); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if err := producer.SendUpdate(userID, changedKey, oldProfile.DisplayName, r.DisplayName); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+func buildMembershipEvents(
+	ctx context.Context,
+	memberships []authtypes.Membership,
+	newProfile authtypes.Profile, userID string, cfg *config.Dendrite,
+	evTime time.Time, queryAPI api.RoomserverQueryAPI,
+) ([]gomatrixserverlib.Event, error) {
+	evs := []gomatrixserverlib.Event{}
+
+	for _, membership := range memberships {
+		builder := gomatrixserverlib.EventBuilder{
+			Sender:   userID,
+			RoomID:   membership.RoomID,
+			Type:     "m.room.member",
+			StateKey: &userID,
+		}
+
+		content := common.MemberContent{
+			Membership: "join",
+		}
+
+		content.DisplayName = newProfile.DisplayName
+		content.AvatarURL = newProfile.AvatarURL
+
+		if err := builder.SetContent(content); err != nil {
+			return nil, err
+		}
+
+		event, err := common.BuildEvent(ctx, &builder, *cfg, evTime, queryAPI, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		evs = append(evs, *event)
+	}
+
+	return evs, nil
+}

clientapi/routing/register.go

@@ -0,0 +1,958 @@
+// Copyright 2017 Vector Creations Ltd
+// Copyright 2017 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/matrix-org/dendrite/common/config"
+
+	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	// Prometheus metrics
+	amtRegUsers = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "dendrite_clientapi_reg_users_total",
+			Help: "Total number of registered users",
+		},
+	)
+)
+
+const (
+	minPasswordLength = 8   // http://matrix.org/docs/spec/client_server/r0.2.0.html#password-based
+	maxPasswordLength = 512 // https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
+	maxUsernameLength = 254 // http://matrix.org/speculator/spec/HEAD/intro.html#user-identifiers TODO account for domain
+	sessionIDLength   = 24
+)
+
+func init() {
+	// Register prometheus metrics. They must be registered to be exposed.
+	prometheus.MustRegister(amtRegUsers)
+}
+
+// sessionsDict keeps track of completed auth stages for each session.
+type sessionsDict struct {
+	sessions map[string][]authtypes.LoginType
+}
+
+// GetCompletedStages returns the completed stages for a session.
+func (d sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType {
+	if completedStages, ok := d.sessions[sessionID]; ok {
+		return completedStages
+	}
+	// Ensure that a empty slice is returned and not nil. See #399.
+	return make([]authtypes.LoginType, 0)
+}
+
+// AddCompletedStage records that a session has completed an auth stage.
+func (d *sessionsDict) AddCompletedStage(sessionID string, stage authtypes.LoginType) {
+	d.sessions[sessionID] = append(d.GetCompletedStages(sessionID), stage)
+}
+
+func newSessionsDict() *sessionsDict {
+	return &sessionsDict{
+		sessions: make(map[string][]authtypes.LoginType),
+	}
+}
+
+var (
+	// TODO: Remove old sessions. Need to do so on a session-specific timeout.
+	// sessions stores the completed flow stages for all sessions. Referenced using their sessionID.
+	sessions           = newSessionsDict()
+	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-./]+$`)
+)
+
+// registerRequest represents the submitted registration request.
+// It can be broken down into 2 sections: the auth dictionary and registration parameters.
+// Registration parameters vary depending on the request, and will need to remembered across
+// sessions. If no parameters are supplied, the server should use the parameters previously
+// remembered. If ANY parameters are supplied, the server should REPLACE all knowledge of
+// previous parameters with the ones supplied. This mean you cannot "build up" request params.
+type registerRequest struct {
+	// registration parameters
+	Password string `json:"password"`
+	Username string `json:"username"`
+	Admin    bool   `json:"admin"`
+	// user-interactive auth params
+	Auth authDict `json:"auth"`
+
+	InitialDisplayName *string `json:"initial_device_display_name"`
+
+	// Prevent this user from logging in
+	InhibitLogin common.WeakBoolean `json:"inhibit_login"`
+
+	// Application Services place Type in the root of their registration
+	// request, whereas clients place it in the authDict struct.
+	Type authtypes.LoginType `json:"type"`
+}
+
+type authDict struct {
+	Type    authtypes.LoginType         `json:"type"`
+	Session string                      `json:"session"`
+	Mac     gomatrixserverlib.HexString `json:"mac"`
+
+	// Recaptcha
+	Response string `json:"response"`
+	// TODO: Lots of custom keys depending on the type
+}
+
+// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api
+type userInteractiveResponse struct {
+	Flows     []authtypes.Flow       `json:"flows"`
+	Completed []authtypes.LoginType  `json:"completed"`
+	Params    map[string]interface{} `json:"params"`
+	Session   string                 `json:"session"`
+}
+
+// legacyRegisterRequest represents the submitted registration request for v1 API.
+type legacyRegisterRequest struct {
+	Password string                      `json:"password"`
+	Username string                      `json:"user"`
+	Admin    bool                        `json:"admin"`
+	Type     authtypes.LoginType         `json:"type"`
+	Mac      gomatrixserverlib.HexString `json:"mac"`
+}
+
+// newUserInteractiveResponse will return a struct to be sent back to the client
+// during registration.
+func newUserInteractiveResponse(
+	sessionID string,
+	fs []authtypes.Flow,
+	params map[string]interface{},
+) userInteractiveResponse {
+	return userInteractiveResponse{
+		fs, sessions.GetCompletedStages(sessionID), params, sessionID,
+	}
+}
+
+// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+type registerResponse struct {
+	UserID      string                       `json:"user_id"`
+	AccessToken string                       `json:"access_token,omitempty"`
+	HomeServer  gomatrixserverlib.ServerName `json:"home_server"`
+	DeviceID    string                       `json:"device_id,omitempty"`
+}
+
+// recaptchaResponse represents the HTTP response from a Google Recaptcha server
+type recaptchaResponse struct {
+	Success     bool      `json:"success"`
+	ChallengeTS time.Time `json:"challenge_ts"`
+	Hostname    string    `json:"hostname"`
+	ErrorCodes  []int     `json:"error-codes"`
+}
+
+// validateUsername returns an error response if the username is invalid
+func validateUsername(username string) *util.JSONResponse {
+	// https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
+	if len(username) > maxUsernameLength {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(fmt.Sprintf("'username' >%d characters", maxUsernameLength)),
+		}
+	} else if !validUsernameRegex.MatchString(username) {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./'"),
+		}
+	} else if username[0] == '_' { // Regex checks its not a zero length string
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidUsername("Username cannot start with a '_'"),
+		}
+	}
+	return nil
+}
+
+// validateApplicationServiceUsername returns an error response if the username is invalid for an application service
+func validateApplicationServiceUsername(username string) *util.JSONResponse {
+	if len(username) > maxUsernameLength {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(fmt.Sprintf("'username' >%d characters", maxUsernameLength)),
+		}
+	} else if !validUsernameRegex.MatchString(username) {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./'"),
+		}
+	}
+	return nil
+}
+
+// validatePassword returns an error response if the password is invalid
+func validatePassword(password string) *util.JSONResponse {
+	// https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
+	if len(password) > maxPasswordLength {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(fmt.Sprintf("'password' >%d characters", maxPasswordLength)),
+		}
+	} else if len(password) > 0 && len(password) < minPasswordLength {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.WeakPassword(fmt.Sprintf("password too weak: min %d chars", minPasswordLength)),
+		}
+	}
+	return nil
+}
+
+// validateRecaptcha returns an error response if the captcha response is invalid
+func validateRecaptcha(
+	cfg *config.Dendrite,
+	response string,
+	clientip string,
+) *util.JSONResponse {
+	if !cfg.Matrix.RecaptchaEnabled {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("Captcha registration is disabled"),
+		}
+	}
+
+	if response == "" {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("Captcha response is required"),
+		}
+	}
+
+	// Make a POST request to Google's API to check the captcha response
+	resp, err := http.PostForm(cfg.Matrix.RecaptchaSiteVerifyAPI,
+		url.Values{
+			"secret":   {cfg.Matrix.RecaptchaPrivateKey},
+			"response": {response},
+			"remoteip": {clientip},
+		},
+	)
+
+	if err != nil {
+		return &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.BadJSON("Error in requesting validation of captcha response"),
+		}
+	}
+
+	// Close the request once we're finishing reading from it
+	defer resp.Body.Close() // nolint: errcheck
+
+	// Grab the body of the response from the captcha server
+	var r recaptchaResponse
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.BadJSON("Error in contacting captcha server" + err.Error()),
+		}
+	}
+	err = json.Unmarshal(body, &r)
+	if err != nil {
+		return &util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.BadJSON("Error in unmarshaling captcha server's response: " + err.Error()),
+		}
+	}
+
+	// Check that we received a "success"
+	if !r.Success {
+		return &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: jsonerror.BadJSON("Invalid captcha response. Please try again."),
+		}
+	}
+	return nil
+}
+
+// UserIDIsWithinApplicationServiceNamespace checks to see if a given userID
+// falls within any of the namespaces of a given Application Service. If no
+// Application Service is given, it will check to see if it matches any
+// Application Service's namespace.
+func UserIDIsWithinApplicationServiceNamespace(
+	cfg *config.Dendrite,
+	userID string,
+	appservice *config.ApplicationService,
+) bool {
+	if appservice != nil {
+		// Loop through given application service's namespaces and see if any match
+		for _, namespace := range appservice.NamespaceMap["users"] {
+			// AS namespaces are checked for validity in config
+			if namespace.RegexpObject.MatchString(userID) {
+				return true
+			}
+		}
+		return false
+	}
+
+	// Loop through all known application service's namespaces and see if any match
+	for _, knownAppService := range cfg.Derived.ApplicationServices {
+		for _, namespace := range knownAppService.NamespaceMap["users"] {
+			// AS namespaces are checked for validity in config
+			if namespace.RegexpObject.MatchString(userID) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// UsernameMatchesMultipleExclusiveNamespaces will check if a given username matches
+// more than one exclusive namespace. More than one is not allowed
+func UsernameMatchesMultipleExclusiveNamespaces(
+	cfg *config.Dendrite,
+	username string,
+) bool {
+	userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
+
+	// Check namespaces and see if more than one match
+	matchCount := 0
+	for _, appservice := range cfg.Derived.ApplicationServices {
+		if appservice.IsInterestedInUserID(userID) {
+			if matchCount++; matchCount > 1 {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// UsernameMatchesExclusiveNamespaces will check if a given username matches any
+// application service's exclusive users namespace
+func UsernameMatchesExclusiveNamespaces(
+	cfg *config.Dendrite,
+	username string,
+) bool {
+	userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
+	return cfg.Derived.ExclusiveApplicationServicesUsernameRegexp.MatchString(userID)
+}
+
+// validateApplicationService checks if a provided application service token
+// corresponds to one that is registered. If so, then it checks if the desired
+// username is within that application service's namespace. As long as these
+// two requirements are met, no error will be returned.
+func validateApplicationService(
+	cfg *config.Dendrite,
+	username string,
+	accessToken string,
+) (string, *util.JSONResponse) {
+	// Check if the token if the application service is valid with one we have
+	// registered in the config.
+	var matchedApplicationService *config.ApplicationService
+	for _, appservice := range cfg.Derived.ApplicationServices {
+		if appservice.ASToken == accessToken {
+			matchedApplicationService = &appservice
+			break
+		}
+	}
+	if matchedApplicationService == nil {
+		return "", &util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: jsonerror.UnknownToken("Supplied access_token does not match any known application service"),
+		}
+	}
+
+	userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
+
+	// Ensure the desired username is within at least one of the application service's namespaces.
+	if !UserIDIsWithinApplicationServiceNamespace(cfg, userID, matchedApplicationService) {
+		// If we didn't find any matches, return M_EXCLUSIVE
+		return "", &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.ASExclusive(fmt.Sprintf(
+				"Supplied username %s did not match any namespaces for application service ID: %s", username, matchedApplicationService.ID)),
+		}
+	}
+
+	// Check this user does not fit multiple application service namespaces
+	if UsernameMatchesMultipleExclusiveNamespaces(cfg, userID) {
+		return "", &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.ASExclusive(fmt.Sprintf(
+				"Supplied username %s matches multiple exclusive application service namespaces. Only 1 match allowed", username)),
+		}
+	}
+
+	// Check username application service is trying to register is valid
+	if err := validateApplicationServiceUsername(username); err != nil {
+		return "", err
+	}
+
+	// No errors, registration valid
+	return matchedApplicationService.ID, nil
+}
+
+// Register processes a /register request.
+// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
+func Register(
+	req *http.Request,
+	accountDB *accounts.Database,
+	deviceDB *devices.Database,
+	cfg *config.Dendrite,
+) util.JSONResponse {
+
+	var r registerRequest
+	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	if resErr != nil {
+		return *resErr
+	}
+
+	// Retrieve or generate the sessionID
+	sessionID := r.Auth.Session
+	if sessionID == "" {
+		// Generate a new, random session ID
+		sessionID = util.RandomString(sessionIDLength)
+	}
+
+	// Don't allow numeric usernames less than MAX_INT64.
+	if _, err := strconv.ParseInt(r.Username, 10, 64); err == nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidUsername("Numeric user IDs are reserved"),
+		}
+	}
+	// Auto generate a numeric username if r.Username is empty
+	if r.Username == "" {
+		id, err := accountDB.GetNewNumericLocalpart(req.Context())
+		if err != nil {
+			return httputil.LogThenError(req, err)
+		}
+
+		r.Username = strconv.FormatInt(id, 10)
+	}
+
+	// Squash username to all lowercase letters
+	r.Username = strings.ToLower(r.Username)
+
+	if resErr = validateUsername(r.Username); resErr != nil {
+		return *resErr
+	}
+	if resErr = validatePassword(r.Password); resErr != nil {
+		return *resErr
+	}
+
+	// Make sure normal user isn't registering under an exclusive application
+	// service namespace. Skip this check if no app services are registered.
+	if r.Auth.Type != authtypes.LoginTypeApplicationService &&
+		len(cfg.Derived.ApplicationServices) != 0 &&
+		UsernameMatchesExclusiveNamespaces(cfg, r.Username) {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.ASExclusive("This username is reserved by an application service."),
+		}
+	}
+
+	logger := util.GetLogger(req.Context())
+	logger.WithFields(log.Fields{
+		"username":   r.Username,
+		"auth.type":  r.Auth.Type,
+		"session_id": r.Auth.Session,
+	}).Info("Processing registration request")
+
+	return handleRegistrationFlow(req, r, sessionID, cfg, accountDB, deviceDB)
+}
+
+// handleRegistrationFlow will direct and complete registration flow stages
+// that the client has requested.
+// nolint: gocyclo
+func handleRegistrationFlow(
+	req *http.Request,
+	r registerRequest,
+	sessionID string,
+	cfg *config.Dendrite,
+	accountDB *accounts.Database,
+	deviceDB *devices.Database,
+) util.JSONResponse {
+	// TODO: Shared secret registration (create new user scripts)
+	// TODO: Enable registration config flag
+	// TODO: Guest account upgrading
+
+	// TODO: Handle loading of previous session parameters from database.
+	// TODO: Handle mapping registrationRequest parameters into session parameters
+
+	// TODO: email / msisdn auth types.
+
+	if cfg.Matrix.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret {
+		return util.MessageResponse(http.StatusForbidden, "Registration has been disabled")
+	}
+
+	switch r.Auth.Type {
+	case authtypes.LoginTypeRecaptcha:
+		// Check given captcha response
+		resErr := validateRecaptcha(cfg, r.Auth.Response, req.RemoteAddr)
+		if resErr != nil {
+			return *resErr
+		}
+
+		// Add Recaptcha to the list of completed registration stages
+		sessions.AddCompletedStage(sessionID, authtypes.LoginTypeRecaptcha)
+
+	case authtypes.LoginTypeSharedSecret:
+		// Check shared secret against config
+		valid, err := isValidMacLogin(cfg, r.Username, r.Password, r.Admin, r.Auth.Mac)
+
+		if err != nil {
+			return httputil.LogThenError(req, err)
+		} else if !valid {
+			return util.MessageResponse(http.StatusForbidden, "HMAC incorrect")
+		}
+
+		// Add SharedSecret to the list of completed registration stages
+		sessions.AddCompletedStage(sessionID, authtypes.LoginTypeSharedSecret)
+
+	case "":
+		// Extract the access token from the request, if there's one to extract
+		// (which we can know by checking whether the error is nil or not).
+		accessToken, err := auth.ExtractAccessToken(req)
+
+		// A missing auth type can mean either the registration is performed by
+		// an AS or the request is made as the first step of a registration
+		// using the User-Interactive Authentication API. This can be determined
+		// by whether the request contains an access token.
+		if err == nil {
+			return handleApplicationServiceRegistration(
+				accessToken, err, req, r, cfg, accountDB, deviceDB,
+			)
+		}
+
+	case authtypes.LoginTypeApplicationService:
+		// Extract the access token from the request.
+		accessToken, err := auth.ExtractAccessToken(req)
+		// Let the AS registration handler handle the process from here. We
+		// don't need a condition on that call since the registration is clearly
+		// stated as being AS-related.
+		return handleApplicationServiceRegistration(
+			accessToken, err, req, r, cfg, accountDB, deviceDB,
+		)
+
+	case authtypes.LoginTypeDummy:
+		// there is nothing to do
+		// Add Dummy to the list of completed registration stages
+		sessions.AddCompletedStage(sessionID, authtypes.LoginTypeDummy)
+
+	default:
+		return util.JSONResponse{
+			Code: http.StatusNotImplemented,
+			JSON: jsonerror.Unknown("unknown/unimplemented auth type"),
+		}
+	}
+
+	// Check if the user's registration flow has been completed successfully
+	// A response with current registration flow and remaining available methods
+	// will be returned if a flow has not been successfully completed yet
+	return checkAndCompleteFlow(sessions.GetCompletedStages(sessionID),
+		req, r, sessionID, cfg, accountDB, deviceDB)
+}
+
+// handleApplicationServiceRegistration handles the registration of an
+// application service's user by validating the AS from its access token and
+// registering the user. Its two first parameters must be the two return values
+// of the auth.ExtractAccessToken function.
+// Returns an error if the access token couldn't be extracted from the request
+// at an earlier step of the registration workflow, or if the provided access
+// token doesn't belong to a valid AS, or if there was an issue completing the
+// registration process.
+func handleApplicationServiceRegistration(
+	accessToken string,
+	tokenErr error,
+	req *http.Request,
+	r registerRequest,
+	cfg *config.Dendrite,
+	accountDB *accounts.Database,
+	deviceDB *devices.Database,
+) util.JSONResponse {
+	// Check if we previously had issues extracting the access token from the
+	// request.
+	if tokenErr != nil {
+		return util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: jsonerror.MissingToken(tokenErr.Error()),
+		}
+	}
+
+	// Check application service register user request is valid.
+	// The application service's ID is returned if so.
+	appserviceID, err := validateApplicationService(
+		cfg, r.Username, accessToken,
+	)
+	if err != nil {
+		return *err
+	}
+
+	// If no error, application service was successfully validated.
+	// Don't need to worry about appending to registration stages as
+	// application service registration is entirely separate.
+	return completeRegistration(
+		req.Context(), accountDB, deviceDB, r.Username, "", appserviceID,
+		r.InhibitLogin, r.InitialDisplayName,
+	)
+}
+
+// checkAndCompleteFlow checks if a given registration flow is completed given
+// a set of allowed flows. If so, registration is completed, otherwise a
+// response with
+func checkAndCompleteFlow(
+	flow []authtypes.LoginType,
+	req *http.Request,
+	r registerRequest,
+	sessionID string,
+	cfg *config.Dendrite,
+	accountDB *accounts.Database,
+	deviceDB *devices.Database,
+) util.JSONResponse {
+	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
+		// This flow was completed, registration can continue
+		return completeRegistration(
+			req.Context(), accountDB, deviceDB, r.Username, r.Password, "",
+			r.InhibitLogin, r.InitialDisplayName,
+		)
+	}
+
+	// There are still more stages to complete.
+	// Return the flows and those that have been completed.
+	return util.JSONResponse{
+		Code: http.StatusUnauthorized,
+		JSON: newUserInteractiveResponse(sessionID,
+			cfg.Derived.Registration.Flows, cfg.Derived.Registration.Params),
+	}
+}
+
+// LegacyRegister process register requests from the legacy v1 API
+func LegacyRegister(
+	req *http.Request,
+	accountDB *accounts.Database,
+	deviceDB *devices.Database,
+	cfg *config.Dendrite,
+) util.JSONResponse {
+	var r legacyRegisterRequest
+	resErr := parseAndValidateLegacyLogin(req, &r)
+	if resErr != nil {
+		return *resErr
+	}
+
+	logger := util.GetLogger(req.Context())
+	logger.WithFields(log.Fields{
+		"username":  r.Username,
+		"auth.type": r.Type,
+	}).Info("Processing registration request")
+
+	if cfg.Matrix.RegistrationDisabled && r.Type != authtypes.LoginTypeSharedSecret {
+		return util.MessageResponse(http.StatusForbidden, "Registration has been disabled")
+	}
+
+	switch r.Type {
+	case authtypes.LoginTypeSharedSecret:
+		if cfg.Matrix.RegistrationSharedSecret == "" {
+			return util.MessageResponse(http.StatusBadRequest, "Shared secret registration is disabled")
+		}
+
+		valid, err := isValidMacLogin(cfg, r.Username, r.Password, r.Admin, r.Mac)
+		if err != nil {
+			return httputil.LogThenError(req, err)
+		}
+
+		if !valid {
+			return util.MessageResponse(http.StatusForbidden, "HMAC incorrect")
+		}
+
+		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil)
+	case authtypes.LoginTypeDummy:
+		// there is nothing to do
+		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil)
+	default:
+		return util.JSONResponse{
+			Code: http.StatusNotImplemented,
+			JSON: jsonerror.Unknown("unknown/unimplemented auth type"),
+		}
+	}
+}
+
+// parseAndValidateLegacyLogin parses the request into r and checks that the
+// request is valid (e.g. valid user names, etc)
+func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *util.JSONResponse {
+	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	if resErr != nil {
+		return resErr
+	}
+
+	// Squash username to all lowercase letters
+	r.Username = strings.ToLower(r.Username)
+
+	if resErr = validateUsername(r.Username); resErr != nil {
+		return resErr
+	}
+	if resErr = validatePassword(r.Password); resErr != nil {
+		return resErr
+	}
+
+	// All registration requests must specify what auth they are using to perform this request
+	if r.Type == "" {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("invalid type"),
+		}
+	}
+
+	return nil
+}
+
+func completeRegistration(
+	ctx context.Context,
+	accountDB *accounts.Database,
+	deviceDB *devices.Database,
+	username, password, appserviceID string,
+	inhibitLogin common.WeakBoolean,
+	displayName *string,
+) util.JSONResponse {
+	if username == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("missing username"),
+		}
+	}
+	// Blank passwords are only allowed by registered application services
+	if password == "" && appserviceID == "" {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("missing password"),
+		}
+	}
+
+	acc, err := accountDB.CreateAccount(ctx, username, password, appserviceID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.Unknown("failed to create account: " + err.Error()),
+		}
+	} else if acc == nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.UserInUse("Desired user ID is already taken."),
+		}
+	}
+
+	// Check whether inhibit_login option is set. If so, don't create an access
+	// token or a device for this user
+	if inhibitLogin {
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: registerResponse{
+				UserID:     userutil.MakeUserID(username, acc.ServerName),
+				HomeServer: acc.ServerName,
+			},
+		}
+	}
+
+	token, err := auth.GenerateAccessToken()
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.Unknown("Failed to generate access token"),
+		}
+	}
+
+	// TODO: Use the device ID in the request.
+	dev, err := deviceDB.CreateDevice(ctx, username, nil, token, displayName)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.Unknown("failed to create device: " + err.Error()),
+		}
+	}
+
+	// Increment prometheus counter for created users
+	amtRegUsers.Inc()
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: registerResponse{
+			UserID:      dev.UserID,
+			AccessToken: dev.AccessToken,
+			HomeServer:  acc.ServerName,
+			DeviceID:    dev.ID,
+		},
+	}
+}
+
+// Used for shared secret registration.
+// Checks if the username, password and isAdmin flag matches the given mac.
+func isValidMacLogin(
+	cfg *config.Dendrite,
+	username, password string,
+	isAdmin bool,
+	givenMac []byte,
+) (bool, error) {
+	sharedSecret := cfg.Matrix.RegistrationSharedSecret
+
+	// Check that shared secret registration isn't disabled.
+	if cfg.Matrix.RegistrationSharedSecret == "" {
+		return false, errors.New("Shared secret registration is disabled")
+	}
+
+	// Double check that username/password don't contain the HMAC delimiters. We should have
+	// already checked this.
+	if strings.Contains(username, "\x00") {
+		return false, errors.New("Username contains invalid character")
+	}
+	if strings.Contains(password, "\x00") {
+		return false, errors.New("Password contains invalid character")
+	}
+	if sharedSecret == "" {
+		return false, errors.New("Shared secret registration is disabled")
+	}
+
+	adminString := "notadmin"
+	if isAdmin {
+		adminString = "admin"
+	}
+	joined := strings.Join([]string{username, password, adminString}, "\x00")
+
+	mac := hmac.New(sha1.New, []byte(sharedSecret))
+	_, err := mac.Write([]byte(joined))
+	if err != nil {
+		return false, err
+	}
+	expectedMAC := mac.Sum(nil)
+
+	return hmac.Equal(givenMac, expectedMAC), nil
+}
+
+// checkFlows checks a single completed flow against another required one. If
+// one contains at least all of the stages that the other does, checkFlows
+// returns true.
+func checkFlows(
+	completedStages []authtypes.LoginType,
+	requiredStages []authtypes.LoginType,
+) bool {
+	// Create temporary slices so they originals will not be modified on sorting
+	completed := make([]authtypes.LoginType, len(completedStages))
+	required := make([]authtypes.LoginType, len(requiredStages))
+	copy(completed, completedStages)
+	copy(required, requiredStages)
+
+	// Sort the slices for simple comparison
+	sort.Slice(completed, func(i, j int) bool { return completed[i] < completed[j] })
+	sort.Slice(required, func(i, j int) bool { return required[i] < required[j] })
+
+	// Iterate through each slice, going to the next required slice only once
+	// we've found a match.
+	i, j := 0, 0
+	for j < len(required) {
+		// Exit if we've reached the end of our input without being able to
+		// match all of the required stages.
+		if i >= len(completed) {
+			return false
+		}
+
+		// If we've found a stage we want, move on to the next required stage.
+		if completed[i] == required[j] {
+			j++
+		}
+		i++
+	}
+	return true
+}
+
+// checkFlowCompleted checks if a registration flow complies with any allowed flow
+// dictated by the server. Order of stages does not matter. A user may complete
+// extra stages as long as the required stages of at least one flow is met.
+func checkFlowCompleted(
+	flow []authtypes.LoginType,
+	allowedFlows []authtypes.Flow,
+) bool {
+	// Iterate through possible flows to check whether any have been fully completed.
+	for _, allowedFlow := range allowedFlows {
+		if checkFlows(flow, allowedFlow.Stages) {
+			return true
+		}
+	}
+	return false
+}
+
+type availableResponse struct {
+	Available bool `json:"available"`
+}
+
+// RegisterAvailable checks if the username is already taken or invalid.
+func RegisterAvailable(
+	req *http.Request,
+	cfg config.Dendrite,
+	accountDB *accounts.Database,
+) util.JSONResponse {
+	username := req.URL.Query().Get("username")
+
+	// Squash username to all lowercase letters
+	username = strings.ToLower(username)
+
+	if err := validateUsername(username); err != nil {
+		return *err
+	}
+
+	// Check if this username is reserved by an application service
+	userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
+	for _, appservice := range cfg.Derived.ApplicationServices {
+		if appservice.IsInterestedInUserID(userID) {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.UserInUse("Desired user ID is reserved by an application service."),
+			}
+		}
+	}
+
+	availability, availabilityErr := accountDB.CheckAccountAvailability(req.Context(), username)
+	if availabilityErr != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: jsonerror.Unknown("failed to check availability: " + availabilityErr.Error()),
+		}
+	}
+	if !availability {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.UserInUse("Desired User ID is already taken."),
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: availableResponse{
+			Available: true,
+		},
+	}
+}

clientapi/routing/register_test.go

@@ -0,0 +1,209 @@
+// Copyright 2017 Andrew Morgan <andrew@amorgan.xyz>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/common/config"
+)
+
+var (
+	// Registration Flows that the server allows.
+	allowedFlows = []authtypes.Flow{
+		{
+			Stages: []authtypes.LoginType{
+				authtypes.LoginType("stage1"),
+				authtypes.LoginType("stage2"),
+			},
+		},
+		{
+			Stages: []authtypes.LoginType{
+				authtypes.LoginType("stage1"),
+				authtypes.LoginType("stage3"),
+			},
+		},
+	}
+)
+
+// Should return true as we're completing all the stages of a single flow in
+// order.
+func TestFlowCheckingCompleteFlowOrdered(t *testing.T) {
+	testFlow := []authtypes.LoginType{
+		authtypes.LoginType("stage1"),
+		authtypes.LoginType("stage3"),
+	}
+
+	if !checkFlowCompleted(testFlow, allowedFlows) {
+		t.Error("Incorrect registration flow verification: ", testFlow, ", from allowed flows: ", allowedFlows, ". Should be true.")
+	}
+}
+
+// Should return false as all stages in a single flow need to be completed.
+func TestFlowCheckingStagesFromDifferentFlows(t *testing.T) {
+	testFlow := []authtypes.LoginType{
+		authtypes.LoginType("stage2"),
+		authtypes.LoginType("stage3"),
+	}
+
+	if checkFlowCompleted(testFlow, allowedFlows) {
+		t.Error("Incorrect registration flow verification: ", testFlow, ", from allowed flows: ", allowedFlows, ". Should be false.")
+	}
+}
+
+// Should return true as we're completing all the stages from a single flow, as
+// well as some extraneous stages.
+func TestFlowCheckingCompleteOrderedExtraneous(t *testing.T) {
+	testFlow := []authtypes.LoginType{
+		authtypes.LoginType("stage1"),
+		authtypes.LoginType("stage3"),
+		authtypes.LoginType("stage4"),
+		authtypes.LoginType("stage5"),
+	}
+	if !checkFlowCompleted(testFlow, allowedFlows) {
+		t.Error("Incorrect registration flow verification: ", testFlow, ", from allowed flows: ", allowedFlows, ". Should be true.")
+	}
+}
+
+// Should return false as we're submitting an empty flow.
+func TestFlowCheckingEmptyFlow(t *testing.T) {
+	testFlow := []authtypes.LoginType{}
+	if checkFlowCompleted(testFlow, allowedFlows) {
+		t.Error("Incorrect registration flow verification: ", testFlow, ", from allowed flows: ", allowedFlows, ". Should be false.")
+	}
+}
+
+// Should return false as we've completed a stage that isn't in any allowed flow.
+func TestFlowCheckingInvalidStage(t *testing.T) {
+	testFlow := []authtypes.LoginType{
+		authtypes.LoginType("stage8"),
+	}
+	if checkFlowCompleted(testFlow, allowedFlows) {
+		t.Error("Incorrect registration flow verification: ", testFlow, ", from allowed flows: ", allowedFlows, ". Should be false.")
+	}
+}
+
+// Should return true as we complete all stages of an allowed flow, though out
+// of order, as well as extraneous stages.
+func TestFlowCheckingExtraneousUnordered(t *testing.T) {
+	testFlow := []authtypes.LoginType{
+		authtypes.LoginType("stage5"),
+		authtypes.LoginType("stage4"),
+		authtypes.LoginType("stage3"),
+		authtypes.LoginType("stage2"),
+		authtypes.LoginType("stage1"),
+	}
+	if !checkFlowCompleted(testFlow, allowedFlows) {
+		t.Error("Incorrect registration flow verification: ", testFlow, ", from allowed flows: ", allowedFlows, ". Should be true.")
+	}
+}
+
+// Should return false as we're providing fewer stages than are required.
+func TestFlowCheckingShortIncorrectInput(t *testing.T) {
+	testFlow := []authtypes.LoginType{
+		authtypes.LoginType("stage8"),
+	}
+	if checkFlowCompleted(testFlow, allowedFlows) {
+		t.Error("Incorrect registration flow verification: ", testFlow, ", from allowed flows: ", allowedFlows, ". Should be false.")
+	}
+}
+
+// Should return false as we're providing different stages than are required.
+func TestFlowCheckingExtraneousIncorrectInput(t *testing.T) {
+	testFlow := []authtypes.LoginType{
+		authtypes.LoginType("stage8"),
+		authtypes.LoginType("stage9"),
+		authtypes.LoginType("stage10"),
+		authtypes.LoginType("stage11"),
+	}
+	if checkFlowCompleted(testFlow, allowedFlows) {
+		t.Error("Incorrect registration flow verification: ", testFlow, ", from allowed flows: ", allowedFlows, ". Should be false.")
+	}
+}
+
+// Completed flows stages should always be a valid slice header.
+// TestEmptyCompletedFlows checks that sessionsDict returns a slice & not nil.
+func TestEmptyCompletedFlows(t *testing.T) {
+	fakeEmptySessions := newSessionsDict()
+	fakeSessionID := "aRandomSessionIDWhichDoesNotExist"
+	ret := fakeEmptySessions.GetCompletedStages(fakeSessionID)
+
+	// check for []
+	if ret == nil || len(ret) != 0 {
+		t.Error("Empty Completed Flow Stages should be a empty slice: returned ", ret, ". Should be []")
+	}
+}
+
+// This method tests validation of the provided Application Service token and
+// username that they're registering
+func TestValidationOfApplicationServices(t *testing.T) {
+	// Set up application service namespaces
+	regex := "@_appservice_.*"
+	regexp, err := regexp.Compile(regex)
+	if err != nil {
+		t.Errorf("Error compiling regex: %s", regex)
+	}
+
+	fakeNamespace := config.ApplicationServiceNamespace{
+		Exclusive:    true,
+		Regex:        regex,
+		RegexpObject: regexp,
+	}
+
+	// Create a fake application service
+	fakeID := "FakeAS"
+	fakeSenderLocalpart := "_appservice_bot"
+	fakeApplicationService := config.ApplicationService{
+		ID:              fakeID,
+		URL:             "null",
+		ASToken:         "1234",
+		HSToken:         "4321",
+		SenderLocalpart: fakeSenderLocalpart,
+		NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+			"users": {fakeNamespace},
+		},
+	}
+
+	// Set up a config
+	fakeConfig := config.Dendrite{}
+	fakeConfig.Matrix.ServerName = "localhost"
+	fakeConfig.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService}
+
+	// Access token is correct, user_id omitted so we are acting as SenderLocalpart
+	asID, resp := validateApplicationService(&fakeConfig, fakeSenderLocalpart, "1234")
+	if resp != nil || asID != fakeID {
+		t.Errorf("appservice should have validated and returned correct ID: %s", resp.JSON)
+	}
+
+	// Access token is incorrect, user_id omitted so we are acting as SenderLocalpart
+	asID, resp = validateApplicationService(&fakeConfig, fakeSenderLocalpart, "xxxx")
+	if resp == nil || asID == fakeID {
+		t.Errorf("access_token should have been marked as invalid")
+	}
+
+	// Access token is correct, acting as valid user_id
+	asID, resp = validateApplicationService(&fakeConfig, "_appservice_bob", "1234")
+	if resp != nil || asID != fakeID {
+		t.Errorf("access_token and user_id should've been valid: %s", resp.JSON)
+	}
+
+	// Access token is correct, acting as invalid user_id
+	asID, resp = validateApplicationService(&fakeConfig, "_something_else", "1234")
+	if resp == nil || asID == fakeID {
+		t.Errorf("user_id should not have been valid: @_something_else:localhost")
+	}
+}

clientapi/routing/routing.go

@@ -0,0 +1,413 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	"github.com/gorilla/mux"
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/common/transactions"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+const pathPrefixV1 = "/_matrix/client/api/v1"
+const pathPrefixR0 = "/_matrix/client/r0"
+const pathPrefixUnstable = "/_matrix/client/unstable"
+
+// Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
+// to clients which need to make outbound HTTP requests.
+func Setup(
+	apiMux *mux.Router, cfg config.Dendrite,
+	producer *producers.RoomserverProducer,
+	queryAPI roomserverAPI.RoomserverQueryAPI,
+	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	asAPI appserviceAPI.AppServiceQueryAPI,
+	accountDB *accounts.Database,
+	deviceDB *devices.Database,
+	federation *gomatrixserverlib.FederationClient,
+	keyRing gomatrixserverlib.KeyRing,
+	userUpdateProducer *producers.UserUpdateProducer,
+	syncProducer *producers.SyncAPIProducer,
+	typingProducer *producers.TypingServerProducer,
+	transactionsCache *transactions.Cache,
+) {
+
+	apiMux.Handle("/_matrix/client/versions",
+		common.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse {
+			return util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: struct {
+					Versions []string `json:"versions"`
+				}{[]string{
+					"r0.0.1",
+					"r0.1.0",
+					"r0.2.0",
+					"r0.3.0",
+				}},
+			}
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
+	v1mux := apiMux.PathPrefix(pathPrefixV1).Subrouter()
+	unstableMux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter()
+
+	authData := auth.Data{
+		AccountDB:   accountDB,
+		DeviceDB:    deviceDB,
+		AppServices: cfg.Derived.ApplicationServices,
+	}
+
+	r0mux.Handle("/createRoom",
+		common.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return CreateRoom(req, device, cfg, producer, accountDB, aliasAPI, asAPI)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/join/{roomIDOrAlias}",
+		common.MakeAuthAPI("join", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return JoinRoomByIDOrAlias(
+				req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB,
+			)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|leave|invite)}",
+		common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/send/{eventType}",
+		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
+		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			txnID := vars["txnID"]
+			return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID,
+				nil, cfg, queryAPI, producer, transactionsCache)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
+		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			emptyString := ""
+			eventType := vars["eventType"]
+			// If there's a trailing slash, remove it
+			if strings.HasSuffix(eventType, "/") {
+				eventType = eventType[:len(eventType)-1]
+			}
+			return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, queryAPI, producer, nil)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
+		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			stateKey := vars["stateKey"]
+			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	r0mux.Handle("/register", common.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
+		return Register(req, accountDB, deviceDB, &cfg)
+	})).Methods(http.MethodPost, http.MethodOptions)
+
+	v1mux.Handle("/register", common.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
+		return LegacyRegister(req, accountDB, deviceDB, &cfg)
+	})).Methods(http.MethodPost, http.MethodOptions)
+
+	r0mux.Handle("/register/available", common.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
+		return RegisterAvailable(req, cfg, accountDB)
+	})).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/directory/room/{roomAlias}",
+		common.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse {
+			vars := mux.Vars(req)
+			return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/directory/room/{roomAlias}",
+		common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return SetLocalAlias(req, device, vars["roomAlias"], &cfg, aliasAPI)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	r0mux.Handle("/directory/room/{roomAlias}",
+		common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI)
+		}),
+	).Methods(http.MethodDelete, http.MethodOptions)
+
+	r0mux.Handle("/logout",
+		common.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return Logout(req, deviceDB, device)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	r0mux.Handle("/logout/all",
+		common.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return LogoutAll(req, deviceDB, device)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	r0mux.Handle("/rooms/{roomID}/typing/{userID}",
+		common.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, typingProducer)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	r0mux.Handle("/account/whoami",
+		common.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return Whoami(req, device)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	// Stub endpoints required by Riot
+
+	r0mux.Handle("/login",
+		common.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse {
+			return Login(req, accountDB, deviceDB, cfg)
+		}),
+	).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
+
+	r0mux.Handle("/pushrules/",
+		common.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse {
+			// TODO: Implement push rules API
+			res := json.RawMessage(`{
+					"global": {
+						"content": [],
+						"override": [],
+						"room": [],
+						"sender": [],
+						"underride": []
+					}
+				}`)
+			return util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: &res,
+			}
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/user/{userId}/filter",
+		common.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return PutFilter(req, device, accountDB, vars["userId"])
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	r0mux.Handle("/user/{userId}/filter/{filterId}",
+		common.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return GetFilter(req, device, accountDB, vars["userId"], vars["filterId"])
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	// Riot user settings
+
+	r0mux.Handle("/profile/{userID}",
+		common.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse {
+			vars := mux.Vars(req)
+			return GetProfile(req, accountDB, vars["userID"], asAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/profile/{userID}/avatar_url",
+		common.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse {
+			vars := mux.Vars(req)
+			return GetAvatarURL(req, accountDB, vars["userID"], asAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/profile/{userID}/avatar_url",
+		common.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
+	// PUT requests, so we need to allow this method
+
+	r0mux.Handle("/profile/{userID}/displayname",
+		common.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse {
+			vars := mux.Vars(req)
+			return GetDisplayName(req, accountDB, vars["userID"], asAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/profile/{userID}/displayname",
+		common.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+	// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
+	// PUT requests, so we need to allow this method
+
+	r0mux.Handle("/account/3pid",
+		common.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return GetAssociated3PIDs(req, accountDB, device)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/account/3pid",
+		common.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return CheckAndSave3PIDAssociation(req, accountDB, device, cfg)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	unstableMux.Handle("/account/3pid/delete",
+		common.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return Forget3PID(req, accountDB)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	r0mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken",
+		common.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse {
+			return RequestEmailToken(req, accountDB, cfg)
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	// Riot logs get flooded unless this is handled
+	r0mux.Handle("/presence/{userID}/status",
+		common.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse {
+			// TODO: Set presence (probably the responsibility of a presence server not clientapi)
+			return util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: struct{}{},
+			}
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	r0mux.Handle("/voip/turnServer",
+		common.MakeAuthAPI("turn_server", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return RequestTurnServer(req, device, cfg)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	unstableMux.Handle("/thirdparty/protocols",
+		common.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse {
+			// TODO: Return the third party protcols
+			return util.JSONResponse{
+				Code: http.StatusOK,
+				JSON: struct{}{},
+			}
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/rooms/{roomID}/initialSync",
+		common.MakeExternalAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse {
+			// TODO: Allow people to peek into rooms.
+			return util.JSONResponse{
+				Code: http.StatusForbidden,
+				JSON: jsonerror.GuestAccessForbidden("Guest access not implemented"),
+			}
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/user/{userID}/account_data/{type}",
+		common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return SaveAccountData(req, accountDB, device, vars["userID"], "", vars["type"], syncProducer)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
+		common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return SaveAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"], syncProducer)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	r0mux.Handle("/rooms/{roomID}/members",
+		common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/rooms/{roomID}/joined_members",
+		common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/rooms/{roomID}/read_markers",
+		common.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse {
+			// TODO: return the read_markers.
+			return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}}
+		}),
+	).Methods(http.MethodPost, http.MethodOptions)
+
+	r0mux.Handle("/devices",
+		common.MakeAuthAPI("get_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return GetDevicesByLocalpart(req, deviceDB, device)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/devices/{deviceID}",
+		common.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return GetDeviceByID(req, deviceDB, device, vars["deviceID"])
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/devices/{deviceID}",
+		common.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars := mux.Vars(req)
+			return UpdateDeviceByID(req, deviceDB, device, vars["deviceID"])
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	// Stub implementations for sytest
+	r0mux.Handle("/events",
+		common.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse {
+			return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{
+				"chunk": []interface{}{},
+				"start": "",
+				"end":   "",
+			}}
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/initialSync",
+		common.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse {
+			return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{
+				"end": "",
+			}}
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+}

clientapi/routing/sendevent.go

@@ -0,0 +1,153 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/common/transactions"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+// http://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-rooms-roomid-send-eventtype-txnid
+// http://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-rooms-roomid-state-eventtype-statekey
+type sendEventResponse struct {
+	EventID string `json:"event_id"`
+}
+
+// SendEvent implements:
+//   /rooms/{roomID}/send/{eventType}
+//   /rooms/{roomID}/send/{eventType}/{txnID}
+//   /rooms/{roomID}/state/{eventType}/{stateKey}
+func SendEvent(
+	req *http.Request,
+	device *authtypes.Device,
+	roomID, eventType string, txnID, stateKey *string,
+	cfg config.Dendrite,
+	queryAPI api.RoomserverQueryAPI,
+	producer *producers.RoomserverProducer,
+	txnCache *transactions.Cache,
+) util.JSONResponse {
+	if txnID != nil {
+		// Try to fetch response from transactionsCache
+		if res, ok := txnCache.FetchTransaction(*txnID); ok {
+			return *res
+		}
+	}
+
+	e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, queryAPI)
+	if resErr != nil {
+		return *resErr
+	}
+
+	var txnAndDeviceID *api.TransactionID
+	if txnID != nil {
+		txnAndDeviceID = &api.TransactionID{
+			TransactionID: *txnID,
+			DeviceID:      device.ID,
+		}
+	}
+
+	// pass the new event to the roomserver and receive the correct event ID
+	// event ID in case of duplicate transaction is discarded
+	eventID, err := producer.SendEvents(
+		req.Context(), []gomatrixserverlib.Event{*e}, cfg.Matrix.ServerName, txnAndDeviceID,
+	)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	res := util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: sendEventResponse{eventID},
+	}
+	// Add response to transactionsCache
+	if txnID != nil {
+		txnCache.AddTransaction(*txnID, &res)
+	}
+
+	return res
+}
+
+func generateSendEvent(
+	req *http.Request,
+	device *authtypes.Device,
+	roomID, eventType string, stateKey *string,
+	cfg config.Dendrite,
+	queryAPI api.RoomserverQueryAPI,
+) (*gomatrixserverlib.Event, *util.JSONResponse) {
+	// parse the incoming http request
+	userID := device.UserID
+	var r map[string]interface{} // must be a JSON object
+	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	if resErr != nil {
+		return nil, resErr
+	}
+
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+
+	// create the new event and set all the fields we can
+	builder := gomatrixserverlib.EventBuilder{
+		Sender:   userID,
+		RoomID:   roomID,
+		Type:     eventType,
+		StateKey: stateKey,
+	}
+	err = builder.SetContent(r)
+	if err != nil {
+		resErr := httputil.LogThenError(req, err)
+		return nil, &resErr
+	}
+
+	var queryRes api.QueryLatestEventsAndStateResponse
+	e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, queryAPI, &queryRes)
+	if err == common.ErrRoomNoExists {
+		return nil, &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Room does not exist"),
+		}
+	} else if err != nil {
+		resErr := httputil.LogThenError(req, err)
+		return nil, &resErr
+	}
+
+	// check to see if this user can perform this operation
+	stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents))
+	for i := range queryRes.StateEvents {
+		stateEvents[i] = &queryRes.StateEvents[i]
+	}
+	provider := gomatrixserverlib.NewAuthEvents(stateEvents)
+	if err = gomatrixserverlib.Allowed(*e, &provider); err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client?
+		}
+	}
+	return e, nil
+}

clientapi/routing/sendtyping.go

@@ -0,0 +1,80 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"database/sql"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/util"
+)
+
+type typingContentJSON struct {
+	Typing  bool  `json:"typing"`
+	Timeout int64 `json:"timeout"`
+}
+
+// SendTyping handles PUT /rooms/{roomID}/typing/{userID}
+// sends the typing events to client API typingProducer
+func SendTyping(
+	req *http.Request, device *authtypes.Device, roomID string,
+	userID string, accountDB *accounts.Database,
+	typingProducer *producers.TypingServerProducer,
+) util.JSONResponse {
+	if device.UserID != userID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Cannot set another user's typing state"),
+		}
+	}
+
+	localpart, err := userutil.ParseUsernameParam(userID, nil)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	// Verify that the user is a member of this room
+	_, err = accountDB.GetMembershipInRoomByLocalpart(req.Context(), localpart, roomID)
+	if err == sql.ErrNoRows {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("User not in this room"),
+		}
+	} else if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	// parse the incoming http request
+	var r typingContentJSON
+	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	if resErr != nil {
+		return *resErr
+	}
+
+	if err = typingProducer.Send(
+		req.Context(), userID, roomID, r.Typing, r.Timeout,
+	); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/threepid.go

@@ -0,0 +1,178 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/threepid"
+	"github.com/matrix-org/dendrite/common/config"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+type reqTokenResponse struct {
+	SID string `json:"sid"`
+}
+
+type threePIDsResponse struct {
+	ThreePIDs []authtypes.ThreePID `json:"threepids"`
+}
+
+// RequestEmailToken implements:
+//     POST /account/3pid/email/requestToken
+//     POST /register/email/requestToken
+func RequestEmailToken(req *http.Request, accountDB *accounts.Database, cfg config.Dendrite) util.JSONResponse {
+	var body threepid.EmailAssociationRequest
+	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
+		return *reqErr
+	}
+
+	var resp reqTokenResponse
+	var err error
+
+	// Check if the 3PID is already in use locally
+	localpart, err := accountDB.GetLocalpartForThreePID(req.Context(), body.Email, "email")
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if len(localpart) > 0 {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MatrixError{
+				ErrCode: "M_THREEPID_IN_USE",
+				Err:     accounts.Err3PIDInUse.Error(),
+			},
+		}
+	}
+
+	resp.SID, err = threepid.CreateSession(req.Context(), body, cfg)
+	if err == threepid.ErrNotTrusted {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.NotTrusted(body.IDServer),
+		}
+	} else if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: resp,
+	}
+}
+
+// CheckAndSave3PIDAssociation implements POST /account/3pid
+func CheckAndSave3PIDAssociation(
+	req *http.Request, accountDB *accounts.Database, device *authtypes.Device,
+	cfg config.Dendrite,
+) util.JSONResponse {
+	var body threepid.EmailAssociationCheckRequest
+	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
+		return *reqErr
+	}
+
+	// Check if the association has been validated
+	verified, address, medium, err := threepid.CheckAssociation(req.Context(), body.Creds, cfg)
+	if err == threepid.ErrNotTrusted {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.NotTrusted(body.Creds.IDServer),
+		}
+	} else if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if !verified {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MatrixError{
+				ErrCode: "M_THREEPID_AUTH_FAILED",
+				Err:     "Failed to auth 3pid",
+			},
+		}
+	}
+
+	if body.Bind {
+		// Publish the association on the identity server if requested
+		err = threepid.PublishAssociation(body.Creds, device.UserID, cfg)
+		if err == threepid.ErrNotTrusted {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.NotTrusted(body.Creds.IDServer),
+			}
+		} else if err != nil {
+			return httputil.LogThenError(req, err)
+		}
+	}
+
+	// Save the association in the database
+	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if err = accountDB.SaveThreePIDAssociation(req.Context(), address, localpart, medium); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+// GetAssociated3PIDs implements GET /account/3pid
+func GetAssociated3PIDs(
+	req *http.Request, accountDB *accounts.Database, device *authtypes.Device,
+) util.JSONResponse {
+	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	threepids, err := accountDB.GetThreePIDsForLocalpart(req.Context(), localpart)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: threePIDsResponse{threepids},
+	}
+}
+
+// Forget3PID implements POST /account/3pid/delete
+func Forget3PID(req *http.Request, accountDB *accounts.Database) util.JSONResponse {
+	var body authtypes.ThreePID
+	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
+		return *reqErr
+	}
+
+	if err := accountDB.RemoveThreePIDAssociation(req.Context(), body.Address, body.Medium); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/voip.go

@@ -0,0 +1,78 @@
+// Copyright 2017 Michael Telatysnki <7t3chguy@gmail.com>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/util"
+)
+
+// RequestTurnServer implements:
+//     GET /voip/turnServer
+func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg config.Dendrite) util.JSONResponse {
+	turnConfig := cfg.TURN
+
+	// TODO Guest Support
+	if len(turnConfig.URIs) == 0 || turnConfig.UserLifetime == "" {
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: struct{}{},
+		}
+	}
+
+	// Duration checked at startup, err not possible
+	duration, _ := time.ParseDuration(turnConfig.UserLifetime)
+
+	resp := gomatrix.RespTurnServer{
+		URIs: turnConfig.URIs,
+		TTL:  int(duration.Seconds()),
+	}
+
+	if turnConfig.SharedSecret != "" {
+		expiry := time.Now().Add(duration).Unix()
+		mac := hmac.New(sha1.New, []byte(turnConfig.SharedSecret))
+		_, err := mac.Write([]byte(resp.Username))
+
+		if err != nil {
+			return httputil.LogThenError(req, err)
+		}
+
+		resp.Username = fmt.Sprintf("%d:%s", expiry, device.UserID)
+		resp.Password = base64.StdEncoding.EncodeToString(mac.Sum(nil))
+	} else if turnConfig.Username != "" && turnConfig.Password != "" {
+		resp.Username = turnConfig.Username
+		resp.Password = turnConfig.Password
+	} else {
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: struct{}{},
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: resp,
+	}
+}

clientapi/routing/whoami.go

@@ -0,0 +1,34 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/util"
+)
+
+// whoamiResponse represents an response for a `whoami` request
+type whoamiResponse struct {
+	UserID string `json:"user_id"`
+}
+
+// Whoami implements `/account/whoami` which enables client to query their account user id.
+// https://matrix.org/docs/spec/client_server/r0.3.0.html#get-matrix-client-r0-account-whoami
+func Whoami(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: whoamiResponse{UserID: device.UserID},
+	}
+}

clientapi/threepid/invites.go

@@ -0,0 +1,364 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package threepid
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// MembershipRequest represents the body of an incoming POST request
+// on /rooms/{roomID}/(join|kick|ban|unban|leave|invite)
+type MembershipRequest struct {
+	UserID   string `json:"user_id"`
+	Reason   string `json:"reason"`
+	IDServer string `json:"id_server"`
+	Medium   string `json:"medium"`
+	Address  string `json:"address"`
+}
+
+// idServerLookupResponse represents the response described at https://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-identity-api-v1-lookup
+type idServerLookupResponse struct {
+	TS         int64                        `json:"ts"`
+	NotBefore  int64                        `json:"not_before"`
+	NotAfter   int64                        `json:"not_after"`
+	Medium     string                       `json:"medium"`
+	Address    string                       `json:"address"`
+	MXID       string                       `json:"mxid"`
+	Signatures map[string]map[string]string `json:"signatures"`
+}
+
+// idServerLookupResponse represents the response described at https://matrix.org/docs/spec/client_server/r0.2.0.html#invitation-storage
+type idServerStoreInviteResponse struct {
+	PublicKey   string             `json:"public_key"`
+	Token       string             `json:"token"`
+	DisplayName string             `json:"display_name"`
+	PublicKeys  []common.PublicKey `json:"public_keys"`
+}
+
+var (
+	// ErrMissingParameter is the error raised if a request for 3PID invite has
+	// an incomplete body
+	ErrMissingParameter = errors.New("'address', 'id_server' and 'medium' must all be supplied")
+	// ErrNotTrusted is the error raised if an identity server isn't in the list
+	// of trusted servers in the configuration file.
+	ErrNotTrusted = errors.New("untrusted server")
+)
+
+// CheckAndProcessInvite analyses the body of an incoming membership request.
+// If the fields relative to a third-party-invite are all supplied, lookups the
+// matching Matrix ID from the given identity server. If no Matrix ID is
+// associated to the given 3PID, asks the identity server to store the invite
+// and emit a "m.room.third_party_invite" event.
+// Returns a representation of the HTTP response to send to the user.
+// Returns a representation of a non-200 HTTP response if something went wrong
+// in the process, or if some 3PID fields aren't supplied but others are.
+// If none of the 3PID-specific fields are supplied, or if a Matrix ID is
+// supplied by the identity server, returns nil to indicate that the request
+// must be processed as a non-3PID membership request. In the latter case,
+// fills the Matrix ID in the request body so a normal invite membership event
+// can be emitted.
+func CheckAndProcessInvite(
+	ctx context.Context,
+	device *authtypes.Device, body *MembershipRequest, cfg config.Dendrite,
+	queryAPI api.RoomserverQueryAPI, db *accounts.Database,
+	producer *producers.RoomserverProducer, membership string, roomID string,
+	evTime time.Time,
+) (inviteStoredOnIDServer bool, err error) {
+	if membership != "invite" || (body.Address == "" && body.IDServer == "" && body.Medium == "") {
+		// If none of the 3PID-specific fields are supplied, it's a standard invite
+		// so return nil for it to be processed as such
+		return
+	} else if body.Address == "" || body.IDServer == "" || body.Medium == "" {
+		// If at least one of the 3PID-specific fields is supplied but not all
+		// of them, return an error
+		err = ErrMissingParameter
+		return
+	}
+
+	lookupRes, storeInviteRes, err := queryIDServer(ctx, db, cfg, device, body, roomID)
+	if err != nil {
+		return
+	}
+
+	if lookupRes.MXID == "" {
+		// No Matrix ID could be found for this 3PID, meaning that a
+		// "m.room.third_party_invite" have to be emitted from the data in
+		// storeInviteRes.
+		err = emit3PIDInviteEvent(
+			ctx, body, storeInviteRes, device, roomID, cfg, queryAPI, producer, evTime,
+		)
+		inviteStoredOnIDServer = err == nil
+
+		return
+	}
+
+	// A Matrix ID have been found: set it in the body request and let the process
+	// continue to create a "m.room.member" event with an "invite" membership
+	body.UserID = lookupRes.MXID
+
+	return
+}
+
+// queryIDServer handles all the requests to the identity server, starting by
+// looking up the given 3PID on the given identity server.
+// If the lookup returned a Matrix ID, checks if the current time is within the
+// time frame in which the 3PID-MXID association is known to be valid, and checks
+// the response's signatures. If one of the checks fails, returns an error.
+// If the lookup didn't return a Matrix ID, asks the identity server to store
+// the invite and to respond with a token.
+// Returns a representation of the response for both cases.
+// Returns an error if a check or a request failed.
+func queryIDServer(
+	ctx context.Context,
+	db *accounts.Database, cfg config.Dendrite, device *authtypes.Device,
+	body *MembershipRequest, roomID string,
+) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) {
+	if err = isTrusted(body.IDServer, cfg); err != nil {
+		return
+	}
+
+	// Lookup the 3PID
+	lookupRes, err = queryIDServerLookup(ctx, body)
+	if err != nil {
+		return
+	}
+
+	if lookupRes.MXID == "" {
+		// No Matrix ID matches with the given 3PID, ask the server to store the
+		// invite and return a token
+		storeInviteRes, err = queryIDServerStoreInvite(ctx, db, cfg, device, body, roomID)
+		return
+	}
+
+	// A Matrix ID matches with the given 3PID
+	// Get timestamp in milliseconds to compare it with the timestamps provided
+	// by the identity server
+	now := time.Now().UnixNano() / 1000000
+	if lookupRes.NotBefore > now || now > lookupRes.NotAfter {
+		// If the current timestamp isn't in the time frame in which the association
+		// is known to be valid, re-run the query
+		return queryIDServer(ctx, db, cfg, device, body, roomID)
+	}
+
+	// Check the request signatures and send an error if one isn't valid
+	if err = checkIDServerSignatures(ctx, body, lookupRes); err != nil {
+		return
+	}
+
+	return
+}
+
+// queryIDServerLookup sends a response to the identity server on /_matrix/identity/api/v1/lookup
+// and returns the response as a structure.
+// Returns an error if the request failed to send or if the response couldn't be parsed.
+func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServerLookupResponse, error) {
+	address := url.QueryEscape(body.Address)
+	requestURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/lookup?medium=%s&address=%s", body.IDServer, body.Medium, address)
+	req, err := http.NewRequest(http.MethodGet, requestURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		// TODO: Log the error supplied with the identity server?
+		errMgs := fmt.Sprintf("Failed to ask %s to store an invite for %s", body.IDServer, body.Address)
+		return nil, errors.New(errMgs)
+	}
+
+	var res idServerLookupResponse
+	err = json.NewDecoder(resp.Body).Decode(&res)
+	return &res, err
+}
+
+// queryIDServerStoreInvite sends a response to the identity server on /_matrix/identity/api/v1/store-invite
+// and returns the response as a structure.
+// Returns an error if the request failed to send or if the response couldn't be parsed.
+func queryIDServerStoreInvite(
+	ctx context.Context,
+	db *accounts.Database, cfg config.Dendrite, device *authtypes.Device,
+	body *MembershipRequest, roomID string,
+) (*idServerStoreInviteResponse, error) {
+	// Retrieve the sender's profile to get their display name
+	localpart, serverName, err := gomatrixserverlib.SplitID('@', device.UserID)
+	if err != nil {
+		return nil, err
+	}
+
+	var profile *authtypes.Profile
+	if serverName == cfg.Matrix.ServerName {
+		profile, err = db.GetProfileByLocalpart(ctx, localpart)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		profile = &authtypes.Profile{}
+	}
+
+	client := http.Client{}
+
+	data := url.Values{}
+	data.Add("medium", body.Medium)
+	data.Add("address", body.Address)
+	data.Add("room_id", roomID)
+	data.Add("sender", device.UserID)
+	data.Add("sender_display_name", profile.DisplayName)
+	// TODO: Also send:
+	//      - The room name (room_name)
+	//      - The room's avatar url (room_avatar_url)
+	//      See https://github.com/matrix-org/sydent/blob/master/sydent/http/servlets/store_invite_servlet.py#L82-L91
+	//      These can be easily retrieved by requesting the public rooms API
+	//      server's database.
+
+	requestURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/store-invite", body.IDServer)
+	req, err := http.NewRequest(http.MethodPost, requestURL, strings.NewReader(data.Encode()))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		errMsg := fmt.Sprintf("Identity server %s responded with a %d error code", body.IDServer, resp.StatusCode)
+		return nil, errors.New(errMsg)
+	}
+
+	var idResp idServerStoreInviteResponse
+	err = json.NewDecoder(resp.Body).Decode(&idResp)
+	return &idResp, err
+}
+
+// queryIDServerPubKey requests a public key identified with a given ID to the
+// a given identity server and returns the matching base64-decoded public key.
+// We assume that the ID server is trusted at this point.
+// Returns an error if the request couldn't be sent, if its body couldn't be parsed
+// or if the key couldn't be decoded from base64.
+func queryIDServerPubKey(ctx context.Context, idServerName string, keyID string) ([]byte, error) {
+	requestURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/pubkey/%s", idServerName, keyID)
+	req, err := http.NewRequest(http.MethodGet, requestURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+
+	var pubKeyRes struct {
+		PublicKey gomatrixserverlib.Base64String `json:"public_key"`
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		errMsg := fmt.Sprintf("Couldn't retrieve key %s from server %s", keyID, idServerName)
+		return nil, errors.New(errMsg)
+	}
+
+	err = json.NewDecoder(resp.Body).Decode(&pubKeyRes)
+	return pubKeyRes.PublicKey, err
+}
+
+// checkIDServerSignatures iterates over the signatures of a requests.
+// If no signature can be found for the ID server's domain, returns an error, else
+// iterates over the signature for the said domain, retrieves the matching public
+// key, and verify it.
+// We assume that the ID server is trusted at this point.
+// Returns nil if all the verifications succeeded.
+// Returns an error if something failed in the process.
+func checkIDServerSignatures(
+	ctx context.Context, body *MembershipRequest, res *idServerLookupResponse,
+) error {
+	// Mashall the body so we can give it to VerifyJSON
+	marshalledBody, err := json.Marshal(*res)
+	if err != nil {
+		return err
+	}
+
+	signatures, ok := res.Signatures[body.IDServer]
+	if !ok {
+		return errors.New("No signature for domain " + body.IDServer)
+	}
+
+	for keyID := range signatures {
+		pubKey, err := queryIDServerPubKey(ctx, body.IDServer, keyID)
+		if err != nil {
+			return err
+		}
+		if err = gomatrixserverlib.VerifyJSON(body.IDServer, gomatrixserverlib.KeyID(keyID), pubKey, marshalledBody); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// emit3PIDInviteEvent builds and sends a "m.room.third_party_invite" event.
+// Returns an error if something failed in the process.
+func emit3PIDInviteEvent(
+	ctx context.Context,
+	body *MembershipRequest, res *idServerStoreInviteResponse,
+	device *authtypes.Device, roomID string, cfg config.Dendrite,
+	queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer,
+	evTime time.Time,
+) error {
+	builder := &gomatrixserverlib.EventBuilder{
+		Sender:   device.UserID,
+		RoomID:   roomID,
+		Type:     "m.room.third_party_invite",
+		StateKey: &res.Token,
+	}
+
+	validityURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/pubkey/isvalid", body.IDServer)
+	content := common.ThirdPartyInviteContent{
+		DisplayName:    res.DisplayName,
+		KeyValidityURL: validityURL,
+		PublicKey:      res.PublicKey,
+		PublicKeys:     res.PublicKeys,
+	}
+
+	if err := builder.SetContent(content); err != nil {
+		return err
+	}
+
+	var queryRes *api.QueryLatestEventsAndStateResponse
+	event, err := common.BuildEvent(ctx, builder, cfg, evTime, queryAPI, queryRes)
+	if err != nil {
+		return err
+	}
+
+	_, err = producer.SendEvents(ctx, []gomatrixserverlib.Event{*event}, cfg.Matrix.ServerName, nil)
+	return err
+}

clientapi/threepid/threepid.go

@@ -0,0 +1,187 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package threepid
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/matrix-org/dendrite/common/config"
+)
+
+// EmailAssociationRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-register-email-requesttoken
+type EmailAssociationRequest struct {
+	IDServer    string `json:"id_server"`
+	Secret      string `json:"client_secret"`
+	Email       string `json:"email"`
+	SendAttempt int    `json:"send_attempt"`
+}
+
+// EmailAssociationCheckRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-account-3pid
+type EmailAssociationCheckRequest struct {
+	Creds Credentials `json:"threePidCreds"`
+	Bind  bool        `json:"bind"`
+}
+
+// Credentials represents the "ThreePidCredentials" structure defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-account-3pid
+type Credentials struct {
+	SID      string `json:"sid"`
+	IDServer string `json:"id_server"`
+	Secret   string `json:"client_secret"`
+}
+
+// CreateSession creates a session on an identity server.
+// Returns the session's ID.
+// Returns an error if there was a problem sending the request or decoding the
+// response, or if the identity server responded with a non-OK status.
+func CreateSession(
+	ctx context.Context, req EmailAssociationRequest, cfg config.Dendrite,
+) (string, error) {
+	if err := isTrusted(req.IDServer, cfg); err != nil {
+		return "", err
+	}
+
+	// Create a session on the ID server
+	postURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/validate/email/requestToken", req.IDServer)
+
+	data := url.Values{}
+	data.Add("client_secret", req.Secret)
+	data.Add("email", req.Email)
+	data.Add("send_attempt", strconv.Itoa(req.SendAttempt))
+
+	request, err := http.NewRequest(http.MethodPost, postURL, strings.NewReader(data.Encode()))
+	if err != nil {
+		return "", err
+	}
+	request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+
+	client := http.Client{}
+	resp, err := client.Do(request.WithContext(ctx))
+	if err != nil {
+		return "", err
+	}
+
+	// Error if the status isn't OK
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("Could not create a session on the server %s", req.IDServer)
+	}
+
+	// Extract the SID from the response and return it
+	var sid struct {
+		SID string `json:"sid"`
+	}
+	err = json.NewDecoder(resp.Body).Decode(&sid)
+
+	return sid.SID, err
+}
+
+// CheckAssociation checks the status of an ongoing association validation on an
+// identity server.
+// Returns a boolean set to true if the association has been validated, false if not.
+// If the association has been validated, also returns the related third-party
+// identifier and its medium.
+// Returns an error if there was a problem sending the request or decoding the
+// response, or if the identity server responded with a non-OK status.
+func CheckAssociation(
+	ctx context.Context, creds Credentials, cfg config.Dendrite,
+) (bool, string, string, error) {
+	if err := isTrusted(creds.IDServer, cfg); err != nil {
+		return false, "", "", err
+	}
+
+	requestURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/3pid/getValidated3pid?sid=%s&client_secret=%s", creds.IDServer, creds.SID, creds.Secret)
+	req, err := http.NewRequest(http.MethodGet, requestURL, nil)
+	if err != nil {
+		return false, "", "", err
+	}
+	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return false, "", "", err
+	}
+
+	var respBody struct {
+		Medium      string `json:"medium"`
+		ValidatedAt int64  `json:"validated_at"`
+		Address     string `json:"address"`
+		ErrCode     string `json:"errcode"`
+		Error       string `json:"error"`
+	}
+
+	if err = json.NewDecoder(resp.Body).Decode(&respBody); err != nil {
+		return false, "", "", err
+	}
+
+	if respBody.ErrCode == "M_SESSION_NOT_VALIDATED" {
+		return false, "", "", nil
+	} else if len(respBody.ErrCode) > 0 {
+		return false, "", "", errors.New(respBody.Error)
+	}
+
+	return true, respBody.Address, respBody.Medium, nil
+}
+
+// PublishAssociation publishes a validated association between a third-party
+// identifier and a Matrix ID.
+// Returns an error if there was a problem sending the request or decoding the
+// response, or if the identity server responded with a non-OK status.
+func PublishAssociation(creds Credentials, userID string, cfg config.Dendrite) error {
+	if err := isTrusted(creds.IDServer, cfg); err != nil {
+		return err
+	}
+
+	postURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/3pid/bind", creds.IDServer)
+
+	data := url.Values{}
+	data.Add("sid", creds.SID)
+	data.Add("client_secret", creds.Secret)
+	data.Add("mxid", userID)
+
+	request, err := http.NewRequest(http.MethodPost, postURL, strings.NewReader(data.Encode()))
+	if err != nil {
+		return err
+	}
+	request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+
+	client := http.Client{}
+	resp, err := client.Do(request)
+	if err != nil {
+		return err
+	}
+
+	// Error if the status isn't OK
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("Could not publish the association on the server %s", creds.IDServer)
+	}
+
+	return nil
+}
+
+// isTrusted checks if a given identity server is part of the list of trusted
+// identity servers in the configuration file.
+// Returns an error if the server isn't trusted.
+func isTrusted(idServer string, cfg config.Dendrite) error {
+	for _, server := range cfg.Matrix.TrustedIDServers {
+		if idServer == server {
+			return nil
+		}
+	}
+	return ErrNotTrusted
+}

clientapi/userutil/userutil.go

@@ -0,0 +1,49 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package userutil
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// ParseUsernameParam extracts localpart from usernameParam.
+// usernameParam can either be a user ID or just the localpart/username.
+// If serverName is passed, it is verified against the domain obtained from usernameParam (if present)
+// Returns error in case of invalid usernameParam.
+func ParseUsernameParam(usernameParam string, expectedServerName *gomatrixserverlib.ServerName) (string, error) {
+	localpart := usernameParam
+
+	if strings.HasPrefix(usernameParam, "@") {
+		lp, domain, err := gomatrixserverlib.SplitID('@', usernameParam)
+
+		if err != nil {
+			return "", errors.New("Invalid username")
+		}
+
+		if expectedServerName != nil && domain != *expectedServerName {
+			return "", errors.New("User ID does not belong to this server")
+		}
+
+		localpart = lp
+	}
+	return localpart, nil
+}
+
+// MakeUserID generates user ID from localpart & server name
+func MakeUserID(localpart string, server gomatrixserverlib.ServerName) string {
+	return fmt.Sprintf("@%s:%s", localpart, string(server))
+}

clientapi/userutil/userutil_test.go

@@ -0,0 +1,71 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package userutil
+
+import (
+	"testing"
+
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+var (
+	localpart                                      = "somelocalpart"
+	serverName        gomatrixserverlib.ServerName = "someservername"
+	invalidServerName gomatrixserverlib.ServerName = "invalidservername"
+	goodUserID                                     = "@" + localpart + ":" + string(serverName)
+	badUserID                                      = "@bad:user:name@noservername:"
+)
+
+// TestGoodUserID checks that correct localpart is returned for a valid user ID.
+func TestGoodUserID(t *testing.T) {
+	lp, err := ParseUsernameParam(goodUserID, &serverName)
+
+	if err != nil {
+		t.Error("User ID Parsing failed for ", goodUserID, " with error: ", err.Error())
+	}
+
+	if lp != localpart {
+		t.Error("Incorrect username, returned: ", lp, " should be: ", localpart)
+	}
+}
+
+// TestWithLocalpartOnly checks that localpart is returned when usernameParam contains only localpart.
+func TestWithLocalpartOnly(t *testing.T) {
+	lp, err := ParseUsernameParam(localpart, &serverName)
+
+	if err != nil {
+		t.Error("User ID Parsing failed for ", localpart, " with error: ", err.Error())
+	}
+
+	if lp != localpart {
+		t.Error("Incorrect username, returned: ", lp, " should be: ", localpart)
+	}
+}
+
+// TestIncorrectDomain checks for error when there's server name mismatch.
+func TestIncorrectDomain(t *testing.T) {
+	_, err := ParseUsernameParam(goodUserID, &invalidServerName)
+
+	if err == nil {
+		t.Error("Invalid Domain should return an error")
+	}
+}
+
+// TestBadUserID checks that ParseUsernameParam fails for invalid user ID
+func TestBadUserID(t *testing.T) {
+	_, err := ParseUsernameParam(badUserID, &serverName)
+
+	if err == nil {
+		t.Error("Illegal User ID should return an error")
+	}
+}