Merge branch 'master' of github.com:matrix-org/dendrite into erikj/sync_txnid

docs/opentracing.md

@@ -0,0 +1,112 @@
+Opentracing
+===========
+
+Dendrite extensively uses the [opentracing.io](http://opentracing.io) framework
+to trace work across the different logical components.
+
+At its most basic opentracing tracks "spans" of work; recording start and end
+times as well as any parent span that caused the piece of work.
+
+A typical example would be a new span being created on an incoming request that
+finishes when the response is sent. When the code needs to hit out to a
+different component a new span is created with the initial span as its parent.
+This would end up looking roughly like:
+
+```
+Received request                             Sent response
+       |<───────────────────────────────────────>|
+                 |<────────────────────>|
+            RPC call            RPC call returns
+```
+
+This is useful to see where the time is being spent processing a request on a
+component. However, opentracing allows tracking of spans across components. This
+makes it possible to see exactly what work goes into processing a request:
+
+
+```
+Component 1       |<─────────────────── HTTP ────────────────────>|
+                     |<──────────────── RPC ─────────────────>|
+Component 2                |<─ SQL ─>|     |<── RPC ───>|
+Component 3                                  |<─ SQL ─>|
+```
+
+This is achieved by serializing span information during all communication
+between components. For HTTP requests, this is achieved by the sender
+serializing the span into a HTTP header, and the receiver deserializing the span
+on receipt. (Generally a new span is then immediately created with the
+deserialized span as the parent).
+
+A collection of spans that are related is called a trace.
+
+
+Spans are passed through the code via contexts, rather than manually. It is
+therefore important that all spans that are created are immediately added to the
+current context. Thankfully the opentracing library gives helper functions for
+doing this:
+
+```golang
+span, ctx := opentracing.StartSpanFromContext(ctx, spanName)
+defer span.Finish()
+```
+
+This will create a new span, adding any span already in `ctx` as a parent to the
+new span.
+
+
+Adding Information
+------------------
+
+Opentracing allows adding information to a trace via three mechanisms:
+- "tags" ─ A span can be tagged with a key/value pair. This is typically
+  information that relates to the span, e.g. for spans created for incoming HTTP
+  requests could include the request path and response codes as tags, spans for
+  SQL could include the query being executed.
+- "logs" ─ Key/value pairs can be looged at a particular instance in a trace.
+  This can be useful to log e.g. any errors that happen.
+- "baggage" ─ Arbitrary key/value pairs can be added to a span to which all
+  child spans have access. Baggage isn't saved and so isn't available when
+  inspecting the traces, but can be used to add context to logs or tags in child
+  spans.
+
+
+See
+[specification.md](https://github.com/opentracing/specification/blob/master/specification.md)
+for some of the common tags and log fields used.
+
+
+Span Relationships
+------------------
+
+Spans can be related to each other. The most common relation is `childOf`, which
+indicates the child span somehow depends on the parent span ─ typically the
+parent span cannot complete until all child spans are completed.
+
+A second relation type is `followsFrom`, where the parent has no dependence on
+the child span. This usually indicates some sort of fire and forget behaviour,
+e.g. adding a message to a pipeline or inserting into a kafka topic.
+
+
+Jaeger
+------
+
+Opentracing is just a framework. We use
+[jaeger](https://github.com/jaegertracing/jaeger) as the actual implementation.
+
+Jaeger is responsible for recording, sending and saving traces, as well as
+giving a UI for viewing and interacting with traces.
+
+To enable jaeger a `Tracer` object must be instansiated from the config (as well
+as having a jaeger server running somewhere, usually locally). A `Tracer` does
+several things:
+- Decides which traces to save and send to the server. There are multiple
+  schemes for doing this, with a simple example being to save a certain fraction
+  of traces.
+- Communicating with the jaeger backend. If not explicitly specified uses the
+  default port on localhost.
+- Associates a service name to all spans created by the tracer. This service
+  name equates to a logical component, e.g. spans created by clientapi will have
+  a different service name than ones created by the syncapi. Database access
+  will also typically use a different service name.
+
+  This means that there is a tracer per service name/component.

src/github.com/matrix-org/dendrite/clientapi/auth/authtypes/logintypes.go

@@ -7,4 +7,5 @@ type LoginType string
 const (
 	LoginTypeDummy        = "m.login.dummy"
 	LoginTypeSharedSecret = "org.matrix.login.shared_secret"
+	LoginTypeRecaptcha    = "m.login.recaptcha"
 )

src/github.com/matrix-org/dendrite/clientapi/routing/register.go

@@ -19,12 +19,16 @@ import (
 	"context"
 	"crypto/hmac"
 	"crypto/sha1"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net/http"
+	"net/url"
 	"regexp"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/matrix-org/dendrite/common/config"
 
@@ -74,6 +78,8 @@ type authDict struct {
 	Session string                      `json:"session"`
 	Mac     gomatrixserverlib.HexString `json:"mac"`
 
+	// Recaptcha
+	Response string `json:"response"`
 	// TODO: Lots of custom keys depending on the type
 }
 
@@ -114,6 +120,14 @@ type registerResponse struct {
 	DeviceID    string                       `json:"device_id"`
 }
 
+// recaptchaResponse represents the HTTP response from a Google Recaptcha server
+type recaptchaResponse struct {
+	Success     bool      `json:"success"`
+	ChallengeTS time.Time `json:"challenge_ts"`
+	Hostname    string    `json:"hostname"`
+	ErrorCodes  []int     `json:"error-codes"`
+}
+
 // validateUserName returns an error response if the username is invalid
 func validateUserName(username string) *util.JSONResponse {
 	// https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
@@ -153,6 +167,72 @@ func validatePassword(password string) *util.JSONResponse {
 	return nil
 }
 
+// validateRecaptcha returns an error response if the captcha response is invalid
+func validateRecaptcha(
+	cfg *config.Dendrite,
+	response string,
+	clientip string,
+) *util.JSONResponse {
+	if !cfg.Matrix.RecaptchaEnabled {
+		return &util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON("Captcha registration is disabled"),
+		}
+	}
+
+	if response == "" {
+		return &util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON("Captcha response is required"),
+		}
+	}
+
+	// Make a POST request to Google's API to check the captcha response
+	resp, err := http.PostForm(cfg.Matrix.RecaptchaSiteVerifyAPI,
+		url.Values{
+			"secret":   {cfg.Matrix.RecaptchaPrivateKey},
+			"response": {response},
+			"remoteip": {clientip},
+		},
+	)
+
+	if err != nil {
+		return &util.JSONResponse{
+			Code: 500,
+			JSON: jsonerror.BadJSON("Error in requesting validation of captcha response"),
+		}
+	}
+
+	// Close the request once we're finishing reading from it
+	defer resp.Body.Close() // nolint: errcheck
+
+	// Grab the body of the response from the captcha server
+	var r recaptchaResponse
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return &util.JSONResponse{
+			Code: 500,
+			JSON: jsonerror.BadJSON("Error in contacting captcha server" + err.Error()),
+		}
+	}
+	err = json.Unmarshal(body, &r)
+	if err != nil {
+		return &util.JSONResponse{
+			Code: 500,
+			JSON: jsonerror.BadJSON("Error in unmarshaling captcha server's response: " + err.Error()),
+		}
+	}
+
+	// Check that we received a "success"
+	if !r.Success {
+		return &util.JSONResponse{
+			Code: 401,
+			JSON: jsonerror.BadJSON("Invalid captcha response. Please try again."),
+		}
+	}
+	return nil
+}
+
 // Register processes a /register request. http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
 func Register(
 	req *http.Request,
@@ -221,26 +301,30 @@ func handleRegistrationFlow(
 	// TODO: Handle loading of previous session parameters from database.
 	// TODO: Handle mapping registrationRequest parameters into session parameters
 
-	// TODO: email / msisdn / recaptcha auth types.
+	// TODO: email / msisdn auth types.
 
 	if cfg.Matrix.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret {
 		return util.MessageResponse(403, "Registration has been disabled")
 	}
 
 	switch r.Auth.Type {
-	case authtypes.LoginTypeSharedSecret:
+	case authtypes.LoginTypeRecaptcha:
-		if cfg.Matrix.RegistrationSharedSecret == "" {
+		// Check given captcha response
-			return util.MessageResponse(400, "Shared secret registration is disabled")
+		resErr := validateRecaptcha(cfg, r.Auth.Response, req.RemoteAddr)
+		if resErr != nil {
+			return *resErr
 		}
 
-		valid, err := isValidMacLogin(r.Username, r.Password, r.Admin,
+		// Add Recaptcha to the list of completed registration stages
-			r.Auth.Mac, cfg.Matrix.RegistrationSharedSecret)
+		sessions[sessionID] = append(sessions[sessionID], authtypes.LoginTypeRecaptcha)
+
+	case authtypes.LoginTypeSharedSecret:
+		// Check shared secret against config
+		valid, err := isValidMacLogin(cfg, r.Username, r.Password, r.Admin, r.Auth.Mac)
 
 		if err != nil {
 			return httputil.LogThenError(req, err)
-		}
+		} else if !valid {
-
-		if !valid {
 			return util.MessageResponse(403, "HMAC incorrect")
 		}
 
@@ -303,7 +387,7 @@ func LegacyRegister(
 			return util.MessageResponse(400, "Shared secret registration is disabled")
 		}
 
-		valid, err := isValidMacLogin(r.Username, r.Password, r.Admin, r.Mac, cfg.Matrix.RegistrationSharedSecret)
+		valid, err := isValidMacLogin(cfg, r.Username, r.Password, r.Admin, r.Mac)
 		if err != nil {
 			return httputil.LogThenError(req, err)
 		}
@@ -412,11 +496,18 @@ func completeRegistration(
 // Used for shared secret registration.
 // Checks if the username, password and isAdmin flag matches the given mac.
 func isValidMacLogin(
+	cfg *config.Dendrite,
 	username, password string,
 	isAdmin bool,
 	givenMac []byte,
-	sharedSecret string,
 ) (bool, error) {
+	sharedSecret := cfg.Matrix.RegistrationSharedSecret
+
+	// Check that shared secret registration isn't disabled.
+	if cfg.Matrix.RegistrationSharedSecret == "" {
+		return false, errors.New("Shared secret registration is disabled")
+	}
+
 	// Double check that username/password don't contain the HMAC delimiters. We should have
 	// already checked this.
 	if strings.Contains(username, "\x00") {

src/github.com/matrix-org/dendrite/clientapi/routing/register_test.go

@@ -22,15 +22,15 @@ import (
 
 var (
 	// Registration Flows that the server allows.
-	allowedFlows []authtypes.Flow = []authtypes.Flow{
+	allowedFlows = []authtypes.Flow{
 		{
-			[]authtypes.LoginType{
+			Stages: []authtypes.LoginType{
 				authtypes.LoginType("stage1"),
 				authtypes.LoginType("stage2"),
 			},
 		},
 		{
-			[]authtypes.LoginType{
+			Stages: []authtypes.LoginType{
 				authtypes.LoginType("stage1"),
 				authtypes.LoginType("stage3"),
 			},

src/github.com/matrix-org/dendrite/cmd/dendrite-client-api-server/main.go

@@ -121,7 +121,7 @@ func main() {
 		queryAPI, aliasAPI, accountDB, deviceDB, federation, keyRing,
 		userUpdateProducer, syncProducer,
 	)
-	common.SetupHTTPAPI(http.DefaultServeMux, api)
+	common.SetupHTTPAPI(http.DefaultServeMux, common.WrapHandlerInCORS(api))
 
 	log.Fatal(http.ListenAndServe(string(cfg.Listen.ClientAPI), nil))
 }

src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go

@@ -70,7 +70,7 @@ func main() {
 
 	api := mux.NewRouter()
 	routing.Setup(api, cfg, db, deviceDB, client)
-	common.SetupHTTPAPI(http.DefaultServeMux, api)
+	common.SetupHTTPAPI(http.DefaultServeMux, common.WrapHandlerInCORS(api))
 
 	log.Fatal(http.ListenAndServe(string(cfg.Listen.MediaAPI), nil))
 }

src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go

@@ -103,7 +103,7 @@ func main() {
 	// Expose the matrix APIs directly rather than putting them under a /api path.
 	go func() {
 		log.Info("Listening on ", *httpBindAddr)
-		log.Fatal(http.ListenAndServe(*httpBindAddr, m.api))
+		log.Fatal(http.ListenAndServe(*httpBindAddr, common.WrapHandlerInCORS(m.api)))
 	}()
 	// Handle HTTPS if certificate and key are provided
 	go func() {

src/github.com/matrix-org/dendrite/cmd/dendrite-public-rooms-api-server/main.go

@@ -85,7 +85,7 @@ func main() {
 
 	api := mux.NewRouter()
 	routing.Setup(api, deviceDB, db)
-	common.SetupHTTPAPI(http.DefaultServeMux, api)
+	common.SetupHTTPAPI(http.DefaultServeMux, common.WrapHandlerInCORS(api))
 
 	log.Fatal(http.ListenAndServe(string(cfg.Listen.PublicRoomsAPI), nil))
 }

src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go

@@ -105,7 +105,7 @@ func main() {
 
 	api := mux.NewRouter()
 	routing.Setup(api, sync.NewRequestPool(db, n, adb), db, deviceDB)
-	common.SetupHTTPAPI(http.DefaultServeMux, api)
+	common.SetupHTTPAPI(http.DefaultServeMux, common.WrapHandlerInCORS(api))
 
 	log.Fatal(http.ListenAndServe(string(cfg.Listen.SyncAPI), nil))
 }

src/github.com/matrix-org/dendrite/common/config/config.go

@@ -83,6 +83,18 @@ type Dendrite struct {
 		// If set, allows registration by anyone who also has the shared
 		// secret, even if registration is otherwise disabled.
 		RegistrationSharedSecret string `yaml:"registration_shared_secret"`
+		// This Home Server's ReCAPTCHA public key.
+		RecaptchaPublicKey string `yaml:"recaptcha_public_key"`
+		// This Home Server's ReCAPTCHA private key.
+		RecaptchaPrivateKey string `yaml:"recaptcha_private_key"`
+		// Boolean stating whether catpcha registration is enabled
+		// and required
+		RecaptchaEnabled bool `yaml:"enable_registration_captcha"`
+		// Secret used to bypass the captcha registration entirely
+		RecaptchaBypassSecret string `yaml:"captcha_bypass_secret"`
+		// HTTP API endpoint used to verify whether the captcha response
+		// was successful
+		RecaptchaSiteVerifyAPI string `yaml:"recaptcha_siteverify_api"`
 		// If set disables new users from registering (except via shared
 		// secrets)
 		RegistrationDisabled bool `yaml:"registration_disabled"`
@@ -339,10 +351,15 @@ func (config *Dendrite) derive() {
 
 	// TODO: Add email auth type
 	// TODO: Add MSISDN auth type
-	// TODO: Add Recaptcha auth type
 
-	config.Derived.Registration.Flows = append(config.Derived.Registration.Flows,
+	if config.Matrix.RecaptchaEnabled {
-		authtypes.Flow{[]authtypes.LoginType{authtypes.LoginTypeDummy}})
+		config.Derived.Registration.Params[authtypes.LoginTypeRecaptcha] = map[string]string{"public_key": config.Matrix.RecaptchaPublicKey}
+		config.Derived.Registration.Flows = append(config.Derived.Registration.Flows,
+			authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeRecaptcha}})
+	} else {
+		config.Derived.Registration.Flows = append(config.Derived.Registration.Flows,
+			authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeDummy}})
+	}
 }
 
 // setDefaults sets default config values if they are not explicitly set.

src/github.com/matrix-org/dendrite/common/httpapi.go

@@ -4,7 +4,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -87,8 +86,29 @@ func MakeFedAPI(
 
 // SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics
 // listener.
-func SetupHTTPAPI(servMux *http.ServeMux, apiMux *mux.Router) {
+func SetupHTTPAPI(servMux *http.ServeMux, apiMux http.Handler) {
 	// This is deprecated.
 	servMux.Handle("/metrics", prometheus.Handler()) // nolint: megacheck, staticcheck
 	servMux.Handle("/api/", http.StripPrefix("/api", apiMux))
 }
+
+// WrapHandlerInCORS adds CORS headers to all responses, including all error
+// responses.
+// Handles OPTIONS requests directly.
+func WrapHandlerInCORS(h http.Handler) http.HandlerFunc {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
+		w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization")
+
+		if r.Method == "OPTIONS" && r.Header.Get("Access-Control-Request-Method") != "" {
+			// Its easiest just to always return a 200 OK for everything. Whether
+			// this is technically correct or not is a question, but in the end this
+			// is what a lot of other people do (including synapse) and the clients
+			// are perfectly happy with it.
+			w.WriteHeader(http.StatusOK)
+		} else {
+			h.ServeHTTP(w, r)
+		}
+	})
+}

src/github.com/matrix-org/dendrite/syncapi/storage/output_room_events_table.go

@@ -46,9 +46,9 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events (
     -- A list of event IDs which represent a delta of added/removed room state. This can be NULL
     -- if there is no delta.
     add_state_ids TEXT[],
-	remove_state_ids TEXT[],
+    remove_state_ids TEXT[],
-	device_id TEXT,  -- The local device that sent the event, if any
+    device_id TEXT,  -- The local device that sent the event, if any
-	transaction_id TEXT  -- The transaction id used to send the event, if any
+    transaction_id TEXT  -- The transaction id used to send the event, if any
 );
 -- for event selection
 CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_output_room_events(event_id);