Add push server component template

pushserver/api/api.go

@@ -0,0 +1,14 @@
+package api
+
+import "context"
+
+type PushserverInternalAPI interface {
+	QueryExample(
+		ctx context.Context,
+		request *QueryExampleRequest,
+		response *QueryExampleResponse,
+	) error
+}
+
+type QueryExampleRequest struct{}
+type QueryExampleResponse struct{}

pushserver/consumers/roomserver.go

@@ -0,0 +1,82 @@
+package consumers
+
+import (
+	"encoding/json"
+
+	"github.com/Shopify/sarama"
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/pushserver/api"
+	"github.com/matrix-org/dendrite/pushserver/storage"
+	rsapi "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/setup/process"
+	log "github.com/sirupsen/logrus"
+)
+
+type OutputRoomEventConsumer struct {
+	cfg        *config.PushServer
+	psAPI      api.PushserverInternalAPI
+	rsConsumer *internal.ContinualConsumer
+	db         storage.Database
+}
+
+func NewOutputRoomEventConsumer(
+	process *process.ProcessContext,
+	cfg *config.PushServer,
+	kafkaConsumer sarama.Consumer,
+	store storage.Database,
+	psAPI api.PushserverInternalAPI,
+) *OutputRoomEventConsumer {
+	consumer := internal.ContinualConsumer{
+		Process:        process,
+		ComponentName:  "pushserver/roomserver",
+		Topic:          string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputRoomEvent)),
+		Consumer:       kafkaConsumer,
+		PartitionStore: store,
+	}
+	s := &OutputRoomEventConsumer{
+		cfg:        cfg,
+		rsConsumer: &consumer,
+		db:         store,
+		psAPI:      psAPI,
+	}
+	consumer.ProcessMessage = s.onMessage
+	return s
+}
+
+func (s *OutputRoomEventConsumer) Start() error {
+	return s.rsConsumer.Start()
+}
+
+func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
+	var output rsapi.OutputEvent
+	if err := json.Unmarshal(msg.Value, &output); err != nil {
+		log.WithError(err).Errorf("roomserver output log: message parse failure")
+		return nil
+	}
+
+	switch output.Type {
+	case rsapi.OutputTypeNewRoomEvent:
+		ev := output.NewRoomEvent.Event
+		if err := s.processMessage(*output.NewRoomEvent); err != nil {
+			// panic rather than continue with an inconsistent database
+			log.WithFields(log.Fields{
+				"event_id":   ev.EventID(),
+				"event":      string(ev.JSON()),
+				log.ErrorKey: err,
+			}).Panicf("roomserver output log: write room event failure")
+			return err
+		}
+
+	default:
+		// Ignore old events, peeks, so on.
+	}
+
+	return nil
+}
+
+func (s *OutputRoomEventConsumer) processMessage(ore rsapi.OutputNewRoomEvent) error {
+	// TODO: New events from the roomserver will be passed here.
+
+	return nil
+}

pushserver/internal/api.go

@@ -0,0 +1,36 @@
+package internal
+
+import (
+	"context"
+
+	"github.com/matrix-org/dendrite/pushserver/api"
+	"github.com/matrix-org/dendrite/pushserver/storage"
+	"github.com/matrix-org/dendrite/setup/config"
+)
+
+// PushserverInternalAPI implements api.PushserverInternalAPI
+type PushserverInternalAPI struct {
+	DB  storage.Database
+	Cfg *config.PushServer
+}
+
+func NewPushserverAPI(
+	cfg *config.PushServer, pushserverDB storage.Database,
+) *PushserverInternalAPI {
+	a := &PushserverInternalAPI{
+		DB:  pushserverDB,
+		Cfg: cfg,
+	}
+	return a
+}
+
+// SetRoomAlias implements RoomserverAliasAPI
+func (p *PushserverInternalAPI) QueryExample(
+	ctx context.Context,
+	request *api.QueryExampleRequest,
+	response *api.QueryExampleResponse,
+) error {
+	// Implement QueryExample here!
+
+	return nil
+}

pushserver/inthttp/client.go

@@ -0,0 +1,48 @@
+package inthttp
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/pushserver/api"
+	"github.com/opentracing/opentracing-go"
+)
+
+type httpPushserverInternalAPI struct {
+	roomserverURL string
+	httpClient    *http.Client
+}
+
+const (
+	PushserverQueryExamplePath = "/pushserver/queryExample"
+)
+
+// NewRoomserverClient creates a PushserverInternalAPI implemented by talking to a HTTP POST API.
+// If httpClient is nil an error is returned
+func NewPushserverClient(
+	pushserverURL string,
+	httpClient *http.Client,
+) (api.PushserverInternalAPI, error) {
+	if httpClient == nil {
+		return nil, errors.New("NewPushserverClient: httpClient is <nil>")
+	}
+	return &httpPushserverInternalAPI{
+		roomserverURL: pushserverURL,
+		httpClient:    httpClient,
+	}, nil
+}
+
+// SetRoomAlias implements RoomserverAliasAPI
+func (h *httpPushserverInternalAPI) QueryExample(
+	ctx context.Context,
+	request *api.QueryExampleRequest,
+	response *api.QueryExampleResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryExample")
+	defer span.Finish()
+
+	apiURL := h.roomserverURL + PushserverQueryExamplePath
+	return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}

pushserver/inthttp/server.go

@@ -0,0 +1,29 @@
+package inthttp
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/internal/httputil"
+	"github.com/matrix-org/dendrite/pushserver/api"
+	"github.com/matrix-org/util"
+)
+
+// AddRoutes adds the RoomserverInternalAPI handlers to the http.ServeMux.
+// nolint: gocyclo
+func AddRoutes(r api.PushserverInternalAPI, internalAPIMux *mux.Router) {
+	internalAPIMux.Handle(PushserverQueryExamplePath,
+		httputil.MakeInternalAPI("queryExample", func(req *http.Request) util.JSONResponse {
+			var request api.QueryExampleRequest
+			var response api.QueryExampleResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.MessageResponse(http.StatusBadRequest, err.Error())
+			}
+			if err := r.QueryExample(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+}

pushserver/pushserver.go

@@ -0,0 +1,47 @@
+package pushserver
+
+import (
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/pushserver/api"
+	"github.com/matrix-org/dendrite/pushserver/consumers"
+	"github.com/matrix-org/dendrite/pushserver/internal"
+	"github.com/matrix-org/dendrite/pushserver/inthttp"
+	"github.com/matrix-org/dendrite/pushserver/storage"
+	"github.com/matrix-org/dendrite/setup"
+	"github.com/matrix-org/dendrite/setup/kafka"
+	"github.com/sirupsen/logrus"
+)
+
+// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
+// on the given input API.
+func AddInternalRoutes(router *mux.Router, intAPI api.PushserverInternalAPI) {
+	inthttp.AddRoutes(intAPI, router)
+}
+
+// NewInternalAPI returns a concerete implementation of the internal API. Callers
+// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
+func NewInternalAPI(
+	base *setup.BaseDendrite,
+) api.PushserverInternalAPI {
+	cfg := &base.Cfg.PushServer
+
+	consumer, _ := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka)
+
+	pushserverDB, err := storage.Open(&cfg.Database)
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to room server db")
+	}
+
+	psAPI := internal.NewPushserverAPI(
+		cfg, pushserverDB,
+	)
+
+	rsConsumer := consumers.NewOutputRoomEventConsumer(
+		base.ProcessContext, cfg, consumer, pushserverDB, psAPI,
+	)
+	if err := rsConsumer.Start(); err != nil {
+		logrus.WithError(err).Panic("failed to start room server consumer")
+	}
+
+	return psAPI
+}

pushserver/storage/interface.go

@@ -0,0 +1,7 @@
+package storage
+
+import "github.com/matrix-org/dendrite/internal"
+
+type Database interface {
+	internal.PartitionStorer
+}

pushserver/storage/postgres/storage.go

@@ -0,0 +1,31 @@
+package postgres
+
+import (
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/pushserver/storage/shared"
+	"github.com/matrix-org/dendrite/setup/config"
+)
+
+type Database struct {
+	shared.Database
+	sqlutil.PartitionOffsetStatements
+}
+
+func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
+	var d Database
+	var err error
+	if d.DB, err = sqlutil.Open(dbProperties); err != nil {
+		return nil, fmt.Errorf("sqlutil.Open: %w", err)
+	}
+	d.Writer = sqlutil.NewDummyWriter()
+
+	if err = d.PartitionOffsetStatements.Prepare(d.DB, d.Writer, "pushserver"); err != nil {
+		return nil, err
+	}
+
+	// Create the tables.
+
+	return &d, nil
+}

pushserver/storage/shared/storage.go

@@ -0,0 +1,12 @@
+package shared
+
+import (
+	"database/sql"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+type Database struct {
+	DB     *sql.DB
+	Writer sqlutil.Writer
+}

pushserver/storage/sqlite3/storage.go

@@ -0,0 +1,31 @@
+package sqlite3
+
+import (
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/pushserver/storage/shared"
+	"github.com/matrix-org/dendrite/setup/config"
+)
+
+type Database struct {
+	shared.Database
+	sqlutil.PartitionOffsetStatements
+}
+
+func Open(dbProperties *config.DatabaseOptions) (*Database, error) {
+	var d Database
+	var err error
+	if d.DB, err = sqlutil.Open(dbProperties); err != nil {
+		return nil, fmt.Errorf("sqlutil.Open: %w", err)
+	}
+	d.Writer = sqlutil.NewExclusiveWriter()
+
+	if err = d.PartitionOffsetStatements.Prepare(d.DB, d.Writer, "pushserver"); err != nil {
+		return nil, err
+	}
+
+	// Create the tables.
+
+	return &d, nil
+}

pushserver/storage/storage.go

@@ -0,0 +1,23 @@
+// +build !wasm
+
+package storage
+
+import (
+	"fmt"
+
+	"github.com/matrix-org/dendrite/pushserver/storage/postgres"
+	"github.com/matrix-org/dendrite/pushserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/setup/config"
+)
+
+// Open opens a database connection.
+func Open(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.Open(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return postgres.Open(dbProperties)
+	default:
+		return nil, fmt.Errorf("unexpected database type")
+	}
+}

pushserver/storage/storage_wasm.go

@@ -0,0 +1,20 @@
+package storage
+
+import (
+	"fmt"
+
+	"github.com/matrix-org/dendrite/pushserver/storage/sqlite3"
+	"github.com/matrix-org/dendrite/setup/config"
+)
+
+// NewDatabase opens a new database
+func Open(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.Open(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return nil, fmt.Errorf("can't use Postgres implementation")
+	default:
+		return nil, fmt.Errorf("unexpected database type")
+	}
+}

pushserver/storage/tables/interface.go

@@ -0,0 +1 @@
+package tables