Docker Hub (#1053)

* Initial Docker Hub test

* Change upstream from for now

* Working polylith setup and build tools

* Some readme and other bits

* Remove the readme for now

* Add readme, rename a couple of things

* COPY instead of ADD

* Simplify component builds

* Fix the docker-compose things

* Build the base image again

* Update readme, add .dockerignore

* Indentation

* Whitespace

* Update images-push.sh

* Update images-push.sh

.dockerignore

@@ -0,0 +1,3 @@
+bin
+*.wasm
+.git

docker/hub/Dockerfile

@@ -0,0 +1,10 @@
+FROM docker.io/golang:1.13.7-alpine3.11 AS builder
+
+RUN apk --update --no-cache add bash build-base
+
+WORKDIR /build
+
+COPY . /build
+
+RUN mkdir -p bin
+RUN sh ./build.sh

docker/hub/Dockerfile.component

@@ -0,0 +1,13 @@
+FROM matrixdotorg/dendrite:latest AS base
+
+FROM alpine:latest
+
+ARG component=monolith
+ENV entrypoint=${component}
+
+COPY --from=base /build/bin/${component} /usr/bin
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT /usr/bin/${entrypoint} $@

docker/hub/README.md

@@ -0,0 +1,68 @@
+# Docker Hub images
+
+These are Docker Hub-friendly images for Dendrite.
+
+## Dockerfiles
+
+The `Dockerfile` builds the base image which contains all of the Dendrite
+components. The `Dockerfile.*` files take components from the base image and
+produce smaller component-specific images, which are substantially smaller
+and do not contain the Go toolchain etc.
+
+## Compose files
+
+There are three sample `docker-compose` files:
+
+- `docker-compose.deps.yml` which runs the Postgres and Kafka prerequisites
+- `docker-compose.monolith.yml` which runs a monolith Dendrite deployment
+- `docker-compose.polylith.yml` which runs a polylith Dendrite deployment
+
+## Configuration
+
+The `docker-compose` files refer to the `/etc/dendrite` volume as where the
+runtime config should come from. The mounted folder must contain:
+
+- `dendrite.yaml` configuration file (based on the sample `dendrite-config.yaml`
+   in the `docker/hub/config` folder in the [Dendrite repository](https://github.com/matrix-org/dendrite)
+- `matrix_key.pem` server key, as generated using `cmd/generate-keys`
+- `server.crt` certificate file
+- `server.key` private key file for the above certificate
+
+To generate keys:
+
+```
+go run github.com/matrix-org/dendrite/cmd/generate-keys \
+  --private-key=matrix_key.pem \
+  --tls-cert=server.crt \
+  --tls-key=server.key
+```
+
+## Starting Dendrite
+
+Once in place, start the dependencies:
+
+```
+docker-compose -f docker-compose.deps.yml up
+```
+
+Wait a few seconds for Kafka and Postgres to finish starting up, and then start a monolith:
+
+```
+docker-compose -f docker-compose.monolith.yml up
+```
+
+... or start the polylith components:
+
+```
+docker-compose -f docker-compose.polylith.yml up
+```
+
+## Building the images
+
+The `docker/hub/images-build.sh` script will build all of the component images.
+
+The `docker/hub/images-push.sh` script will push them to Docker Hub (subject
+to permissions). 
+
+If you wish to build and push your own images, rename `matrixdotorg/dendrite` to
+the name of another Docker Hub repository in `images-build.sh` and `images-push.sh`.

docker/hub/config/dendrite-config.yaml

@@ -0,0 +1,129 @@
+# The config file format version
+# This is used by dendrite to tell if it understands the config format.
+# This will change if the structure of the config file changes or if the meaning
+# of an existing config key changes.
+version: 0
+
+# The matrix specific config
+matrix:
+    # The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
+    server_name: "example.com"
+    # The path to the PEM formatted matrix private key.
+    private_key: "matrix_key.pem"
+    # The x509 certificates used by the federation listeners for this server
+    federation_certificates: ["server.crt"]
+    # The list of identity servers trusted to verify third party identifiers by this server.
+    # Defaults to no trusted servers.
+    trusted_third_party_id_servers:
+      - vector.im
+      - matrix.org
+
+# The media repository config
+media:
+    # The base path to where the media files will be stored. May be relative or absolute.
+    base_path: /var/dendrite/media
+
+    # The maximum file size in bytes that is allowed to be stored on this server.
+    # Note: if max_file_size_bytes is set to 0, the size is unlimited.
+    # Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
+    max_file_size_bytes: 10485760
+
+    # Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
+    # NOTE: This is a possible denial-of-service attack vector - use at your own risk
+    dynamic_thumbnails: false
+
+    # A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
+    # method is one of crop or scale. If omitted, it will default to scale.
+    # crop scales to fill the requested dimensions and crops the excess.
+    # scale scales to fit the requested dimensions and one dimension may be smaller than requested.
+    thumbnail_sizes:
+      - width: 32
+        height: 32
+        method: crop
+      - width: 96
+        height: 96
+        method: crop
+      - width: 320
+        height: 240
+        method: scale
+      - width: 640
+        height: 480
+        method: scale
+      - width: 800
+        height: 600
+        method: scale
+
+# The config for the TURN server
+turn:
+    # Whether or not guests can request TURN credentials
+    turn_allow_guests: true
+    # How long the authorization should last
+    turn_user_lifetime: "1h"
+    # The list of TURN URIs to pass to clients
+    turn_uris: []
+
+    # Authorization via Shared Secret
+    # The shared secret from coturn
+    turn_shared_secret: "<SECRET STRING GOES HERE>"
+
+    # Authorization via Static Username & Password
+    # Hardcoded Username and Password
+    turn_username: ""
+    turn_password: ""
+
+# The config for communicating with kafka
+kafka:
+    # Where the kafka servers are running.
+    addresses: ["kafka:9092"]
+    # Whether to use naffka instead of kafka.
+    # Naffka can only be used when running dendrite as a single monolithic server.
+    # Kafka can be used both with a monolithic server and when running the
+    # components as separate servers.
+    # If enabled database.naffka must also be specified.
+    use_naffka: false
+    # The names of the kafka topics to use.
+    topics:
+        output_room_event: roomserverOutput
+        output_client_data: clientapiOutput
+        output_typing_event: eduServerOutput
+        user_updates: userUpdates
+
+
+# The postgres connection configs for connecting to the databases e.g a postgres:// URI
+database:
+    account: "postgres://dendrite:itsasecret@postgres/dendrite_account?sslmode=disable"
+    device: "postgres://dendrite:itsasecret@postgres/dendrite_device?sslmode=disable"
+    media_api: "postgres://dendrite:itsasecret@postgres/dendrite_mediaapi?sslmode=disable"
+    sync_api: "postgres://dendrite:itsasecret@postgres/dendrite_syncapi?sslmode=disable"
+    room_server: "postgres://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable"
+    server_key: "postgres://dendrite:itsasecret@postgres/dendrite_serverkey?sslmode=disable"
+    federation_sender: "postgres://dendrite:itsasecret@postgres/dendrite_federationsender?sslmode=disable"
+    public_rooms_api: "postgres://dendrite:itsasecret@postgres/dendrite_publicroomsapi?sslmode=disable"
+    appservice: "postgres://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable"
+    # If using naffka you need to specify a naffka database
+    #naffka: "postgres://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable"
+
+# The TCP host:port pairs to bind the internal HTTP APIs to.
+# These shouldn't be exposed to the public internet.
+# These aren't needed when running dendrite as a monolithic server.
+listen:
+    room_server: "room_server:7770"
+    client_api: "client_api:7771"
+    federation_api: "federation_api:7772"
+    sync_api: "sync_api:7773"
+    media_api: "media_api:7774"
+    public_rooms_api: "public_rooms_api:7775"
+    federation_sender: "federation_sender:7776"
+    edu_server: "edu_server:7777"
+
+# The configuration for tracing the dendrite components.
+tracing:
+    # Config for the jaeger opentracing reporter.
+    # See https://godoc.org/github.com/uber/jaeger-client-go/config#Configuration
+    # for documentation.
+    jaeger:
+        disabled: true
+
+# A list of application service config files to use
+application_services:
+    config_files: []

docker/hub/docker-compose.deps.yml

@@ -0,0 +1,36 @@
+version: "3.4"
+services:
+  postgres:
+    hostname: postgres
+    image: postgres:9.5
+    restart: always
+    volumes:
+      - ../postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh
+    environment:
+      POSTGRES_PASSWORD: itsasecret
+      POSTGRES_USER: dendrite
+    networks:
+      - internal
+
+  zookeeper:
+    hostname: zookeeper
+    image: zookeeper
+    networks:
+      - internal
+
+  kafka:
+    container_name: dendrite_kafka
+    hostname: kafka
+    image: wurstmeister/kafka
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: "kafka"
+      KAFKA_DELETE_TOPIC_ENABLE: "true"
+      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
+    depends_on:
+      - zookeeper
+    networks:
+      - internal
+
+networks:
+  internal:
+    attachable: true

docker/hub/docker-compose.monolith.yml

@@ -0,0 +1,18 @@
+version: "3.4"
+services:
+  monolith:
+    hostname: monolith
+    image: matrixdotorg/dendrite:monolith
+    command: [
+      "--config=dendrite.yaml",
+      "--tls-cert=server.crt",
+      "--tls-key=server.key"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+networks:
+  internal:
+    attachable: true

docker/hub/docker-compose.polylith.yml

@@ -0,0 +1,146 @@
+version: "3.4"
+services:
+  client_api_proxy:
+    hostname: client_api_proxy
+    image: matrixdotorg/dendrite:clientproxy
+    command: [
+      "--bind-address=:8008",
+      "--client-api-server-url=http://client_api:7771",
+      "--sync-api-server-url=http://sync_api:7773",
+      "--media-api-server-url=http://media_api:7774",
+      "--public-rooms-api-server-url=http://public_rooms_api:7775"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+    depends_on:
+      - sync_api
+      - client_api
+      - media_api
+      - public_rooms_api
+    ports:
+      - "8008:8008"
+
+  client_api:
+    hostname: client_api
+    image: matrixdotorg/dendrite:clientapi
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+      - room_server
+    networks:
+      - internal
+
+  media_api:
+    hostname: media_api
+    image: matrixdotorg/dendrite:mediaapi
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  public_rooms_api:
+    hostname: public_rooms_api
+    image: matrixdotorg/dendrite:publicroomsapi
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  sync_api:
+    hostname: sync_api
+    image: matrixdotorg/dendrite:syncapi
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  room_server:
+    hostname: room_server
+    image: matrixdotorg/dendrite:roomserver
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  edu_server:
+    hostname: edu_server
+    image: matrixdotorg/dendrite:eduserver
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  federation_api_proxy:
+    hostname: federation_api_proxy
+    image: matrixdotorg/dendrite:federationproxy
+    command: [
+      "--bind-address=:8448",
+      "--federation-api-url=http://federation_api_server:7772",
+      "--media-api-server-url=http://media_api:7774"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    depends_on:
+      - federation_api
+      - federation_sender
+      - media_api
+    networks:
+      - internal
+    ports:
+      - "8448:8448"
+
+  federation_api:
+    hostname: federation_api
+    image: matrixdotorg/dendrite:federationapi
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+
+  federation_sender:
+    hostname: federation_sender
+    image: matrixdotorg/dendrite:federationsender
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+      - internal
+      
+  key_server:
+    hostname: key_serverde
+    image: matrixdotorg/dendrite:keyserver
+    command: [
+      "--config=dendrite.yaml"
+    ]
+    volumes:
+      - ./config:/etc/dendrite
+    networks:
+        - internal
+
+networks:
+  internal:
+    attachable: true

docker/hub/images-build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+cd $(git rev-parse --show-toplevel)
+
+docker build -f docker/hub/Dockerfile -t matrixdotorg/dendrite:latest .
+
+docker build -t matrixdotorg/dendrite:clientapi         --build-arg component=dendrite-client-api-server        -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:clientproxy       --build-arg component=client-api-proxy                  -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:eduserver         --build-arg component=dendrite-edu-server               -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:federationapi     --build-arg component=dendrite-federation-api-server    -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:federationsender  --build-arg component=dendrite-federation-sender-server -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:federationproxy   --build-arg component=federation-api-proxy              -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:keyserver         --build-arg component=dendrite-key-server               -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:mediaapi          --build-arg component=dendrite-media-api-server         -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:publicroomsapi    --build-arg component=dendrite-public-rooms-api-server  -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:roomserver        --build-arg component=dendrite-room-server              -f docker/hub/Dockerfile.component .
+docker build -t matrixdotorg/dendrite:syncapi           --build-arg component=dendrite-sync-api-server          -f docker/hub/Dockerfile.component .

docker/hub/images-push.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+docker push matrixdotorg/dendrite:clientapi
+docker push matrixdotorg/dendrite:clientproxy
+docker push matrixdotorg/dendrite:eduserver
+docker push matrixdotorg/dendrite:federationapi
+docker push matrixdotorg/dendrite:federationsender
+docker push matrixdotorg/dendrite:federationproxy
+docker push matrixdotorg/dendrite:keyserver
+docker push matrixdotorg/dendrite:mediaapi
+docker push matrixdotorg/dendrite:publicroomsapi
+docker push matrixdotorg/dendrite:roomserver
+docker push matrixdotorg/dendrite:syncapi