Untangle precommit and travis test scripts (#288)

scripts/README.md

@@ -0,0 +1,7 @@
+# Dev Scripts
+
+These are a collection of scripts that should be helpful for those developing
+on dendrite.
+
+See `find-lint.sh` for environment variables that control linter resource
+usage.

scripts/build-test-lint.sh

@@ -0,0 +1,26 @@
+#! /bin/bash
+
+# Builds, tests and lints dendrite, and should be run before pushing commits
+
+set -eu
+
+export GOPATH="$(pwd):$(pwd)/vendor"
+export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin"
+
+echo "Checking that it builds"
+gb build
+
+# Check that all the packages can build.
+# When `go build` is given multiple packages it won't output anything, and just
+# checks that everything builds. This seems to do a better job of handling
+# missing imports than `gb build` does.
+echo "Double checking it builds..."
+go build github.com/matrix-org/dendrite/cmd/...
+
+./scripts/find-lint.sh
+
+echo "Double checking spelling..."
+misspell -error src *.md
+
+echo "Testing..."
+gb test

scripts/find-lint.sh

@@ -0,0 +1,41 @@
+#! /bin/bash
+
+# Runs the linters against dendrite
+
+# The linters can take a lot of resources and are slow, so they can be
+# configured using two environment variables:
+#
+# - `DENDRITE_LINT_CONCURRENCY` - number of concurrent linters to run,
+#   gometalinter defaults this to 8
+# - `DENDRITE_LINT_DISABLE_GC` - if set then the the go gc will be disabled
+#   when running the linters, speeding them up but using much more memory.
+
+
+set -eu
+
+export GOPATH="$(pwd):$(pwd)/vendor"
+export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin"
+
+args=""
+if [ ${1:-""} = "fast" ]
+then args="--config=linter-fast.json"
+else args="--config=linter.json"
+fi
+
+if [ -n "${DENDRITE_LINT_CONCURRENCY:-}" ]
+then args="$args --concurrency=$DENDRITE_LINT_CONCURRENCY"
+fi
+
+if [ -z "${DENDRITE_LINT_DISABLE_GC:-}" ]
+then args="$args --enable-gc"
+fi
+
+echo "Installing lint search engine..."
+go install github.com/alecthomas/gometalinter/
+gometalinter --config=linter.json ./... --install
+
+echo "Looking for lint..."
+gometalinter ./... $args
+
+echo "Double checking spelling..."
+misspell -error src *.md

scripts/install-local-kafka.sh

@@ -0,0 +1,24 @@
+# /bin/bash
+
+# Downloads, installs and runs a kafka instance
+
+set -eu
+
+# The mirror to download kafka from is picked from the list of mirrors at
+# https://www.apache.org/dyn/closer.cgi?path=/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz
+# TODO: Check the signature since we are downloading over HTTP.
+MIRROR=http://apache.mirror.anlx.net/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz
+
+# Only download the kafka if it isn't already downloaded.
+test -f kafka.tgz || wget $MIRROR -O kafka.tgz
+# Unpack the kafka over the top of any existing installation
+mkdir -p kafka && tar xzf kafka.tgz -C kafka --strip-components 1
+# Start the zookeeper running in the background.
+# By default the zookeeper listens on localhost:2181
+kafka/bin/zookeeper-server-start.sh -daemon kafka/config/zookeeper.properties
+# Enable topic deletion so that the integration tests can create a fresh topic
+# for each test run.
+echo "delete.topic.enable=true" >> kafka/config/server.properties
+# Start the kafka server running in the background.
+# By default the kafka listens on localhost:9092
+kafka/bin/kafka-server-start.sh -daemon kafka/config/server.properties

scripts/travis-test.sh

@@ -0,0 +1,27 @@
+#! /bin/bash
+
+# The entry point for travis tests
+
+set -eu
+
+# Tune the GC to use more memory to reduce the number of garbage collections
+export GOGC=400
+export DENDRITE_LINT_DISABLE_GC=1
+
+# Check that the servers build (this is done explicitly because `gb build` can silently fail (exit 0) and then we'd test a stale binary)
+gb build github.com/matrix-org/dendrite/cmd/dendrite-room-server
+gb build github.com/matrix-org/dendrite/cmd/roomserver-integration-tests
+gb build github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server
+gb build github.com/matrix-org/dendrite/cmd/syncserver-integration-tests
+gb build github.com/matrix-org/dendrite/cmd/create-account
+gb build github.com/matrix-org/dendrite/cmd/dendrite-media-api-server
+gb build github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests
+gb build github.com/matrix-org/dendrite/cmd/client-api-proxy
+
+# Run unit tests and linters
+./scripts/build-test-lint.sh
+
+# Run the integration tests
+bin/roomserver-integration-tests
+bin/syncserver-integration-tests
+bin/mediaapi-integration-tests