Fix #897 and shuffle directory around (#1054)

* Fix #897 and shuffle directory around

* Update find-lint

* goimports

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>

build/scripts/README.md

@@ -0,0 +1,7 @@
+# Dev Scripts
+
+These are a collection of scripts that should be helpful for those developing
+on dendrite.
+
+See `find-lint.sh` for environment variables that control linter resource
+usage.

build/scripts/build-test-lint.sh

@@ -0,0 +1,16 @@
+#! /bin/bash
+
+# Builds, tests and lints dendrite, and should be run before pushing commits
+
+set -eu
+
+# Check that all the packages can build.
+# When `go build` is given multiple packages it won't output anything, and just
+# checks that everything builds.
+echo "Checking that it builds..."
+go build ./cmd/...
+
+./scripts/find-lint.sh
+
+echo "Testing..."
+go test ./...

build/scripts/find-lint.sh

@@ -0,0 +1,43 @@
+#! /bin/bash
+
+# Runs the linters against dendrite
+
+# The linters can take a lot of resources and are slow, so they can be
+# configured using the following environment variables:
+#
+# - `DENDRITE_LINT_CONCURRENCY` - number of concurrent linters to run,
+#   golangci-lint defaults this to NumCPU
+# - `GOGC` - how often to perform garbage collection during golangci-lint runs.
+#   Essentially a ratio of memory/speed. See https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint
+#   for more info.
+
+
+set -eux
+
+cd `dirname $0`/../..
+
+args=""
+if [ ${1:-""} = "fast" ]
+then args="--fast"
+fi
+
+echo "Installing golangci-lint..."
+
+# Make a backup of go.{mod,sum} first
+# TODO: Once go 1.13 is out, use go get's -mod=readonly option
+# https://github.com/golang/go/issues/30667
+cp go.mod go.mod.bak && cp go.sum go.sum.bak
+go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1
+
+# Run linting
+echo "Looking for lint..."
+
+# Capture exit code to ensure go.{mod,sum} is restored before exiting
+exit_code=0
+
+golangci-lint run $args || exit_code=1
+
+# Restore go.{mod,sum}
+mv go.mod.bak go.mod && mv go.sum.bak go.sum
+
+exit $exit_code

build/scripts/install-local-kafka.sh

@@ -0,0 +1,25 @@
+#! /bin/bash
+
+# Downloads, installs and runs a kafka instance
+
+set -eu
+
+cd `dirname $0`/..
+
+mkdir -p .downloads
+
+KAFKA_URL=http://archive.apache.org/dist/kafka/2.1.0/kafka_2.11-2.1.0.tgz
+
+# Only download the kafka if it isn't already downloaded.
+test -f .downloads/kafka.tgz || wget $KAFKA_URL -O .downloads/kafka.tgz
+# Unpack the kafka over the top of any existing installation
+mkdir -p kafka && tar xzf .downloads/kafka.tgz -C kafka --strip-components 1
+# Start the zookeeper running in the background.
+# By default the zookeeper listens on localhost:2181
+kafka/bin/zookeeper-server-start.sh -daemon kafka/config/zookeeper.properties
+# Enable topic deletion so that the integration tests can create a fresh topic
+# for each test run.
+echo -e "\n\ndelete.topic.enable=true" >> kafka/config/server.properties
+# Start the kafka server running in the background.
+# By default the kafka listens on localhost:9092
+kafka/bin/kafka-server-start.sh -daemon kafka/config/server.properties