Add opt-in anonymous stats reporting (#2249)

* Initial phone home stats queries

* Add userAgent to UpdateDeviceLastSeen
Add new Table for tracking daily user vists

* Add user_daily_visits table

* Fix queries

* userapi stats tables & queries

* userapi interface and internal api

* sycnapi stats queries

* testing phone home stats

* Add complete config to syncapi

* add missing files

* Fix queries

* Send empty request

* Add version & monolith stats

* Add configuration for phone home stats

* Move WASM to its own file, add config and comments

* Add tracing methods

* Add total rooms

* Add more fields, actually send data somewhere

* Move stats to the userapi

* Move phone home stats to util package

* Cleanup

* Linter & parts of GH comments

* More GH comments changes
- Move comments to SQL statements
- Shrink interface, add struct for stats
- No fatal errors, use defaults

* Be more explicit when querying

* Fix wrong calculation & wrong query params
Add tests

* Add Windows stats

* ADd build constraint

* Use new testing structure
Fix issues with getting values when using SQLite
Fix wrong AddDate value
Export UpdateUserDailyVisits

* Fix query params

* Fix test

* Add comment about countR30UsersSQL and countR30UsersV2SQL; fix test

* Update config

* Also update example config file

* Use OS level proxy, update logging

Co-authored-by: kegsay <kegan@matrix.org>

userapi/util/phonehomestats.go

@@ -0,0 +1,160 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"math"
+	"net/http"
+	"runtime"
+	"syscall"
+	"time"
+
+	"github.com/matrix-org/dendrite/internal"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/dendrite/userapi/storage"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+)
+
+type phoneHomeStats struct {
+	prevData   timestampToRUUsage
+	stats      map[string]interface{}
+	serverName gomatrixserverlib.ServerName
+	startTime  time.Time
+	cfg        *config.Dendrite
+	db         storage.Statistics
+	isMonolith bool
+	client     *http.Client
+}
+
+type timestampToRUUsage struct {
+	timestamp int64
+	usage     syscall.Rusage
+}
+
+func StartPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, statsDB storage.Statistics) {
+
+	p := phoneHomeStats{
+		startTime:  startTime,
+		serverName: cfg.Global.ServerName,
+		cfg:        cfg,
+		db:         statsDB,
+		isMonolith: cfg.IsMonolith,
+		client: &http.Client{
+			Timeout:   time.Second * 30,
+			Transport: http.DefaultTransport,
+		},
+	}
+
+	// start initial run after 5min
+	time.AfterFunc(time.Minute*5, p.collect)
+
+	// run every 3 hours
+	ticker := time.NewTicker(time.Hour * 3)
+	for range ticker.C {
+		p.collect()
+	}
+}
+
+func (p *phoneHomeStats) collect() {
+	p.stats = make(map[string]interface{})
+	// general information
+	p.stats["homeserver"] = p.serverName
+	p.stats["monolith"] = p.isMonolith
+	p.stats["version"] = internal.VersionString()
+	p.stats["timestamp"] = time.Now().Unix()
+	p.stats["go_version"] = runtime.Version()
+	p.stats["go_arch"] = runtime.GOARCH
+	p.stats["go_os"] = runtime.GOOS
+	p.stats["num_cpu"] = runtime.NumCPU()
+	p.stats["num_go_routine"] = runtime.NumGoroutine()
+	p.stats["uptime_seconds"] = math.Floor(time.Since(p.startTime).Seconds())
+
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute*1)
+	defer cancel()
+
+	// cpu and memory usage information
+	err := getMemoryStats(p)
+	if err != nil {
+		logrus.WithError(err).Warn("unable to get memory/cpu stats, using defaults")
+	}
+
+	// configuration information
+	p.stats["federation_disabled"] = p.cfg.Global.DisableFederation
+	p.stats["nats_embedded"] = true
+	p.stats["nats_in_memory"] = p.cfg.Global.JetStream.InMemory
+	if len(p.cfg.Global.JetStream.Addresses) > 0 {
+		p.stats["nats_embedded"] = false
+		p.stats["nats_in_memory"] = false // probably
+	}
+	if len(p.cfg.Logging) > 0 {
+		p.stats["log_level"] = p.cfg.Logging[0].Level
+	} else {
+		p.stats["log_level"] = "info"
+	}
+
+	// message and room stats
+	// TODO: Find a solution to actually set these values
+	p.stats["total_room_count"] = 0
+	p.stats["daily_messages"] = 0
+	p.stats["daily_sent_messages"] = 0
+	p.stats["daily_e2ee_messages"] = 0
+	p.stats["daily_sent_e2ee_messages"] = 0
+
+	// user stats and DB engine
+	userStats, db, err := p.db.UserStatistics(ctx)
+	if err != nil {
+		logrus.WithError(err).Warn("unable to query userstats, using default values")
+	}
+	p.stats["database_engine"] = db.Engine
+	p.stats["database_server_version"] = db.Version
+	p.stats["total_users"] = userStats.AllUsers
+	p.stats["total_nonbridged_users"] = userStats.NonBridgedUsers
+	p.stats["daily_active_users"] = userStats.DailyUsers
+	p.stats["monthly_active_users"] = userStats.MonthlyUsers
+	for t, c := range userStats.RegisteredUsersByType {
+		p.stats["daily_user_type_"+t] = c
+	}
+	for t, c := range userStats.R30Users {
+		p.stats["r30_users_"+t] = c
+	}
+	for t, c := range userStats.R30UsersV2 {
+		p.stats["r30v2_users_"+t] = c
+	}
+
+	output := bytes.Buffer{}
+	if err = json.NewEncoder(&output).Encode(p.stats); err != nil {
+		logrus.WithError(err).Error("unable to encode anonymous stats")
+		return
+	}
+
+	logrus.Infof("Reporting stats to %s: %s", p.cfg.Global.ReportStats.Endpoint, output.String())
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, p.cfg.Global.ReportStats.Endpoint, &output)
+	if err != nil {
+		logrus.WithError(err).Error("unable to create anonymous stats request")
+		return
+	}
+	request.Header.Set("User-Agent", "Dendrite/"+internal.VersionString())
+
+	_, err = p.client.Do(request)
+	if err != nil {
+		logrus.WithError(err).Error("unable to send anonymous stats")
+		return
+	}
+}

userapi/util/stats.go

@@ -0,0 +1,47 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !wasm && !windows
+// +build !wasm,!windows
+
+package util
+
+import (
+	"syscall"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+func getMemoryStats(p *phoneHomeStats) error {
+	oldUsage := p.prevData
+	newUsage := syscall.Rusage{}
+	if err := syscall.Getrusage(syscall.RUSAGE_SELF, &newUsage); err != nil {
+		logrus.WithError(err).Error("unable to get usage")
+		return err
+	}
+	newData := timestampToRUUsage{timestamp: time.Now().Unix(), usage: newUsage}
+	p.prevData = newData
+
+	usedCPUTime := (newUsage.Utime.Sec + newUsage.Stime.Sec) - (oldUsage.usage.Utime.Sec + oldUsage.usage.Stime.Sec)
+
+	if usedCPUTime == 0 || newData.timestamp == oldUsage.timestamp {
+		p.stats["cpu_average"] = 0
+	} else {
+		// conversion to int64 required for GOARCH=386
+		p.stats["cpu_average"] = int64(usedCPUTime) / (newData.timestamp - oldUsage.timestamp) * 100
+	}
+	p.stats["memory_rss"] = newUsage.Maxrss
+	return nil
+}

userapi/util/stats_wasm.go

@@ -0,0 +1,20 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+// stub, since WASM doesn't support syscall.Getrusage
+func getMemoryStats(p *phoneHomeStats) error {
+	return nil
+}

userapi/util/stats_windows.go

@@ -0,0 +1,29 @@
+// Copyright 2022 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !wasm
+// +build !wasm
+
+package util
+
+import (
+	"runtime"
+)
+
+func getMemoryStats(p *phoneHomeStats) error {
+	var memStats runtime.MemStats
+	runtime.ReadMemStats(&memStats)
+	p.stats["memory_rss"] = memStats.Alloc
+	return nil
+}