Stream performance metrics (#1785)

* WIP playback metrics

* Playback metrics collecting + APIs. Closes #793

* Cleanup console messages

* Update test

* Increase browser test timeout

* Update browser tests to not fail

metrics/metrics.go

@@ -6,22 +6,32 @@ import (
 	"github.com/owncast/owncast/config"
 	"github.com/owncast/owncast/core/data"
 	"github.com/owncast/owncast/models"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 // How often we poll for updates.
-const metricsPollingInterval = 1 * time.Minute
+const hardwareMetricsPollingInterval = 1 * time.Minute
+
+const (
+	// How often we poll for updates.
+	viewerMetricsPollingInterval = 2 * time.Minute
+	activeChatClientCountKey     = "chat_client_count"
+	activeViewerCountKey         = "viewer_count"
+)
 
 // CollectedMetrics stores different collected + timestamped values.
 type CollectedMetrics struct {
-	CPUUtilizations  []timestampedValue `json:"cpu"`
-	RAMUtilizations  []timestampedValue `json:"memory"`
-	DiskUtilizations []timestampedValue `json:"disk"`
+	CPUUtilizations        []TimestampedValue `json:"cpu"`
+	RAMUtilizations        []TimestampedValue `json:"memory"`
+	DiskUtilizations       []TimestampedValue `json:"disk"`
+	errorCount             []TimestampedValue `json:"-"`
+	lowestBitrate          []TimestampedValue `json:"-"`
+	segmentDownloadSeconds []TimestampedValue `json:"-"`
+	averageLatency         []TimestampedValue `json:"-"`
+	qualityVariantChanges  []TimestampedValue `json:"-"`
 }
 
 // Metrics is the shared Metrics instance.
-var Metrics *CollectedMetrics
+var metrics *CollectedMetrics
 
 // Start will begin the metrics collection and alerting.
 func Start(getStatus func() models.Status) {
@@ -34,40 +44,12 @@ func Start(getStatus func() models.Status) {
 		"host":    host,
 	}
 
-	activeViewerCount = promauto.NewGauge(prometheus.GaugeOpts{
-		Name:        "owncast_instance_active_viewer_count",
-		Help:        "The number of viewers.",
-		ConstLabels: labels,
-	})
+	setupPrometheusCollectors()
 
-	activeChatClientCount = promauto.NewGauge(prometheus.GaugeOpts{
-		Name:        "owncast_instance_active_chat_client_count",
-		Help:        "The number of connected chat clients.",
-		ConstLabels: labels,
-	})
-
-	chatUserCount = promauto.NewGauge(prometheus.GaugeOpts{
-		Name:        "owncast_instance_total_chat_users",
-		Help:        "The total number of chat users on this Owncast instance.",
-		ConstLabels: labels,
-	})
-
-	currentChatMessageCount = promauto.NewGauge(prometheus.GaugeOpts{
-		Name:        "owncast_instance_current_chat_message_count",
-		Help:        "The number of chat messages currently saved before cleanup.",
-		ConstLabels: labels,
-	})
-
-	cpuUsage = promauto.NewGauge(prometheus.GaugeOpts{
-		Name:        "owncast_instance_cpu_use_pct",
-		Help:        "CPU percentage used as seen within Owncast",
-		ConstLabels: labels,
-	})
-
-	Metrics = new(CollectedMetrics)
+	metrics = new(CollectedMetrics)
 	go startViewerCollectionMetrics()
 
-	for range time.Tick(metricsPollingInterval) {
+	for range time.Tick(hardwareMetricsPollingInterval) {
 		handlePolling()
 	}
 }
@@ -78,6 +60,17 @@ func handlePolling() {
 	collectRAMUtilization()
 	collectDiskUtilization()
 
+	collectPlaybackErrorCount()
+	collectLatencyValues()
+	collectSegmentDownloadDuration()
+	collectLowestBandwidth()
+	collectQualityVariantChanges()
+
 	// Alerting
 	handleAlerting()
 }
+
+// GetMetrics will return the collected metrics.
+func GetMetrics() *CollectedMetrics {
+	return metrics
+}