HLS video handling/storage/state refactor (#151)

* WIP with new transcoder progress monitor * A whole different WIP in progress monitoring via local PUTs * Use an actual hls playlist parser to rewrite master playlist * Cleanup * Private vs public path for thumbnail generation * Allow each storage provider to make decisions of how to store different types of files * Simplify inbound file writes * Revert * Split out set stream as connected/disconnected state methods * Update videojs * Add comment about the hls handler * Rework of the offline stream state. For #85 * Delete old unreferenced video segment files from disk * Cleanup all segments and revert to a completely offline state after 5min * Stop thumbnail generation on stream stop. Copy logo to thumbnail on cleanup. * Update transcoder test * Add comment * Return http 200 on success to transcoder. Tweak how files are written to disk * Force pixel color format in transcoder * Add debugging info for S3 transfers. Add default ACL. * Fix cleanup timer * Reset session stats when we cleanup the session. * Put log file back * Update test * File should not be a part of this commit * Add centralized shared performance timer for use anywhere * Post-rebase cleanup * Support returning nil from storage provider save * Updates to reflect package changes + other updates in master * Fix storage providers being overwritten * Do not return pointer in save. Support cache headers with S3 providers * Split out videojs + vhs and point to specific working versions of them * Bump vjs and vhs versions * Fix test * Remove unused * Update upload warning message * No longer valid comment * Pin videojs and vhs versions
2020-10-14 14:07:38 -07:00
parent 57f2e4b567
commit 6ea9affce0
43 changed files with 30296 additions and 56701 deletions
--- a/core/ffmpeg/fileWriterReceiverService.go
+++ b/core/ffmpeg/fileWriterReceiverService.go
@@ -0,0 +1,103 @@
+package ffmpeg
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"net/http"
+
+	"github.com/owncast/owncast/config"
+	"github.com/owncast/owncast/utils"
+	log "github.com/sirupsen/logrus"
+)
+
+// FileWriterReceiverServiceCallback are to be fired when transcoder responses are written to disk
+type FileWriterReceiverServiceCallback interface {
+	SegmentWritten(localFilePath string)
+	VariantPlaylistWritten(localFilePath string)
+	MasterPlaylistWritten(localFilePath string)
+}
+
+// FileWriterReceiverService accepts transcoder responses via HTTP and fires the callbacks
+type FileWriterReceiverService struct {
+	callbacks FileWriterReceiverServiceCallback
+}
+
+// SetupFileWriterReceiverService will start listening for transcoder responses
+func (s *FileWriterReceiverService) SetupFileWriterReceiverService(callbacks FileWriterReceiverServiceCallback) {
+	s.callbacks = callbacks
+
+	httpServer := http.NewServeMux()
+	httpServer.HandleFunc("/", s.uploadHandler)
+
+	localListenerAddress := "127.0.0.1:" + strconv.Itoa(config.Config.GetPublicWebServerPort()+1)
+	go http.ListenAndServe(localListenerAddress, httpServer)
+	log.Traceln("Transcoder response listening on: " + localListenerAddress)
+}
+
+func (s *FileWriterReceiverService) uploadHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "PUT" {
+		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		return
+	}
+
+	path := r.URL.Path
+	writePath := filepath.Join(config.PrivateHLSStoragePath, path)
+
+	var buf bytes.Buffer
+	io.Copy(&buf, r.Body)
+	data := buf.Bytes()
+	f, err := os.Create(writePath)
+	if err != nil {
+		returnError(err, w, r)
+		return
+	}
+
+	defer f.Close()
+	_, err = f.Write(data)
+	if err != nil {
+		returnError(err, w, r)
+		return
+	}
+
+	s.fileWritten(writePath)
+	w.WriteHeader(http.StatusOK)
+}
+
+var _inWarningState = false
+
+func (s *FileWriterReceiverService) fileWritten(path string) {
+	index := utils.GetIndexFromFilePath(path)
+
+	if utils.GetRelativePathFromAbsolutePath(path) == "hls/stream.m3u8" {
+		s.callbacks.MasterPlaylistWritten(path)
+
+	} else if strings.HasSuffix(path, ".ts") {
+		performanceMonitorKey := "segmentWritten-" + index
+		averagePerformance := utils.GetAveragePerformance(performanceMonitorKey)
+
+		utils.StartPerformanceMonitor(performanceMonitorKey)
+		s.callbacks.SegmentWritten(path)
+
+		if averagePerformance != 0 && averagePerformance > float64(float64(config.Config.GetVideoSegmentSecondsLength())) {
+			if !_inWarningState {
+				log.Warnln("slow encoding for variant", index, "if this continues you may see buffering or errors. troubleshoot this issue by visiting https://owncast.online/docs/troubleshooting/")
+				_inWarningState = true
+			}
+		} else {
+			_inWarningState = false
+		}
+
+	} else if strings.HasSuffix(path, ".m3u8") {
+		s.callbacks.VariantPlaylistWritten(path)
+	}
+}
+
+func returnError(err error, w http.ResponseWriter, r *http.Request) {
+	log.Errorln(err)
+	http.Error(w, http.StatusText(http.StatusInternalServerError)+": "+err.Error(), http.StatusInternalServerError)
+}