HLS video handling/storage/state refactor (#151)

* WIP with new transcoder progress monitor

* A whole different WIP in progress monitoring via local PUTs

* Use an actual hls playlist parser to rewrite master playlist

* Cleanup

* Private vs public path for thumbnail generation

* Allow each storage provider to make decisions of how to store different types of files

* Simplify inbound file writes

* Revert

* Split out set stream as connected/disconnected state methods

* Update videojs

* Add comment about the hls handler

* Rework of the offline stream state.  For #85

* Delete old unreferenced video segment files from disk

* Cleanup all segments and revert to a completely offline state after 5min

* Stop thumbnail generation on stream stop. Copy logo to thumbnail on cleanup.

* Update transcoder test

* Add comment

* Return http 200 on success to transcoder. Tweak how files are written to disk

* Force pixel color format in transcoder

* Add debugging info for S3 transfers. Add default ACL.

* Fix cleanup timer

* Reset session stats when we cleanup the session.

* Put log file back

* Update test

* File should not be a part of this commit

* Add centralized shared performance timer for use anywhere

* Post-rebase cleanup

* Support returning nil from storage provider save

* Updates to reflect package changes + other updates in master

* Fix storage providers being overwritten

* Do not return pointer in save. Support cache headers with S3 providers

* Split out videojs + vhs and point to specific working versions of them

* Bump vjs and vhs versions

* Fix test

* Remove unused

* Update upload warning message

* No longer valid comment

* Pin videojs and vhs versions

core/ffmpeg/ffmpeg.go

@@ -1,14 +0,0 @@
-package ffmpeg
-
-import (
-	"github.com/owncast/owncast/config"
-)
-
-//ShowStreamOfflineState generates and shows the stream's offline state
-func ShowStreamOfflineState() {
-	transcoder := NewTranscoder()
-	transcoder.SetSegmentLength(10)
-	transcoder.SetAppendToStream(true)
-	transcoder.SetInput(config.Config.GetOfflineContentPath())
-	transcoder.Start()
-}

core/ffmpeg/fileWriterReceiverService.go

@@ -0,0 +1,103 @@
+package ffmpeg
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"net/http"
+
+	"github.com/owncast/owncast/config"
+	"github.com/owncast/owncast/utils"
+	log "github.com/sirupsen/logrus"
+)
+
+// FileWriterReceiverServiceCallback are to be fired when transcoder responses are written to disk
+type FileWriterReceiverServiceCallback interface {
+	SegmentWritten(localFilePath string)
+	VariantPlaylistWritten(localFilePath string)
+	MasterPlaylistWritten(localFilePath string)
+}
+
+// FileWriterReceiverService accepts transcoder responses via HTTP and fires the callbacks
+type FileWriterReceiverService struct {
+	callbacks FileWriterReceiverServiceCallback
+}
+
+// SetupFileWriterReceiverService will start listening for transcoder responses
+func (s *FileWriterReceiverService) SetupFileWriterReceiverService(callbacks FileWriterReceiverServiceCallback) {
+	s.callbacks = callbacks
+
+	httpServer := http.NewServeMux()
+	httpServer.HandleFunc("/", s.uploadHandler)
+
+	localListenerAddress := "127.0.0.1:" + strconv.Itoa(config.Config.GetPublicWebServerPort()+1)
+	go http.ListenAndServe(localListenerAddress, httpServer)
+	log.Traceln("Transcoder response listening on: " + localListenerAddress)
+}
+
+func (s *FileWriterReceiverService) uploadHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "PUT" {
+		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		return
+	}
+
+	path := r.URL.Path
+	writePath := filepath.Join(config.PrivateHLSStoragePath, path)
+
+	var buf bytes.Buffer
+	io.Copy(&buf, r.Body)
+	data := buf.Bytes()
+	f, err := os.Create(writePath)
+	if err != nil {
+		returnError(err, w, r)
+		return
+	}
+
+	defer f.Close()
+	_, err = f.Write(data)
+	if err != nil {
+		returnError(err, w, r)
+		return
+	}
+
+	s.fileWritten(writePath)
+	w.WriteHeader(http.StatusOK)
+}
+
+var _inWarningState = false
+
+func (s *FileWriterReceiverService) fileWritten(path string) {
+	index := utils.GetIndexFromFilePath(path)
+
+	if utils.GetRelativePathFromAbsolutePath(path) == "hls/stream.m3u8" {
+		s.callbacks.MasterPlaylistWritten(path)
+
+	} else if strings.HasSuffix(path, ".ts") {
+		performanceMonitorKey := "segmentWritten-" + index
+		averagePerformance := utils.GetAveragePerformance(performanceMonitorKey)
+
+		utils.StartPerformanceMonitor(performanceMonitorKey)
+		s.callbacks.SegmentWritten(path)
+
+		if averagePerformance != 0 && averagePerformance > float64(float64(config.Config.GetVideoSegmentSecondsLength())) {
+			if !_inWarningState {
+				log.Warnln("slow encoding for variant", index, "if this continues you may see buffering or errors. troubleshoot this issue by visiting https://owncast.online/docs/troubleshooting/")
+				_inWarningState = true
+			}
+		} else {
+			_inWarningState = false
+		}
+
+	} else if strings.HasSuffix(path, ".m3u8") {
+		s.callbacks.VariantPlaylistWritten(path)
+	}
+}
+
+func returnError(err error, w http.ResponseWriter, r *http.Request) {
+	log.Errorln(err)
+	http.Error(w, http.StatusText(http.StatusInternalServerError)+": "+err.Error(), http.StatusInternalServerError)
+}

core/ffmpeg/hlsFilesystemCleanup.go

@@ -0,0 +1,63 @@
+package ffmpeg
+
+import (
+	log "github.com/sirupsen/logrus"
+
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/owncast/owncast/config"
+)
+
+// Cleanup will delete old files off disk that are no longer being referenced
+// in the stream.
+func Cleanup(directoryPath string) {
+	// Determine how many files we should keep on disk
+	maxNumber := config.Config.GetMaxNumberOfReferencedSegmentsInPlaylist()
+	buffer := 10
+
+	files, err := getSegmentFiles(directoryPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if len(files) < maxNumber+buffer {
+		return
+	}
+
+	// Delete old files on disk
+	filesToDelete := files[maxNumber+buffer:]
+	for _, file := range filesToDelete {
+		os.Remove(filepath.Join(directoryPath, file.Name()))
+	}
+}
+
+func getSegmentFiles(dirname string) ([]os.FileInfo, error) {
+	f, err := os.Open(dirname)
+	if err != nil {
+		return nil, err
+	}
+	list, err := f.Readdir(-1) // -1 says to get a list of all files
+	f.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	filteredList := make([]os.FileInfo, 0)
+
+	// Filter out playlists because we don't want to clean them up
+	for _, file := range list {
+		if filepath.Ext(file.Name()) == ".m3u8" {
+			continue
+		}
+		filteredList = append(filteredList, file)
+	}
+
+	// Sort by date so we can delete old files
+	sort.Slice(filteredList, func(i, j int) bool {
+		return filteredList[i].ModTime().UnixNano() > filteredList[j].ModTime().UnixNano()
+	})
+
+	return filteredList, nil
+}

core/ffmpeg/hlsHandler.go

@@ -0,0 +1,25 @@
+package ffmpeg
+
+import (
+	"github.com/owncast/owncast/models"
+)
+
+// HLSHandler gets told about available HLS playlists and segments
+type HLSHandler struct {
+	Storage models.StorageProvider
+}
+
+// SegmentWritten is fired when a HLS segment is written to disk
+func (h *HLSHandler) SegmentWritten(localFilePath string) {
+	h.Storage.SegmentWritten(localFilePath)
+}
+
+// VariantPlaylistWritten is fired when a HLS variant playlist is written to disk
+func (h *HLSHandler) VariantPlaylistWritten(localFilePath string) {
+	h.Storage.VariantPlaylistWritten(localFilePath)
+}
+
+// MasterPlaylistWritten is fired when a HLS master playlist is written to disk
+func (h *HLSHandler) MasterPlaylistWritten(localFilePath string) {
+	h.Storage.MasterPlaylistWritten(localFilePath)
+}

core/ffmpeg/thumbnailGenerator.go

@@ -13,36 +13,44 @@ import (
 	"github.com/owncast/owncast/config"
 )
 
+var _timer *time.Ticker
+
+func StopThumbnailGenerator() {
+	if _timer != nil {
+		_timer.Stop()
+	}
+}
+
 //StartThumbnailGenerator starts generating thumbnails
 func StartThumbnailGenerator(chunkPath string, variantIndex int) {
 	// Every 20 seconds create a thumbnail from the most
 	// recent video segment.
-	ticker := time.NewTicker(20 * time.Second)
+	_timer = time.NewTicker(20 * time.Second)
 	quit := make(chan struct{})
 
 	go func() {
 		for {
 			select {
-			case <-ticker.C:
+			case <-_timer.C:
 				if err := fireThumbnailGenerator(chunkPath, variantIndex); err != nil {
 					log.Errorln("Unable to generate thumbnail:", err)
 				}
 			case <-quit:
 				//TODO: evaluate if this is ever stopped
 				log.Debug("thumbnail generator has stopped")
-				ticker.Stop()
+				_timer.Stop()
 				return
 			}
 		}
 	}()
 }
 
-func fireThumbnailGenerator(chunkPath string, variantIndex int) error {
+func fireThumbnailGenerator(segmentPath string, variantIndex int) error {
 	// JPG takes less time to encode than PNG
 	outputFile := path.Join(config.WebRoot, "thumbnail.jpg")
 	previewGifFile := path.Join(config.WebRoot, "preview.gif")
 
-	framePath := path.Join(chunkPath, strconv.Itoa(variantIndex))
+	framePath := path.Join(segmentPath, strconv.Itoa(variantIndex))
 	files, err := ioutil.ReadDir(framePath)
 	if err != nil {
 		return err

core/ffmpeg/transcoder.go

@@ -3,7 +3,6 @@ package ffmpeg
 import (
 	"fmt"
 	"os/exec"
-	"path"
 	"strconv"
 	"strings"
 
@@ -27,6 +26,8 @@ type Transcoder struct {
 	appendToStream       bool
 	ffmpegPath           string
 	segmentIdentifier    string
+	internalListenerPort int
+	TranscoderCompleted  func(error)
 }
 
 // HLSVariant is a combination of settings that results in a single HLS stream
@@ -91,16 +92,27 @@ func (t *Transcoder) Start() {
 		log.Panicln(err, command)
 	}
 
+	err = _commandExec.Wait()
+	if t.TranscoderCompleted != nil {
+		t.TranscoderCompleted(err)
+	}
 	return
 }
 
 func (t *Transcoder) getString() string {
-	hlsOptionFlags := []string{
-		"delete_segments",
-		"program_date_time",
-		"temp_file",
+	var port int
+	if config.Config != nil {
+		port = config.Config.GetPublicWebServerPort() + 1
+	} else if t.internalListenerPort != 0 {
+		port = t.internalListenerPort
+	} else {
+		log.Panicln("A internal port must be set for transcoder callback")
 	}
 
+	localListenerAddress := "http://127.0.0.1:" + strconv.Itoa(port)
+
+	hlsOptionFlags := []string{}
+
 	if t.appendToStream {
 		hlsOptionFlags = append(hlsOptionFlags, "append_list")
 	}
@@ -109,32 +121,43 @@ func (t *Transcoder) getString() string {
 		t.segmentIdentifier = shortid.MustGenerate()
 	}
 
+	hlsOptionsString := ""
+	if len(hlsOptionFlags) > 0 {
+		hlsOptionsString = "-hls_flags " + strings.Join(hlsOptionFlags, "+")
+	}
 	ffmpegFlags := []string{
-		"cat", t.input, "|",
 		t.ffmpegPath,
 		"-hide_banner",
-		"-i pipe:",
+		"-loglevel warning",
+		"-i ", t.input,
+
 		t.getVariantsString(),
 
 		// HLS Output
 		"-f", "hls",
+
 		"-hls_time", strconv.Itoa(t.segmentLengthSeconds), // Length of each segment
 		"-hls_list_size", strconv.Itoa(t.hlsPlaylistLength), // Max # in variant playlist
 		"-hls_delete_threshold", "10", // Start deleting files after hls_list_size + 10
-		"-hls_flags", strings.Join(hlsOptionFlags, "+"), // Specific options in HLS generation
+		hlsOptionsString,
 
 		// Video settings
 		"-tune", "zerolatency", // Option used for good for fast encoding and low-latency streaming (always includes iframes in each segment)
-		// "-profile:v", "high", // Main – for standard definition (SD) to 640×480, High – for high definition (HD) to 1920×1080
+		"-pix_fmt", "yuv420p", // Force yuv420p color format
+		"-profile:v", "high", // Main – for standard definition (SD) to 640×480, High – for high definition (HD) to 1920×1080
 		"-sc_threshold", "0", // Disable scene change detection for creating segments
 
 		// Filenames
 		"-master_pl_name", "stream.m3u8",
-		"-strftime 1",                                                               // Support the use of strftime in filenames
-		"-hls_segment_filename", path.Join(t.segmentOutputPath, "/%v/stream-%s-"+t.segmentIdentifier+".ts"), // Each segment's filename
+		"-strftime 1", // Support the use of strftime in filenames
+
+		"-hls_segment_filename", localListenerAddress + "/%v/stream-" + t.segmentIdentifier + "%s.ts", // Send HLS segments back to us over HTTP
 		"-max_muxing_queue_size", "400", // Workaround for Too many packets error: https://trac.ffmpeg.org/ticket/6375?cversion=0
-		path.Join(t.segmentOutputPath, "/%v/stream.m3u8"), // Each variant's playlist
-		"2> transcoder.log",
+
+		"-method PUT -http_persistent 1",         // HLS results sent back to us will be over PUTs
+		"-fflags +genpts",                        // Generate presentation time stamp if missing
+		localListenerAddress + "/%v/stream.m3u8", // Send HLS playlists back to us over HTTP
+		"2> transcoder.log",                      // Log to a file for debugging
 	}
 
 	return strings.Join(ffmpegFlags, " ")
@@ -180,7 +203,7 @@ func getVariantFromConfigQuality(quality config.StreamQuality, index int) HLSVar
 }
 
 // NewTranscoder will return a new Transcoder, populated by the config
-func NewTranscoder() Transcoder {
+func NewTranscoder() *Transcoder {
 	transcoder := new(Transcoder)
 	transcoder.ffmpegPath = config.Config.GetFFMpegPath()
 	transcoder.hlsPlaylistLength = config.Config.GetMaxNumberOfReferencedSegmentsInPlaylist()
@@ -207,7 +230,7 @@ func NewTranscoder() Transcoder {
 		transcoder.AddVariant(variant)
 	}
 
-	return *transcoder
+	return transcoder
 }
 
 // Uses `map` https://www.ffmpeg.org/ffmpeg-all.html#Stream-specifiers-1 https://www.ffmpeg.org/ffmpeg-all.html#Advanced-options
@@ -364,3 +387,7 @@ func (t *Transcoder) SetAppendToStream(append bool) {
 func (t *Transcoder) SetIdentifier(output string) {
 	t.segmentIdentifier = output
 }
+
+func (t *Transcoder) SetInternalHTTPPort(port int) {
+	t.internalListenerPort = port
+}

core/ffmpeg/transcoder_test.go

@@ -12,6 +12,7 @@ func TestFFmpegCommand(t *testing.T) {
 	transcoder.SetOutputPath("fakeOutput")
 	transcoder.SetHLSPlaylistLength(10)
 	transcoder.SetIdentifier("jdofFGg")
+	transcoder.SetInternalHTTPPort(8123)
 
 	variant := HLSVariant{}
 	variant.videoBitrate = 1200
@@ -23,7 +24,7 @@ func TestFFmpegCommand(t *testing.T) {
 
 	cmd := transcoder.getString()
 
-	expected := `cat fakecontent.flv | /fake/path/ffmpeg -hide_banner -i pipe:  -map v:0 -c:v:0 libx264 -b:v:0 1200k -maxrate:v:0 1272k -bufsize:v:0 1440k -g:v:0 119 -x264-params:v:0 "scenecut=0:open_gop=0:min-keyint=119:keyint=119" -map a:0 -c:a:0 copy -r 30 -preset veryfast  -var_stream_map "v:0,a:0 " -f hls -hls_time 4 -hls_list_size 10 -hls_delete_threshold 10 -hls_flags delete_segments+program_date_time+temp_file -tune zerolatency -sc_threshold 0 -master_pl_name stream.m3u8 -strftime 1 -hls_segment_filename fakeOutput/%v/stream-%s-jdofFGg.ts -max_muxing_queue_size 400 fakeOutput/%v/stream.m3u8 2> transcoder.log`
+	expected := `/fake/path/ffmpeg -hide_banner -loglevel warning -i  fakecontent.flv  -map v:0 -c:v:0 libx264 -b:v:0 1200k -maxrate:v:0 1272k -bufsize:v:0 1440k -g:v:0 119 -x264-params:v:0 "scenecut=0:open_gop=0:min-keyint=119:keyint=119" -map a:0 -c:a:0 copy -r 30 -preset veryfast  -var_stream_map "v:0,a:0 " -f hls -hls_time 4 -hls_list_size 10 -hls_delete_threshold 10  -tune zerolatency -pix_fmt yuv420p -profile:v high -sc_threshold 0 -master_pl_name stream.m3u8 -strftime 1 -hls_segment_filename http://127.0.0.1:8123/%v/stream-jdofFGg%s.ts -max_muxing_queue_size 400 -method PUT -http_persistent 1 -fflags +genpts http://127.0.0.1:8123/%v/stream.m3u8 2> transcoder.log`
 
 	if cmd != expected {
 		t.Errorf("ffmpeg command does not match expected.  Got %s, want: %s", cmd, expected)