0.0.6 -> Master (#731)

* Implement webhook events for external integrations (#574)

* Implement webhook events for external integrations

Reference #556

* move message type to models and remove duplicate

* add json header so content type can be determined

* Pass at migrating webhooks to datastore + management apis (#589)

* Pass at migrating webhooks to datastore + management apis

* Support nil lastUsed timestamps and return back the new webhook on create

* Cleanup from review feedback

* Simplify a bit

Co-authored-by: Aaron Ogle <aaron@geekgonecrazy.com>

Co-authored-by: Gabe Kangas <gabek@real-ity.com>

* Webhook query cleanup

* Access tokens + Send system message external API (#585)

* New add, get and delete access token APIs

* Create auth token middleware

* Update last_used timestamp when using an access token

* Add auth'ed endpoint for sending system messages

* Cleanup

* Update api spec for new apis

* Commit updated API documentation

* Add auth'ed endpoint for sending user chat messages

* Return access token string

* Commit updated API documentation

* Fix route

* Support nil lastUsed time

* Commit updated Javascript packages

* Remove duplicate function post rebase

* Fix msg id generation

* Update controllers/admin/chat.go

Co-authored-by: Aaron Ogle <geekgonecrazy@users.noreply.github.com>

* Webhook query cleanup

* Add SystemMessageSent to EventType

Co-authored-by: Owncast <owncast@owncast.online>
Co-authored-by: Aaron Ogle <geekgonecrazy@users.noreply.github.com>

* Set webhook as used on completion. Closes #610

* Display webhook errors as errors

* Commit updated API documentation

* Add user joined chat event

* Change integration API paths. Update API spec

* Update development version of admin that supports integration apis

* Commit updated API documentation

* Add automated tests for external integration APIs

* check error

* quiet this test for now

* Route up some additional 3rd party apis. #638

* Commit updated API documentation

* Save username on user joined event

* Add missing scope to valid scopes list

* Add generic chat action event API for 3rd parties. Closes #666

* Commit updated API documentation

* First pass at moving WIP config framework into project for #234

* Only support exported fields in custom types

* Using YP get/set key as a first pass at using the data layer. Fixes + integration.

* Ignore test db

* Start adding getters and setters for config values

* More get/set config work. Starting to populate api with data

* Wire up some config edit endpoints

* More endpoints

* Disable cors middleware

* Add more endpoints and add test to test them

* Remove the in-memory change APIs

* Add endpoint for changing tags

* Add more config endpoints

* Starting to point more things away from config file and to the datastore

* Populate YP with db data

* Create new util method for parsing page body markdown and return it in api

* Verify proposed path to ffmpeg

* For development purposes show the config key in logs

* Move stats values to datastore

* Moving over more values to the datastore

* Move S3 config to datastore

* First pass the config -> db migrator

* Add the start of the video config apis

* It builds pointing everything away from the config

* Tweak ffmpeg path error message

* Backup database every hour. Closes #549

* Config + defaults + migration work for db

* Cleanup logging

* Remove all the old config structs

* Add descriptive info about migration

* Tweak ffmpeg validation logic

* Fix db backup path. backup on db version migration

* Set video and s3 configurations

* Update api spec with new config endpoints

* Add migrator for stats file

* Commit updated API documentation

* Use a dynamic system port for internal HLS writes. Closes #577 (#626)

* Use a dynamic system port for internal HLS writes. Closes #577

* Cleanup

* YP key migration to datastore

* Create a backup directory if needed before migrations

* Remove config test that no longer makes sense. Cleanup.

* Change number types from float32 to float64

* Update automated test suite

* Allow restoring a database backup via command line flags. Closes #549

* Add new hls segment config api

* Commit updated API documentation

* Update apis to require a value container property

* add socialHandles api

* Commit updated API documentation

* Add new latancy level setting to replace segment settings

* Commit updated API documentation

* Fix spelling

* Commit updated API documentation

* hardcode a json api of available social platforms

* Add additional icons

* Return social handles in server config api

* Add socialhandles validation to test

* Move list of hard coded social platforms to an api

* Remove audio only code from transcoder since we do not use it

* Add latency levels api + snapshot of video settings as current broadcast

* Add config/serverurl endpoint

* Return 404 on YP api if disabled

* Surface stream title in YP response

* Add stream title to web ui

* Cleanup log message. Closes #520

* Rename ffmpeg package to transcoder

* Add ws package for testing

* Reduce chat backlog to past 5hrs, max 50. Closes #548

* Fix error formatting

* Add endpoint for resetting yp registration

* Add yp/reset to api spec. return status in response

* Return zero viewer count if stream is offline. Closes #422

* Post-rebase fixes

* Fix merge conflict in openapi file

* Commit updated API documentation

* Standardize controller names

* Support setting the stream key via the command line. Closes #665

* Return social handles with YP data. First half of https://github.com/owncast/owncast-yp/issues/28

* Give the YP package access to server status regardless if enabled or not

* Change delay in automated tests

* Add stream title integration API. For #638

* Commit updated API documentation

* Add storage to the migrator

* Missing returning NSFW value in server config

* Add flag to ignore websocket client. Closes #537

* Add error for parsing broadcaster metadata

* Add support for a cli specified http server port. Closes #674

* Add cpu usage levels and a temporary mapping between it and libx264 presets

* Test for valid url endpoint when saving s3 config

* Re-configure storage on every stream to allow changing storage providers

* After 5 minutes of a stream being stopped clear the stream title

* Hide viewer count once stream goes offline instead of when player stops

* Pull steamTitle from the status that gets updated instead of the config

* Commit updated API documentation

* Optionally show stream title in the header

* Reset stream title when server starts

* Show chat action when stream title is updated

* Allow system messages to come back in persistence

* Split out getting chat history for moderation + fix tests

* Remove server title and standardize on name only

* Commit updated API documentation

* Bump github.com/aws/aws-sdk-go from 1.37.1 to 1.37.2 (#680)

Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.37.1 to 1.37.2.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Changelog](https://github.com/aws/aws-sdk-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.37.1...v1.37.2)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Add video variant and stream latency config file migrator

* Remove mostly unused disable upgrade check bool

* Commit updated API documentation

* Allow bundling the admin from the 0.0.6 branch

* Fix saving port numbers

* Use name instead of old title on window focus

* Work on latency levels. Fix test to use levels. Clean up transcoder to only reference levels

* Another place where title -> name

* Fix test

* Bump github.com/aws/aws-sdk-go from 1.37.2 to 1.37.3 (#690)

Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.37.2 to 1.37.3.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Changelog](https://github.com/aws/aws-sdk-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.37.2...v1.37.3)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Update dependabot config

* Bump github.com/aws/aws-sdk-go from 1.37.3 to 1.37.5 (#693)

Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.37.3 to 1.37.5.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Changelog](https://github.com/aws/aws-sdk-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.37.3...v1.37.5)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Bump video.js from 7.10.2 to 7.11.4 in /build/javascript (#694)

* Bump video.js from 7.10.2 to 7.11.4 in /build/javascript

Bumps [video.js](https://github.com/videojs/video.js) from 7.10.2 to 7.11.4.
- [Release notes](https://github.com/videojs/video.js/releases)
- [Changelog](https://github.com/videojs/video.js/blob/main/CHANGELOG.md)
- [Commits](https://github.com/videojs/video.js/compare/v7.10.2...v7.11.4)

Signed-off-by: dependabot[bot] <support@github.com>

* Commit updated Javascript packages

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Owncast <owncast@owncast.online>

* Make the latency migrator dynamic so I can tweak values easier

* Split out fetching ffmpeg path from validating the path so it can be changed in the admin

* Some commenting and linter cleanup

* Validate the path for a logo change and throw an error if it does not exist

* Logo change requests have to be a real file now

* Cleanup, making linter happy

* Format javascript on push

* Only format js in master

* Tweak latency level values

* Remove unused config file examples

* Fix thumbnail generation after messing with the ffmpeg path getter

* Reduce how often we report high hardware utilization warnings

* Bundle the 0.0.6 branch version of the admin

* Return validated ffmpeg path in admin server config

* Change the logo to be stored in the data directory instead of webroot

* Bump postcss from 8.2.4 to 8.2.5 in /build/javascript (#702)

Bumps [postcss](https://github.com/postcss/postcss) from 8.2.4 to 8.2.5.
- [Release notes](https://github.com/postcss/postcss/releases)
- [Changelog](https://github.com/postcss/postcss/blob/main/CHANGELOG.md)
- [Commits](https://github.com/postcss/postcss/compare/8.2.4...8.2.5)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Default config file no longer used

* don't show stream title when offline

addresses https://github.com/owncast/owncast/issues/677

* Remove auto-clearing stream title. #677

* webroot -> data when using logo as thumbnail

* Do not list websocket/access token create/delete as integration APIs

* Commit updated API documentation

* Bundle updated admin

* Remove pointing to the 0.0.6 admin branch

* Linter cleanup

* Linter cleanup

* Add donations and follow links to show up under social handles

* Prettified Code!

* More linter cleanup

* Update admin bundle

* Remove use of platforms.js and return icons with social handles. Closes #732

* Update admin bundle

* Support custom config path for use in migration

* Remove unused platform-logos.gif

* Reduce log level of message

* Remove unused logo files in static dir

* Handle dev vs. release build info

* Restore logo.png for initial thumbnail

* Cleanup some files from the build process that are not needed

* Fix incorrect build-time injection var

* Fix missing file getting copied to the build

* Remove console directory message.

* Update admin bundle

* Fix comment

* Report storage setup error

* add some value set error checking

* Use validated dynamic ffmpeg path for animated gif preview

* Make chat message links be white so they don't hide in the bg. Closes #599

* Restore conditional that was accidentally removed

Co-authored-by: Aaron Ogle <geekgonecrazy@users.noreply.github.com>
Co-authored-by: Owncast <owncast@owncast.online>
Co-authored-by: Ginger Wong <omqmail@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: nebunez <uoj2y7wak869@opayq.net>
Co-authored-by: gabek <gabek@users.noreply.github.com>

core/chat/chat.go

@@ -60,12 +60,16 @@ func SendMessage(message models.ChatEvent) {
 }
 
 // GetMessages gets all of the messages.
-func GetMessages(filtered bool) []models.ChatEvent {
+func GetMessages() []models.ChatEvent {
 	if _server == nil {
 		return []models.ChatEvent{}
 	}
 
-	return getChatHistory(filtered)
+	return getChatHistory()
+}
+
+func GetModerationChatMessages() []models.ChatEvent {
+	return getChatModerationHistory()
 }
 
 func GetClient(clientID string) *Client {

core/chat/client.go

@@ -28,36 +28,38 @@ type Client struct {
 	Username     *string
 	ClientID     string            // How we identify unique viewers when counting viewer counts.
 	Geo          *geoip.GeoDetails `json:"geo"`
+	Ignore       bool              // If set to true this will not be treated as a viewer
 
 	socketID              string // How we identify a single websocket client.
 	ws                    *websocket.Conn
 	ch                    chan models.ChatEvent
 	pingch                chan models.PingMessage
 	usernameChangeChannel chan models.NameChangeEvent
+	userJoinedChannel     chan models.UserJoinedEvent
 
 	doneCh chan bool
 
 	rateLimiter *rate.Limiter
 }
 
-const (
-	CHAT             = "CHAT"
-	NAMECHANGE       = "NAME_CHANGE"
-	PING             = "PING"
-	PONG             = "PONG"
-	VISIBILITYUPDATE = "VISIBILITY-UPDATE"
-)
-
 // NewClient creates a new chat client.
 func NewClient(ws *websocket.Conn) *Client {
 	if ws == nil {
 		log.Panicln("ws cannot be nil")
 	}
 
+	var ignoreClient = false
+	for _, extraData := range ws.Config().Protocol {
+		if extraData == "IGNORE_CLIENT" {
+			ignoreClient = true
+		}
+	}
+
 	ch := make(chan models.ChatEvent, channelBufSize)
 	doneCh := make(chan bool)
 	pingch := make(chan models.PingMessage)
 	usernameChangeChannel := make(chan models.NameChangeEvent)
+	userJoinedChannel := make(chan models.UserJoinedEvent)
 
 	ipAddress := utils.GetIPAddressFromRequest(ws.Request())
 	userAgent := ws.Request().UserAgent()
@@ -66,7 +68,7 @@ func NewClient(ws *websocket.Conn) *Client {
 
 	rateLimiter := rate.NewLimiter(0.6, 5)
 
-	return &Client{time.Now(), 0, userAgent, ipAddress, nil, clientID, nil, socketID, ws, ch, pingch, usernameChangeChannel, doneCh, rateLimiter}
+	return &Client{time.Now(), 0, userAgent, ipAddress, nil, clientID, nil, ignoreClient, socketID, ws, ch, pingch, usernameChangeChannel, userJoinedChannel, doneCh, rateLimiter}
 }
 
 func (c *Client) write(msg models.ChatEvent) {
@@ -105,6 +107,12 @@ func (c *Client) listenWrite() {
 			if err != nil {
 				c.handleClientSocketError(err)
 			}
+		case msg := <-c.userJoinedChannel:
+			err := websocket.JSON.Send(c.ws, msg)
+			if err != nil {
+				c.handleClientSocketError(err)
+			}
+
 		// receive done request
 		case <-c.doneCh:
 			_server.removeClient(c)
@@ -157,28 +165,46 @@ func (c *Client) listenRead() {
 				log.Errorln(err)
 			}
 
-			messageType := messageTypeCheck["type"]
+			messageType := messageTypeCheck["type"].(string)
 
 			if !c.passesRateLimit() {
 				continue
 			}
 
-			if messageType == CHAT {
+			if messageType == models.MessageSent {
 				c.chatMessageReceived(data)
-			} else if messageType == NAMECHANGE {
+			} else if messageType == models.UserNameChanged {
 				c.userChangedName(data)
+			} else if messageType == models.UserJoined {
+				c.userJoined(data)
 			}
 		}
 	}
 }
 
+func (c *Client) userJoined(data []byte) {
+	var msg models.UserJoinedEvent
+	if err := json.Unmarshal(data, &msg); err != nil {
+		log.Errorln(err)
+		return
+	}
+
+	msg.ID = shortid.MustGenerate()
+	msg.Type = models.UserJoined
+	msg.Timestamp = time.Now()
+
+	c.Username = &msg.Username
+
+	_server.userJoined(msg)
+}
+
 func (c *Client) userChangedName(data []byte) {
 	var msg models.NameChangeEvent
 	err := json.Unmarshal(data, &msg)
 	if err != nil {
 		log.Errorln(err)
 	}
-	msg.Type = NAMECHANGE
+	msg.Type = models.UserNameChanged
 	msg.ID = shortid.MustGenerate()
 	_server.usernameChanged(msg)
 	c.Username = &msg.NewName
@@ -191,10 +217,7 @@ func (c *Client) chatMessageReceived(data []byte) {
 		log.Errorln(err)
 	}
 
-	id, _ := shortid.Generate()
-	msg.ID = id
-	msg.Timestamp = time.Now()
-	msg.Visible = true
+	msg.SetDefaults()
 
 	c.MessageCount++
 	c.Username = &msg.Author

core/chat/messages.go

@@ -1,6 +1,8 @@
 package chat
 
 import (
+	"github.com/owncast/owncast/core/webhooks"
+	"github.com/owncast/owncast/models"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -20,8 +22,10 @@ func SetMessagesVisibility(messageIDs []string, visibility bool) error {
 			log.Errorln(err)
 			continue
 		}
-		message.MessageType = VISIBILITYUPDATE
+		message.MessageType = models.VisibiltyToggled
 		_server.sendAll(message)
+
+		go webhooks.SendChatEvent(message)
 	}
 
 	return nil

core/chat/persistence.go

@@ -61,15 +61,8 @@ func addMessage(message models.ChatEvent) {
 	}
 }
 
-func getChatHistory(filtered bool) []models.ChatEvent {
+func getChat(query string) []models.ChatEvent {
 	history := make([]models.ChatEvent, 0)
-
-	// Get all messages sent within the past day
-	var query = "SELECT * FROM messages WHERE messageType != 'SYSTEM' AND datetime(timestamp) >=datetime('now', '-1 Day')"
-	if filtered {
-		query = query + " AND visible = 1"
-	}
-
 	rows, err := _db.Query(query)
 	if err != nil {
 		log.Fatal(err)
@@ -80,7 +73,7 @@ func getChatHistory(filtered bool) []models.ChatEvent {
 		var id string
 		var author string
 		var body string
-		var messageType string
+		var messageType models.EventType
 		var visible int
 		var timestamp time.Time
 
@@ -109,6 +102,17 @@ func getChatHistory(filtered bool) []models.ChatEvent {
 	return history
 }
 
+func getChatModerationHistory() []models.ChatEvent {
+	var query = "SELECT * FROM messages WHERE messageType == 'CHAT' AND datetime(timestamp) >=datetime('now', '-5 Hour')"
+	return getChat(query)
+}
+
+func getChatHistory() []models.ChatEvent {
+	// Get all messages sent within the past 5hrs, max 50
+	var query = "SELECT * FROM messages WHERE datetime(timestamp) >=datetime('now', '-5 Hour') AND visible = 1 LIMIT 50"
+	return getChat(query)
+}
+
 func saveMessageVisibility(messageIDs []string, visible bool) error {
 	tx, err := _db.Begin()
 	if err != nil {
@@ -150,7 +154,7 @@ func getMessageById(messageID string) (models.ChatEvent, error) {
 	var id string
 	var author string
 	var body string
-	var messageType string
+	var messageType models.EventType
 	var visible int
 	var timestamp time.Time
 

core/chat/server.go

@@ -9,7 +9,8 @@ import (
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/net/websocket"
 
-	"github.com/owncast/owncast/config"
+	"github.com/owncast/owncast/core/data"
+	"github.com/owncast/owncast/core/webhooks"
 	"github.com/owncast/owncast/models"
 )
 
@@ -61,7 +62,7 @@ func (s *server) sendAll(msg models.ChatEvent) {
 }
 
 func (s *server) ping() {
-	ping := models.PingMessage{MessageType: PING}
+	ping := models.PingMessage{MessageType: models.PING}
 	for _, c := range s.Clients {
 		c.pingch <- ping
 	}
@@ -71,6 +72,16 @@ func (s *server) usernameChanged(msg models.NameChangeEvent) {
 	for _, c := range s.Clients {
 		c.usernameChangeChannel <- msg
 	}
+
+	go webhooks.SendChatEventUsernameChanged(msg)
+}
+
+func (s *server) userJoined(msg models.UserJoinedEvent) {
+	for _, c := range s.Clients {
+		c.userJoinedChannel <- msg
+	}
+
+	go webhooks.SendChatEventUserJoined(msg)
 }
 
 func (s *server) onConnection(ws *websocket.Conn) {
@@ -103,8 +114,10 @@ func (s *server) Listen() {
 			s.Clients[c.socketID] = c
 			l.Unlock()
 
-			s.listener.ClientAdded(c.GetViewerClientFromChatClient())
-			s.sendWelcomeMessageToClient(c)
+			if !c.Ignore {
+				s.listener.ClientAdded(c.GetViewerClientFromChatClient())
+				s.sendWelcomeMessageToClient(c)
+			}
 
 		// remove a client
 		case c := <-s.delCh:
@@ -122,7 +135,11 @@ func (s *server) Listen() {
 				s.sendAll(msg)
 
 				// Store in the message history
+				msg.SetDefaults()
 				addMessage(msg)
+
+				// Send webhooks
+				go webhooks.SendChatEvent(msg)
 			}
 		case ping := <-s.pingCh:
 			fmt.Println("PING?", ping)
@@ -154,8 +171,8 @@ func (s *server) sendWelcomeMessageToClient(c *Client) {
 		// Add an artificial delay so people notice this message come in.
 		time.Sleep(7 * time.Second)
 
-		initialChatMessageText := fmt.Sprintf("Welcome to %s! %s", config.Config.InstanceDetails.Title, config.Config.InstanceDetails.Summary)
-		initialMessage := models.ChatEvent{ClientID: "owncast-server", Author: config.Config.InstanceDetails.Name, Body: initialChatMessageText, ID: "initial-message-1", MessageType: "SYSTEM", Visible: true, Timestamp: time.Now()}
+		initialChatMessageText := fmt.Sprintf("Welcome to %s! %s", data.GetServerName(), data.GetServerSummary())
+		initialMessage := models.ChatEvent{ClientID: "owncast-server", Author: data.GetServerName(), Body: initialChatMessageText, ID: "initial-message-1", MessageType: "SYSTEM", Visible: true, Timestamp: time.Now()}
 		c.write(initialMessage)
 	}()
 }