LogalDeveloper/owncast
0
Fork 0
Code Activity

Config repository (#3988)

Browse Source 
* WIP

* fix(test): fix ap test failing

* fix: fix unkeyed fields being used

* chore(tests): clean up browser tests by splitting out federation UI tests
This commit is contained in:
Gabe Kangas
2024-11-15 19:20:58 -08:00
committed by GitHub
parent 56d52c283c
commit 0b5d7c8a4d
88 changed files with 2078 additions and 1643 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
core/data/activitypub.go
View File

@@ -1,13 +0,0 @@
package data
// GetFederatedInboxMap is a mapping between account names and their outbox.
func GetFederatedInboxMap() map[string]string {
return map[string]string{
GetDefaultFederationUsername(): GetDefaultFederationUsername(),
}
}
// GetDefaultFederationUsername will return the username used for sending federation activities.
func GetDefaultFederationUsername() string {
return GetFederationUsername()
}

1027
core/data/config.go
View File

File diff suppressed because it is too large Load Diff

60
core/data/configEntry.go
View File

@@ -1,60 +0,0 @@
package data
import (
"bytes"
"encoding/gob"
)
// ConfigEntry is the actual object saved to the database.
// The Value is encoded using encoding/gob.
type ConfigEntry struct {
Value interface{}
Key string
}
func (c *ConfigEntry) getStringSlice() ([]string, error) {
decoder := c.getDecoder()
var result []string
err := decoder.Decode(&result)
return result, err
}
func (c *ConfigEntry) getStringMap() (map[string]string, error) {
decoder := c.getDecoder()
var result map[string]string
err := decoder.Decode(&result)
return result, err
}
func (c *ConfigEntry) getString() (string, error) {
decoder := c.getDecoder()
var result string
err := decoder.Decode(&result)
return result, err
}
func (c *ConfigEntry) getNumber() (float64, error) {
decoder := c.getDecoder()
var result float64
err := decoder.Decode(&result)
return result, err
}
func (c *ConfigEntry) getBool() (bool, error) {
decoder := c.getDecoder()
var result bool
err := decoder.Decode(&result)
return result, err
}
func (c *ConfigEntry) getObject(result interface{}) error {
decoder := c.getDecoder()
err := decoder.Decode(result)
return err
}
func (c *ConfigEntry) getDecoder() *gob.Decoder {
valueBytes := c.Value.([]byte)
decoder := gob.NewDecoder(bytes.NewBuffer(valueBytes))
return decoder
}

23
core/data/crypto.go
View File

@@ -1,23 +0,0 @@
package data
// GetPublicKey will return the public key.
func GetPublicKey() string {
value, _ := _datastore.GetString(publicKeyKey)
return value
}
// SetPublicKey will save the public key.
func SetPublicKey(key string) error {
return _datastore.SetString(publicKeyKey, key)
}
// GetPrivateKey will return the private key.
func GetPrivateKey() string {
value, _ := _datastore.GetString(privateKeyKey)
return value
}
// SetPrivateKey will save the private key.
func SetPrivateKey(key string) error {
return _datastore.SetString(privateKeyKey, key)
}

10
core/data/data_test.go
View File

@@ -4,6 +4,8 @@ import (
"fmt"
"os"
"testing"
"github.com/owncast/owncast/models"
)
func TestMain(m *testing.M) {
@@ -89,7 +91,7 @@ func TestCustomType(t *testing.T) {
}
// Save config entry to the database
if err := _datastore.Save(ConfigEntry{&testStruct, testKey}); err != nil {
if err := _datastore.Save(models.ConfigEntry{&testStruct, testKey}); err != nil {
t.Error(err)
}
@@ -101,7 +103,7 @@ func TestCustomType(t *testing.T) {
// Get a typed struct out of it
var testResult TestStruct
if err := entryResult.getObject(&testResult); err != nil {
if err := entryResult.GetObject(&testResult); err != nil {
t.Error(err)
}
@@ -121,7 +123,7 @@ func TestStringMap(t *testing.T) {
}
// Save config entry to the database
if err := _datastore.Save(ConfigEntry{&testMap, testKey}); err != nil {
if err := _datastore.Save(models.ConfigEntry{Value: &testMap, Key: testKey}); err != nil {
t.Error(err)
}
@@ -131,7 +133,7 @@ func TestStringMap(t *testing.T) {
t.Error(err)
}
testResult, err := entryResult.getStringMap()
testResult, err := entryResult.GetStringMap()
if err != nil {
t.Error(err)
}

39
core/data/persistence.go → core/data/datastore.go
View File

@@ -5,12 +5,11 @@ import (
"database/sql"
"encoding/gob"
"sync"
"time"
// sqlite requires a blank import.
_ "github.com/mattn/go-sqlite3"
"github.com/owncast/owncast/config"
"github.com/owncast/owncast/db"
"github.com/owncast/owncast/models"
log "github.com/sirupsen/logrus"
)
@@ -21,7 +20,8 @@ type Datastore struct {
DbLock *sync.Mutex
}
func (ds *Datastore) warmCache() {
// WarmCache pre-caches all configuration values in memory.
func (ds *Datastore) WarmCache() {
log.Traceln("Warming config value cache")
res, err := ds.DB.Query("SELECT key, value FROM datastore")
@@ -46,10 +46,10 @@ func (ds *Datastore) GetQueries() *db.Queries {
}
// Get will query the database for the key and return the entry.
func (ds *Datastore) Get(key string) (ConfigEntry, error) {
func (ds *Datastore) Get(key string) (models.ConfigEntry, error) {
cachedValue, err := ds.GetCachedValue(key)
if err == nil {
return ConfigEntry{
return models.ConfigEntry{
Key: key,
Value: cachedValue,
}, nil
@@ -60,10 +60,10 @@ func (ds *Datastore) Get(key string) (ConfigEntry, error) {
row := ds.DB.QueryRow("SELECT key, value FROM datastore WHERE key = ? LIMIT 1", key)
if err := row.Scan(&resultKey, &resultValue); err != nil {
return ConfigEntry{}, err
return models.ConfigEntry{}, err
}
result := ConfigEntry{
result := models.ConfigEntry{
Key: resultKey,
Value: resultValue,
}
@@ -73,7 +73,7 @@ func (ds *Datastore) Get(key string) (ConfigEntry, error) {
}
// Save will save the ConfigEntry to the database.
func (ds *Datastore) Save(e ConfigEntry) error {
func (ds *Datastore) Save(e models.ConfigEntry) error {
ds.DbLock.Lock()
defer ds.DbLock.Unlock()
@@ -93,7 +93,6 @@ func (ds *Datastore) Save(e ConfigEntry) error {
return err
}
_, err = stmt.Exec(e.Key, dataGob.Bytes())
if err != nil {
return err
}
@@ -121,26 +120,6 @@ func (ds *Datastore) Setup() {
);`
ds.MustExec(createTableSQL)
if !HasPopulatedDefaults() {
PopulateDefaults()
}
if !hasPopulatedFederationDefaults() {
if err := SetFederationGoLiveMessage(config.GetDefaults().FederationGoLiveMessage); err != nil {
log.Errorln(err)
}
if err := _datastore.SetBool("HAS_POPULATED_FEDERATION_DEFAULTS", true); err != nil {
log.Errorln(err)
}
}
// Set the server initialization date if needed.
if hasSetInitDate, _ := GetServerInitTime(); hasSetInitDate == nil || !hasSetInitDate.Valid {
_ = SetServerInitTime(time.Now())
}
migrateDatastoreValues(_datastore)
}
// Reset will delete all config entries in the datastore and start over.
@@ -156,8 +135,6 @@ func (ds *Datastore) Reset() {
if _, err = stmt.Exec(); err != nil {
log.Fatalln(err)
}
PopulateDefaults()
}
// GetDatastore returns the shared instance of the owncast datastore.

86
core/data/datastoreMigrations.go
View File

@@ -1,86 +0,0 @@
package data
import (
"strings"
"github.com/owncast/owncast/models"
log "github.com/sirupsen/logrus"
)
const (
datastoreValuesVersion = 4
datastoreValueVersionKey = "DATA_STORE_VERSION"
)
func migrateDatastoreValues(datastore *Datastore) {
currentVersion, _ := _datastore.GetNumber(datastoreValueVersionKey)
if currentVersion == 0 {
currentVersion = datastoreValuesVersion
}
for v := currentVersion; v < datastoreValuesVersion; v++ {
log.Infof("Migration datastore values from %d to %d\n", int(v), int(v+1))
switch v {
case 0:
migrateToDatastoreValues1(datastore)
case 1:
migrateToDatastoreValues2(datastore)
case 2:
migrateToDatastoreValues3ServingEndpoint3(datastore)
case 3:
migrateToDatastoreValues4(datastore)
default:
log.Fatalln("missing datastore values migration step")
}
}
if err := _datastore.SetNumber(datastoreValueVersionKey, datastoreValuesVersion); err != nil {
log.Errorln("error setting datastore value version:", err)
}
}
func migrateToDatastoreValues1(datastore *Datastore) {
// Migrate the forbidden usernames to be a slice instead of a string.
forbiddenUsernamesString, _ := datastore.GetString(blockedUsernamesKey)
if forbiddenUsernamesString != "" {
forbiddenUsernamesSlice := strings.Split(forbiddenUsernamesString, ",")
if err := datastore.SetStringSlice(blockedUsernamesKey, forbiddenUsernamesSlice); err != nil {
log.Errorln("error migrating blocked username list:", err)
}
}
// Migrate the suggested usernames to be a slice instead of a string.
suggestedUsernamesString, _ := datastore.GetString(suggestedUsernamesKey)
if suggestedUsernamesString != "" {
suggestedUsernamesSlice := strings.Split(suggestedUsernamesString, ",")
if err := datastore.SetStringSlice(suggestedUsernamesKey, suggestedUsernamesSlice); err != nil {
log.Errorln("error migrating suggested username list:", err)
}
}
}
func migrateToDatastoreValues2(datastore *Datastore) {
oldAdminPassword, _ := datastore.GetString("stream_key")
// Avoids double hashing the password
_ = datastore.SetString("admin_password_key", oldAdminPassword)
_ = SetStreamKeys([]models.StreamKey{
{Key: oldAdminPassword, Comment: "Default stream key"},
})
}
func migrateToDatastoreValues3ServingEndpoint3(_ *Datastore) {
s3Config := GetS3Config()
if !s3Config.Enabled {
return
}
_ = SetVideoServingEndpoint(s3Config.ServingEndpoint)
}
func migrateToDatastoreValues4(datastore *Datastore) {
unhashed_pass, _ := datastore.GetString("admin_password_key")
err := SetAdminPassword(unhashed_pass)
if err != nil {
log.Fatalln("error migrating admin password:", err)
}
}

54
core/data/defaults.go
View File

@@ -1,54 +0,0 @@
package data
import (
"github.com/owncast/owncast/config"
"github.com/owncast/owncast/models"
)
// HasPopulatedDefaults will determine if the defaults have been inserted into the database.
func HasPopulatedDefaults() bool {
hasPopulated, err := _datastore.GetBool("HAS_POPULATED_DEFAULTS")
if err != nil {
return false
}
return hasPopulated
}
func hasPopulatedFederationDefaults() bool {
hasPopulated, err := _datastore.GetBool("HAS_POPULATED_FEDERATION_DEFAULTS")
if err != nil {
return false
}
return hasPopulated
}
// PopulateDefaults will set default values in the database.
func PopulateDefaults() {
_datastore.warmCache()
defaults := config.GetDefaults()
if HasPopulatedDefaults() {
return
}
_ = SetAdminPassword(defaults.AdminPassword)
_ = SetStreamKeys(defaults.StreamKeys)
_ = SetHTTPPortNumber(float64(defaults.WebServerPort))
_ = SetRTMPPortNumber(float64(defaults.RTMPServerPort))
_ = SetLogoPath(defaults.Logo)
_ = SetServerMetadataTags([]string{"owncast", "streaming"})
_ = SetServerSummary(defaults.Summary)
_ = SetServerWelcomeMessage("")
_ = SetServerName(defaults.Name)
_ = SetExtraPageBodyContent(defaults.PageBodyContent)
_ = SetFederationGoLiveMessage(defaults.FederationGoLiveMessage)
_ = SetSocialHandles([]models.SocialHandle{
{
Platform: "github",
URL: "https://github.com/owncast/owncast",
},
})
_ = _datastore.SetBool("HAS_POPULATED_DEFAULTS", true)
}

64
core/data/messages.go
View File

@@ -1,14 +1,5 @@
package data
import (
"context"
"database/sql"
"github.com/owncast/owncast/db"
"github.com/owncast/owncast/models"
log "github.com/sirupsen/logrus"
)
// GetMessagesCount will return the number of messages in the database.
func GetMessagesCount() int64 {
query := `SELECT COUNT(*) FROM messages`
@@ -25,58 +16,3 @@ func GetMessagesCount() int64 {
}
return count
}
// CreateBanIPTable will create the IP ban table if needed.
func CreateBanIPTable(db *sql.DB) {
createTableSQL := ` CREATE TABLE IF NOT EXISTS ip_bans (
"ip_address" TEXT NOT NULL PRIMARY KEY,
"notes" TEXT,
"created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);`
stmt, err := db.Prepare(createTableSQL)
if err != nil {
log.Fatal("error creating ip ban table", err)
}
defer stmt.Close()
if _, err := stmt.Exec(); err != nil {
log.Fatal("error creating ip ban table", err)
}
}
// BanIPAddress will persist a new IP address ban to the datastore.
func BanIPAddress(address, note string) error {
return _datastore.GetQueries().BanIPAddress(context.Background(), db.BanIPAddressParams{
IpAddress: address,
Notes: sql.NullString{String: note, Valid: true},
})
}
// IsIPAddressBanned will return if an IP address has been previously blocked.
func IsIPAddressBanned(address string) (bool, error) {
blocked, error := _datastore.GetQueries().IsIPAddressBlocked(context.Background(), address)
return blocked > 0, error
}
// GetIPAddressBans will return all the banned IP addresses.
func GetIPAddressBans() ([]models.IPAddress, error) {
result, err := _datastore.GetQueries().GetIPAddressBans(context.Background())
if err != nil {
return nil, err
}
response := []models.IPAddress{}
for _, ip := range result {
response = append(response, models.IPAddress{
IPAddress: ip.IpAddress,
Notes: ip.Notes.String,
CreatedAt: ip.CreatedAt.Time,
})
}
return response, err
}
// RemoveIPAddressBan will remove a previously banned IP address.
func RemoveIPAddressBan(address string) error {
return _datastore.GetQueries().RemoveIPAddressBan(context.Background(), address)
}

22
core/data/types.go
View File

@@ -1,17 +1,19 @@
package data
import "github.com/owncast/owncast/models"
// GetStringSlice will return the string slice value for a key.
func (ds *Datastore) GetStringSlice(key string) ([]string, error) {
configEntry, err := ds.Get(key)
if err != nil {
return []string{}, err
}
return configEntry.getStringSlice()
return configEntry.GetStringSlice()
}
// SetStringSlice will set the string slice value for a key.
func (ds *Datastore) SetStringSlice(key string, value []string) error {
configEntry := ConfigEntry{value, key}
configEntry := models.ConfigEntry{Value: value, Key: key}
return ds.Save(configEntry)
}
@@ -21,12 +23,12 @@ func (ds *Datastore) GetString(key string) (string, error) {
if err != nil {
return "", err
}
return configEntry.getString()
return configEntry.GetString()
}
// SetString will set the string value for a key.
func (ds *Datastore) SetString(key string, value string) error {
configEntry := ConfigEntry{value, key}
configEntry := models.ConfigEntry{Value: value, Key: key}
return ds.Save(configEntry)
}
@@ -36,12 +38,12 @@ func (ds *Datastore) GetNumber(key string) (float64, error) {
if err != nil {
return 0, err
}
return configEntry.getNumber()
return configEntry.GetNumber()
}
// SetNumber will set the numeric value for a key.
func (ds *Datastore) SetNumber(key string, value float64) error {
configEntry := ConfigEntry{value, key}
configEntry := models.ConfigEntry{Value: value, Key: key}
return ds.Save(configEntry)
}
@@ -51,12 +53,12 @@ func (ds *Datastore) GetBool(key string) (bool, error) {
if err != nil {
return false, err
}
return configEntry.getBool()
return configEntry.GetBool()
}
// SetBool will set the boolean value for a key.
func (ds *Datastore) SetBool(key string, value bool) error {
configEntry := ConfigEntry{value, key}
configEntry := models.ConfigEntry{Value: value, Key: key}
return ds.Save(configEntry)
}
@@ -66,11 +68,11 @@ func (ds *Datastore) GetStringMap(key string) (map[string]string, error) {
if err != nil {
return map[string]string{}, err
}
return configEntry.getStringMap()
return configEntry.GetStringMap()
}
// SetStringMap will set the string map value for a key.
func (ds *Datastore) SetStringMap(key string, value map[string]string) error {
configEntry := ConfigEntry{value, key}
configEntry := models.ConfigEntry{Value: value, Key: key}
return ds.Save(configEntry)
}