ActivityPub background actor validation, update and cleanup process (#4765)

* feat(ap): add validation+update+cleanup AP actor job

* fix(test): use time.Time instead of sql.NullTime

* feat(test): add integration test for AP follower cleanup/validation job
This commit is contained in:
Gabe Kangas
2026-02-02 17:34:25 -08:00
committed by GitHub
parent 1a14b266aa
commit 5a4af3a294
16 changed files with 1315 additions and 25 deletions
@@ -0,0 +1,47 @@
name: ActivityPub Follower Validation Tests
on:
push:
paths:
- "activitypub/**"
- "config/**"
- "main.go"
- "test/automated/activitypub/**"
- ".github/workflows/activitypub-follower-validation-tests.yaml"
pull_request:
paths:
- "activitypub/**"
- "config/**"
- "main.go"
- "test/automated/activitypub/**"
- ".github/workflows/activitypub-follower-validation-tests.yaml"
jobs:
follower-validation-test:
runs-on: ubuntu-latest
steps:
- id: skip_check
uses: fkirc/skip-duplicate-actions@v5
with:
concurrent_skipping: "same_content_newer"
- name: Check out repository code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: "^1.23"
cache: true
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y sqlite3 ffmpeg jq
- name: Run follower validation test
timeout-minutes: 15
run: |
cd test/automated/activitypub
./test-follower-validation.sh
+4
View File
@@ -3,6 +3,7 @@ package activitypub
import (
"github.com/owncast/owncast/activitypub/crypto"
"github.com/owncast/owncast/activitypub/inbox"
"github.com/owncast/owncast/activitypub/jobs"
"github.com/owncast/owncast/activitypub/outbox"
"github.com/owncast/owncast/activitypub/persistence"
"github.com/owncast/owncast/activitypub/persistence/followersrepository"
@@ -32,6 +33,9 @@ func Start(datastore *data.Datastore) {
log.Errorln("Unable to get private key", err)
}
}
// Start follower validation background job.
jobs.StartFollowerValidationJob()
}
func getOutboundWorkerPoolSize() int {
+117
View File
@@ -0,0 +1,117 @@
package jobs
import (
"time"
"github.com/owncast/owncast/activitypub/persistence/followersrepository"
"github.com/owncast/owncast/activitypub/resolvers"
"github.com/owncast/owncast/config"
"github.com/owncast/owncast/models"
"github.com/owncast/owncast/persistence/configrepository"
log "github.com/sirupsen/logrus"
)
const (
// ValidationInterval is how often the validation job runs.
ValidationInterval = 1 * time.Hour
// FollowersPerRun is how many followers to validate per job run.
FollowersPerRun = 5
// FailureDurationThreshold is how long a follower must be unreachable before removal.
FailureDurationThreshold = 7 * 24 * time.Hour // 7 days
// DelayBetweenFollowers is the delay between validating individual followers.
DelayBetweenFollowers = 2 * time.Second
)
// GetValidationInterval returns the configured validation interval, or the default.
func GetValidationInterval() time.Duration {
if config.FollowerValidationInterval > 0 {
return config.FollowerValidationInterval
}
return ValidationInterval
}
// StartFollowerValidationJob starts the background job that periodically validates followers.
func StartFollowerValidationJob() {
interval := GetValidationInterval()
ticker := time.NewTicker(interval)
go func() {
for range ticker.C {
runFollowerValidation()
}
}()
log.Debugf("Follower validation job scheduled with interval %v", interval)
}
func runFollowerValidation() {
configRepo := configrepository.Get()
if !configRepo.GetFederationEnabled() {
return
}
followersRepo := followersrepository.Get()
followers, err := followersRepo.GetFollowersToValidate(FollowersPerRun)
if err != nil {
log.Errorln("Failed to get followers for validation:", err)
return
}
for _, follower := range followers {
validateAndUpdateFollower(followersRepo, follower)
time.Sleep(DelayBetweenFollowers)
}
}
func validateAndUpdateFollower(repo followersrepository.FollowersRepository, follower models.Follower) {
resolvedActor, err := resolvers.GetResolvedActorFromIRI(follower.ActorIRI)
if err != nil {
handleValidationFailure(repo, follower, err)
return
}
// Success - clear failure timestamp and update data
if err := repo.UpdateFollowerValidationSuccess(follower.ActorIRI); err != nil {
log.Errorln("Failed to update validation success:", err)
}
// Update follower data
if err := repo.Update(
resolvedActor.ActorIriString(),
resolvedActor.InboxString(),
resolvedActor.SharedInboxString(),
resolvedActor.Name,
resolvedActor.FullUsername,
resolvedActor.ImageString(),
); err != nil {
log.Errorln("Failed to update follower data:", err)
}
}
func handleValidationFailure(repo followersrepository.FollowersRepository, follower models.Follower, resolveErr error) {
log.Debugf("Follower validation failed for %s: %v", follower.ActorIRI, resolveErr)
// Check removal eligibility BEFORE updating failure timestamp.
// We use the existing FirstValidationFailureAt from the database to determine
// if this follower has been failing long enough to be removed.
// If FirstValidationFailureAt is not set, this is a new failure and we just record it.
shouldRemove := false
if follower.FirstValidationFailureAt.Valid {
failureDuration := time.Since(follower.FirstValidationFailureAt.Time)
shouldRemove = failureDuration >= FailureDurationThreshold
}
// Update failure timestamp (sets first_validation_failure_at if not already set)
if err := repo.UpdateFollowerValidationFailure(follower.ActorIRI); err != nil {
log.Errorln("Failed to update validation failure:", err)
return
}
// Remove follower if they've exceeded the failure threshold
if shouldRemove {
failureDuration := time.Since(follower.FirstValidationFailureAt.Time)
log.Infof("Removing follower %s after %v of consecutive failures",
follower.ActorIRI, failureDuration.Round(time.Hour))
if err := repo.RemoveByIRI(follower.ActorIRI); err != nil {
log.Errorln("Failed to remove invalid follower:", err)
}
}
}
+351
View File
@@ -0,0 +1,351 @@
package jobs
import (
"net/url"
"os"
"testing"
"time"
"github.com/go-fed/activity/streams"
"github.com/owncast/owncast/activitypub/apmodels"
"github.com/owncast/owncast/activitypub/persistence"
"github.com/owncast/owncast/activitypub/persistence/followersrepository"
"github.com/owncast/owncast/core/data"
"github.com/owncast/owncast/utils"
)
func TestMain(m *testing.M) {
setup()
code := m.Run()
os.Exit(code)
}
var datastore *data.Datastore
func setup() {
resetTestDatabase()
}
// resetTestDatabase initializes a fresh in-memory database for testing.
func resetTestDatabase() {
data.SetupPersistence(":memory:")
datastore = data.GetDatastore()
persistence.Setup(datastore)
}
// setupTestWithRepo resets the database and returns a new repository instance.
func setupTestWithRepo(t *testing.T) followersrepository.FollowersRepository {
t.Helper()
resetTestDatabase()
return followersrepository.New(datastore)
}
func createTestFollower(repo followersrepository.FollowersRepository, iri, inbox, name, username string) {
actorIRI, _ := url.Parse(iri)
inboxURL, _ := url.Parse(inbox)
requestIRI, _ := url.Parse("https://fake.server/follow/request")
fakeRequest := streams.NewActivityStreamsFollow()
repo.Add(apmodels.ActivityPubActor{
ActorIri: actorIRI,
Inbox: inboxURL,
Name: name,
Username: username,
FullUsername: username + "@fake.server",
FollowRequestIri: requestIRI,
RequestObject: fakeRequest,
}, true)
}
func TestGetFollowersToValidate(t *testing.T) {
repo := setupTestWithRepo(t)
// Create some test followers
for i := 0; i < 10; i++ {
user, _ := utils.GenerateRandomString(10)
createTestFollower(repo, "https://fake.server/user/"+user, "https://fake.server/user/"+user+"/inbox", user, user)
}
// Get followers to validate
followers, err := repo.GetFollowersToValidate(5)
if err != nil {
t.Fatalf("Error getting followers to validate: %s", err)
}
if len(followers) != 5 {
t.Errorf("Expected 5 followers to validate, got %d", len(followers))
}
}
func TestUpdateFollowerValidationSuccess(t *testing.T) {
repo := setupTestWithRepo(t)
// Create a test follower
testIRI := "https://fake.server/user/testuser"
createTestFollower(repo, testIRI, "https://fake.server/user/testuser/inbox", "Test User", "testuser")
// Mark validation as successful
err := repo.UpdateFollowerValidationSuccess(testIRI)
if err != nil {
t.Fatalf("Error updating follower validation success: %s", err)
}
// Verify the follower was updated
followers, err := repo.GetFollowersToValidate(10)
if err != nil {
t.Fatalf("Error getting followers: %s", err)
}
// After validation success, FirstValidationFailureAt should be NULL/invalid
for _, f := range followers {
if f.ActorIRI == testIRI {
if f.FirstValidationFailureAt.Valid {
t.Error("Expected FirstValidationFailureAt to be NULL after success")
}
}
}
}
func TestUpdateFollowerValidationFailure(t *testing.T) {
repo := setupTestWithRepo(t)
// Create a test follower
testIRI := "https://fake.server/user/testuser"
createTestFollower(repo, testIRI, "https://fake.server/user/testuser/inbox", "Test User", "testuser")
// Mark validation as failed
err := repo.UpdateFollowerValidationFailure(testIRI)
if err != nil {
t.Fatalf("Error updating follower validation failure: %s", err)
}
// Verify the FirstValidationFailureAt is set
followers, err := repo.GetFollowersToValidate(10)
if err != nil {
t.Fatalf("Error getting followers: %s", err)
}
found := false
for _, f := range followers {
if f.ActorIRI == testIRI {
found = true
if !f.FirstValidationFailureAt.Valid {
t.Error("Expected FirstValidationFailureAt to be set after failure")
}
}
}
if !found {
t.Error("Test follower not found in results")
}
}
func TestValidationFailureClearedOnSuccess(t *testing.T) {
repo := setupTestWithRepo(t)
// Create a test follower
testIRI := "https://fake.server/user/testuser"
createTestFollower(repo, testIRI, "https://fake.server/user/testuser/inbox", "Test User", "testuser")
// Mark validation as failed first
err := repo.UpdateFollowerValidationFailure(testIRI)
if err != nil {
t.Fatalf("Error updating follower validation failure: %s", err)
}
// Then mark as successful
err = repo.UpdateFollowerValidationSuccess(testIRI)
if err != nil {
t.Fatalf("Error updating follower validation success: %s", err)
}
// Verify FirstValidationFailureAt is cleared
followers, err := repo.GetFollowersToValidate(10)
if err != nil {
t.Fatalf("Error getting followers: %s", err)
}
for _, f := range followers {
if f.ActorIRI == testIRI {
if f.FirstValidationFailureAt.Valid {
t.Error("Expected FirstValidationFailureAt to be NULL after success")
}
}
}
}
func TestRemoveByIRI(t *testing.T) {
repo := setupTestWithRepo(t)
// Create a test follower
testIRI := "https://fake.server/user/testuser"
createTestFollower(repo, testIRI, "https://fake.server/user/testuser/inbox", "Test User", "testuser")
// Verify follower exists
count, err := repo.GetCount()
if err != nil {
t.Fatalf("Error getting count: %s", err)
}
if count != 1 {
t.Errorf("Expected 1 follower, got %d", count)
}
// Remove the follower
err = repo.RemoveByIRI(testIRI)
if err != nil {
t.Fatalf("Error removing follower: %s", err)
}
// Verify follower is removed
count, err = repo.GetCount()
if err != nil {
t.Fatalf("Error getting count: %s", err)
}
if count != 0 {
t.Errorf("Expected 0 followers after removal, got %d", count)
}
}
func TestFailureDurationThresholdLogic(t *testing.T) {
repo := setupTestWithRepo(t)
// Create a test follower
testIRI := "https://fake.server/user/testfailure"
createTestFollower(repo, testIRI, "https://fake.server/user/testfailure/inbox", "Test User", "testfailure")
// Set first_validation_failure_at to 8 days ago (past threshold)
eightDaysAgo := time.Now().Add(-8 * 24 * time.Hour)
_, err := datastore.DB.Exec(
"UPDATE ap_followers SET first_validation_failure_at = ? WHERE iri = ?",
eightDaysAgo, testIRI,
)
if err != nil {
t.Fatalf("Error setting first_validation_failure_at: %s", err)
}
// Fetch the follower
followers, err := repo.GetFollowersToValidate(1)
if err != nil {
t.Fatalf("Error getting followers: %s", err)
}
if len(followers) != 1 {
t.Fatalf("Expected 1 follower, got %d", len(followers))
}
// Verify the failure duration is calculated correctly
failureDuration := time.Since(followers[0].FirstValidationFailureAt.Time)
if failureDuration < FailureDurationThreshold {
t.Errorf("Expected failure duration (%v) to be >= threshold (%v)", failureDuration, FailureDurationThreshold)
}
// Test removal - this directly tests the removal logic without network calls
err = repo.RemoveByIRI(testIRI)
if err != nil {
t.Fatalf("Error removing follower: %s", err)
}
// Verify the follower was removed
count, err := repo.GetCount()
if err != nil {
t.Fatalf("Error getting count: %s", err)
}
if count != 0 {
t.Errorf("Expected follower to be removed, but count is %d", count)
}
}
func TestFailureNotRemovedBeforeThreshold(t *testing.T) {
repo := setupTestWithRepo(t)
// Create a test follower
testIRI := "https://fake.server/user/testnotremoved"
createTestFollower(repo, testIRI, "https://fake.server/user/testnotremoved/inbox", "Test User", "testnotremoved")
// Set first_validation_failure_at to 1 day ago (not past threshold)
oneDayAgo := time.Now().Add(-1 * 24 * time.Hour)
_, err := datastore.DB.Exec(
"UPDATE ap_followers SET first_validation_failure_at = ? WHERE iri = ?",
oneDayAgo, testIRI,
)
if err != nil {
t.Fatalf("Error setting first_validation_failure_at: %s", err)
}
// Fetch the follower
followers, err := repo.GetFollowersToValidate(1)
if err != nil {
t.Fatalf("Error getting followers: %s", err)
}
if len(followers) != 1 {
t.Fatalf("Expected 1 follower, got %d", len(followers))
}
// Verify the failure duration is below threshold
failureDuration := time.Since(followers[0].FirstValidationFailureAt.Time)
if failureDuration >= FailureDurationThreshold {
t.Errorf("Expected failure duration (%v) to be < threshold (%v)", failureDuration, FailureDurationThreshold)
}
// Follower should NOT be removed yet
count, err := repo.GetCount()
if err != nil {
t.Fatalf("Error getting count: %s", err)
}
if count != 1 {
t.Errorf("Expected follower to NOT be removed yet, but count is %d", count)
}
}
func TestFollowersOrderedByOldestValidatedFirst(t *testing.T) {
repo := setupTestWithRepo(t)
// Create followers
iri1 := "https://fake.server/user/first"
iri2 := "https://fake.server/user/second"
iri3 := "https://fake.server/user/third"
createTestFollower(repo, iri1, "https://fake.server/user/first/inbox", "First", "first")
createTestFollower(repo, iri2, "https://fake.server/user/second/inbox", "Second", "second")
createTestFollower(repo, iri3, "https://fake.server/user/third/inbox", "Third", "third")
// Set different last_validated_at times
now := time.Now()
_, err := datastore.DB.Exec("UPDATE ap_followers SET last_validated_at = ? WHERE iri = ?",
now.Add(-3*time.Hour), iri1)
if err != nil {
t.Fatalf("Error updating: %s", err)
}
_, err = datastore.DB.Exec("UPDATE ap_followers SET last_validated_at = ? WHERE iri = ?",
now.Add(-1*time.Hour), iri2)
if err != nil {
t.Fatalf("Error updating: %s", err)
}
// iri3 has NULL last_validated_at (never validated)
// Get followers - should return in order: iri3 (NULL), iri1 (oldest), iri2 (newest)
followers, err := repo.GetFollowersToValidate(3)
if err != nil {
t.Fatalf("Error getting followers: %s", err)
}
if len(followers) != 3 {
t.Fatalf("Expected 3 followers, got %d", len(followers))
}
// NULL values should come first (NULLS FIRST in query)
if followers[0].ActorIRI != iri3 {
t.Errorf("Expected first follower to be %s (never validated), got %s", iri3, followers[0].ActorIRI)
}
// Then oldest validated
if followers[1].ActorIRI != iri1 {
t.Errorf("Expected second follower to be %s (oldest validated), got %s", iri1, followers[1].ActorIRI)
}
// Then newest validated
if followers[2].ActorIRI != iri2 {
t.Errorf("Expected third follower to be %s (newest validated), got %s", iri2, followers[2].ActorIRI)
}
}
+2
View File
@@ -19,6 +19,8 @@ func createFederationFollowersTable() {
"approved_at" TIMESTAMP,
"disabled_at" TIMESTAMP,
"request_object" BLOB,
"last_validated_at" TIMESTAMP,
"first_validation_failure_at" TIMESTAMP,
PRIMARY KEY (iri));`
_datastore.MustExec(createTableSQL)
_datastore.MustExec(`CREATE INDEX IF NOT EXISTS idx_iri ON ap_followers (iri);`)
@@ -42,6 +42,14 @@ type FollowersRepository interface {
BlockOrReject(iri string) error
// Update updates the details of a stored follower.
Update(actorIRI string, inbox string, sharedInbox string, name string, username string, image string) error
// GetFollowersToValidate returns followers needing validation, ordered by oldest validated first.
GetFollowersToValidate(limit int) ([]models.Follower, error)
// UpdateFollowerValidationSuccess marks a follower as successfully validated and clears failure timestamp.
UpdateFollowerValidationSuccess(iri string) error
// UpdateFollowerValidationFailure marks a validation failure, setting first failure time if not already set.
UpdateFollowerValidationFailure(iri string) error
// RemoveByIRI removes a follower directly by IRI string.
RemoveByIRI(iri string) error
}
// SqlFollowersRepository is the SQL-based implementation of FollowersRepository.
@@ -367,3 +375,98 @@ func (r *SqlFollowersRepository) removeFollow(actor *url.URL) error {
return tx.Commit()
}
// GetFollowersToValidate returns followers needing validation, ordered by oldest validated first.
func (r *SqlFollowersRepository) GetFollowersToValidate(limit int) ([]models.Follower, error) {
ctx := context.Background()
followersResult, err := r.datastore.GetQueries().GetFollowersToValidate(ctx, utils.SafeIntToInt32(limit))
if err != nil {
return nil, err
}
followers := make([]models.Follower, 0)
for _, row := range followersResult {
singleFollower := models.Follower{
Name: row.Name.String,
Username: row.Username,
Image: row.Image.String,
ActorIRI: row.Iri,
Inbox: row.Inbox,
SharedInbox: row.SharedInbox.String,
FirstValidationFailureAt: utils.NullTime(row.FirstValidationFailureAt),
}
followers = append(followers, singleFollower)
}
return followers, nil
}
// UpdateFollowerValidationSuccess marks a follower as successfully validated and clears failure timestamp.
func (r *SqlFollowersRepository) UpdateFollowerValidationSuccess(iri string) error {
r.datastore.DbLock.Lock()
defer r.datastore.DbLock.Unlock()
ctx := context.Background()
tx, err := r.datastore.DB.Begin()
if err != nil {
return errors.Wrap(err, "error beginning transaction")
}
defer func() {
_ = tx.Rollback()
}()
if err := r.datastore.GetQueries().WithTx(tx).UpdateFollowerValidationSuccess(ctx, db.UpdateFollowerValidationSuccessParams{
LastValidatedAt: sql.NullTime{Time: time.Now(), Valid: true},
Iri: iri,
}); err != nil {
return errors.Wrap(err, "error updating follower validation success")
}
return tx.Commit()
}
// UpdateFollowerValidationFailure marks a validation failure, setting first failure time if not already set.
func (r *SqlFollowersRepository) UpdateFollowerValidationFailure(iri string) error {
r.datastore.DbLock.Lock()
defer r.datastore.DbLock.Unlock()
ctx := context.Background()
tx, err := r.datastore.DB.Begin()
if err != nil {
return errors.Wrap(err, "error beginning transaction")
}
defer func() {
_ = tx.Rollback()
}()
if err := r.datastore.GetQueries().WithTx(tx).UpdateFollowerValidationFailure(ctx, db.UpdateFollowerValidationFailureParams{
LastValidatedAt: sql.NullTime{Time: time.Now(), Valid: true},
Iri: iri,
}); err != nil {
return errors.Wrap(err, "error updating follower validation failure")
}
return tx.Commit()
}
// RemoveByIRI removes a follower directly by IRI string.
func (r *SqlFollowersRepository) RemoveByIRI(iri string) error {
r.datastore.DbLock.Lock()
defer r.datastore.DbLock.Unlock()
tx, err := r.datastore.DB.Begin()
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
if err := r.datastore.GetQueries().WithTx(tx).RemoveFollowerByIRI(context.Background(), iri); err != nil {
return err
}
return tx.Commit()
}
+4
View File
@@ -43,6 +43,10 @@ var EnableAutoUpdate = false
// A temporary stream key that can be set via the command line.
var TemporaryStreamKey = ""
// FollowerValidationInterval is how often the follower validation job runs.
// Defaults to 0 which means use the default (1 hour).
var FollowerValidationInterval time.Duration = 0
// GetCommit will return an identifier used for identifying the point in time this build took place.
func GetCommit() string {
if GitCommit == "" {
+1 -1
View File
@@ -19,7 +19,7 @@ import (
)
const (
schemaVersion = 8
schemaVersion = 9
)
var (
+13 -11
View File
@@ -18,17 +18,19 @@ type ApAcceptedActivity struct {
}
type ApFollower struct {
Iri string
Inbox string
SharedInbox sql.NullString
Name sql.NullString
Username string
Image sql.NullString
Request string
RequestObject []byte
CreatedAt sql.NullTime
ApprovedAt sql.NullTime
DisabledAt sql.NullTime
Iri string
Inbox string
SharedInbox sql.NullString
Name sql.NullString
Username string
Image sql.NullString
Request string
RequestObject []byte
CreatedAt sql.NullTime
ApprovedAt sql.NullTime
DisabledAt sql.NullTime
LastValidatedAt sql.NullTime
FirstValidationFailureAt sql.NullTime
}
type ApOutbox struct {
+17
View File
@@ -57,6 +57,23 @@ SELECT count(*) FROM ap_accepted_activities WHERE iri = $1 AND actor = $2 AND TY
-- name: UpdateFollowerByIRI :exec
UPDATE ap_followers SET inbox = $1, shared_inbox = $2, name = $3, username = $4, image = $5 WHERE iri = $6;
-- name: GetFollowersToValidate :many
SELECT iri, inbox, shared_inbox, name, username, image, first_validation_failure_at
FROM ap_followers
WHERE approved_at IS NOT NULL AND disabled_at IS NULL
ORDER BY last_validated_at ASC NULLS FIRST
LIMIT $1;
-- name: UpdateFollowerValidationSuccess :exec
UPDATE ap_followers
SET last_validated_at = $1, first_validation_failure_at = NULL
WHERE iri = $2;
-- name: UpdateFollowerValidationFailure :exec
UPDATE ap_followers
SET last_validated_at = $1, first_validation_failure_at = COALESCE(first_validation_failure_at, $1)
WHERE iri = $2;
-- name: GetUniqueDeliveryInboxes :many
SELECT COALESCE(shared_inbox, inbox) as delivery_inbox FROM ap_followers WHERE approved_at is not null GROUP BY delivery_inbox;
+97 -2
View File
@@ -306,9 +306,23 @@ const getFollowerByIRI = `-- name: GetFollowerByIRI :one
SELECT iri, inbox, shared_inbox, name, username, image, request, request_object, created_at, approved_at, disabled_at FROM ap_followers WHERE iri = $1
`
func (q *Queries) GetFollowerByIRI(ctx context.Context, iri string) (ApFollower, error) {
type GetFollowerByIRIRow struct {
Iri string
Inbox string
SharedInbox sql.NullString
Name sql.NullString
Username string
Image sql.NullString
Request string
RequestObject []byte
CreatedAt sql.NullTime
ApprovedAt sql.NullTime
DisabledAt sql.NullTime
}
func (q *Queries) GetFollowerByIRI(ctx context.Context, iri string) (GetFollowerByIRIRow, error) {
row := q.db.QueryRowContext(ctx, getFollowerByIRI, iri)
var i ApFollower
var i GetFollowerByIRIRow
err := row.Scan(
&i.Iri,
&i.Inbox,
@@ -340,6 +354,55 @@ func (q *Queries) GetFollowerCount(ctx context.Context) (int64, error) {
return count, err
}
const getFollowersToValidate = `-- name: GetFollowersToValidate :many
SELECT iri, inbox, shared_inbox, name, username, image, first_validation_failure_at
FROM ap_followers
WHERE approved_at IS NOT NULL AND disabled_at IS NULL
ORDER BY last_validated_at ASC NULLS FIRST
LIMIT $1
`
type GetFollowersToValidateRow struct {
Iri string
Inbox string
SharedInbox sql.NullString
Name sql.NullString
Username string
Image sql.NullString
FirstValidationFailureAt sql.NullTime
}
func (q *Queries) GetFollowersToValidate(ctx context.Context, limit int32) ([]GetFollowersToValidateRow, error) {
rows, err := q.db.QueryContext(ctx, getFollowersToValidate, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []GetFollowersToValidateRow
for rows.Next() {
var i GetFollowersToValidateRow
if err := rows.Scan(
&i.Iri,
&i.Inbox,
&i.SharedInbox,
&i.Name,
&i.Username,
&i.Image,
&i.FirstValidationFailureAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getIPAddressBans = `-- name: GetIPAddressBans :many
SELECT ip_address, notes, created_at FROM ip_bans
`
@@ -815,3 +878,35 @@ func (q *Queries) UpdateFollowerByIRI(ctx context.Context, arg UpdateFollowerByI
)
return err
}
const updateFollowerValidationFailure = `-- name: UpdateFollowerValidationFailure :exec
UPDATE ap_followers
SET last_validated_at = $1, first_validation_failure_at = COALESCE(first_validation_failure_at, $1)
WHERE iri = $2
`
type UpdateFollowerValidationFailureParams struct {
LastValidatedAt sql.NullTime
Iri string
}
func (q *Queries) UpdateFollowerValidationFailure(ctx context.Context, arg UpdateFollowerValidationFailureParams) error {
_, err := q.db.ExecContext(ctx, updateFollowerValidationFailure, arg.LastValidatedAt, arg.Iri)
return err
}
const updateFollowerValidationSuccess = `-- name: UpdateFollowerValidationSuccess :exec
UPDATE ap_followers
SET last_validated_at = $1, first_validation_failure_at = NULL
WHERE iri = $2
`
type UpdateFollowerValidationSuccessParams struct {
LastValidatedAt sql.NullTime
Iri string
}
func (q *Queries) UpdateFollowerValidationSuccess(ctx context.Context, arg UpdateFollowerValidationSuccessParams) error {
_, err := q.db.ExecContext(ctx, updateFollowerValidationSuccess, arg.LastValidatedAt, arg.Iri)
return err
}
+2
View File
@@ -13,6 +13,8 @@ CREATE TABLE IF NOT EXISTS ap_followers (
"created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
"approved_at" TIMESTAMP,
"disabled_at" TIMESTAMP,
"last_validated_at" TIMESTAMP,
"first_validation_failure_at" TIMESTAMP,
PRIMARY KEY (iri));
CREATE INDEX iri_index ON ap_followers (iri);
CREATE INDEX approved_at_index ON ap_followers (approved_at);
+19 -11
View File
@@ -4,6 +4,7 @@ import (
"flag"
"os"
"strconv"
"time"
"github.com/owncast/owncast/logging"
"github.com/owncast/owncast/persistence/configrepository"
@@ -18,17 +19,18 @@ import (
)
var (
dbFile = flag.String("database", "", "Path to the database file.")
logDirectory = flag.String("logdir", "", "Directory where logs will be written to")
backupDirectory = flag.String("backupdir", "", "Directory where backups will be written to")
enableDebugOptions = flag.Bool("enableDebugFeatures", false, "Enable additional debugging options.")
enableVerboseLogging = flag.Bool("enableVerboseLogging", false, "Enable additional logging.")
restoreDatabaseFile = flag.String("restoreDatabase", "", "Restore an Owncast database backup")
newAdminPassword = flag.String("adminpassword", "", "Set your admin password")
newStreamKey = flag.String("streamkey", "", "Set a temporary stream key for this session")
webServerPortOverride = flag.String("webserverport", "", "Force the web server to listen on a specific port")
webServerIPOverride = flag.String("webserverip", "", "Force web server to listen on this IP address")
rtmpPortOverride = flag.Int("rtmpport", 0, "Set listen port for the RTMP server")
dbFile = flag.String("database", "", "Path to the database file.")
logDirectory = flag.String("logdir", "", "Directory where logs will be written to")
backupDirectory = flag.String("backupdir", "", "Directory where backups will be written to")
enableDebugOptions = flag.Bool("enableDebugFeatures", false, "Enable additional debugging options.")
enableVerboseLogging = flag.Bool("enableVerboseLogging", false, "Enable additional logging.")
restoreDatabaseFile = flag.String("restoreDatabase", "", "Restore an Owncast database backup")
newAdminPassword = flag.String("adminpassword", "", "Set your admin password")
newStreamKey = flag.String("streamkey", "", "Set a temporary stream key for this session")
webServerPortOverride = flag.String("webserverport", "", "Force the web server to listen on a specific port")
webServerIPOverride = flag.String("webserverip", "", "Force web server to listen on this IP address")
rtmpPortOverride = flag.Int("rtmpport", 0, "Set listen port for the RTMP server")
followerValidationIntervalSecs = flag.Int("followervalidationinterval", 0, "Set follower validation interval in seconds")
)
// nolint:cyclop
@@ -159,6 +161,12 @@ func handleCommandLineFlags() {
log.Errorln(err)
}
}
// Set the follower validation interval
if *followerValidationIntervalSecs > 0 {
config.FollowerValidationInterval = time.Duration(*followerValidationIntervalSecs) * time.Second
log.Printf("Follower validation interval set to %v", config.FollowerValidationInterval)
}
}
func configureLogging(enableDebugFeatures bool, enableVerboseLogging bool) {
+2
View File
@@ -20,4 +20,6 @@ type Follower struct {
Timestamp utils.NullTime `json:"timestamp,omitempty"`
// DisabledAt is when this follower was rejected or disabled.
DisabledAt utils.NullTime `json:"disabledAt,omitempty"`
// FirstValidationFailureAt is when consecutive validation failures started.
FirstValidationFailureAt utils.NullTime `json:"-"`
}
+17
View File
@@ -35,6 +35,8 @@ func MigrateDatabaseSchema(db *sql.DB, from, to int) error {
migrateToSchema7(db)
case 7:
migrateToSchema8(db)
case 8:
migrateToSchema9(db)
default:
log.Fatalln("missing database migration step")
}
@@ -48,6 +50,21 @@ func MigrateDatabaseSchema(db *sql.DB, from, to int) error {
return nil
}
func migrateToSchema9(db *sql.DB) {
// Add columns for tracking follower validation status.
// last_validated_at: when the follower was last successfully validated
// first_validation_failure_at: when consecutive validation failures started (for removal threshold)
alterStmts := []string{
"ALTER TABLE ap_followers ADD COLUMN last_validated_at TIMESTAMP",
"ALTER TABLE ap_followers ADD COLUMN first_validation_failure_at TIMESTAMP",
}
for _, stmt := range alterStmts {
if _, err := db.Exec(stmt); err != nil {
log.Warnln("Migration statement may have already been applied:", err)
}
}
}
func migrateToSchema8(db *sql.DB) {
// Add shared_inbox column to ap_followers for ActivityPub shared inbox support.
// This allows delivering messages to a server's shared inbox instead of each
+519
View File
@@ -0,0 +1,519 @@
#!/bin/bash
# shellcheck disable=SC2317 # cleanup() is invoked via trap, not direct call
# Follower Validation Test
#
# This test verifies that the follower validation job correctly:
# 1. Removes invalid/fake followers that have been failing for > 7 days
# 2. Keeps valid real followers
#
# The test:
# 1. Builds and starts Owncast with a 20-second validation interval
# 2. Configures federation via admin API
# 3. Inserts 5 fake followers and 5 real followers directly into the database
# 4. Sets first_validation_failure_at to 8+ days ago for fake followers
# 5. Waits for the validation job to run
# 6. Verifies fake followers were removed and real followers remain
set -e
_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(git rev-parse --show-toplevel)"
# Configuration
OWNCAST_PORT="${OWNCAST_PORT:-8080}"
ADMIN_USER="admin"
ADMIN_PASS="abc123"
FEDERATION_USERNAME="streamer"
VALIDATION_INTERVAL=20 # seconds
MAX_TEST_DURATION=600 # Maximum test duration in seconds (10 minutes)
TEST_START_TIME=""
# Directories
TEMP_DIR=""
OWNCAST_DB=""
# PIDs
OWNCAST_PID=""
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m'
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_test() { echo -e "${CYAN}[TEST]${NC} $1"; }
# Check if test has exceeded maximum duration
check_timeout() {
if [[ -z "${TEST_START_TIME}" ]]; then
return 0
fi
local now
now=$(date +%s)
local elapsed=$((now - TEST_START_TIME))
if [[ ${elapsed} -ge ${MAX_TEST_DURATION} ]]; then
log_error "Test exceeded maximum duration of ${MAX_TEST_DURATION} seconds"
exit 1
fi
}
# Real ActivityPub accounts (these should resolve successfully)
REAL_ACCOUNTS=(
# Original 5
"https://social.gabekangas.com/users/gabek"
"https://social.owncast.online/users/owncast"
"https://sfba.social/users/dnalounge"
"https://mastodon.social/users/docpop"
"https://mastodon.social/users/Gargron"
# Additional 20 real accounts from various instances
"https://mastodon.social/users/Mastodon"
"https://mastodon.social/users/joinmastodon"
"https://mastodon.social/users/thunderbird"
"https://mastodon.social/users/mozilla"
"https://mastodon.social/users/pixelfed"
"https://fosstodon.org/users/fosstodon"
"https://fosstodon.org/users/kde"
"https://fosstodon.org/users/gnome"
"https://hachyderm.io/users/hachyderm"
"https://hachyderm.io/users/nova"
"https://infosec.exchange/users/jerry"
"https://mas.to/users/Mer__edith"
"https://mastodon.online/users/mastodonusercount"
"https://mstdn.social/users/stux"
"https://techhub.social/users/techhub"
"https://universeodon.com/users/Universeodon"
"https://mastodon.world/users/mastodonworld"
"https://social.vivaldi.net/users/user_vivaldi"
"https://c.im/users/publicvoit"
"https://mastodon.social/users/w3c"
)
# Fake follower IRIs (these will fail validation)
FAKE_ACCOUNTS=(
# Original 5
"https://fake-server-1.invalid/users/fakeuser1"
"https://fake-server-2.invalid/users/fakeuser2"
"https://fake-server-3.invalid/users/fakeuser3"
"https://nonexistent-instance.fake/users/nobody"
"https://totally-fake.example/actors/testuser"
# Additional 20 fake accounts
"https://fake-server-4.invalid/users/fakeuser4"
"https://fake-server-5.invalid/users/fakeuser5"
"https://fake-server-6.invalid/users/fakeuser6"
"https://fake-server-7.invalid/users/fakeuser7"
"https://fake-server-8.invalid/users/fakeuser8"
"https://fake-server-9.invalid/users/fakeuser9"
"https://fake-server-10.invalid/users/fakeuser10"
"https://nonexistent-1.fake/users/ghost1"
"https://nonexistent-2.fake/users/ghost2"
"https://nonexistent-3.fake/users/ghost3"
"https://nonexistent-4.fake/users/ghost4"
"https://nonexistent-5.fake/users/ghost5"
"https://imaginary-instance.test/actors/phantom1"
"https://imaginary-instance.test/actors/phantom2"
"https://imaginary-instance.test/actors/phantom3"
"https://madeup-server.invalid/users/invented1"
"https://madeup-server.invalid/users/invented2"
"https://madeup-server.invalid/users/invented3"
"https://fictional-fedi.fake/users/notreal1"
"https://fictional-fedi.fake/users/notreal2"
)
cleanup() {
log_info "Cleaning up..."
if [[ -n "${OWNCAST_PID}" ]] && kill -0 "${OWNCAST_PID}" 2>/dev/null; then
kill "${OWNCAST_PID}" 2>/dev/null || true
wait "${OWNCAST_PID}" 2>/dev/null || true
fi
if [[ -n "${TEMP_DIR}" ]] && [[ -d "${TEMP_DIR}" ]]; then
rm -rf "${TEMP_DIR}"
fi
log_info "Cleanup complete."
}
trap cleanup EXIT
setup_temp_dir() {
TEMP_DIR=$(mktemp -d)
OWNCAST_DB="${TEMP_DIR}/owncast.db"
log_info "Temp directory: ${TEMP_DIR}"
log_info "Database: ${OWNCAST_DB}"
}
build_owncast() {
log_info "Building Owncast..."
pushd "${REPO_ROOT}" > /dev/null
CGO_ENABLED=1 go build -o owncast main.go
popd > /dev/null
log_info "Owncast built"
}
start_owncast() {
log_info "Starting Owncast with ${VALIDATION_INTERVAL}s validation interval..."
# Start Owncast with test configuration
OWNCAST_ALLOW_INTERNAL_FEDERATION=true \
OWNCAST_INSECURE_SKIP_VERIFY=true \
"${REPO_ROOT}/owncast" \
-database "${OWNCAST_DB}" \
-followervalidationinterval "${VALIDATION_INTERVAL}" \
-enableVerboseLogging &
OWNCAST_PID=$!
log_info "Owncast started with PID ${OWNCAST_PID}"
# Wait for Owncast to be ready
local max_attempts=30
local attempt=0
while [[ ${attempt} -lt ${max_attempts} ]]; do
if curl -s "http://localhost:${OWNCAST_PORT}/api/status" > /dev/null 2>&1; then
log_info "Owncast is ready"
return 0
fi
attempt=$((attempt + 1))
sleep 1
done
log_error "Owncast did not become ready"
return 1
}
stop_owncast() {
log_info "Stopping Owncast..."
if [[ -n "${OWNCAST_PID}" ]] && kill -0 "${OWNCAST_PID}" 2>/dev/null; then
kill "${OWNCAST_PID}" 2>/dev/null || true
wait "${OWNCAST_PID}" 2>/dev/null || true
OWNCAST_PID=""
fi
# Give the database time to fully close
sleep 2
log_info "Owncast stopped"
}
configure_owncast() {
log_info "Configuring Owncast federation..."
local base_url="http://localhost:${OWNCAST_PORT}"
local auth
auth=$(echo -n "${ADMIN_USER}:${ADMIN_PASS}" | base64)
# Set server URL
curl -s -X POST "${base_url}/api/admin/config/serverurl" \
-H "Authorization: Basic ${auth}" \
-H "Content-Type: application/json" \
-d '{"value": "http://localhost:8080"}' > /dev/null
# Set federation username
curl -s -X POST "${base_url}/api/admin/config/federation/username" \
-H "Authorization: Basic ${auth}" \
-H "Content-Type: application/json" \
-d "{\"value\": \"${FEDERATION_USERNAME}\"}" > /dev/null
# Enable federation
curl -s -X POST "${base_url}/api/admin/config/federation/enable" \
-H "Authorization: Basic ${auth}" \
-H "Content-Type: application/json" \
-d '{"value": true}' > /dev/null
# Disable private mode (auto-accept follows)
curl -s -X POST "${base_url}/api/admin/config/federation/private" \
-H "Authorization: Basic ${auth}" \
-H "Content-Type: application/json" \
-d '{"value": false}' > /dev/null
log_info "Owncast federation configured"
}
# Get the domain from an actor IRI
get_domain_from_iri() {
local iri="$1"
echo "${iri}" | sed -E 's|https?://([^/]+)/.*|\1|'
}
# Get the username from an actor IRI
get_username_from_iri() {
local iri="$1"
echo "${iri}" | sed -E 's|.*/([^/]+)$|\1|'
}
insert_followers() {
log_info "Inserting test followers into database..."
local now
now=$(date -u +"%Y-%m-%d %H:%M:%S")
# 8 days ago - past the 7-day threshold for removal
local eight_days_ago
eight_days_ago=$(date -u -d "8 days ago" +"%Y-%m-%d %H:%M:%S" 2>/dev/null || date -u -v-8d +"%Y-%m-%d %H:%M:%S")
# Insert real followers (these should validate successfully and remain)
log_info "Inserting ${#REAL_ACCOUNTS[@]} real followers..."
for iri in "${REAL_ACCOUNTS[@]}"; do
local domain
domain=$(get_domain_from_iri "${iri}")
local username
username=$(get_username_from_iri "${iri}")
local inbox="https://${domain}/inbox"
local shared_inbox="https://${domain}/inbox"
local request_iri="https://localhost:8080/federation/follow-request-${username}"
sqlite3 "${OWNCAST_DB}" <<EOF
INSERT INTO ap_followers (iri, inbox, shared_inbox, name, username, image, request, request_object, created_at, approved_at)
VALUES (
'${iri}',
'${inbox}',
'${shared_inbox}',
'${username}',
'${username}@${domain}',
'',
'${request_iri}',
'{}',
'${now}',
'${now}'
);
EOF
log_info " Inserted real follower: ${iri}"
done
# Insert fake followers with first_validation_failure_at set to 8 days ago
# These should be removed after validation fails
log_info "Inserting ${#FAKE_ACCOUNTS[@]} fake followers (with 8-day-old failure timestamp)..."
for iri in "${FAKE_ACCOUNTS[@]}"; do
local domain
domain=$(get_domain_from_iri "${iri}")
local username
username=$(get_username_from_iri "${iri}")
local inbox="https://${domain}/inbox"
local request_iri="https://localhost:8080/federation/follow-request-${username}"
sqlite3 "${OWNCAST_DB}" <<EOF
INSERT INTO ap_followers (iri, inbox, name, username, image, request, request_object, created_at, approved_at, first_validation_failure_at)
VALUES (
'${iri}',
'${inbox}',
'${username}',
'${username}@${domain}',
'',
'${request_iri}',
'{}',
'${now}',
'${now}',
'${eight_days_ago}'
);
EOF
log_info " Inserted fake follower: ${iri}"
done
log_info "Inserted ${#REAL_ACCOUNTS[@]} real and ${#FAKE_ACCOUNTS[@]} fake followers"
}
# Get all follower IRIs via the admin API
# Returns a newline-separated list of follower IRIs
get_followers_via_api() {
local base_url="http://localhost:${OWNCAST_PORT}"
local auth
auth=$(echo -n "${ADMIN_USER}:${ADMIN_PASS}" | base64)
# Fetch all followers (using high limit to get all at once)
local response
response=$(curl -s "${base_url}/api/admin/followers?limit=1000" \
-H "Authorization: Basic ${auth}" 2>/dev/null || echo '{"results":[]}')
# Extract IRIs from the JSON response
echo "${response}" | jq -r '.results[]?.link // empty' 2>/dev/null || echo ""
}
get_follower_count() {
local followers
followers=$(get_followers_via_api)
if [[ -z "${followers}" ]]; then
echo "0"
else
echo "${followers}" | wc -l | tr -d ' '
fi
}
get_real_follower_count() {
# Count followers that match real account patterns
local followers
followers=$(get_followers_via_api)
local count=0
for iri in "${REAL_ACCOUNTS[@]}"; do
if echo "${followers}" | grep -qF "${iri}"; then
count=$((count + 1))
fi
done
echo "${count}"
}
get_fake_follower_count() {
# Count followers that match fake account patterns
local followers
followers=$(get_followers_via_api)
local count=0
for iri in "${FAKE_ACCOUNTS[@]}"; do
if echo "${followers}" | grep -qF "${iri}"; then
count=$((count + 1))
fi
done
echo "${count}"
}
wait_for_validation() {
log_info "Waiting for follower validation job to run..."
local _initial_fake_count
_initial_fake_count=$(get_fake_follower_count)
# Calculate wait time based on number of followers
# The job validates 5 followers per run (FollowersPerRun)
# With 2-second delay between followers and VALIDATION_INTERVAL between runs
local total_followers=$((${#REAL_ACCOUNTS[@]} + ${#FAKE_ACCOUNTS[@]}))
local cycles_needed=$(( (total_followers + 4) / 5 )) # Round up
local wait_time=$(( (VALIDATION_INTERVAL + 12) * cycles_needed + 30 )) # Extra buffer for network delays
log_info "Total followers: ${total_followers}, estimated cycles needed: ${cycles_needed}"
log_info "Waiting up to ${wait_time} seconds for validation cycles..."
local elapsed=0
local check_interval=10
while [[ ${elapsed} -lt ${wait_time} ]]; do
# Check if we've exceeded the maximum test duration
check_timeout
sleep ${check_interval}
elapsed=$((elapsed + check_interval))
local current_fake_count
current_fake_count=$(get_fake_follower_count)
local current_real_count
current_real_count=$(get_real_follower_count)
log_info " ${elapsed}s: Real followers: ${current_real_count}/${#REAL_ACCOUNTS[@]}, Fake followers: ${current_fake_count}/${#FAKE_ACCOUNTS[@]}"
# If all fake followers are gone and all real remain, we can stop early
if [[ ${current_fake_count} -eq 0 ]] && [[ ${current_real_count} -eq ${#REAL_ACCOUNTS[@]} ]]; then
log_info "Validation complete early - all fake followers removed"
return 0
fi
done
return 0
}
verify_results() {
log_info "Verifying validation results..."
local total_count
total_count=$(get_follower_count)
local real_count
real_count=$(get_real_follower_count)
local fake_count
fake_count=$(get_fake_follower_count)
log_test "Total followers: ${total_count}"
log_test "Real followers remaining: ${real_count}/${#REAL_ACCOUNTS[@]}"
log_test "Fake followers remaining: ${fake_count}/${#FAKE_ACCOUNTS[@]}"
# List remaining followers for debugging
log_info "Remaining followers:"
get_followers_via_api | while read -r iri; do
if [[ -n "${iri}" ]]; then
log_info " - ${iri}"
fi
done
echo ""
echo "========================================"
echo "Follower Validation Test Results"
echo "========================================"
echo "Expected real followers: ${#REAL_ACCOUNTS[@]}"
echo "Actual real followers: ${real_count}"
echo "Expected fake followers: 0"
echo "Actual fake followers: ${fake_count}"
echo "========================================"
local passed=true
# Check that all fake followers were removed
if [[ ${fake_count} -ne 0 ]]; then
log_error "FAIL: ${fake_count} fake followers should have been removed"
passed=false
else
log_test "PASS: All fake followers were removed"
fi
# Check that all real followers remain
if [[ ${real_count} -ne ${#REAL_ACCOUNTS[@]} ]]; then
log_error "FAIL: Only ${real_count}/${#REAL_ACCOUNTS[@]} real followers remain"
passed=false
else
log_test "PASS: All real followers remain"
fi
echo ""
if [[ "${passed}" == "true" ]]; then
echo -e "${GREEN}TEST PASSED${NC}"
return 0
else
echo -e "${RED}TEST FAILED${NC}"
return 1
fi
}
main() {
# Record test start time for timeout checking
TEST_START_TIME=$(date +%s)
echo ""
echo "========================================"
echo "Follower Validation Test"
echo "========================================"
echo ""
# Setup
setup_temp_dir
build_owncast
start_owncast
configure_owncast
# Stop Owncast before inserting test data to avoid database contention
# The database schema is now created, so we can safely insert data
stop_owncast
# Insert test data while Owncast is stopped
insert_followers
# Restart Owncast to run the validation job
start_owncast
# Verify initial state
log_info "Initial state:"
log_info " Total followers: $(get_follower_count)"
log_info " Real followers: $(get_real_follower_count)"
log_info " Fake followers: $(get_fake_follower_count)"
# Wait for validation
wait_for_validation
# Verify results
if verify_results; then
exit 0
else
exit 1
fi
}
main "$@"