ActivityPub background actor validation, update and cleanup process (#4765)
* feat(ap): add validation+update+cleanup AP actor job * fix(test): use time.Time instead of sql.NullTime * feat(test): add integration test for AP follower cleanup/validation job
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
name: ActivityPub Follower Validation Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "activitypub/**"
|
||||
- "config/**"
|
||||
- "main.go"
|
||||
- "test/automated/activitypub/**"
|
||||
- ".github/workflows/activitypub-follower-validation-tests.yaml"
|
||||
pull_request:
|
||||
paths:
|
||||
- "activitypub/**"
|
||||
- "config/**"
|
||||
- "main.go"
|
||||
- "test/automated/activitypub/**"
|
||||
- ".github/workflows/activitypub-follower-validation-tests.yaml"
|
||||
|
||||
jobs:
|
||||
follower-validation-test:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- id: skip_check
|
||||
uses: fkirc/skip-duplicate-actions@v5
|
||||
with:
|
||||
concurrent_skipping: "same_content_newer"
|
||||
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: "^1.23"
|
||||
cache: true
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y sqlite3 ffmpeg jq
|
||||
|
||||
- name: Run follower validation test
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
cd test/automated/activitypub
|
||||
./test-follower-validation.sh
|
||||
@@ -3,6 +3,7 @@ package activitypub
|
||||
import (
|
||||
"github.com/owncast/owncast/activitypub/crypto"
|
||||
"github.com/owncast/owncast/activitypub/inbox"
|
||||
"github.com/owncast/owncast/activitypub/jobs"
|
||||
"github.com/owncast/owncast/activitypub/outbox"
|
||||
"github.com/owncast/owncast/activitypub/persistence"
|
||||
"github.com/owncast/owncast/activitypub/persistence/followersrepository"
|
||||
@@ -32,6 +33,9 @@ func Start(datastore *data.Datastore) {
|
||||
log.Errorln("Unable to get private key", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Start follower validation background job.
|
||||
jobs.StartFollowerValidationJob()
|
||||
}
|
||||
|
||||
func getOutboundWorkerPoolSize() int {
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/owncast/owncast/activitypub/persistence/followersrepository"
|
||||
"github.com/owncast/owncast/activitypub/resolvers"
|
||||
"github.com/owncast/owncast/config"
|
||||
"github.com/owncast/owncast/models"
|
||||
"github.com/owncast/owncast/persistence/configrepository"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
// ValidationInterval is how often the validation job runs.
|
||||
ValidationInterval = 1 * time.Hour
|
||||
// FollowersPerRun is how many followers to validate per job run.
|
||||
FollowersPerRun = 5
|
||||
// FailureDurationThreshold is how long a follower must be unreachable before removal.
|
||||
FailureDurationThreshold = 7 * 24 * time.Hour // 7 days
|
||||
// DelayBetweenFollowers is the delay between validating individual followers.
|
||||
DelayBetweenFollowers = 2 * time.Second
|
||||
)
|
||||
|
||||
// GetValidationInterval returns the configured validation interval, or the default.
|
||||
func GetValidationInterval() time.Duration {
|
||||
if config.FollowerValidationInterval > 0 {
|
||||
return config.FollowerValidationInterval
|
||||
}
|
||||
return ValidationInterval
|
||||
}
|
||||
|
||||
// StartFollowerValidationJob starts the background job that periodically validates followers.
|
||||
func StartFollowerValidationJob() {
|
||||
interval := GetValidationInterval()
|
||||
ticker := time.NewTicker(interval)
|
||||
go func() {
|
||||
for range ticker.C {
|
||||
runFollowerValidation()
|
||||
}
|
||||
}()
|
||||
log.Debugf("Follower validation job scheduled with interval %v", interval)
|
||||
}
|
||||
|
||||
func runFollowerValidation() {
|
||||
configRepo := configrepository.Get()
|
||||
if !configRepo.GetFederationEnabled() {
|
||||
return
|
||||
}
|
||||
|
||||
followersRepo := followersrepository.Get()
|
||||
followers, err := followersRepo.GetFollowersToValidate(FollowersPerRun)
|
||||
if err != nil {
|
||||
log.Errorln("Failed to get followers for validation:", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, follower := range followers {
|
||||
validateAndUpdateFollower(followersRepo, follower)
|
||||
time.Sleep(DelayBetweenFollowers)
|
||||
}
|
||||
}
|
||||
|
||||
func validateAndUpdateFollower(repo followersrepository.FollowersRepository, follower models.Follower) {
|
||||
resolvedActor, err := resolvers.GetResolvedActorFromIRI(follower.ActorIRI)
|
||||
if err != nil {
|
||||
handleValidationFailure(repo, follower, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Success - clear failure timestamp and update data
|
||||
if err := repo.UpdateFollowerValidationSuccess(follower.ActorIRI); err != nil {
|
||||
log.Errorln("Failed to update validation success:", err)
|
||||
}
|
||||
|
||||
// Update follower data
|
||||
if err := repo.Update(
|
||||
resolvedActor.ActorIriString(),
|
||||
resolvedActor.InboxString(),
|
||||
resolvedActor.SharedInboxString(),
|
||||
resolvedActor.Name,
|
||||
resolvedActor.FullUsername,
|
||||
resolvedActor.ImageString(),
|
||||
); err != nil {
|
||||
log.Errorln("Failed to update follower data:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func handleValidationFailure(repo followersrepository.FollowersRepository, follower models.Follower, resolveErr error) {
|
||||
log.Debugf("Follower validation failed for %s: %v", follower.ActorIRI, resolveErr)
|
||||
|
||||
// Check removal eligibility BEFORE updating failure timestamp.
|
||||
// We use the existing FirstValidationFailureAt from the database to determine
|
||||
// if this follower has been failing long enough to be removed.
|
||||
// If FirstValidationFailureAt is not set, this is a new failure and we just record it.
|
||||
shouldRemove := false
|
||||
if follower.FirstValidationFailureAt.Valid {
|
||||
failureDuration := time.Since(follower.FirstValidationFailureAt.Time)
|
||||
shouldRemove = failureDuration >= FailureDurationThreshold
|
||||
}
|
||||
|
||||
// Update failure timestamp (sets first_validation_failure_at if not already set)
|
||||
if err := repo.UpdateFollowerValidationFailure(follower.ActorIRI); err != nil {
|
||||
log.Errorln("Failed to update validation failure:", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Remove follower if they've exceeded the failure threshold
|
||||
if shouldRemove {
|
||||
failureDuration := time.Since(follower.FirstValidationFailureAt.Time)
|
||||
log.Infof("Removing follower %s after %v of consecutive failures",
|
||||
follower.ActorIRI, failureDuration.Round(time.Hour))
|
||||
if err := repo.RemoveByIRI(follower.ActorIRI); err != nil {
|
||||
log.Errorln("Failed to remove invalid follower:", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-fed/activity/streams"
|
||||
"github.com/owncast/owncast/activitypub/apmodels"
|
||||
"github.com/owncast/owncast/activitypub/persistence"
|
||||
"github.com/owncast/owncast/activitypub/persistence/followersrepository"
|
||||
"github.com/owncast/owncast/core/data"
|
||||
"github.com/owncast/owncast/utils"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
setup()
|
||||
code := m.Run()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
var datastore *data.Datastore
|
||||
|
||||
func setup() {
|
||||
resetTestDatabase()
|
||||
}
|
||||
|
||||
// resetTestDatabase initializes a fresh in-memory database for testing.
|
||||
func resetTestDatabase() {
|
||||
data.SetupPersistence(":memory:")
|
||||
datastore = data.GetDatastore()
|
||||
persistence.Setup(datastore)
|
||||
}
|
||||
|
||||
// setupTestWithRepo resets the database and returns a new repository instance.
|
||||
func setupTestWithRepo(t *testing.T) followersrepository.FollowersRepository {
|
||||
t.Helper()
|
||||
resetTestDatabase()
|
||||
return followersrepository.New(datastore)
|
||||
}
|
||||
|
||||
func createTestFollower(repo followersrepository.FollowersRepository, iri, inbox, name, username string) {
|
||||
actorIRI, _ := url.Parse(iri)
|
||||
inboxURL, _ := url.Parse(inbox)
|
||||
requestIRI, _ := url.Parse("https://fake.server/follow/request")
|
||||
fakeRequest := streams.NewActivityStreamsFollow()
|
||||
|
||||
repo.Add(apmodels.ActivityPubActor{
|
||||
ActorIri: actorIRI,
|
||||
Inbox: inboxURL,
|
||||
Name: name,
|
||||
Username: username,
|
||||
FullUsername: username + "@fake.server",
|
||||
FollowRequestIri: requestIRI,
|
||||
RequestObject: fakeRequest,
|
||||
}, true)
|
||||
}
|
||||
|
||||
func TestGetFollowersToValidate(t *testing.T) {
|
||||
repo := setupTestWithRepo(t)
|
||||
|
||||
// Create some test followers
|
||||
for i := 0; i < 10; i++ {
|
||||
user, _ := utils.GenerateRandomString(10)
|
||||
createTestFollower(repo, "https://fake.server/user/"+user, "https://fake.server/user/"+user+"/inbox", user, user)
|
||||
}
|
||||
|
||||
// Get followers to validate
|
||||
followers, err := repo.GetFollowersToValidate(5)
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting followers to validate: %s", err)
|
||||
}
|
||||
|
||||
if len(followers) != 5 {
|
||||
t.Errorf("Expected 5 followers to validate, got %d", len(followers))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateFollowerValidationSuccess(t *testing.T) {
|
||||
repo := setupTestWithRepo(t)
|
||||
|
||||
// Create a test follower
|
||||
testIRI := "https://fake.server/user/testuser"
|
||||
createTestFollower(repo, testIRI, "https://fake.server/user/testuser/inbox", "Test User", "testuser")
|
||||
|
||||
// Mark validation as successful
|
||||
err := repo.UpdateFollowerValidationSuccess(testIRI)
|
||||
if err != nil {
|
||||
t.Fatalf("Error updating follower validation success: %s", err)
|
||||
}
|
||||
|
||||
// Verify the follower was updated
|
||||
followers, err := repo.GetFollowersToValidate(10)
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting followers: %s", err)
|
||||
}
|
||||
|
||||
// After validation success, FirstValidationFailureAt should be NULL/invalid
|
||||
for _, f := range followers {
|
||||
if f.ActorIRI == testIRI {
|
||||
if f.FirstValidationFailureAt.Valid {
|
||||
t.Error("Expected FirstValidationFailureAt to be NULL after success")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateFollowerValidationFailure(t *testing.T) {
|
||||
repo := setupTestWithRepo(t)
|
||||
|
||||
// Create a test follower
|
||||
testIRI := "https://fake.server/user/testuser"
|
||||
createTestFollower(repo, testIRI, "https://fake.server/user/testuser/inbox", "Test User", "testuser")
|
||||
|
||||
// Mark validation as failed
|
||||
err := repo.UpdateFollowerValidationFailure(testIRI)
|
||||
if err != nil {
|
||||
t.Fatalf("Error updating follower validation failure: %s", err)
|
||||
}
|
||||
|
||||
// Verify the FirstValidationFailureAt is set
|
||||
followers, err := repo.GetFollowersToValidate(10)
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting followers: %s", err)
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, f := range followers {
|
||||
if f.ActorIRI == testIRI {
|
||||
found = true
|
||||
if !f.FirstValidationFailureAt.Valid {
|
||||
t.Error("Expected FirstValidationFailureAt to be set after failure")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Error("Test follower not found in results")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationFailureClearedOnSuccess(t *testing.T) {
|
||||
repo := setupTestWithRepo(t)
|
||||
|
||||
// Create a test follower
|
||||
testIRI := "https://fake.server/user/testuser"
|
||||
createTestFollower(repo, testIRI, "https://fake.server/user/testuser/inbox", "Test User", "testuser")
|
||||
|
||||
// Mark validation as failed first
|
||||
err := repo.UpdateFollowerValidationFailure(testIRI)
|
||||
if err != nil {
|
||||
t.Fatalf("Error updating follower validation failure: %s", err)
|
||||
}
|
||||
|
||||
// Then mark as successful
|
||||
err = repo.UpdateFollowerValidationSuccess(testIRI)
|
||||
if err != nil {
|
||||
t.Fatalf("Error updating follower validation success: %s", err)
|
||||
}
|
||||
|
||||
// Verify FirstValidationFailureAt is cleared
|
||||
followers, err := repo.GetFollowersToValidate(10)
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting followers: %s", err)
|
||||
}
|
||||
|
||||
for _, f := range followers {
|
||||
if f.ActorIRI == testIRI {
|
||||
if f.FirstValidationFailureAt.Valid {
|
||||
t.Error("Expected FirstValidationFailureAt to be NULL after success")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveByIRI(t *testing.T) {
|
||||
repo := setupTestWithRepo(t)
|
||||
|
||||
// Create a test follower
|
||||
testIRI := "https://fake.server/user/testuser"
|
||||
createTestFollower(repo, testIRI, "https://fake.server/user/testuser/inbox", "Test User", "testuser")
|
||||
|
||||
// Verify follower exists
|
||||
count, err := repo.GetCount()
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting count: %s", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Errorf("Expected 1 follower, got %d", count)
|
||||
}
|
||||
|
||||
// Remove the follower
|
||||
err = repo.RemoveByIRI(testIRI)
|
||||
if err != nil {
|
||||
t.Fatalf("Error removing follower: %s", err)
|
||||
}
|
||||
|
||||
// Verify follower is removed
|
||||
count, err = repo.GetCount()
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting count: %s", err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("Expected 0 followers after removal, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailureDurationThresholdLogic(t *testing.T) {
|
||||
repo := setupTestWithRepo(t)
|
||||
|
||||
// Create a test follower
|
||||
testIRI := "https://fake.server/user/testfailure"
|
||||
createTestFollower(repo, testIRI, "https://fake.server/user/testfailure/inbox", "Test User", "testfailure")
|
||||
|
||||
// Set first_validation_failure_at to 8 days ago (past threshold)
|
||||
eightDaysAgo := time.Now().Add(-8 * 24 * time.Hour)
|
||||
_, err := datastore.DB.Exec(
|
||||
"UPDATE ap_followers SET first_validation_failure_at = ? WHERE iri = ?",
|
||||
eightDaysAgo, testIRI,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Error setting first_validation_failure_at: %s", err)
|
||||
}
|
||||
|
||||
// Fetch the follower
|
||||
followers, err := repo.GetFollowersToValidate(1)
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting followers: %s", err)
|
||||
}
|
||||
|
||||
if len(followers) != 1 {
|
||||
t.Fatalf("Expected 1 follower, got %d", len(followers))
|
||||
}
|
||||
|
||||
// Verify the failure duration is calculated correctly
|
||||
failureDuration := time.Since(followers[0].FirstValidationFailureAt.Time)
|
||||
if failureDuration < FailureDurationThreshold {
|
||||
t.Errorf("Expected failure duration (%v) to be >= threshold (%v)", failureDuration, FailureDurationThreshold)
|
||||
}
|
||||
|
||||
// Test removal - this directly tests the removal logic without network calls
|
||||
err = repo.RemoveByIRI(testIRI)
|
||||
if err != nil {
|
||||
t.Fatalf("Error removing follower: %s", err)
|
||||
}
|
||||
|
||||
// Verify the follower was removed
|
||||
count, err := repo.GetCount()
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting count: %s", err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("Expected follower to be removed, but count is %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailureNotRemovedBeforeThreshold(t *testing.T) {
|
||||
repo := setupTestWithRepo(t)
|
||||
|
||||
// Create a test follower
|
||||
testIRI := "https://fake.server/user/testnotremoved"
|
||||
createTestFollower(repo, testIRI, "https://fake.server/user/testnotremoved/inbox", "Test User", "testnotremoved")
|
||||
|
||||
// Set first_validation_failure_at to 1 day ago (not past threshold)
|
||||
oneDayAgo := time.Now().Add(-1 * 24 * time.Hour)
|
||||
_, err := datastore.DB.Exec(
|
||||
"UPDATE ap_followers SET first_validation_failure_at = ? WHERE iri = ?",
|
||||
oneDayAgo, testIRI,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Error setting first_validation_failure_at: %s", err)
|
||||
}
|
||||
|
||||
// Fetch the follower
|
||||
followers, err := repo.GetFollowersToValidate(1)
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting followers: %s", err)
|
||||
}
|
||||
|
||||
if len(followers) != 1 {
|
||||
t.Fatalf("Expected 1 follower, got %d", len(followers))
|
||||
}
|
||||
|
||||
// Verify the failure duration is below threshold
|
||||
failureDuration := time.Since(followers[0].FirstValidationFailureAt.Time)
|
||||
if failureDuration >= FailureDurationThreshold {
|
||||
t.Errorf("Expected failure duration (%v) to be < threshold (%v)", failureDuration, FailureDurationThreshold)
|
||||
}
|
||||
|
||||
// Follower should NOT be removed yet
|
||||
count, err := repo.GetCount()
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting count: %s", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Errorf("Expected follower to NOT be removed yet, but count is %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFollowersOrderedByOldestValidatedFirst(t *testing.T) {
|
||||
repo := setupTestWithRepo(t)
|
||||
|
||||
// Create followers
|
||||
iri1 := "https://fake.server/user/first"
|
||||
iri2 := "https://fake.server/user/second"
|
||||
iri3 := "https://fake.server/user/third"
|
||||
|
||||
createTestFollower(repo, iri1, "https://fake.server/user/first/inbox", "First", "first")
|
||||
createTestFollower(repo, iri2, "https://fake.server/user/second/inbox", "Second", "second")
|
||||
createTestFollower(repo, iri3, "https://fake.server/user/third/inbox", "Third", "third")
|
||||
|
||||
// Set different last_validated_at times
|
||||
now := time.Now()
|
||||
_, err := datastore.DB.Exec("UPDATE ap_followers SET last_validated_at = ? WHERE iri = ?",
|
||||
now.Add(-3*time.Hour), iri1)
|
||||
if err != nil {
|
||||
t.Fatalf("Error updating: %s", err)
|
||||
}
|
||||
_, err = datastore.DB.Exec("UPDATE ap_followers SET last_validated_at = ? WHERE iri = ?",
|
||||
now.Add(-1*time.Hour), iri2)
|
||||
if err != nil {
|
||||
t.Fatalf("Error updating: %s", err)
|
||||
}
|
||||
// iri3 has NULL last_validated_at (never validated)
|
||||
|
||||
// Get followers - should return in order: iri3 (NULL), iri1 (oldest), iri2 (newest)
|
||||
followers, err := repo.GetFollowersToValidate(3)
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting followers: %s", err)
|
||||
}
|
||||
|
||||
if len(followers) != 3 {
|
||||
t.Fatalf("Expected 3 followers, got %d", len(followers))
|
||||
}
|
||||
|
||||
// NULL values should come first (NULLS FIRST in query)
|
||||
if followers[0].ActorIRI != iri3 {
|
||||
t.Errorf("Expected first follower to be %s (never validated), got %s", iri3, followers[0].ActorIRI)
|
||||
}
|
||||
|
||||
// Then oldest validated
|
||||
if followers[1].ActorIRI != iri1 {
|
||||
t.Errorf("Expected second follower to be %s (oldest validated), got %s", iri1, followers[1].ActorIRI)
|
||||
}
|
||||
|
||||
// Then newest validated
|
||||
if followers[2].ActorIRI != iri2 {
|
||||
t.Errorf("Expected third follower to be %s (newest validated), got %s", iri2, followers[2].ActorIRI)
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,8 @@ func createFederationFollowersTable() {
|
||||
"approved_at" TIMESTAMP,
|
||||
"disabled_at" TIMESTAMP,
|
||||
"request_object" BLOB,
|
||||
"last_validated_at" TIMESTAMP,
|
||||
"first_validation_failure_at" TIMESTAMP,
|
||||
PRIMARY KEY (iri));`
|
||||
_datastore.MustExec(createTableSQL)
|
||||
_datastore.MustExec(`CREATE INDEX IF NOT EXISTS idx_iri ON ap_followers (iri);`)
|
||||
|
||||
@@ -42,6 +42,14 @@ type FollowersRepository interface {
|
||||
BlockOrReject(iri string) error
|
||||
// Update updates the details of a stored follower.
|
||||
Update(actorIRI string, inbox string, sharedInbox string, name string, username string, image string) error
|
||||
// GetFollowersToValidate returns followers needing validation, ordered by oldest validated first.
|
||||
GetFollowersToValidate(limit int) ([]models.Follower, error)
|
||||
// UpdateFollowerValidationSuccess marks a follower as successfully validated and clears failure timestamp.
|
||||
UpdateFollowerValidationSuccess(iri string) error
|
||||
// UpdateFollowerValidationFailure marks a validation failure, setting first failure time if not already set.
|
||||
UpdateFollowerValidationFailure(iri string) error
|
||||
// RemoveByIRI removes a follower directly by IRI string.
|
||||
RemoveByIRI(iri string) error
|
||||
}
|
||||
|
||||
// SqlFollowersRepository is the SQL-based implementation of FollowersRepository.
|
||||
@@ -367,3 +375,98 @@ func (r *SqlFollowersRepository) removeFollow(actor *url.URL) error {
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// GetFollowersToValidate returns followers needing validation, ordered by oldest validated first.
|
||||
func (r *SqlFollowersRepository) GetFollowersToValidate(limit int) ([]models.Follower, error) {
|
||||
ctx := context.Background()
|
||||
followersResult, err := r.datastore.GetQueries().GetFollowersToValidate(ctx, utils.SafeIntToInt32(limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
followers := make([]models.Follower, 0)
|
||||
|
||||
for _, row := range followersResult {
|
||||
singleFollower := models.Follower{
|
||||
Name: row.Name.String,
|
||||
Username: row.Username,
|
||||
Image: row.Image.String,
|
||||
ActorIRI: row.Iri,
|
||||
Inbox: row.Inbox,
|
||||
SharedInbox: row.SharedInbox.String,
|
||||
FirstValidationFailureAt: utils.NullTime(row.FirstValidationFailureAt),
|
||||
}
|
||||
|
||||
followers = append(followers, singleFollower)
|
||||
}
|
||||
|
||||
return followers, nil
|
||||
}
|
||||
|
||||
// UpdateFollowerValidationSuccess marks a follower as successfully validated and clears failure timestamp.
|
||||
func (r *SqlFollowersRepository) UpdateFollowerValidationSuccess(iri string) error {
|
||||
r.datastore.DbLock.Lock()
|
||||
defer r.datastore.DbLock.Unlock()
|
||||
|
||||
ctx := context.Background()
|
||||
tx, err := r.datastore.DB.Begin()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error beginning transaction")
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
if err := r.datastore.GetQueries().WithTx(tx).UpdateFollowerValidationSuccess(ctx, db.UpdateFollowerValidationSuccessParams{
|
||||
LastValidatedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
Iri: iri,
|
||||
}); err != nil {
|
||||
return errors.Wrap(err, "error updating follower validation success")
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// UpdateFollowerValidationFailure marks a validation failure, setting first failure time if not already set.
|
||||
func (r *SqlFollowersRepository) UpdateFollowerValidationFailure(iri string) error {
|
||||
r.datastore.DbLock.Lock()
|
||||
defer r.datastore.DbLock.Unlock()
|
||||
|
||||
ctx := context.Background()
|
||||
tx, err := r.datastore.DB.Begin()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error beginning transaction")
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
if err := r.datastore.GetQueries().WithTx(tx).UpdateFollowerValidationFailure(ctx, db.UpdateFollowerValidationFailureParams{
|
||||
LastValidatedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
||||
Iri: iri,
|
||||
}); err != nil {
|
||||
return errors.Wrap(err, "error updating follower validation failure")
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// RemoveByIRI removes a follower directly by IRI string.
|
||||
func (r *SqlFollowersRepository) RemoveByIRI(iri string) error {
|
||||
r.datastore.DbLock.Lock()
|
||||
defer r.datastore.DbLock.Unlock()
|
||||
|
||||
tx, err := r.datastore.DB.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
if err := r.datastore.GetQueries().WithTx(tx).RemoveFollowerByIRI(context.Background(), iri); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@@ -43,6 +43,10 @@ var EnableAutoUpdate = false
|
||||
// A temporary stream key that can be set via the command line.
|
||||
var TemporaryStreamKey = ""
|
||||
|
||||
// FollowerValidationInterval is how often the follower validation job runs.
|
||||
// Defaults to 0 which means use the default (1 hour).
|
||||
var FollowerValidationInterval time.Duration = 0
|
||||
|
||||
// GetCommit will return an identifier used for identifying the point in time this build took place.
|
||||
func GetCommit() string {
|
||||
if GitCommit == "" {
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
schemaVersion = 8
|
||||
schemaVersion = 9
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
+13
-11
@@ -18,17 +18,19 @@ type ApAcceptedActivity struct {
|
||||
}
|
||||
|
||||
type ApFollower struct {
|
||||
Iri string
|
||||
Inbox string
|
||||
SharedInbox sql.NullString
|
||||
Name sql.NullString
|
||||
Username string
|
||||
Image sql.NullString
|
||||
Request string
|
||||
RequestObject []byte
|
||||
CreatedAt sql.NullTime
|
||||
ApprovedAt sql.NullTime
|
||||
DisabledAt sql.NullTime
|
||||
Iri string
|
||||
Inbox string
|
||||
SharedInbox sql.NullString
|
||||
Name sql.NullString
|
||||
Username string
|
||||
Image sql.NullString
|
||||
Request string
|
||||
RequestObject []byte
|
||||
CreatedAt sql.NullTime
|
||||
ApprovedAt sql.NullTime
|
||||
DisabledAt sql.NullTime
|
||||
LastValidatedAt sql.NullTime
|
||||
FirstValidationFailureAt sql.NullTime
|
||||
}
|
||||
|
||||
type ApOutbox struct {
|
||||
|
||||
@@ -57,6 +57,23 @@ SELECT count(*) FROM ap_accepted_activities WHERE iri = $1 AND actor = $2 AND TY
|
||||
-- name: UpdateFollowerByIRI :exec
|
||||
UPDATE ap_followers SET inbox = $1, shared_inbox = $2, name = $3, username = $4, image = $5 WHERE iri = $6;
|
||||
|
||||
-- name: GetFollowersToValidate :many
|
||||
SELECT iri, inbox, shared_inbox, name, username, image, first_validation_failure_at
|
||||
FROM ap_followers
|
||||
WHERE approved_at IS NOT NULL AND disabled_at IS NULL
|
||||
ORDER BY last_validated_at ASC NULLS FIRST
|
||||
LIMIT $1;
|
||||
|
||||
-- name: UpdateFollowerValidationSuccess :exec
|
||||
UPDATE ap_followers
|
||||
SET last_validated_at = $1, first_validation_failure_at = NULL
|
||||
WHERE iri = $2;
|
||||
|
||||
-- name: UpdateFollowerValidationFailure :exec
|
||||
UPDATE ap_followers
|
||||
SET last_validated_at = $1, first_validation_failure_at = COALESCE(first_validation_failure_at, $1)
|
||||
WHERE iri = $2;
|
||||
|
||||
-- name: GetUniqueDeliveryInboxes :many
|
||||
SELECT COALESCE(shared_inbox, inbox) as delivery_inbox FROM ap_followers WHERE approved_at is not null GROUP BY delivery_inbox;
|
||||
|
||||
|
||||
+97
-2
@@ -306,9 +306,23 @@ const getFollowerByIRI = `-- name: GetFollowerByIRI :one
|
||||
SELECT iri, inbox, shared_inbox, name, username, image, request, request_object, created_at, approved_at, disabled_at FROM ap_followers WHERE iri = $1
|
||||
`
|
||||
|
||||
func (q *Queries) GetFollowerByIRI(ctx context.Context, iri string) (ApFollower, error) {
|
||||
type GetFollowerByIRIRow struct {
|
||||
Iri string
|
||||
Inbox string
|
||||
SharedInbox sql.NullString
|
||||
Name sql.NullString
|
||||
Username string
|
||||
Image sql.NullString
|
||||
Request string
|
||||
RequestObject []byte
|
||||
CreatedAt sql.NullTime
|
||||
ApprovedAt sql.NullTime
|
||||
DisabledAt sql.NullTime
|
||||
}
|
||||
|
||||
func (q *Queries) GetFollowerByIRI(ctx context.Context, iri string) (GetFollowerByIRIRow, error) {
|
||||
row := q.db.QueryRowContext(ctx, getFollowerByIRI, iri)
|
||||
var i ApFollower
|
||||
var i GetFollowerByIRIRow
|
||||
err := row.Scan(
|
||||
&i.Iri,
|
||||
&i.Inbox,
|
||||
@@ -340,6 +354,55 @@ func (q *Queries) GetFollowerCount(ctx context.Context) (int64, error) {
|
||||
return count, err
|
||||
}
|
||||
|
||||
const getFollowersToValidate = `-- name: GetFollowersToValidate :many
|
||||
SELECT iri, inbox, shared_inbox, name, username, image, first_validation_failure_at
|
||||
FROM ap_followers
|
||||
WHERE approved_at IS NOT NULL AND disabled_at IS NULL
|
||||
ORDER BY last_validated_at ASC NULLS FIRST
|
||||
LIMIT $1
|
||||
`
|
||||
|
||||
type GetFollowersToValidateRow struct {
|
||||
Iri string
|
||||
Inbox string
|
||||
SharedInbox sql.NullString
|
||||
Name sql.NullString
|
||||
Username string
|
||||
Image sql.NullString
|
||||
FirstValidationFailureAt sql.NullTime
|
||||
}
|
||||
|
||||
func (q *Queries) GetFollowersToValidate(ctx context.Context, limit int32) ([]GetFollowersToValidateRow, error) {
|
||||
rows, err := q.db.QueryContext(ctx, getFollowersToValidate, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []GetFollowersToValidateRow
|
||||
for rows.Next() {
|
||||
var i GetFollowersToValidateRow
|
||||
if err := rows.Scan(
|
||||
&i.Iri,
|
||||
&i.Inbox,
|
||||
&i.SharedInbox,
|
||||
&i.Name,
|
||||
&i.Username,
|
||||
&i.Image,
|
||||
&i.FirstValidationFailureAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const getIPAddressBans = `-- name: GetIPAddressBans :many
|
||||
SELECT ip_address, notes, created_at FROM ip_bans
|
||||
`
|
||||
@@ -815,3 +878,35 @@ func (q *Queries) UpdateFollowerByIRI(ctx context.Context, arg UpdateFollowerByI
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const updateFollowerValidationFailure = `-- name: UpdateFollowerValidationFailure :exec
|
||||
UPDATE ap_followers
|
||||
SET last_validated_at = $1, first_validation_failure_at = COALESCE(first_validation_failure_at, $1)
|
||||
WHERE iri = $2
|
||||
`
|
||||
|
||||
type UpdateFollowerValidationFailureParams struct {
|
||||
LastValidatedAt sql.NullTime
|
||||
Iri string
|
||||
}
|
||||
|
||||
func (q *Queries) UpdateFollowerValidationFailure(ctx context.Context, arg UpdateFollowerValidationFailureParams) error {
|
||||
_, err := q.db.ExecContext(ctx, updateFollowerValidationFailure, arg.LastValidatedAt, arg.Iri)
|
||||
return err
|
||||
}
|
||||
|
||||
const updateFollowerValidationSuccess = `-- name: UpdateFollowerValidationSuccess :exec
|
||||
UPDATE ap_followers
|
||||
SET last_validated_at = $1, first_validation_failure_at = NULL
|
||||
WHERE iri = $2
|
||||
`
|
||||
|
||||
type UpdateFollowerValidationSuccessParams struct {
|
||||
LastValidatedAt sql.NullTime
|
||||
Iri string
|
||||
}
|
||||
|
||||
func (q *Queries) UpdateFollowerValidationSuccess(ctx context.Context, arg UpdateFollowerValidationSuccessParams) error {
|
||||
_, err := q.db.ExecContext(ctx, updateFollowerValidationSuccess, arg.LastValidatedAt, arg.Iri)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ CREATE TABLE IF NOT EXISTS ap_followers (
|
||||
"created_at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
"approved_at" TIMESTAMP,
|
||||
"disabled_at" TIMESTAMP,
|
||||
"last_validated_at" TIMESTAMP,
|
||||
"first_validation_failure_at" TIMESTAMP,
|
||||
PRIMARY KEY (iri));
|
||||
CREATE INDEX iri_index ON ap_followers (iri);
|
||||
CREATE INDEX approved_at_index ON ap_followers (approved_at);
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"flag"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/owncast/owncast/logging"
|
||||
"github.com/owncast/owncast/persistence/configrepository"
|
||||
@@ -18,17 +19,18 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
dbFile = flag.String("database", "", "Path to the database file.")
|
||||
logDirectory = flag.String("logdir", "", "Directory where logs will be written to")
|
||||
backupDirectory = flag.String("backupdir", "", "Directory where backups will be written to")
|
||||
enableDebugOptions = flag.Bool("enableDebugFeatures", false, "Enable additional debugging options.")
|
||||
enableVerboseLogging = flag.Bool("enableVerboseLogging", false, "Enable additional logging.")
|
||||
restoreDatabaseFile = flag.String("restoreDatabase", "", "Restore an Owncast database backup")
|
||||
newAdminPassword = flag.String("adminpassword", "", "Set your admin password")
|
||||
newStreamKey = flag.String("streamkey", "", "Set a temporary stream key for this session")
|
||||
webServerPortOverride = flag.String("webserverport", "", "Force the web server to listen on a specific port")
|
||||
webServerIPOverride = flag.String("webserverip", "", "Force web server to listen on this IP address")
|
||||
rtmpPortOverride = flag.Int("rtmpport", 0, "Set listen port for the RTMP server")
|
||||
dbFile = flag.String("database", "", "Path to the database file.")
|
||||
logDirectory = flag.String("logdir", "", "Directory where logs will be written to")
|
||||
backupDirectory = flag.String("backupdir", "", "Directory where backups will be written to")
|
||||
enableDebugOptions = flag.Bool("enableDebugFeatures", false, "Enable additional debugging options.")
|
||||
enableVerboseLogging = flag.Bool("enableVerboseLogging", false, "Enable additional logging.")
|
||||
restoreDatabaseFile = flag.String("restoreDatabase", "", "Restore an Owncast database backup")
|
||||
newAdminPassword = flag.String("adminpassword", "", "Set your admin password")
|
||||
newStreamKey = flag.String("streamkey", "", "Set a temporary stream key for this session")
|
||||
webServerPortOverride = flag.String("webserverport", "", "Force the web server to listen on a specific port")
|
||||
webServerIPOverride = flag.String("webserverip", "", "Force web server to listen on this IP address")
|
||||
rtmpPortOverride = flag.Int("rtmpport", 0, "Set listen port for the RTMP server")
|
||||
followerValidationIntervalSecs = flag.Int("followervalidationinterval", 0, "Set follower validation interval in seconds")
|
||||
)
|
||||
|
||||
// nolint:cyclop
|
||||
@@ -159,6 +161,12 @@ func handleCommandLineFlags() {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Set the follower validation interval
|
||||
if *followerValidationIntervalSecs > 0 {
|
||||
config.FollowerValidationInterval = time.Duration(*followerValidationIntervalSecs) * time.Second
|
||||
log.Printf("Follower validation interval set to %v", config.FollowerValidationInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func configureLogging(enableDebugFeatures bool, enableVerboseLogging bool) {
|
||||
|
||||
@@ -20,4 +20,6 @@ type Follower struct {
|
||||
Timestamp utils.NullTime `json:"timestamp,omitempty"`
|
||||
// DisabledAt is when this follower was rejected or disabled.
|
||||
DisabledAt utils.NullTime `json:"disabledAt,omitempty"`
|
||||
// FirstValidationFailureAt is when consecutive validation failures started.
|
||||
FirstValidationFailureAt utils.NullTime `json:"-"`
|
||||
}
|
||||
|
||||
@@ -35,6 +35,8 @@ func MigrateDatabaseSchema(db *sql.DB, from, to int) error {
|
||||
migrateToSchema7(db)
|
||||
case 7:
|
||||
migrateToSchema8(db)
|
||||
case 8:
|
||||
migrateToSchema9(db)
|
||||
default:
|
||||
log.Fatalln("missing database migration step")
|
||||
}
|
||||
@@ -48,6 +50,21 @@ func MigrateDatabaseSchema(db *sql.DB, from, to int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateToSchema9(db *sql.DB) {
|
||||
// Add columns for tracking follower validation status.
|
||||
// last_validated_at: when the follower was last successfully validated
|
||||
// first_validation_failure_at: when consecutive validation failures started (for removal threshold)
|
||||
alterStmts := []string{
|
||||
"ALTER TABLE ap_followers ADD COLUMN last_validated_at TIMESTAMP",
|
||||
"ALTER TABLE ap_followers ADD COLUMN first_validation_failure_at TIMESTAMP",
|
||||
}
|
||||
for _, stmt := range alterStmts {
|
||||
if _, err := db.Exec(stmt); err != nil {
|
||||
log.Warnln("Migration statement may have already been applied:", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func migrateToSchema8(db *sql.DB) {
|
||||
// Add shared_inbox column to ap_followers for ActivityPub shared inbox support.
|
||||
// This allows delivering messages to a server's shared inbox instead of each
|
||||
|
||||
+519
@@ -0,0 +1,519 @@
|
||||
#!/bin/bash
|
||||
# shellcheck disable=SC2317 # cleanup() is invoked via trap, not direct call
|
||||
|
||||
# Follower Validation Test
|
||||
#
|
||||
# This test verifies that the follower validation job correctly:
|
||||
# 1. Removes invalid/fake followers that have been failing for > 7 days
|
||||
# 2. Keeps valid real followers
|
||||
#
|
||||
# The test:
|
||||
# 1. Builds and starts Owncast with a 20-second validation interval
|
||||
# 2. Configures federation via admin API
|
||||
# 3. Inserts 5 fake followers and 5 real followers directly into the database
|
||||
# 4. Sets first_validation_failure_at to 8+ days ago for fake followers
|
||||
# 5. Waits for the validation job to run
|
||||
# 6. Verifies fake followers were removed and real followers remain
|
||||
|
||||
set -e
|
||||
|
||||
_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(git rev-parse --show-toplevel)"
|
||||
|
||||
# Configuration
|
||||
OWNCAST_PORT="${OWNCAST_PORT:-8080}"
|
||||
ADMIN_USER="admin"
|
||||
ADMIN_PASS="abc123"
|
||||
FEDERATION_USERNAME="streamer"
|
||||
VALIDATION_INTERVAL=20 # seconds
|
||||
MAX_TEST_DURATION=600 # Maximum test duration in seconds (10 minutes)
|
||||
TEST_START_TIME=""
|
||||
|
||||
# Directories
|
||||
TEMP_DIR=""
|
||||
OWNCAST_DB=""
|
||||
|
||||
# PIDs
|
||||
OWNCAST_PID=""
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
||||
log_test() { echo -e "${CYAN}[TEST]${NC} $1"; }
|
||||
|
||||
# Check if test has exceeded maximum duration
|
||||
check_timeout() {
|
||||
if [[ -z "${TEST_START_TIME}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
local now
|
||||
now=$(date +%s)
|
||||
local elapsed=$((now - TEST_START_TIME))
|
||||
if [[ ${elapsed} -ge ${MAX_TEST_DURATION} ]]; then
|
||||
log_error "Test exceeded maximum duration of ${MAX_TEST_DURATION} seconds"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Real ActivityPub accounts (these should resolve successfully)
|
||||
REAL_ACCOUNTS=(
|
||||
# Original 5
|
||||
"https://social.gabekangas.com/users/gabek"
|
||||
"https://social.owncast.online/users/owncast"
|
||||
"https://sfba.social/users/dnalounge"
|
||||
"https://mastodon.social/users/docpop"
|
||||
"https://mastodon.social/users/Gargron"
|
||||
# Additional 20 real accounts from various instances
|
||||
"https://mastodon.social/users/Mastodon"
|
||||
"https://mastodon.social/users/joinmastodon"
|
||||
"https://mastodon.social/users/thunderbird"
|
||||
"https://mastodon.social/users/mozilla"
|
||||
"https://mastodon.social/users/pixelfed"
|
||||
"https://fosstodon.org/users/fosstodon"
|
||||
"https://fosstodon.org/users/kde"
|
||||
"https://fosstodon.org/users/gnome"
|
||||
"https://hachyderm.io/users/hachyderm"
|
||||
"https://hachyderm.io/users/nova"
|
||||
"https://infosec.exchange/users/jerry"
|
||||
"https://mas.to/users/Mer__edith"
|
||||
"https://mastodon.online/users/mastodonusercount"
|
||||
"https://mstdn.social/users/stux"
|
||||
"https://techhub.social/users/techhub"
|
||||
"https://universeodon.com/users/Universeodon"
|
||||
"https://mastodon.world/users/mastodonworld"
|
||||
"https://social.vivaldi.net/users/user_vivaldi"
|
||||
"https://c.im/users/publicvoit"
|
||||
"https://mastodon.social/users/w3c"
|
||||
)
|
||||
|
||||
# Fake follower IRIs (these will fail validation)
|
||||
FAKE_ACCOUNTS=(
|
||||
# Original 5
|
||||
"https://fake-server-1.invalid/users/fakeuser1"
|
||||
"https://fake-server-2.invalid/users/fakeuser2"
|
||||
"https://fake-server-3.invalid/users/fakeuser3"
|
||||
"https://nonexistent-instance.fake/users/nobody"
|
||||
"https://totally-fake.example/actors/testuser"
|
||||
# Additional 20 fake accounts
|
||||
"https://fake-server-4.invalid/users/fakeuser4"
|
||||
"https://fake-server-5.invalid/users/fakeuser5"
|
||||
"https://fake-server-6.invalid/users/fakeuser6"
|
||||
"https://fake-server-7.invalid/users/fakeuser7"
|
||||
"https://fake-server-8.invalid/users/fakeuser8"
|
||||
"https://fake-server-9.invalid/users/fakeuser9"
|
||||
"https://fake-server-10.invalid/users/fakeuser10"
|
||||
"https://nonexistent-1.fake/users/ghost1"
|
||||
"https://nonexistent-2.fake/users/ghost2"
|
||||
"https://nonexistent-3.fake/users/ghost3"
|
||||
"https://nonexistent-4.fake/users/ghost4"
|
||||
"https://nonexistent-5.fake/users/ghost5"
|
||||
"https://imaginary-instance.test/actors/phantom1"
|
||||
"https://imaginary-instance.test/actors/phantom2"
|
||||
"https://imaginary-instance.test/actors/phantom3"
|
||||
"https://madeup-server.invalid/users/invented1"
|
||||
"https://madeup-server.invalid/users/invented2"
|
||||
"https://madeup-server.invalid/users/invented3"
|
||||
"https://fictional-fedi.fake/users/notreal1"
|
||||
"https://fictional-fedi.fake/users/notreal2"
|
||||
)
|
||||
|
||||
cleanup() {
|
||||
log_info "Cleaning up..."
|
||||
|
||||
if [[ -n "${OWNCAST_PID}" ]] && kill -0 "${OWNCAST_PID}" 2>/dev/null; then
|
||||
kill "${OWNCAST_PID}" 2>/dev/null || true
|
||||
wait "${OWNCAST_PID}" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if [[ -n "${TEMP_DIR}" ]] && [[ -d "${TEMP_DIR}" ]]; then
|
||||
rm -rf "${TEMP_DIR}"
|
||||
fi
|
||||
|
||||
log_info "Cleanup complete."
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
setup_temp_dir() {
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
OWNCAST_DB="${TEMP_DIR}/owncast.db"
|
||||
log_info "Temp directory: ${TEMP_DIR}"
|
||||
log_info "Database: ${OWNCAST_DB}"
|
||||
}
|
||||
|
||||
build_owncast() {
|
||||
log_info "Building Owncast..."
|
||||
|
||||
pushd "${REPO_ROOT}" > /dev/null
|
||||
CGO_ENABLED=1 go build -o owncast main.go
|
||||
popd > /dev/null
|
||||
|
||||
log_info "Owncast built"
|
||||
}
|
||||
|
||||
start_owncast() {
|
||||
log_info "Starting Owncast with ${VALIDATION_INTERVAL}s validation interval..."
|
||||
|
||||
# Start Owncast with test configuration
|
||||
OWNCAST_ALLOW_INTERNAL_FEDERATION=true \
|
||||
OWNCAST_INSECURE_SKIP_VERIFY=true \
|
||||
"${REPO_ROOT}/owncast" \
|
||||
-database "${OWNCAST_DB}" \
|
||||
-followervalidationinterval "${VALIDATION_INTERVAL}" \
|
||||
-enableVerboseLogging &
|
||||
OWNCAST_PID=$!
|
||||
|
||||
log_info "Owncast started with PID ${OWNCAST_PID}"
|
||||
|
||||
# Wait for Owncast to be ready
|
||||
local max_attempts=30
|
||||
local attempt=0
|
||||
|
||||
while [[ ${attempt} -lt ${max_attempts} ]]; do
|
||||
if curl -s "http://localhost:${OWNCAST_PORT}/api/status" > /dev/null 2>&1; then
|
||||
log_info "Owncast is ready"
|
||||
return 0
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
log_error "Owncast did not become ready"
|
||||
return 1
|
||||
}
|
||||
|
||||
stop_owncast() {
|
||||
log_info "Stopping Owncast..."
|
||||
if [[ -n "${OWNCAST_PID}" ]] && kill -0 "${OWNCAST_PID}" 2>/dev/null; then
|
||||
kill "${OWNCAST_PID}" 2>/dev/null || true
|
||||
wait "${OWNCAST_PID}" 2>/dev/null || true
|
||||
OWNCAST_PID=""
|
||||
fi
|
||||
# Give the database time to fully close
|
||||
sleep 2
|
||||
log_info "Owncast stopped"
|
||||
}
|
||||
|
||||
configure_owncast() {
|
||||
log_info "Configuring Owncast federation..."
|
||||
|
||||
local base_url="http://localhost:${OWNCAST_PORT}"
|
||||
local auth
|
||||
auth=$(echo -n "${ADMIN_USER}:${ADMIN_PASS}" | base64)
|
||||
|
||||
# Set server URL
|
||||
curl -s -X POST "${base_url}/api/admin/config/serverurl" \
|
||||
-H "Authorization: Basic ${auth}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"value": "http://localhost:8080"}' > /dev/null
|
||||
|
||||
# Set federation username
|
||||
curl -s -X POST "${base_url}/api/admin/config/federation/username" \
|
||||
-H "Authorization: Basic ${auth}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"value\": \"${FEDERATION_USERNAME}\"}" > /dev/null
|
||||
|
||||
# Enable federation
|
||||
curl -s -X POST "${base_url}/api/admin/config/federation/enable" \
|
||||
-H "Authorization: Basic ${auth}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"value": true}' > /dev/null
|
||||
|
||||
# Disable private mode (auto-accept follows)
|
||||
curl -s -X POST "${base_url}/api/admin/config/federation/private" \
|
||||
-H "Authorization: Basic ${auth}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"value": false}' > /dev/null
|
||||
|
||||
log_info "Owncast federation configured"
|
||||
}
|
||||
|
||||
# Get the domain from an actor IRI
|
||||
get_domain_from_iri() {
|
||||
local iri="$1"
|
||||
echo "${iri}" | sed -E 's|https?://([^/]+)/.*|\1|'
|
||||
}
|
||||
|
||||
# Get the username from an actor IRI
|
||||
get_username_from_iri() {
|
||||
local iri="$1"
|
||||
echo "${iri}" | sed -E 's|.*/([^/]+)$|\1|'
|
||||
}
|
||||
|
||||
insert_followers() {
|
||||
log_info "Inserting test followers into database..."
|
||||
|
||||
local now
|
||||
now=$(date -u +"%Y-%m-%d %H:%M:%S")
|
||||
|
||||
# 8 days ago - past the 7-day threshold for removal
|
||||
local eight_days_ago
|
||||
eight_days_ago=$(date -u -d "8 days ago" +"%Y-%m-%d %H:%M:%S" 2>/dev/null || date -u -v-8d +"%Y-%m-%d %H:%M:%S")
|
||||
|
||||
# Insert real followers (these should validate successfully and remain)
|
||||
log_info "Inserting ${#REAL_ACCOUNTS[@]} real followers..."
|
||||
for iri in "${REAL_ACCOUNTS[@]}"; do
|
||||
local domain
|
||||
domain=$(get_domain_from_iri "${iri}")
|
||||
local username
|
||||
username=$(get_username_from_iri "${iri}")
|
||||
local inbox="https://${domain}/inbox"
|
||||
local shared_inbox="https://${domain}/inbox"
|
||||
local request_iri="https://localhost:8080/federation/follow-request-${username}"
|
||||
|
||||
sqlite3 "${OWNCAST_DB}" <<EOF
|
||||
INSERT INTO ap_followers (iri, inbox, shared_inbox, name, username, image, request, request_object, created_at, approved_at)
|
||||
VALUES (
|
||||
'${iri}',
|
||||
'${inbox}',
|
||||
'${shared_inbox}',
|
||||
'${username}',
|
||||
'${username}@${domain}',
|
||||
'',
|
||||
'${request_iri}',
|
||||
'{}',
|
||||
'${now}',
|
||||
'${now}'
|
||||
);
|
||||
EOF
|
||||
log_info " Inserted real follower: ${iri}"
|
||||
done
|
||||
|
||||
# Insert fake followers with first_validation_failure_at set to 8 days ago
|
||||
# These should be removed after validation fails
|
||||
log_info "Inserting ${#FAKE_ACCOUNTS[@]} fake followers (with 8-day-old failure timestamp)..."
|
||||
for iri in "${FAKE_ACCOUNTS[@]}"; do
|
||||
local domain
|
||||
domain=$(get_domain_from_iri "${iri}")
|
||||
local username
|
||||
username=$(get_username_from_iri "${iri}")
|
||||
local inbox="https://${domain}/inbox"
|
||||
local request_iri="https://localhost:8080/federation/follow-request-${username}"
|
||||
|
||||
sqlite3 "${OWNCAST_DB}" <<EOF
|
||||
INSERT INTO ap_followers (iri, inbox, name, username, image, request, request_object, created_at, approved_at, first_validation_failure_at)
|
||||
VALUES (
|
||||
'${iri}',
|
||||
'${inbox}',
|
||||
'${username}',
|
||||
'${username}@${domain}',
|
||||
'',
|
||||
'${request_iri}',
|
||||
'{}',
|
||||
'${now}',
|
||||
'${now}',
|
||||
'${eight_days_ago}'
|
||||
);
|
||||
EOF
|
||||
log_info " Inserted fake follower: ${iri}"
|
||||
done
|
||||
|
||||
log_info "Inserted ${#REAL_ACCOUNTS[@]} real and ${#FAKE_ACCOUNTS[@]} fake followers"
|
||||
}
|
||||
|
||||
# Get all follower IRIs via the admin API
|
||||
# Returns a newline-separated list of follower IRIs
|
||||
get_followers_via_api() {
|
||||
local base_url="http://localhost:${OWNCAST_PORT}"
|
||||
local auth
|
||||
auth=$(echo -n "${ADMIN_USER}:${ADMIN_PASS}" | base64)
|
||||
|
||||
# Fetch all followers (using high limit to get all at once)
|
||||
local response
|
||||
response=$(curl -s "${base_url}/api/admin/followers?limit=1000" \
|
||||
-H "Authorization: Basic ${auth}" 2>/dev/null || echo '{"results":[]}')
|
||||
|
||||
# Extract IRIs from the JSON response
|
||||
echo "${response}" | jq -r '.results[]?.link // empty' 2>/dev/null || echo ""
|
||||
}
|
||||
|
||||
get_follower_count() {
|
||||
local followers
|
||||
followers=$(get_followers_via_api)
|
||||
if [[ -z "${followers}" ]]; then
|
||||
echo "0"
|
||||
else
|
||||
echo "${followers}" | wc -l | tr -d ' '
|
||||
fi
|
||||
}
|
||||
|
||||
get_real_follower_count() {
|
||||
# Count followers that match real account patterns
|
||||
local followers
|
||||
followers=$(get_followers_via_api)
|
||||
local count=0
|
||||
for iri in "${REAL_ACCOUNTS[@]}"; do
|
||||
if echo "${followers}" | grep -qF "${iri}"; then
|
||||
count=$((count + 1))
|
||||
fi
|
||||
done
|
||||
echo "${count}"
|
||||
}
|
||||
|
||||
get_fake_follower_count() {
|
||||
# Count followers that match fake account patterns
|
||||
local followers
|
||||
followers=$(get_followers_via_api)
|
||||
local count=0
|
||||
for iri in "${FAKE_ACCOUNTS[@]}"; do
|
||||
if echo "${followers}" | grep -qF "${iri}"; then
|
||||
count=$((count + 1))
|
||||
fi
|
||||
done
|
||||
echo "${count}"
|
||||
}
|
||||
|
||||
wait_for_validation() {
|
||||
log_info "Waiting for follower validation job to run..."
|
||||
|
||||
local _initial_fake_count
|
||||
_initial_fake_count=$(get_fake_follower_count)
|
||||
|
||||
# Calculate wait time based on number of followers
|
||||
# The job validates 5 followers per run (FollowersPerRun)
|
||||
# With 2-second delay between followers and VALIDATION_INTERVAL between runs
|
||||
local total_followers=$((${#REAL_ACCOUNTS[@]} + ${#FAKE_ACCOUNTS[@]}))
|
||||
local cycles_needed=$(( (total_followers + 4) / 5 )) # Round up
|
||||
local wait_time=$(( (VALIDATION_INTERVAL + 12) * cycles_needed + 30 )) # Extra buffer for network delays
|
||||
|
||||
log_info "Total followers: ${total_followers}, estimated cycles needed: ${cycles_needed}"
|
||||
log_info "Waiting up to ${wait_time} seconds for validation cycles..."
|
||||
|
||||
local elapsed=0
|
||||
local check_interval=10
|
||||
|
||||
while [[ ${elapsed} -lt ${wait_time} ]]; do
|
||||
# Check if we've exceeded the maximum test duration
|
||||
check_timeout
|
||||
|
||||
sleep ${check_interval}
|
||||
elapsed=$((elapsed + check_interval))
|
||||
|
||||
local current_fake_count
|
||||
current_fake_count=$(get_fake_follower_count)
|
||||
local current_real_count
|
||||
current_real_count=$(get_real_follower_count)
|
||||
|
||||
log_info " ${elapsed}s: Real followers: ${current_real_count}/${#REAL_ACCOUNTS[@]}, Fake followers: ${current_fake_count}/${#FAKE_ACCOUNTS[@]}"
|
||||
|
||||
# If all fake followers are gone and all real remain, we can stop early
|
||||
if [[ ${current_fake_count} -eq 0 ]] && [[ ${current_real_count} -eq ${#REAL_ACCOUNTS[@]} ]]; then
|
||||
log_info "Validation complete early - all fake followers removed"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
verify_results() {
|
||||
log_info "Verifying validation results..."
|
||||
|
||||
local total_count
|
||||
total_count=$(get_follower_count)
|
||||
local real_count
|
||||
real_count=$(get_real_follower_count)
|
||||
local fake_count
|
||||
fake_count=$(get_fake_follower_count)
|
||||
|
||||
log_test "Total followers: ${total_count}"
|
||||
log_test "Real followers remaining: ${real_count}/${#REAL_ACCOUNTS[@]}"
|
||||
log_test "Fake followers remaining: ${fake_count}/${#FAKE_ACCOUNTS[@]}"
|
||||
|
||||
# List remaining followers for debugging
|
||||
log_info "Remaining followers:"
|
||||
get_followers_via_api | while read -r iri; do
|
||||
if [[ -n "${iri}" ]]; then
|
||||
log_info " - ${iri}"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Follower Validation Test Results"
|
||||
echo "========================================"
|
||||
echo "Expected real followers: ${#REAL_ACCOUNTS[@]}"
|
||||
echo "Actual real followers: ${real_count}"
|
||||
echo "Expected fake followers: 0"
|
||||
echo "Actual fake followers: ${fake_count}"
|
||||
echo "========================================"
|
||||
|
||||
local passed=true
|
||||
|
||||
# Check that all fake followers were removed
|
||||
if [[ ${fake_count} -ne 0 ]]; then
|
||||
log_error "FAIL: ${fake_count} fake followers should have been removed"
|
||||
passed=false
|
||||
else
|
||||
log_test "PASS: All fake followers were removed"
|
||||
fi
|
||||
|
||||
# Check that all real followers remain
|
||||
if [[ ${real_count} -ne ${#REAL_ACCOUNTS[@]} ]]; then
|
||||
log_error "FAIL: Only ${real_count}/${#REAL_ACCOUNTS[@]} real followers remain"
|
||||
passed=false
|
||||
else
|
||||
log_test "PASS: All real followers remain"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [[ "${passed}" == "true" ]]; then
|
||||
echo -e "${GREEN}TEST PASSED${NC}"
|
||||
return 0
|
||||
else
|
||||
echo -e "${RED}TEST FAILED${NC}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
# Record test start time for timeout checking
|
||||
TEST_START_TIME=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Follower Validation Test"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Setup
|
||||
setup_temp_dir
|
||||
build_owncast
|
||||
start_owncast
|
||||
configure_owncast
|
||||
|
||||
# Stop Owncast before inserting test data to avoid database contention
|
||||
# The database schema is now created, so we can safely insert data
|
||||
stop_owncast
|
||||
|
||||
# Insert test data while Owncast is stopped
|
||||
insert_followers
|
||||
|
||||
# Restart Owncast to run the validation job
|
||||
start_owncast
|
||||
|
||||
# Verify initial state
|
||||
log_info "Initial state:"
|
||||
log_info " Total followers: $(get_follower_count)"
|
||||
log_info " Real followers: $(get_real_follower_count)"
|
||||
log_info " Fake followers: $(get_fake_follower_count)"
|
||||
|
||||
# Wait for validation
|
||||
wait_for_validation
|
||||
|
||||
# Verify results
|
||||
if verify_results; then
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user