feat: add resilient updates and access controls

internal/scheduler/scheduler.go

@@ -21,24 +21,42 @@ var (
 	ErrDisabled      = errors.New("the updater is disabled")
 )
 
+const defaultRetryInitial = 5 * time.Second
+
+type ipResolver interface {
+	ResolveWithIPv6(ctx context.Context, ipv4Endpoints, ipv6Endpoints []string, ipv6Enabled bool) ipresolve.Result
+	ValidateEndpoint(ctx context.Context, raw string) error
+}
+
 type Scheduler struct {
 	store    *store.Store
 	provider *hetzner.Client
-	resolver *ipresolve.Resolver
+	resolver ipResolver
 	cipher   *secure.Cipher
 	logger   *slog.Logger
 	running  atomic.Bool
 	nextRun  atomic.Int64
 	wake     chan struct{}
+	retryMin time.Duration
+	retryMax time.Duration
 }
 
 func New(database *store.Store, provider *hetzner.Client, resolver *ipresolve.Resolver, cipher *secure.Cipher, logger *slog.Logger) *Scheduler {
-	return &Scheduler{store: database, provider: provider, resolver: resolver, cipher: cipher, logger: logger, wake: make(chan struct{}, 1)}
+	return &Scheduler{
+		store:    database,
+		provider: provider,
+		resolver: resolver,
+		cipher:   cipher,
+		logger:   logger,
+		wake:     make(chan struct{}, 1),
+		retryMin: defaultRetryInitial,
+	}
 }
 
 func (s *Scheduler) Start(ctx context.Context) {
 	go func() {
 		defer s.nextRun.Store(0)
+		consecutiveFailures := 0
 		for {
 			settings, err := s.store.GetSettings(ctx)
 			if err != nil {
@@ -60,11 +78,22 @@ func (s *Scheduler) Start(ctx context.Context) {
 				}
 				continue
 			}
+			wait := time.Duration(settings.UpdateIntervalSeconds) * time.Second
+			if wait < time.Minute {
+				wait = time.Minute
+			}
+			retryAfter := time.Duration(0)
 			if settings.UpdaterEnabled {
-				if _, runErr := s.Run(ctx); runErr != nil && !errors.Is(runErr, ErrRunInProgress) && !errors.Is(runErr, ErrDisabled) {
-					s.logger.Warn("scheduled update run failed", "error", runErr)
+				_, runErr := s.Run(ctx)
+				if runErr == nil {
+					consecutiveFailures = 0
+				} else if !errors.Is(runErr, ErrRunInProgress) && !errors.Is(runErr, ErrDisabled) {
+					consecutiveFailures++
+					retryAfter = s.retryDelay(consecutiveFailures, wait)
+					s.logger.Warn("scheduled update run failed", "error", runErr, "retry_in", retryAfter)
 				}
 			} else {
+				consecutiveFailures = 0
 				s.nextRun.Store(0)
 				select {
 				case <-ctx.Done():
@@ -73,9 +102,8 @@ func (s *Scheduler) Start(ctx context.Context) {
 				}
 				continue
 			}
-			wait := time.Duration(settings.UpdateIntervalSeconds) * time.Second
-			if wait < time.Minute {
-				wait = time.Minute
+			if retryAfter > 0 {
+				wait = retryAfter
 			}
 			s.nextRun.Store(time.Now().UTC().Add(wait).Unix())
 			timer := time.NewTimer(wait)
@@ -84,6 +112,7 @@ func (s *Scheduler) Start(ctx context.Context) {
 				timer.Stop()
 				return
 			case <-s.wake:
+				consecutiveFailures = 0
 				if !timer.Stop() {
 					select {
 					case <-timer.C:
@@ -96,6 +125,30 @@ func (s *Scheduler) Start(ctx context.Context) {
 	}()
 }
 
+func (s *Scheduler) retryDelay(failures int, scheduledWait time.Duration) time.Duration {
+	minimum, maximum := s.retryMin, scheduledWait
+	if minimum <= 0 {
+		minimum = defaultRetryInitial
+	}
+	if s.retryMax > 0 && s.retryMax < maximum {
+		maximum = s.retryMax
+	}
+	if maximum < minimum {
+		maximum = minimum
+	}
+	wait := minimum
+	for attempt := 1; attempt < failures && wait < maximum; attempt++ {
+		if wait > maximum/2 {
+			return maximum
+		}
+		wait *= 2
+	}
+	if wait > maximum {
+		return maximum
+	}
+	return wait
+}
+
 func (s *Scheduler) Trigger() {
 	s.nextRun.Store(0)
 	select {
@@ -131,22 +184,43 @@ func (s *Scheduler) Run(ctx context.Context) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	ipResult := s.resolver.ResolveWithIPv6(ctx, settings.IPv4Endpoints, settings.IPv6Endpoints, settings.IPv6Enabled)
-	if ipResult.IPv4 == "" && ipResult.IPv6 == "" {
-		message := "Keine öffentliche IP ermittelt"
-		if ipResult.IPv4Err != nil {
-			message += ": " + ipResult.IPv4Err.Error()
-		}
-		return s.finishRun(ctx, runID, "error", "", "", message, errors.New(message))
-	}
-
 	targets, err := s.store.ListManagedTargets(ctx)
 	if err != nil {
-		return s.finishRun(ctx, runID, "error", ipResult.IPv4, ipResult.IPv6, "Aktive Ziele konnten nicht geladen werden", err)
+		return s.finishRun(ctx, runID, "error", "", "", "Aktive Ziele konnten nicht geladen werden", err)
 	}
+	needsIPv6 := false
+	for _, target := range targets {
+		if target.Target.UpdateIPv6 {
+			needsIPv6 = true
+			break
+		}
+	}
+	ipResult := s.resolver.ResolveWithIPv6(ctx, settings.IPv4Endpoints, settings.IPv6Endpoints, settings.IPv6Enabled && needsIPv6)
+	var resolutionErr error
+	if ipResult.IPv4Err != nil {
+		resolutionErr = errors.Join(resolutionErr, fmt.Errorf("IPv4-Auflösung fehlgeschlagen: %w", ipResult.IPv4Err))
+	}
+	if settings.IPv6Enabled && needsIPv6 && ipResult.IPv6Err != nil {
+		resolutionErr = errors.Join(resolutionErr, fmt.Errorf("IPv6-Auflösung fehlgeschlagen: %w", ipResult.IPv6Err))
+	}
+	if ipResult.IPv4 == "" && ipResult.IPv6 == "" {
+		message := "Keine öffentliche IP ermittelt"
+		if resolutionErr == nil {
+			resolutionErr = errors.New(message)
+		} else {
+			message += ": " + resolutionErr.Error()
+		}
+		return s.finishRun(ctx, runID, "error", "", "", message, resolutionErr)
+	}
+
 	if len(targets) == 0 {
 		summary := "Keine aktiven DynDNS-Ziele"
-		return s.finishRun(ctx, runID, "ok", ipResult.IPv4, ipResult.IPv6, summary, nil)
+		status := "ok"
+		if resolutionErr != nil {
+			status = "partial"
+			summary += "; eine IP-Familie war nicht verfügbar"
+		}
+		return s.finishRun(ctx, runID, status, ipResult.IPv4, ipResult.IPv6, summary, resolutionErr)
 	}
 
 	grouped := make(map[int64][]store.ManagedTarget)
@@ -172,13 +246,18 @@ func (s *Scheduler) Run(ctx context.Context) (string, error) {
 		status = "partial"
 	}
 	summary := fmt.Sprintf("%d Ziel(e) aktualisiert, %d Fehler", updated, failed)
-	if ipResult.IPv4Err != nil || (settings.IPv6Enabled && ipResult.IPv6Err != nil) {
+	if resolutionErr != nil {
 		summary += "; eine IP-Familie war nicht verfügbar"
 		if status == "ok" {
 			status = "partial"
 		}
 	}
-	return s.finishRun(ctx, runID, status, ipResult.IPv4, ipResult.IPv6, summary, nil)
+	var runErr error
+	if failed > 0 {
+		runErr = fmt.Errorf("%d DynDNS-Ziel(e) konnten nicht synchronisiert werden", failed)
+	}
+	runErr = errors.Join(runErr, resolutionErr)
+	return s.finishRun(ctx, runID, status, ipResult.IPv4, ipResult.IPv6, summary, runErr)
 }
 
 func (s *Scheduler) finishRun(ctx context.Context, runID int64, status, ipv4, ipv6, summary string, runErr error) (string, error) {