hetzner-api-dyndns-records-webinterface/internal/scheduler/scheduler.go

package scheduler

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"strings"
	"sync/atomic"
	"time"

	"github.com/root-kdw12/hetzner-api-dyndns-records-webinterface/internal/diagnostics"
	"github.com/root-kdw12/hetzner-api-dyndns-records-webinterface/internal/hetzner"
	"github.com/root-kdw12/hetzner-api-dyndns-records-webinterface/internal/ipresolve"
	"github.com/root-kdw12/hetzner-api-dyndns-records-webinterface/internal/model"
	"github.com/root-kdw12/hetzner-api-dyndns-records-webinterface/internal/secure"
	"github.com/root-kdw12/hetzner-api-dyndns-records-webinterface/internal/store"
)

var (
	ErrRunInProgress = errors.New("an update run is already in progress")
	ErrDisabled      = errors.New("the updater is disabled")
)

const (
	defaultRetryInitial = 5 * time.Second
	defaultRetryMaximum = time.Minute
)

type ipResolver interface {
	ResolveWithIPv6(ctx context.Context, ipv4Endpoints, ipv6Endpoints []string, ipv6Enabled bool) ipresolve.Result
	ValidateEndpoint(ctx context.Context, raw string) error
}

type Scheduler struct {
	store    *store.Store
	provider *hetzner.Client
	resolver ipResolver
	cipher   *secure.Cipher
	logger   *slog.Logger
	running  atomic.Bool
	nextRun  atomic.Int64
	wake     chan struct{}
	changes  chan struct{}
	done     chan struct{}
	retryMin time.Duration
	retryMax time.Duration
}

func New(database *store.Store, provider *hetzner.Client, resolver *ipresolve.Resolver, cipher *secure.Cipher, logger *slog.Logger) *Scheduler {
	return &Scheduler{
		store:    database,
		provider: provider,
		resolver: resolver,
		cipher:   cipher,
		logger:   logger,
		wake:     make(chan struct{}, 1),
		changes:  make(chan struct{}, 1),
		done:     make(chan struct{}),
		retryMin: defaultRetryInitial,
	}
}

func (s *Scheduler) Start(ctx context.Context) {
	go func() {
		defer close(s.done)
		defer s.nextRun.Store(0)
		consecutiveFailures := 0
		var failureSince, networkLoggedAt time.Time
		for {
			settings, err := s.store.GetSettings(ctx)
			if err != nil {
				s.logger.Error("read scheduler settings", "error", err)
				s.nextRun.Store(0)
				timer := time.NewTimer(time.Minute)
				select {
				case <-ctx.Done():
					timer.Stop()
					return
				case <-s.wake:
					if !timer.Stop() {
						select {
						case <-timer.C:
						default:
						}
					}
				case <-timer.C:
				}
				continue
			}
			wait := time.Duration(settings.UpdateIntervalSeconds) * time.Second
			if wait < time.Minute {
				wait = time.Minute
			}
			retryAfter := time.Duration(0)
			if settings.UpdaterEnabled {
				_, runErr := s.Run(ctx)
				if runErr == nil {
					if consecutiveFailures > 0 {
						diagnostics.Record(diagnostics.Event{Code: "update_recovered", Target: "updater", Attempt: consecutiveFailures, DurationMS: time.Since(failureSince).Milliseconds()})
					}
					consecutiveFailures = 0
				} else if errors.Is(runErr, ErrRunInProgress) {
					retryAfter = time.Second
				} else if !errors.Is(runErr, ErrDisabled) {
					consecutiveFailures++
					if consecutiveFailures == 1 {
						failureSince = time.Now()
					}
					retryAfter = s.retryDelay(consecutiveFailures, wait)
					event := diagnostics.Event{Code: "update_failed", Target: "updater", Failure: diagnostics.Failure(runErr), Attempt: consecutiveFailures, RetrySeconds: int64(retryAfter / time.Second)}
					if consecutiveFailures == 1 || time.Since(networkLoggedAt) >= 15*time.Minute {
						event.Network = diagnostics.CurrentNetwork()
						networkLoggedAt = time.Now()
					}
					diagnostics.Record(event)
					s.logger.Warn("scheduled update run failed", "error", runErr, "retry_in", retryAfter)
				}
			} else {
				consecutiveFailures = 0
				s.nextRun.Store(0)
				select {
				case <-ctx.Done():
					return
				case <-s.wake:
				}
				continue
			}
			if retryAfter > 0 {
				wait = retryAfter
			}
			s.nextRun.Store(time.Now().UTC().Add(wait).Unix())
			timer := time.NewTimer(wait)
			select {
			case <-ctx.Done():
				timer.Stop()
				return
			case <-s.wake:
				if !timer.Stop() {
					select {
					case <-timer.C:
					default:
					}
				}
			case <-timer.C:
			}
		}
	}()
}

func (s *Scheduler) retryDelay(failures int, scheduledWait time.Duration) time.Duration {
	minimum, maximum := s.retryMin, scheduledWait
	if minimum <= 0 {
		minimum = defaultRetryInitial
	}
	retryMax := s.retryMax
	if retryMax <= 0 {
		retryMax = defaultRetryMaximum
	}
	if retryMax < maximum {
		maximum = retryMax
	}
	if maximum < minimum {
		maximum = minimum
	}
	wait := minimum
	for attempt := 1; attempt < failures && wait < maximum; attempt++ {
		if wait > maximum/2 {
			return maximum
		}
		wait *= 2
	}
	if wait > maximum {
		return maximum
	}
	return wait
}

func (s *Scheduler) Trigger() {
	s.nextRun.Store(0)
	select {
	case s.wake <- struct{}{}:
	default:
	}
}

func (s *Scheduler) IsRunning() bool { return s.running.Load() }

func (s *Scheduler) NextRunAt() time.Time {
	value := s.nextRun.Load()
	if value == 0 {
		return time.Time{}
	}
	return time.Unix(value, 0).UTC()
}

// LockChanges serializes a complete WebUI mutation (provider and database)
// with updater snapshots. Callers must hold it until the local change is saved.
func (s *Scheduler) LockChanges(ctx context.Context) (func(), error) {
	if err := ctx.Err(); err != nil {
		return nil, err
	}
	select {
	case s.changes <- struct{}{}:
		return func() { <-s.changes }, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

func (s *Scheduler) Wait(ctx context.Context) error {
	select {
	case <-s.done:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

func (s *Scheduler) Run(ctx context.Context) (string, error) {
	if !s.running.CompareAndSwap(false, true) {
		return "", ErrRunInProgress
	}
	defer s.running.Store(false)
	select {
	case s.changes <- struct{}{}:
		defer func() { <-s.changes }()
	default:
		return "", ErrRunInProgress
	}

	settings, err := s.store.GetSettings(ctx)
	if err != nil {
		return "", err
	}
	if !settings.UpdaterEnabled {
		return "", ErrDisabled
	}
	runID, err := s.store.CreateUpdateRun(ctx)
	if err != nil {
		return "", err
	}
	targets, err := s.store.ListManagedTargets(ctx)
	if err != nil {
		return s.finishRun(ctx, runID, "error", "", "", "Aktive Ziele konnten nicht geladen werden", err)
	}
	needsIPv6 := false
	for _, target := range targets {
		if target.Target.UpdateIPv6 {
			needsIPv6 = true
			break
		}
	}
	ipResult := s.resolver.ResolveWithIPv6(ctx, settings.IPv4Endpoints, settings.IPv6Endpoints, settings.IPv6Enabled && needsIPv6)
	var resolutionErr error
	if ipResult.IPv4Err != nil {
		resolutionErr = errors.Join(resolutionErr, fmt.Errorf("IPv4-Auflösung fehlgeschlagen: %w", ipResult.IPv4Err))
	}
	if settings.IPv6Enabled && needsIPv6 && ipResult.IPv6Err != nil {
		resolutionErr = errors.Join(resolutionErr, fmt.Errorf("IPv6-Auflösung fehlgeschlagen: %w", ipResult.IPv6Err))
	}
	if ipResult.IPv4 == "" && ipResult.IPv6 == "" {
		message := "Keine öffentliche IP ermittelt"
		if resolutionErr == nil {
			resolutionErr = errors.New(message)
		} else {
			message += ": " + resolutionErr.Error()
		}
		return s.finishRun(ctx, runID, "error", "", "", message, resolutionErr)
	}

	if len(targets) == 0 {
		summary := "Keine aktiven DynDNS-Ziele"
		status := "ok"
		if resolutionErr != nil {
			status = "partial"
			summary += "; eine IP-Familie war nicht verfügbar"
		}
		return s.finishRun(ctx, runID, status, ipResult.IPv4, ipResult.IPv6, summary, resolutionErr)
	}

	grouped := make(map[int64][]store.ManagedTarget)
	for _, target := range targets {
		grouped[target.Zone.ID] = append(grouped[target.Zone.ID], target)
	}
	updated, failed := 0, 0
	for _, group := range grouped {
		if len(group) == 0 {
			continue
		}
		if err := s.syncZoneTargets(ctx, group, ipResult.IPv4, ipResult.IPv6, settings.IPv6Enabled, &updated, &failed); err != nil {
			failed += len(group)
			diagnostics.Record(diagnostics.Event{Code: "provider_update_failed", Target: "api.hetzner.cloud", RunID: runID, Failure: diagnostics.Failure(err), HTTPStatus: diagnostics.HTTPStatus(err)})
			s.logger.Warn("sync DNS zone", "zone", group[0].Zone.Name, "error", err)
			for _, target := range group {
				_ = s.store.UpdateTargetResult(ctx, target.Target.ID, "Fehler", safeError(err), false)
			}
			_ = s.store.MarkZoneSynced(ctx, group[0].Zone.ID, safeError(err))
		}
	}
	status := "ok"
	if failed > 0 {
		status = "partial"
	}
	summary := fmt.Sprintf("%d Ziel(e) aktualisiert, %d Fehler", updated, failed)
	if resolutionErr != nil {
		summary += "; eine IP-Familie war nicht verfügbar"
		if status == "ok" {
			status = "partial"
		}
	}
	var runErr error
	if failed > 0 {
		runErr = fmt.Errorf("%d DynDNS-Ziel(e) konnten nicht synchronisiert werden", failed)
	}
	runErr = errors.Join(runErr, resolutionErr)
	return s.finishRun(ctx, runID, status, ipResult.IPv4, ipResult.IPv6, summary, runErr)
}

func (s *Scheduler) finishRun(ctx context.Context, runID int64, status, ipv4, ipv6, summary string, runErr error) (string, error) {
	// Persist cancellation too; using the canceled run context leaves history
	// stuck at "running". Keep cleanup bounded during shutdown.
	finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
	defer cancel()
	if err := s.store.CompleteUpdateRun(finishCtx, runID, status, ipv4, ipv6, summary); err != nil {
		s.logger.Error("persist update result", "run_id", runID, "error", err)
		runErr = errors.Join(runErr, fmt.Errorf("Updater-Ergebnis konnte nicht gespeichert werden: %w", err))
		status = "error"
	}
	diagnostics.Record(diagnostics.Event{Code: "update_run_finished", Target: "updater", RunID: runID, Status: status, Failure: diagnostics.Failure(runErr), HTTPStatus: diagnostics.HTTPStatus(runErr)})
	return summary, runErr
}

func (s *Scheduler) syncZoneTargets(ctx context.Context, targets []store.ManagedTarget, ipv4, ipv6 string, ipv6Enabled bool, updated, failed *int) error {
	zone, credential := targets[0].Zone, targets[0].Credential
	token, err := s.cipher.Decrypt(credential.Ciphertext, credential.Nonce)
	if err != nil {
		return err
	}
	rrsets, err := s.provider.ListRRSets(ctx, token, zone.ProviderID)
	if err != nil {
		return err
	}
	byKey := make(map[string]hetzner.RRSet, len(rrsets))
	for _, rrset := range rrsets {
		if rrset.Type == "A" || rrset.Type == "AAAA" {
			byKey[rrKey(rrset.Name, rrset.Type)] = rrset
		}
	}
	for _, item := range targets {
		targetUpdated, status, targetErr := s.syncTarget(ctx, token, zone, item.Target, byKey, ipv4, ipv6, ipv6Enabled)
		if targetUpdated {
			*updated = *updated + 1
		}
		if targetErr != nil {
			*failed = *failed + 1
			diagnostics.Record(diagnostics.Event{Code: "provider_update_failed", Target: "api.hetzner.cloud", TargetID: item.Target.ID, ActionID: diagnostics.ActionID(targetErr), Failure: diagnostics.Failure(targetErr), HTTPStatus: diagnostics.HTTPStatus(targetErr)})
			_ = s.store.UpdateTargetResult(ctx, item.Target.ID, "Fehler", safeError(targetErr), targetUpdated)
			continue
		}
		if err := s.store.UpdateTargetResult(ctx, item.Target.ID, status, "", targetUpdated); err != nil {
			return err
		}
	}
	return s.store.MarkZoneSynced(ctx, zone.ID, "")
}

func (s *Scheduler) syncTarget(ctx context.Context, token string, zone model.Zone, target model.Target, rrsets map[string]hetzner.RRSet, ipv4, ipv6 string, ipv6Enabled bool) (bool, string, error) {
	var states []string
	var familyErrors []error
	updated := false
	for _, family := range []struct {
		enabled        bool
		recordType, ip string
	}{
		{target.UpdateIPv4, "A", ipv4},
		{target.UpdateIPv6, "AAAA", ipv6},
	} {
		if !family.enabled {
			continue
		}
		if family.recordType == "AAAA" && !ipv6Enabled {
			states = append(states, "AAAA global deaktiviert")
			continue
		}
		changed, state, err := s.syncFamily(ctx, token, zone, target, rrsets[rrKey(target.RecordName, family.recordType)], family.recordType, family.ip)
		updated = updated || changed
		if err != nil {
			familyErrors = append(familyErrors, fmt.Errorf("%s: %w", family.recordType, err))
			states = append(states, family.recordType+" Fehler")
		} else {
			states = append(states, state)
		}
	}
	if len(states) == 0 {
		return false, "Deaktivierte IP-Familien", nil
	}
	return updated, strings.Join(states, " · "), errors.Join(familyErrors...)
}

func (s *Scheduler) syncFamily(ctx context.Context, token string, zone model.Zone, target model.Target, rrset hetzner.RRSet, recordType, ip string) (bool, string, error) {
	if ip == "" {
		return false, recordType + " nicht verfügbar", nil
	}
	if rrset.Name == "" {
		ttl := target.TTL
		if ttl < 60 {
			ttl = 60
		}
		if err := s.provider.CreateRRSet(ctx, token, zone.ProviderID, target.RecordName, recordType, ttl, []hetzner.Record{{Value: ip, Comment: target.Description}}); err != nil {
			return false, "", err
		}
		return true, recordType + " angelegt", nil
	}
	if rrset.Protection.Change {
		return false, "", errors.New("das RRset ist bei Hetzner gegen Änderungen geschützt")
	}
	if len(rrset.Records) != 1 {
		return false, "", fmt.Errorf("%s enthält %d Records und wird aus Sicherheitsgründen nicht geändert", rrset.Name, len(rrset.Records))
	}
	ttlChanged := false
	if rrset.TTL == nil || *rrset.TTL != targetTTL(target.TTL) {
		if err := s.provider.ChangeRRSetTTL(ctx, token, zone.ProviderID, target.RecordName, recordType, targetTTL(target.TTL)); err != nil {
			return false, "", err
		}
		ttlChanged = true
	}
	current := rrset.Records[0]
	if current.Value == ip {
		if current.Comment != target.Description {
			if err := s.provider.UpdateRecordComments(ctx, token, zone.ProviderID, target.RecordName, recordType, []hetzner.Record{{Value: current.Value, Comment: target.Description}}); err != nil {
				return ttlChanged, "", err
			}
			status := recordType + " Kommentar synchronisiert"
			if ttlChanged {
				status += " · TTL aktualisiert"
			}
			return true, status, nil
		}
		if ttlChanged {
			return true, recordType + " TTL aktualisiert", nil
		}
		return false, recordType + " unverändert", nil
	}
	if err := s.provider.SetRRSetRecords(ctx, token, zone.ProviderID, target.RecordName, recordType, []hetzner.Record{{Value: ip, Comment: target.Description}}); err != nil {
		return ttlChanged, "", err
	}
	status := recordType + " aktualisiert"
	if ttlChanged {
		status += " · TTL aktualisiert"
	}
	return true, status, nil
}

func (s *Scheduler) SyncZone(ctx context.Context, zoneID int64) (int, error) {
	zone, credential, err := s.store.GetZoneWithCredential(ctx, zoneID)
	if err != nil {
		return 0, err
	}
	token, err := s.cipher.Decrypt(credential.Ciphertext, credential.Nonce)
	if err != nil {
		return 0, err
	}
	rrsets, err := s.provider.ListRRSets(ctx, token, zone.ProviderID)
	if err != nil {
		_ = s.store.MarkZoneSynced(ctx, zone.ID, safeError(err))
		return 0, err
	}
	type imported struct {
		ipv4, ipv6  bool
		description string
		ttl         int
	}
	grouped := map[string]*imported{}
	var cnames []model.CNAMERecord
	for _, rrset := range rrsets {
		if rrset.Type == "CNAME" {
			if len(rrset.Records) != 1 {
				continue
			}
			ttl := targetTTL(60)
			if rrset.TTL != nil {
				ttl = targetTTL(*rrset.TTL)
			}
			cnames = append(cnames, model.CNAMERecord{
				ZoneID:      zone.ID,
				Active:      true,
				RecordName:  normalRecordName(rrset.Name),
				FQDN:        fqdn(rrset.Name, zone.Name),
				Destination: rrset.Records[0].Value,
				Description: rrset.Records[0].Comment,
				TTL:         ttl,
			})
			continue
		}
		if rrset.Type != "A" && rrset.Type != "AAAA" {
			continue
		}
		entry := grouped[rrset.Name]
		if entry == nil {
			entry = &imported{ttl: 60}
			grouped[rrset.Name] = entry
		}
		if rrset.Type == "A" {
			entry.ipv4 = true
		} else {
			entry.ipv6 = true
		}
		if rrset.TTL != nil && *rrset.TTL >= 60 {
			entry.ttl = *rrset.TTL
		}
		if entry.description == "" && len(rrset.Records) > 0 {
			entry.description = rrset.Records[0].Comment
		}
	}
	added := 0
	for name, entry := range grouped {
		target := model.Target{ZoneID: zone.ID, RecordName: normalRecordName(name), FQDN: fqdn(name, zone.Name), UpdateIPv4: entry.ipv4, UpdateIPv6: entry.ipv6, Description: entry.description, TTL: entry.ttl}
		isNew, err := s.store.UpsertImportedTarget(ctx, target)
		if err != nil {
			return added, err
		}
		if isNew {
			added++
			continue
		}
		if err := s.store.UpdateImportedTargetTTL(ctx, zone.ID, target.RecordName, target.TTL); err != nil {
			return added, err
		}
	}
	if err := s.store.ReplaceCNAMERecords(ctx, zone.ID, cnames); err != nil {
		return 0, err
	}
	_ = s.store.MarkZoneSynced(ctx, zone.ID, "")
	_ = s.store.AddAudit(ctx, "zone.import", fmt.Sprintf("%s: %d neue A/AAAA-Ziele importiert", zone.Name, added))
	return added, nil
}

func (s *Scheduler) SyncTargetComment(ctx context.Context, target model.Target) error {
	zone, credential, err := s.store.GetZoneWithCredential(ctx, target.ZoneID)
	if err != nil {
		return err
	}
	token, err := s.cipher.Decrypt(credential.Ciphertext, credential.Nonce)
	if err != nil {
		return err
	}
	rrsets, err := s.provider.ListRRSets(ctx, token, zone.ProviderID)
	if err != nil {
		return err
	}
	for _, rrset := range rrsets {
		if rrset.Name != target.RecordName || (rrset.Type != "A" && rrset.Type != "AAAA") {
			continue
		}
		if (rrset.Type == "A" && !target.UpdateIPv4) || (rrset.Type == "AAAA" && !target.UpdateIPv6) || len(rrset.Records) != 1 {
			continue
		}
		if rrset.Records[0].Comment != target.Description {
			if err := s.provider.UpdateRecordComments(ctx, token, zone.ProviderID, target.RecordName, rrset.Type, []hetzner.Record{{Value: rrset.Records[0].Value, Comment: target.Description}}); err != nil {
				return err
			}
		}
	}
	return nil
}

// SyncTargetTTL applies a changed TTL immediately when the selected A/AAAA
// RRsets already exist. Missing RRsets will receive the same TTL when the
// DynDNS worker creates them on its next run.
func (s *Scheduler) SyncTargetTTL(ctx context.Context, target model.Target) error {
	zone, token, err := s.zoneToken(ctx, target.ZoneID)
	if err != nil {
		return err
	}
	rrsets, err := s.provider.ListRRSets(ctx, token, zone.ProviderID)
	if err != nil {
		return err
	}
	ttl := targetTTL(target.TTL)
	var recordTypes []string
	for _, rrset := range rrsets {
		if rrset.Name != target.RecordName {
			continue
		}
		if (rrset.Type == "A" && !target.UpdateIPv4) || (rrset.Type == "AAAA" && !target.UpdateIPv6) || (rrset.Type != "A" && rrset.Type != "AAAA") {
			continue
		}
		if rrset.TTL != nil && *rrset.TTL == ttl {
			continue
		}
		if rrset.Protection.Change {
			return errors.New("das RRset ist bei Hetzner gegen Änderungen geschützt")
		}
		recordTypes = append(recordTypes, rrset.Type)
	}
	for _, recordType := range recordTypes {
		if err := s.provider.ChangeRRSetTTL(ctx, token, zone.ProviderID, target.RecordName, recordType, ttl); err != nil {
			return err
		}
	}
	return nil
}

// SyncStaticTarget writes manually entered addresses while DynDNS is disabled.
// It deliberately reuses the guarded RRSet update path used by the updater:
// protected or multi-record RRSets are left untouched.
func (s *Scheduler) SyncStaticTarget(ctx context.Context, target model.Target) (bool, string, error) {
	zone, token, err := s.zoneToken(ctx, target.ZoneID)
	if err != nil {
		return false, "", err
	}
	rrsets, err := s.provider.ListRRSets(ctx, token, zone.ProviderID)
	if err != nil {
		return false, "", err
	}
	byKey := make(map[string]hetzner.RRSet, len(rrsets))
	for _, rrset := range rrsets {
		if rrset.Type == "A" || rrset.Type == "AAAA" {
			byKey[rrKey(rrset.Name, rrset.Type)] = rrset
		}
	}
	target.UpdateIPv4 = target.UpdateIPv4 && target.StaticIPv4 != ""
	target.UpdateIPv6 = target.UpdateIPv6 && target.StaticIPv6 != ""
	if !target.UpdateIPv4 && !target.UpdateIPv6 {
		return false, "Keine feste IP eingetragen", nil
	}
	return s.syncTarget(ctx, token, zone, target, byKey, target.StaticIPv4, target.StaticIPv6, true)
}

// DeleteTargetFamilies removes selected A/AAAA RRSets while keeping the local
// target itself. This lets users temporarily disable every family without
// losing its configuration.
func (s *Scheduler) DeleteTargetFamilies(ctx context.Context, target model.Target, deleteIPv4, deleteIPv6 bool) (int, error) {
	if !deleteIPv4 && !deleteIPv6 {
		return 0, nil
	}
	zone, token, err := s.zoneToken(ctx, target.ZoneID)
	if err != nil {
		return 0, err
	}
	rrsets, err := s.provider.ListRRSets(ctx, token, zone.ProviderID)
	if err != nil {
		return 0, err
	}
	var recordTypes []string
	for _, rrset := range rrsets {
		if rrset.Name != target.RecordName || (rrset.Type != "A" && rrset.Type != "AAAA") {
			continue
		}
		if (rrset.Type == "A" && !deleteIPv4) || (rrset.Type == "AAAA" && !deleteIPv6) {
			continue
		}
		if rrset.Protection.Change {
			return 0, errors.New("das RRset ist bei Hetzner gegen Änderungen geschützt")
		}
		recordTypes = append(recordTypes, rrset.Type)
	}
	for _, recordType := range recordTypes {
		if err := s.provider.DeleteRRSet(ctx, token, zone.ProviderID, target.RecordName, recordType); err != nil {
			return 0, err
		}
	}
	return len(recordTypes), nil
}

// DeleteZoneRecords removes every A, AAAA and CNAME RRSet represented by the
// supplied local records. Other provider records in the zone remain untouched.
// Protected RRsets are rejected before the first remote deletion.
func (s *Scheduler) DeleteZoneRecords(ctx context.Context, zoneID int64, targets []model.Target, cnames []model.CNAMERecord) (int, error) {
	zone, token, err := s.zoneToken(ctx, zoneID)
	if err != nil {
		return 0, err
	}
	rrsets, err := s.provider.ListRRSets(ctx, token, zone.ProviderID)
	if err != nil {
		return 0, err
	}
	managed := make(map[string]struct{}, len(targets)*2+len(cnames))
	for _, target := range targets {
		managed[rrKey(target.RecordName, "A")] = struct{}{}
		managed[rrKey(target.RecordName, "AAAA")] = struct{}{}
	}
	for _, record := range cnames {
		managed[rrKey(record.RecordName, "CNAME")] = struct{}{}
	}
	deletions := make([]hetzner.RRSet, 0, len(managed))
	for _, rrset := range rrsets {
		if _, found := managed[rrKey(rrset.Name, rrset.Type)]; !found {
			continue
		}
		if rrset.Protection.Change {
			return 0, fmt.Errorf("das RRset %s (%s) ist bei Hetzner gegen Änderungen geschützt", fqdn(rrset.Name, zone.Name), rrset.Type)
		}
		deletions = append(deletions, rrset)
	}
	deleted := 0
	for _, rrset := range deletions {
		if err := s.provider.DeleteRRSet(ctx, token, zone.ProviderID, rrset.Name, rrset.Type); err != nil {
			return deleted, err
		}
		deleted++
	}
	return deleted, nil
}

// DeleteIPv6Targets removes all managed AAAA RRSets before global IPv6 is
// disabled. All protected RRSets are detected before the first deletion.
func (s *Scheduler) DeleteIPv6Targets(ctx context.Context, targets []model.Target) (int, error) {
	type zoneState struct {
		zone   model.Zone
		token  string
		rrsets map[string]hetzner.RRSet
	}
	type deletion struct {
		zone  model.Zone
		token string
		name  string
	}
	states := make(map[int64]zoneState)
	deletions := make([]deletion, 0, len(targets))
	for _, target := range targets {
		state, found := states[target.ZoneID]
		if !found {
			zone, token, err := s.zoneToken(ctx, target.ZoneID)
			if err != nil {
				return 0, err
			}
			rrsets, err := s.provider.ListRRSets(ctx, token, zone.ProviderID)
			if err != nil {
				return 0, err
			}
			state = zoneState{zone: zone, token: token, rrsets: make(map[string]hetzner.RRSet, len(rrsets))}
			for _, rrset := range rrsets {
				state.rrsets[rrKey(rrset.Name, rrset.Type)] = rrset
			}
			states[target.ZoneID] = state
		}
		rrset, exists := state.rrsets[rrKey(target.RecordName, "AAAA")]
		if !exists {
			continue
		}
		if rrset.Protection.Change {
			return 0, errors.New("mindestens ein AAAA-Record ist bei Hetzner gegen Änderungen geschützt")
		}
		deletions = append(deletions, deletion{zone: state.zone, token: state.token, name: target.RecordName})
	}
	for _, item := range deletions {
		if err := s.provider.DeleteRRSet(ctx, item.token, item.zone.ProviderID, item.name, "AAAA"); err != nil {
			return 0, err
		}
	}
	return len(deletions), nil
}

// DisableIPv6 removes managed AAAA RRSets and disables IPv6 on every linked
// local target. It is safe to run when IPv6 is already disabled.
func (s *Scheduler) DisableIPv6(ctx context.Context) (int, error) {
	targets, err := s.store.ListTargetsWithIPv6(ctx)
	if err != nil {
		return 0, err
	}
	deleted, err := s.DeleteIPv6Targets(ctx, targets)
	if err != nil {
		return 0, err
	}
	if err := s.store.DisableIPv6ForTargets(ctx); err != nil {
		return 0, err
	}
	return deleted, nil
}

// SyncZoneTTL updates the TTL of every RRSet this application manages in a
// zone. It preflights protected or stale CNAME RRSets before changing any
// remote value, while missing DynDNS RRSets keep the new TTL locally for their
// eventual creation.
func (s *Scheduler) SyncZoneTTL(ctx context.Context, zoneID int64, targets []model.Target, cnames []model.CNAMERecord, ttl int) (int, error) {
	zone, token, err := s.zoneToken(ctx, zoneID)
	if err != nil {
		return 0, err
	}
	rrsets, err := s.provider.ListRRSets(ctx, token, zone.ProviderID)
	if err != nil {
		return 0, err
	}
	byKey := make(map[string]hetzner.RRSet, len(rrsets))
	for _, rrset := range rrsets {
		byKey[rrKey(rrset.Name, rrset.Type)] = rrset
	}
	type change struct {
		name       string
		recordType string
	}
	changes := make([]change, 0, len(targets)*2+len(cnames))
	seen := make(map[string]bool, cap(changes))
	queue := func(name, recordType string, required bool) error {
		key := rrKey(name, recordType)
		rrset, found := byKey[key]
		if !found {
			if required {
				return fmt.Errorf("der CNAME %s wurde bei Hetzner nicht gefunden; lies die Zone erneut ein", fqdn(name, zone.Name))
			}
			return nil
		}
		if rrset.TTL != nil && *rrset.TTL == targetTTL(ttl) {
			return nil
		}
		if rrset.Protection.Change {
			return errors.New("mindestens ein RRset ist bei Hetzner gegen Änderungen geschützt")
		}
		if !seen[key] {
			seen[key] = true
			changes = append(changes, change{name: name, recordType: recordType})
		}
		return nil
	}
	for _, target := range targets {
		if target.UpdateIPv4 {
			if err := queue(target.RecordName, "A", false); err != nil {
				return 0, err
			}
		}
		if target.UpdateIPv6 {
			if err := queue(target.RecordName, "AAAA", false); err != nil {
				return 0, err
			}
		}
	}
	for _, record := range cnames {
		if !record.Active {
			continue
		}
		if err := queue(record.RecordName, "CNAME", true); err != nil {
			return 0, err
		}
	}
	for _, item := range changes {
		if err := s.provider.ChangeRRSetTTL(ctx, token, zone.ProviderID, item.name, item.recordType, targetTTL(ttl)); err != nil {
			return 0, err
		}
	}
	return len(changes), nil
}

func (s *Scheduler) CreateCNAME(ctx context.Context, record model.CNAMERecord) error {
	zone, token, err := s.zoneToken(ctx, record.ZoneID)
	if err != nil {
		return err
	}
	return s.provider.CreateRRSet(ctx, token, zone.ProviderID, record.RecordName, "CNAME", targetTTL(record.TTL), []hetzner.Record{{Value: record.Destination, Comment: record.Description}})
}

func (s *Scheduler) UpdateCNAME(ctx context.Context, previous, record model.CNAMERecord) error {
	if previous.ZoneID != record.ZoneID || previous.RecordName != record.RecordName {
		return errors.New("der Name eines CNAME-Records kann nicht geändert werden")
	}
	if previous.Destination == record.Destination && previous.Description == record.Description && previous.TTL == record.TTL {
		return nil
	}
	zone, token, err := s.zoneToken(ctx, record.ZoneID)
	if err != nil {
		return err
	}
	rrsets, err := s.provider.ListRRSets(ctx, token, zone.ProviderID)
	if err != nil {
		return err
	}
	var current *hetzner.RRSet
	for index := range rrsets {
		if rrsets[index].Name == record.RecordName && rrsets[index].Type == "CNAME" {
			current = &rrsets[index]
			break
		}
	}
	if current == nil {
		return errors.New("der CNAME-Record wurde bei Hetzner nicht gefunden; lies die Zone erneut ein")
	}
	if current.Protection.Change {
		return errors.New("das RRset ist bei Hetzner gegen Änderungen geschützt")
	}
	if previous.TTL != record.TTL {
		if err := s.provider.ChangeRRSetTTL(ctx, token, zone.ProviderID, record.RecordName, "CNAME", targetTTL(record.TTL)); err != nil {
			return err
		}
	}
	if previous.Destination != record.Destination || previous.Description != record.Description {
		if err := s.provider.SetRRSetRecords(ctx, token, zone.ProviderID, record.RecordName, "CNAME", []hetzner.Record{{Value: record.Destination, Comment: record.Description}}); err != nil {
			return err
		}
	}
	return nil
}

func (s *Scheduler) DeleteCNAME(ctx context.Context, record model.CNAMERecord) error {
	zone, token, err := s.zoneToken(ctx, record.ZoneID)
	if err != nil {
		return err
	}
	return s.provider.DeleteRRSet(ctx, token, zone.ProviderID, record.RecordName, "CNAME")
}

func (s *Scheduler) zoneToken(ctx context.Context, zoneID int64) (model.Zone, string, error) {
	zone, credential, err := s.store.GetZoneWithCredential(ctx, zoneID)
	if err != nil {
		return model.Zone{}, "", err
	}
	token, err := s.cipher.Decrypt(credential.Ciphertext, credential.Nonce)
	if err != nil {
		return model.Zone{}, "", err
	}
	return zone, token, nil
}

func (s *Scheduler) ValidateEndpoints(ctx context.Context, endpoints []string) error {
	ctx, cancel := context.WithTimeout(ctx, 8*time.Second)
	defer cancel()
	if len(endpoints) > 8 {
		return errors.New("höchstens acht IP-Endpoints pro Familie erlaubt")
	}
	for _, endpoint := range endpoints {
		if err := s.resolver.ValidateEndpoint(ctx, endpoint); err != nil {
			return fmt.Errorf("%s: %w", endpoint, err)
		}
	}
	return nil
}

func (s *Scheduler) ValidateToken(ctx context.Context, token string) ([]hetzner.Zone, error) {
	return s.provider.ListZones(ctx, token)
}

func (s *Scheduler) Provider() *hetzner.Client { return s.provider }

func rrKey(name, recordType string) string { return normalRecordName(name) + "|" + recordType }

func targetTTL(ttl int) int {
	if ttl < 60 {
		return 60
	}
	return ttl
}

func normalRecordName(name string) string {
	name = strings.TrimSuffix(strings.TrimSpace(name), ".")
	if name == "" || name == "@" {
		return "@"
	}
	return name
}

func fqdn(name, zone string) string {
	if normalRecordName(name) == "@" {
		return zone
	}
	return normalRecordName(name) + "." + zone
}

func safeError(err error) string {
	if err == nil {
		return ""
	}
	message := err.Error()
	if len(message) > 300 {
		return message[:300]
	}
	return message
}