feat: add resilient container restart controls

internal/updateagent/docker.go

@@ -64,7 +64,7 @@ func (r *DockerRunner) Update(ctx context.Context, request updateprotocol.Reques
 	if !updateprotocol.ValidVersion(request.Version) {
 		return errors.New("invalid target version")
 	}
-	if err := r.docker.negotiate(ctx); err != nil {
+	if err := waitForDockerAPI(ctx, 30*time.Second, time.Second, r.docker.negotiate); err != nil {
 		return &updateError{message: "Containerverwaltung ist nicht erreichbar; der Dienst läuft unverändert weiter", err: err}
 	}
 	target, err := r.docker.findTarget(ctx, r.targetLabel)
@@ -75,7 +75,18 @@ func (r *DockerRunner) Update(ctx context.Context, request updateprotocol.Reques
 	if err != nil {
 		return err
 	}
-	if inspected.State.Health == nil || inspected.State.Health.Status != "healthy" {
+	if inspected.State.Health == nil {
+		return &updateError{message: "Die laufende Instanz ist nicht healthy; das Update wurde sicherheitshalber nicht gestartet", err: errors.New("current container is not healthy")}
+	}
+	switch inspected.State.Health.Status {
+	case "healthy":
+	case "starting":
+		progress(updateprotocol.PhaseQueued, "Laufender Container beendet noch seinen Start-Healthcheck")
+		inspected, err = r.docker.waitCurrentHealthy(ctx, target.ID, inspected.Image, 100*time.Second, time.Second)
+		if err != nil {
+			return &updateError{message: "Die laufende Instanz wurde nach dem Host-Start nicht rechtzeitig healthy; das Update wurde nicht gestartet", err: err}
+		}
+	default:
 		return &updateError{message: "Die laufende Instanz ist nicht healthy; das Update wurde sicherheitshalber nicht gestartet", err: errors.New("current container is not healthy")}
 	}
 	currentVersion, err := resolveCurrentVersion(request.CurrentVersion, inspected.Config, r.imageRepository)
@@ -847,6 +858,72 @@ func (c *dockerClient) waitHealthy(ctx context.Context, id, expectedImageID, exp
 	}
 }
 
+func (c *dockerClient) waitCurrentHealthy(ctx context.Context, id, expectedImageID string, timeout, pollInterval time.Duration) (containerInspect, error) {
+	deadline := time.NewTimer(timeout)
+	ticker := time.NewTicker(pollInterval)
+	defer deadline.Stop()
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return containerInspect{}, ctx.Err()
+		case <-deadline.C:
+			return containerInspect{}, errors.New("current container did not become healthy in time")
+		case <-ticker.C:
+			inspected, err := c.inspectContainer(ctx, id)
+			if err != nil {
+				return containerInspect{}, err
+			}
+			if !inspected.State.Running {
+				return containerInspect{}, fmt.Errorf("current container stopped with exit code %d", inspected.State.ExitCode)
+			}
+			if expectedImageID != "" && inspected.Image != expectedImageID {
+				return containerInspect{}, fmt.Errorf("current container image changed from %s to %s", expectedImageID, inspected.Image)
+			}
+			if inspected.State.Health == nil {
+				return containerInspect{}, errors.New("current container has no healthcheck")
+			}
+			switch inspected.State.Health.Status {
+			case "healthy":
+				return inspected, nil
+			case "starting":
+				continue
+			default:
+				return containerInspect{}, fmt.Errorf("current container health is %q", inspected.State.Health.Status)
+			}
+		}
+	}
+}
+
+func waitForDockerAPI(ctx context.Context, timeout, interval time.Duration, negotiate func(context.Context) error) error {
+	retryCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	var lastErr error
+	for {
+		attemptCtx, attemptCancel := context.WithTimeout(retryCtx, 3*time.Second)
+		lastErr = negotiate(attemptCtx)
+		attemptCancel()
+		if lastErr == nil {
+			return nil
+		}
+		timer := time.NewTimer(interval)
+		select {
+		case <-retryCtx.Done():
+			if !timer.Stop() {
+				select {
+				case <-timer.C:
+				default:
+				}
+			}
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+			return lastErr
+		case <-timer.C:
+		}
+	}
+}
+
 func (c *dockerClient) request(ctx context.Context, method, path string, body io.Reader, headers http.Header, versioned bool) (*http.Response, error) {
 	if versioned {
 		path = c.apiPrefix + path

internal/updateagent/docker_test.go

@@ -1,13 +1,26 @@
 package updateagent
 
 import (
+	"context"
 	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
 	"strconv"
+	"strings"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/root-kdw12/hetzner-api-dyndns-records-webinterface/internal/buildinfo"
 )
 
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
+	return f(request)
+}
+
 func TestValidateUpdateCompatibilityAllowsDirectMajorJump(t *testing.T) {
 	t.Parallel()
 	current := compatibleImage("v1.4.0", "v1.4.0")
@@ -17,6 +30,40 @@ func TestValidateUpdateCompatibilityAllowsDirectMajorJump(t *testing.T) {
 	}
 }
 
+func TestWaitForDockerAPIRetriesBootTransient(t *testing.T) {
+	t.Parallel()
+	var calls atomic.Int32
+	err := waitForDockerAPI(context.Background(), 100*time.Millisecond, time.Millisecond, func(context.Context) error {
+		if calls.Add(1) < 3 {
+			return errors.New("Docker is still starting")
+		}
+		return nil
+	})
+	if err != nil || calls.Load() != 3 {
+		t.Fatalf("Docker boot transient was not retried: calls=%d err=%v", calls.Load(), err)
+	}
+}
+
+func TestWaitCurrentHealthyAllowsStartingButRejectsUnhealthy(t *testing.T) {
+	t.Parallel()
+	var calls atomic.Int32
+	client := dockerTestClient(func() string {
+		if calls.Add(1) == 1 {
+			return "starting"
+		}
+		return "healthy"
+	})
+	inspected, err := client.waitCurrentHealthy(context.Background(), "container-id", "sha256:current", 100*time.Millisecond, time.Millisecond)
+	if err != nil || inspected.State.Health == nil || inspected.State.Health.Status != "healthy" {
+		t.Fatalf("starting container did not become eligible: inspect=%#v err=%v", inspected, err)
+	}
+
+	unhealthy := dockerTestClient(func() string { return "unhealthy" })
+	if _, err := unhealthy.waitCurrentHealthy(context.Background(), "container-id", "sha256:current", 100*time.Millisecond, time.Millisecond); err == nil || !strings.Contains(err.Error(), "unhealthy") {
+		t.Fatalf("unhealthy container was not rejected immediately: %v", err)
+	}
+}
+
 func TestResolveCurrentVersionRequiresRuntimeAndImageMetadataToAgree(t *testing.T) {
 	t.Parallel()
 	config := json.RawMessage(`{"Image":"registry.example/app:v1.4.0","Labels":{"org.opencontainers.image.version":"v1.4.0"}}`)
@@ -203,3 +250,11 @@ func contains(values []string, expected string) bool {
 	}
 	return false
 }
+
+func dockerTestClient(health func() string) *dockerClient {
+	transport := roundTripFunc(func(*http.Request) (*http.Response, error) {
+		body := `{"Id":"container-id","Image":"sha256:current","Config":{"Labels":{}},"State":{"Running":true,"ExitCode":0,"Health":{"Status":"` + health() + `"}}}`
+		return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}, nil
+	})
+	return &dockerClient{client: &http.Client{Transport: transport}, apiPrefix: "/v1.0"}
+}