feat: add resilient updates and access controls

2026-08-28 14:02:07 +02:00
1 parent a000152423
commit f02ec9977c
@@ -0,0 +1,452 @@
+package updateagent
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/root-kdw12/hetzner-api-dyndns-records-webinterface/internal/updateprotocol"
+)
+
+const defaultImageRepository = "git.j-mahler.de/admin/domain-control-webinterface-prod"
+
+type DockerRunner struct {
+	docker          *dockerClient
+	imageRepository string
+	targetLabel     string
+	logger          *slog.Logger
+}
+
+type updateError struct {
+	message string
+	err     error
+}
+
+func (e *updateError) Error() string         { return e.err.Error() }
+func (e *updateError) Unwrap() error         { return e.err }
+func (e *updateError) PublicMessage() string { return e.message }
+
+func NewDockerRunner(socket, imageRepository, targetLabel string, logger *slog.Logger) *DockerRunner {
+	if imageRepository == "" {
+		imageRepository = defaultImageRepository
+	}
+	return &DockerRunner{docker: newDockerClient(socket), imageRepository: imageRepository, targetLabel: targetLabel, logger: logger}
+}
+
+func (r *DockerRunner) Update(ctx context.Context, request updateprotocol.Request, progress func(phase, message string)) (resultErr error) {
+	if !updateprotocol.ValidVersion(request.Version) {
+		return errors.New("invalid target version")
+	}
+	if err := r.docker.negotiate(ctx); err != nil {
+		return &updateError{message: "Containerverwaltung ist nicht erreichbar; der Dienst läuft unverändert weiter", err: err}
+	}
+	progress(updateprotocol.PhasePulling, "Container-Image wird geladen")
+	if err := r.docker.pullImage(ctx, r.imageRepository, request.Version, request.RegistryUsername, request.RegistryToken); err != nil {
+		return &updateError{message: "Image konnte nicht geladen werden; der Dienst läuft unverändert weiter", err: err}
+	}
+	target, err := r.docker.findTarget(ctx, r.targetLabel)
+	if err != nil {
+		return &updateError{message: "Laufender Anwendungscontainer wurde nicht eindeutig gefunden", err: err}
+	}
+	inspected, err := r.docker.inspectContainer(ctx, target.ID)
+	if err != nil {
+		return err
+	}
+	originalName := strings.TrimPrefix(inspected.Name, "/")
+	if originalName == "" {
+		return errors.New("target container has no name")
+	}
+	backupName := originalName + "-rollback-" + fmt.Sprint(time.Now().UTC().Unix())
+	var oldStopped, oldRenamed bool
+	var newID string
+	defer func() {
+		if resultErr == nil || (!oldStopped && !oldRenamed && newID == "") {
+			return
+		}
+		rollbackCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+		defer cancel()
+		if rollbackErr := r.rollback(rollbackCtx, target.ID, originalName, oldStopped, oldRenamed, newID); rollbackErr != nil {
+			r.logger.Error("software update rollback failed", "error", rollbackErr)
+			resultErr = &updateError{message: "Update und automatische Wiederherstellung sind fehlgeschlagen; Containerstatus am Host prüfen", err: fmt.Errorf("%w; rollback: %v", resultErr, rollbackErr)}
+		}
+	}()
+
+	progress(updateprotocol.PhaseRestarting, "Anwendung wird kontrolliert neu gestartet")
+	if err := r.docker.stopContainer(ctx, target.ID, 20); err != nil {
+		return err
+	}
+	oldStopped = true
+	if err := r.docker.renameContainer(ctx, target.ID, backupName); err != nil {
+		return err
+	}
+	oldRenamed = true
+	newID, err = r.docker.createReplacement(ctx, inspected, originalName, r.imageRepository+":"+request.Version)
+	if err != nil {
+		return err
+	}
+	if err := r.docker.startContainer(ctx, newID); err != nil {
+		return err
+	}
+	progress(updateprotocol.PhaseVerifying, "Neuer Container wird geprüft")
+	if err := r.docker.waitHealthy(ctx, newID, 100*time.Second); err != nil {
+		return err
+	}
+	if err := r.docker.removeContainer(ctx, target.ID, false); err != nil {
+		r.logger.Warn("old update rollback container could not be removed", "container", target.ID, "error", err)
+	}
+	oldStopped, oldRenamed, newID = false, false, ""
+	return nil
+}
+
+func (r *DockerRunner) rollback(ctx context.Context, oldID, originalName string, oldStopped, oldRenamed bool, newID string) error {
+	var failures []string
+	if newID != "" {
+		if err := r.docker.stopContainer(ctx, newID, 5); err != nil && !isDockerNotModified(err) {
+			failures = append(failures, "stop replacement: "+err.Error())
+		}
+		if err := r.docker.removeContainer(ctx, newID, true); err != nil {
+			failures = append(failures, "remove replacement: "+err.Error())
+		}
+	}
+	if oldRenamed {
+		if err := r.docker.renameContainer(ctx, oldID, originalName); err != nil {
+			failures = append(failures, "restore name: "+err.Error())
+		}
+	}
+	if oldStopped {
+		if err := r.docker.startContainer(ctx, oldID); err != nil && !isDockerNotModified(err) {
+			failures = append(failures, "restart previous container: "+err.Error())
+		}
+	}
+	if len(failures) > 0 {
+		return errors.New(strings.Join(failures, "; "))
+	}
+	return nil
+}
+
+type dockerClient struct {
+	socket    string
+	client    *http.Client
+	apiPrefix string
+}
+
+type containerSummary struct {
+	ID    string   `json:"Id"`
+	Names []string `json:"Names"`
+	State string   `json:"State"`
+}
+
+type containerInspect struct {
+	ID              string          `json:"Id"`
+	Name            string          `json:"Name"`
+	Config          json.RawMessage `json:"Config"`
+	HostConfig      json.RawMessage `json:"HostConfig"`
+	NetworkSettings struct {
+		Networks map[string]endpointSettings `json:"Networks"`
+	} `json:"NetworkSettings"`
+	State containerState `json:"State"`
+}
+
+type endpointSettings struct {
+	IPAMConfig json.RawMessage   `json:"IPAMConfig,omitempty"`
+	Links      []string          `json:"Links,omitempty"`
+	Aliases    []string          `json:"Aliases,omitempty"`
+	MacAddress string            `json:"MacAddress,omitempty"`
+	DriverOpts map[string]string `json:"DriverOpts,omitempty"`
+	GwPriority int               `json:"GwPriority,omitempty"`
+}
+
+type containerState struct {
+	Running  bool `json:"Running"`
+	ExitCode int  `json:"ExitCode"`
+	Health   *struct {
+		Status string `json:"Status"`
+	} `json:"Health"`
+}
+
+func newDockerClient(socket string) *dockerClient {
+	transport := &http.Transport{
+		DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+			return (&net.Dialer{Timeout: 2 * time.Second}).DialContext(ctx, "unix", socket)
+		},
+		MaxIdleConns:        2,
+		MaxIdleConnsPerHost: 2,
+		IdleConnTimeout:     30 * time.Second,
+	}
+	return &dockerClient{socket: socket, client: &http.Client{Transport: transport}}
+}
+
+func (c *dockerClient) negotiate(ctx context.Context) error {
+	response, err := c.request(ctx, http.MethodGet, "/version", nil, nil, false)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return dockerStatusError(response)
+	}
+	var version struct {
+		APIVersion string `json:"ApiVersion"`
+	}
+	if err := json.NewDecoder(io.LimitReader(response.Body, 64<<10)).Decode(&version); err != nil {
+		return fmt.Errorf("decode Docker API version: %w", err)
+	}
+	for _, character := range version.APIVersion {
+		if (character < '0' || character > '9') && character != '.' {
+			return errors.New("Docker returned an invalid API version")
+		}
+	}
+	if version.APIVersion == "" {
+		return errors.New("Docker returned an empty API version")
+	}
+	c.apiPrefix = "/v" + version.APIVersion
+	return nil
+}
+
+func (c *dockerClient) findTarget(ctx context.Context, label string) (containerSummary, error) {
+	filters, _ := json.Marshal(map[string][]string{"label": {label}})
+	query := url.Values{"all": {"1"}, "filters": {string(filters)}}
+	response, err := c.request(ctx, http.MethodGet, "/containers/json?"+query.Encode(), nil, nil, true)
+	if err != nil {
+		return containerSummary{}, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return containerSummary{}, dockerStatusError(response)
+	}
+	var containers []containerSummary
+	if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&containers); err != nil {
+		return containerSummary{}, err
+	}
+	var running []containerSummary
+	for _, container := range containers {
+		if container.State == "running" {
+			running = append(running, container)
+		}
+	}
+	if len(running) != 1 {
+		return containerSummary{}, fmt.Errorf("expected exactly one running target container, found %d", len(running))
+	}
+	return running[0], nil
+}
+
+func (c *dockerClient) pullImage(ctx context.Context, repository, tag, username, token string) error {
+	authJSON, _ := json.Marshal(map[string]string{"username": username, "password": token, "serveraddress": strings.Split(repository, "/")[0]})
+	headers := http.Header{"X-Registry-Auth": {base64.URLEncoding.EncodeToString(authJSON)}}
+	query := url.Values{"fromImage": {repository}, "tag": {tag}}
+	response, err := c.request(ctx, http.MethodPost, "/images/create?"+query.Encode(), nil, headers, true)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return dockerStatusError(response)
+	}
+	scanner := bufio.NewScanner(response.Body)
+	scanner.Buffer(make([]byte, 64<<10), 1<<20)
+	for scanner.Scan() {
+		var event struct {
+			Error string `json:"error"`
+		}
+		if err := json.Unmarshal(scanner.Bytes(), &event); err == nil && event.Error != "" {
+			return errors.New(event.Error)
+		}
+	}
+	return scanner.Err()
+}
+
+func (c *dockerClient) inspectContainer(ctx context.Context, id string) (containerInspect, error) {
+	response, err := c.request(ctx, http.MethodGet, "/containers/"+url.PathEscape(id)+"/json", nil, nil, true)
+	if err != nil {
+		return containerInspect{}, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return containerInspect{}, dockerStatusError(response)
+	}
+	var inspected containerInspect
+	if err := json.NewDecoder(io.LimitReader(response.Body, 4<<20)).Decode(&inspected); err != nil {
+		return containerInspect{}, err
+	}
+	return inspected, nil
+}
+
+func (c *dockerClient) stopContainer(ctx context.Context, id string, seconds int) error {
+	query := url.Values{"t": {fmt.Sprint(seconds)}}
+	response, err := c.request(ctx, http.MethodPost, "/containers/"+url.PathEscape(id)+"/stop?"+query.Encode(), nil, nil, true)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode == http.StatusNoContent || response.StatusCode == http.StatusNotModified {
+		return nil
+	}
+	return dockerStatusError(response)
+}
+
+func (c *dockerClient) renameContainer(ctx context.Context, id, name string) error {
+	query := url.Values{"name": {name}}
+	response, err := c.request(ctx, http.MethodPost, "/containers/"+url.PathEscape(id)+"/rename?"+query.Encode(), nil, nil, true)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode == http.StatusNoContent {
+		return nil
+	}
+	return dockerStatusError(response)
+}
+
+func (c *dockerClient) createReplacement(ctx context.Context, inspected containerInspect, name, image string) (string, error) {
+	var body map[string]json.RawMessage
+	if err := json.Unmarshal(inspected.Config, &body); err != nil {
+		return "", fmt.Errorf("decode container config: %w", err)
+	}
+	imageJSON, _ := json.Marshal(image)
+	body["Image"] = imageJSON
+	body["HostConfig"] = inspected.HostConfig
+	endpoints := make(map[string]endpointSettings, len(inspected.NetworkSettings.Networks))
+	oldIDPrefix := inspected.ID
+	if len(oldIDPrefix) > 12 {
+		oldIDPrefix = oldIDPrefix[:12]
+	}
+	for network, endpoint := range inspected.NetworkSettings.Networks {
+		aliases := endpoint.Aliases[:0]
+		for _, alias := range endpoint.Aliases {
+			if alias != inspected.ID && alias != oldIDPrefix {
+				aliases = append(aliases, alias)
+			}
+		}
+		endpoint.Aliases = aliases
+		endpoints[network] = endpoint
+	}
+	networkingJSON, _ := json.Marshal(map[string]any{"EndpointsConfig": endpoints})
+	body["NetworkingConfig"] = networkingJSON
+	encoded, err := json.Marshal(body)
+	if err != nil {
+		return "", err
+	}
+	query := url.Values{"name": {name}}
+	response, err := c.request(ctx, http.MethodPost, "/containers/create?"+query.Encode(), bytes.NewReader(encoded), http.Header{"Content-Type": {"application/json"}}, true)
+	if err != nil {
+		return "", err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusCreated {
+		return "", dockerStatusError(response)
+	}
+	var created struct {
+		ID string `json:"Id"`
+	}
+	if err := json.NewDecoder(io.LimitReader(response.Body, 64<<10)).Decode(&created); err != nil {
+		return "", err
+	}
+	if created.ID == "" {
+		return "", errors.New("Docker created a replacement without an ID")
+	}
+	return created.ID, nil
+}
+
+func (c *dockerClient) startContainer(ctx context.Context, id string) error {
+	response, err := c.request(ctx, http.MethodPost, "/containers/"+url.PathEscape(id)+"/start", nil, nil, true)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode == http.StatusNoContent || response.StatusCode == http.StatusNotModified {
+		return nil
+	}
+	return dockerStatusError(response)
+}
+
+func (c *dockerClient) removeContainer(ctx context.Context, id string, force bool) error {
+	query := url.Values{"force": {fmt.Sprint(force)}, "v": {"0"}}
+	response, err := c.request(ctx, http.MethodDelete, "/containers/"+url.PathEscape(id)+"?"+query.Encode(), nil, nil, true)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode == http.StatusNoContent || response.StatusCode == http.StatusNotFound {
+		return nil
+	}
+	return dockerStatusError(response)
+}
+
+func (c *dockerClient) waitHealthy(ctx context.Context, id string, timeout time.Duration) error {
+	deadline := time.NewTimer(timeout)
+	ticker := time.NewTicker(time.Second)
+	defer deadline.Stop()
+	defer ticker.Stop()
+	consecutiveHealthy := 0
+	startedAt := time.Now()
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-deadline.C:
+			return errors.New("replacement container did not become healthy in time")
+		case <-ticker.C:
+			inspected, err := c.inspectContainer(ctx, id)
+			if err != nil {
+				return err
+			}
+			if !inspected.State.Running {
+				return fmt.Errorf("replacement container stopped with exit code %d", inspected.State.ExitCode)
+			}
+			if inspected.State.Health == nil {
+				if time.Since(startedAt) >= 8*time.Second {
+					return nil
+				}
+				continue
+			}
+			if inspected.State.Health.Status == "healthy" {
+				consecutiveHealthy++
+				if consecutiveHealthy >= 3 {
+					return nil
+				}
+			} else {
+				consecutiveHealthy = 0
+			}
+		}
+	}
+}
+
+func (c *dockerClient) request(ctx context.Context, method, path string, body io.Reader, headers http.Header, versioned bool) (*http.Response, error) {
+	if versioned {
+		path = c.apiPrefix + path
+	}
+	request, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, body)
+	if err != nil {
+		return nil, err
+	}
+	for name, values := range headers {
+		for _, value := range values {
+			request.Header.Add(name, value)
+		}
+	}
+	return c.client.Do(request)
+}
+
+func dockerStatusError(response *http.Response) error {
+	message, _ := io.ReadAll(io.LimitReader(response.Body, 8<<10))
+	trimmed := strings.TrimSpace(string(message))
+	if trimmed == "" {
+		trimmed = http.StatusText(response.StatusCode)
+	}
+	return fmt.Errorf("Docker API HTTP %d: %s", response.StatusCode, trimmed)
+}
+
+func isDockerNotModified(err error) bool {
+	return err != nil && strings.Contains(err.Error(), "HTTP 304")
+}