feat: add resilient container restart controls
This commit is contained in:
admin committed
1 parent
1bbb14470c
commit
7bb33e528c
13 files changed
+747
-28
No files matched your search
@@ -3,12 +3,14 @@ package appupdate
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -19,6 +21,48 @@ import (
|
||||
"github.com/root-kdw12/hetzner-api-dyndns-records-webinterface/internal/updateprotocol"
|
||||
)
|
||||
|
||||
type fakeUpdateAgent struct {
|
||||
mu sync.Mutex
|
||||
status updateprotocol.Status
|
||||
statusErr error
|
||||
startErr error
|
||||
statusCalls int
|
||||
startCalls int
|
||||
requests []updateprotocol.Request
|
||||
}
|
||||
|
||||
func (a *fakeUpdateAgent) Status(context.Context) (updateprotocol.Status, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.statusCalls++
|
||||
return a.status, a.statusErr
|
||||
}
|
||||
|
||||
func (a *fakeUpdateAgent) Start(_ context.Context, request updateprotocol.Request) error {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.startCalls++
|
||||
a.requests = append(a.requests, request)
|
||||
if a.startErr == nil {
|
||||
a.status.Phase = updateprotocol.PhaseQueued
|
||||
a.status.TargetVersion = request.Version
|
||||
}
|
||||
return a.startErr
|
||||
}
|
||||
|
||||
func (a *fakeUpdateAgent) setStatus(status updateprotocol.Status, err error) {
|
||||
a.mu.Lock()
|
||||
a.status = status
|
||||
a.statusErr = err
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
func (a *fakeUpdateAgent) counts() (status, start int) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.statusCalls, a.startCalls
|
||||
}
|
||||
|
||||
func TestManagerKeepsUpdateReminderWhenGiteaBecomesUnavailable(t *testing.T) {
|
||||
t.Parallel()
|
||||
const credential = " test-password-or-token-that-must-stay-secret "
|
||||
@@ -138,6 +182,116 @@ func TestAgentCompatibilityStartsAtStableV14Protocol(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestUpdateRefreshesStaleAgentStateBeforeStarting(t *testing.T) {
|
||||
t.Parallel()
|
||||
agent := &fakeUpdateAgent{status: compatibleAgentStatus(updateprotocol.PhaseIdle)}
|
||||
manager, database := updateReadyManager(t, agent)
|
||||
if manager.State().AgentAvailable {
|
||||
t.Fatal("test precondition failed: cached agent state is already available")
|
||||
}
|
||||
if err := manager.RequestUpdate(context.Background(), "v1.4.4"); err != nil {
|
||||
t.Fatalf("live agent was rejected because of stale cached state: %v", err)
|
||||
}
|
||||
statusCalls, startCalls := agent.counts()
|
||||
if statusCalls != 1 || startCalls != 1 {
|
||||
t.Fatalf("agent was not checked immediately before start: status=%d start=%d", statusCalls, startCalls)
|
||||
}
|
||||
settings, err := database.GetSoftwareUpdateSettings(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if settings.DesiredVersion != "v1.4.4" {
|
||||
t.Fatalf("target version was not persisted: %#v", settings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestUpdateRejectsUnavailableIncompatibleAndBusyAgents(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
name string
|
||||
status updateprotocol.Status
|
||||
statusErr error
|
||||
}{
|
||||
{name: "unavailable", statusErr: errors.New("offline")},
|
||||
{name: "incompatible", status: updateprotocol.Status{Phase: updateprotocol.PhaseIdle, AgentVersion: "v1.3.0", Protocol: 1}},
|
||||
{name: "busy", status: compatibleAgentStatus(updateprotocol.PhasePulling)},
|
||||
}
|
||||
for _, test := range tests {
|
||||
test := test
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
agent := &fakeUpdateAgent{status: test.status, statusErr: test.statusErr}
|
||||
manager, database := updateReadyManager(t, agent)
|
||||
if err := manager.RequestUpdate(context.Background(), "v1.4.4"); err == nil {
|
||||
t.Fatal("unsafe update request was accepted")
|
||||
}
|
||||
_, startCalls := agent.counts()
|
||||
if startCalls != 0 {
|
||||
t.Fatalf("agent start was called %d times", startCalls)
|
||||
}
|
||||
settings, err := database.GetSoftwareUpdateSettings(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if settings.DesiredVersion != "" {
|
||||
t.Fatalf("rejected update changed desired version: %#v", settings)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestUpdateSerializesParallelClicks(t *testing.T) {
|
||||
t.Parallel()
|
||||
agent := &fakeUpdateAgent{status: compatibleAgentStatus(updateprotocol.PhaseIdle)}
|
||||
manager, _ := updateReadyManager(t, agent)
|
||||
results := make(chan error, 2)
|
||||
start := make(chan struct{})
|
||||
for index := 0; index < 2; index++ {
|
||||
go func() {
|
||||
<-start
|
||||
results <- manager.RequestUpdate(context.Background(), "v1.4.4")
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
var successes int
|
||||
for index := 0; index < 2; index++ {
|
||||
if err := <-results; err == nil {
|
||||
successes++
|
||||
}
|
||||
}
|
||||
_, startCalls := agent.counts()
|
||||
if successes != 1 || startCalls != 1 {
|
||||
t.Fatalf("parallel clicks started an unexpected number of updates: successes=%d starts=%d", successes, startCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentMonitorRecoversAfterDelayedAgentStartup(t *testing.T) {
|
||||
t.Parallel()
|
||||
agent := &fakeUpdateAgent{statusErr: errors.New("not listening yet")}
|
||||
manager := &Manager{agent: agent}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
manager.monitorAgent(ctx, 2*time.Millisecond, 20*time.Millisecond)
|
||||
close(done)
|
||||
}()
|
||||
waitForCondition(t, time.Second, func() bool {
|
||||
statusCalls, _ := agent.counts()
|
||||
return statusCalls > 0
|
||||
})
|
||||
agent.setStatus(compatibleAgentStatus(updateprotocol.PhaseIdle), nil)
|
||||
waitForCondition(t, time.Second, func() bool {
|
||||
state := manager.State()
|
||||
return state.AgentAvailable && state.AgentCompatible
|
||||
})
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("agent monitor did not stop after context cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagerClearsCompletedDesiredVersionOnStartup(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
@@ -176,3 +330,58 @@ func TestManagerClearsCompletedDesiredVersionOnStartup(t *testing.T) {
|
||||
t.Fatalf("completed desired version remained persisted: %#v", settings)
|
||||
}
|
||||
}
|
||||
|
||||
func compatibleAgentStatus(phase string) updateprotocol.Status {
|
||||
return updateprotocol.Status{Phase: phase, AgentVersion: buildinfo.MinimumAgentVersion, Protocol: buildinfo.MinimumAgentProtocol}
|
||||
}
|
||||
|
||||
func updateReadyManager(t *testing.T, agent updateAgent) (*Manager, *store.Store) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
database, err := store.Open(ctx, filepath.Join(t.TempDir(), "app.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = database.Close() })
|
||||
keyFile := filepath.Join(t.TempDir(), "master-key")
|
||||
key := make([]byte, 32)
|
||||
for index := range key {
|
||||
key[index] = byte(index + 1)
|
||||
}
|
||||
if err := os.WriteFile(keyFile, []byte(base64.RawStdEncoding.EncodeToString(key)), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cipher, err := secure.LoadCipher(keyFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manager, err := New(database, cipher, "https://git.example/api/releases/latest", filepath.Join(t.TempDir(), "agent.sock"), time.Hour, slog.New(slog.NewTextHandler(os.Stderr, nil)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manager.agent = agent
|
||||
if err := manager.SaveCredential(ctx, "registry-user", "registry-password"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.RecordSoftwareUpdateSuccess(ctx, "v1.4.4", "https://git.example/releases/v1.4.4", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings, err := database.GetSoftwareUpdateSettings(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manager.setStoredState(settings)
|
||||
return manager, database
|
||||
}
|
||||
|
||||
func waitForCondition(t *testing.T, timeout time.Duration, condition func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if condition() {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
t.Fatal("condition did not become true in time")
|
||||
}
|
||||
Reference in new issue
Block a user