330 lines
9.2 KiB
Go
330 lines
9.2 KiB
Go
// Package diagnostics records only explicitly selected technical fields. Raw
|
|
// errors, request headers, URLs, credentials and arbitrary log text are excluded.
|
|
package diagnostics
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"log/slog"
|
|
"net"
|
|
"net/netip"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
const maxFileSize = 4 << 20
|
|
|
|
type Event struct {
|
|
Time time.Time `json:"time"`
|
|
Code string `json:"event"`
|
|
Address string `json:"address,omitempty"`
|
|
DNSResolver string `json:"dns_resolver,omitempty"`
|
|
Status string `json:"status,omitempty"`
|
|
Target string `json:"target,omitempty"`
|
|
Family string `json:"family,omitempty"`
|
|
Failure string `json:"failure,omitempty"`
|
|
RunID int64 `json:"run_id,omitempty"`
|
|
TargetID int64 `json:"target_id,omitempty"`
|
|
ActionID int64 `json:"action_id,omitempty"`
|
|
Attempt int `json:"attempt,omitempty"`
|
|
DurationMS int64 `json:"duration_ms,omitempty"`
|
|
RetrySeconds int64 `json:"retry_seconds,omitempty"`
|
|
HTTPStatus int `json:"http_status,omitempty"`
|
|
Network *Network `json:"network,omitempty"`
|
|
}
|
|
|
|
type Route struct {
|
|
Interface string `json:"interface"`
|
|
Destination string `json:"destination"`
|
|
Gateway string `json:"gateway"`
|
|
Mask string `json:"mask"`
|
|
}
|
|
|
|
type Network struct {
|
|
Nameservers []string `json:"nameservers"`
|
|
DockerUpstreams []string `json:"docker_upstreams,omitempty"`
|
|
Routes []Route `json:"ipv4_routes"`
|
|
}
|
|
|
|
type Recorder struct {
|
|
mu sync.Mutex
|
|
path string
|
|
logger *slog.Logger
|
|
}
|
|
|
|
var active atomic.Pointer[Recorder]
|
|
|
|
func Configure(path string, logger *slog.Logger) error {
|
|
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
|
return err
|
|
}
|
|
file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = file.Chmod(0600)
|
|
closeErr := file.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if closeErr != nil {
|
|
return closeErr
|
|
}
|
|
active.Store(&Recorder{path: path, logger: logger})
|
|
return nil
|
|
}
|
|
|
|
func Record(event Event) {
|
|
if recorder := active.Load(); recorder != nil {
|
|
recorder.Record(event)
|
|
}
|
|
}
|
|
|
|
func Snapshot() []Event {
|
|
if recorder := active.Load(); recorder != nil {
|
|
return recorder.Snapshot()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Recorder) Record(event Event) {
|
|
event.Time = time.Now().UTC()
|
|
event = sanitize(event)
|
|
if event.Code == "" {
|
|
return
|
|
}
|
|
r.logger.Info("diagnostic", "event", event.Code, "target", event.Target, "family", event.Family, "failure", event.Failure, "run_id", event.RunID, "target_id", event.TargetID, "action_id", event.ActionID, "attempt", event.Attempt, "duration_ms", event.DurationMS, "retry_seconds", event.RetrySeconds, "http_status", event.HTTPStatus, "address", event.Address, "dns_resolver", event.DNSResolver, "status", event.Status, "network", event.Network)
|
|
encoded, err := json.Marshal(event)
|
|
if err != nil {
|
|
return
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if info, err := os.Stat(r.path); err == nil && info.Size()+int64(len(encoded)+1) > maxFileSize {
|
|
if err := os.Rename(r.path, r.path+".1"); err != nil {
|
|
r.logger.Warn("diagnostic log rotation failed")
|
|
return
|
|
}
|
|
}
|
|
file, err := os.OpenFile(r.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
|
|
if err != nil {
|
|
r.logger.Warn("diagnostic log unavailable")
|
|
return
|
|
}
|
|
_, err = file.Write(append(encoded, '\n'))
|
|
closeErr := file.Close()
|
|
if err != nil || closeErr != nil {
|
|
r.logger.Warn("diagnostic log write failed")
|
|
}
|
|
}
|
|
|
|
func (r *Recorder) Snapshot() []Event {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
var events []Event
|
|
for _, path := range []string{r.path + ".1", r.path} {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
scanner := bufio.NewScanner(io.LimitReader(file, maxFileSize))
|
|
scanner.Buffer(make([]byte, 4096), 64<<10)
|
|
for scanner.Scan() {
|
|
var event Event
|
|
if json.Unmarshal(scanner.Bytes(), &event) == nil {
|
|
event = sanitize(event)
|
|
if event.Code != "" {
|
|
events = append(events, event)
|
|
}
|
|
}
|
|
}
|
|
file.Close()
|
|
}
|
|
return events
|
|
}
|
|
|
|
func allowed(value string, values ...string) string {
|
|
for _, v := range values {
|
|
if value == v {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func sanitize(e Event) Event {
|
|
if ip, err := netip.ParseAddr(e.Address); err == nil && ip.Zone() == "" {
|
|
e.Address = ip.String()
|
|
} else {
|
|
e.Address = ""
|
|
}
|
|
if ip, err := netip.ParseAddr(e.DNSResolver); err == nil && ip.Zone() == "" {
|
|
e.DNSResolver = ip.String()
|
|
} else {
|
|
e.DNSResolver = ""
|
|
}
|
|
e.Status = allowed(e.Status, "ok", "partial", "error")
|
|
e.Code = allowed(e.Code, "startup", "dns_fallback_query", "update_run_finished", "provider_update_failed", "dns_fallback", "dns_fallback_ok", "dns_failed", "ip_fetch_ok", "ip_fetch_failed", "update_failed", "update_recovered", "pihole_refresh_ok", "pihole_refresh_failed", "release_check_ok", "release_check_failed")
|
|
e.Target = allowed(e.Target, "api.ipify.org", "api6.ipify.org", "ifconfig.co", "api.hetzner.cloud", "custom_endpoint", "pihole", "git", "updater")
|
|
e.Family = allowed(e.Family, "ip", "ip4", "ip6")
|
|
e.Failure = allowed(e.Failure, "dns_timeout", "dns_not_found", "dns_server_failure", "dns_error", "no_route", "connection_refused", "connection_reset", "timeout", "canceled", "tls", "http", "provider_action", "other")
|
|
if e.Network != nil {
|
|
n := *e.Network
|
|
n.Nameservers = validIPs(n.Nameservers)
|
|
n.DockerUpstreams = validIPs(n.DockerUpstreams)
|
|
n.Routes = nil
|
|
for _, route := range e.Network.Routes {
|
|
if validInterface(route.Interface) && len(validIPs([]string{route.Destination, route.Gateway, route.Mask})) == 3 {
|
|
n.Routes = append(n.Routes, route)
|
|
}
|
|
}
|
|
e.Network = &n
|
|
}
|
|
return e
|
|
}
|
|
|
|
func Target(host string) string {
|
|
host = strings.ToLower(strings.TrimSuffix(host, "."))
|
|
if allowed(host, "api.ipify.org", "api6.ipify.org", "ifconfig.co", "api.hetzner.cloud") != "" {
|
|
return host
|
|
}
|
|
return "custom_endpoint"
|
|
}
|
|
|
|
func Failure(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
if ActionID(err) > 0 {
|
|
return "provider_action"
|
|
}
|
|
if HTTPStatus(err) != 0 {
|
|
return "http"
|
|
}
|
|
var unknown x509.UnknownAuthorityError
|
|
var hostname x509.HostnameError
|
|
var invalid x509.CertificateInvalidError
|
|
var record tls.RecordHeaderError
|
|
if errors.As(err, &unknown) || errors.As(err, &hostname) || errors.As(err, &invalid) || errors.As(err, &record) {
|
|
return "tls"
|
|
}
|
|
var dns *net.DNSError
|
|
if errors.As(err, &dns) {
|
|
if dns.IsNotFound {
|
|
return "dns_not_found"
|
|
}
|
|
if dns.IsTimeout {
|
|
return "dns_timeout"
|
|
}
|
|
if dns.IsTemporary {
|
|
return "dns_server_failure"
|
|
}
|
|
return "dns_error"
|
|
}
|
|
switch {
|
|
case errors.Is(err, syscall.EHOSTUNREACH), errors.Is(err, syscall.ENETUNREACH):
|
|
return "no_route"
|
|
case errors.Is(err, syscall.ECONNREFUSED):
|
|
return "connection_refused"
|
|
case errors.Is(err, syscall.ECONNRESET):
|
|
return "connection_reset"
|
|
case errors.Is(err, context.Canceled):
|
|
return "canceled"
|
|
}
|
|
var timeout net.Error
|
|
if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &timeout) && timeout.Timeout()) {
|
|
return "timeout"
|
|
}
|
|
return "other"
|
|
}
|
|
|
|
func ActionID(err error) int64 {
|
|
var action interface{ ProviderActionID() int64 }
|
|
if errors.As(err, &action) {
|
|
return action.ProviderActionID()
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func HTTPStatus(err error) int {
|
|
var status interface{ HTTPStatusCode() int }
|
|
if errors.As(err, &status) {
|
|
return status.HTTPStatusCode()
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func CurrentNetwork() *Network {
|
|
n := &Network{}
|
|
if data, err := os.ReadFile("/etc/resolv.conf"); err == nil {
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) == 2 && fields[0] == "nameserver" {
|
|
n.Nameservers = append(n.Nameservers, validIPs(fields[1:])...)
|
|
}
|
|
if strings.HasPrefix(line, "# ExtServers:") {
|
|
n.DockerUpstreams = append(n.DockerUpstreams, validIPs(strings.Fields(strings.NewReplacer("[", " ", "]", " ", ",", " ").Replace(strings.TrimPrefix(line, "# ExtServers:"))))...)
|
|
}
|
|
}
|
|
}
|
|
if data, err := os.ReadFile("/proc/net/route"); err == nil {
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
f := strings.Fields(line)
|
|
if len(f) < 8 || !validInterface(f[0]) {
|
|
continue
|
|
}
|
|
flags, _ := strconv.ParseUint(f[3], 16, 32)
|
|
if flags&1 == 0 {
|
|
continue
|
|
}
|
|
route := Route{f[0], routeIP(f[1]), routeIP(f[2]), routeIP(f[7])}
|
|
if route.Destination != "" && route.Gateway != "" && route.Mask != "" {
|
|
n.Routes = append(n.Routes, route)
|
|
}
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func routeIP(raw string) string {
|
|
b, err := hex.DecodeString(raw)
|
|
if err != nil || len(b) != 4 {
|
|
return ""
|
|
}
|
|
value := binary.LittleEndian.Uint32(b)
|
|
return netip.AddrFrom4([4]byte{byte(value >> 24), byte(value >> 16), byte(value >> 8), byte(value)}).String()
|
|
}
|
|
func validInterface(s string) bool {
|
|
if s == "" || len(s) > 32 {
|
|
return false
|
|
}
|
|
for _, c := range s {
|
|
if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_' || c == '-' || c == '.') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
func validIPs(values []string) []string {
|
|
var result []string
|
|
for _, v := range values {
|
|
if ip, err := netip.ParseAddr(v); err == nil && ip.Zone() == "" {
|
|
result = append(result, ip.String())
|
|
}
|
|
}
|
|
return result
|
|
}
|