feat(linux): implement systemd watchdog, add status updates to heartbeat, use SdNotify daemon constants

- Log unhandled, unexpected signals
This commit is contained in:
2025-06-23 17:43:21 -05:00
parent 6bb8d4a3cb
commit 34caa5c5a3
2 changed files with 30 additions and 12 deletions

View File

@@ -9,6 +9,7 @@ Type=notify
NotifyAccess=main NotifyAccess=main
ExecStart=$BINARY_PATH ExecStart=$BINARY_PATH
ExecReload=/bin/kill -HUP $MAINPID ExecReload=/bin/kill -HUP $MAINPID
WatchdogSec=10
Restart=on-failure Restart=on-failure
RestartSec=5 RestartSec=5

View File

@@ -3,6 +3,7 @@
package service package service
import ( import (
"fmt"
"log/slog" "log/slog"
"os" "os"
"os/signal" "os/signal"
@@ -34,18 +35,30 @@ func NewService(logger *slog.Logger) Service {
// Run implements the Service interface for Linux // Run implements the Service interface for Linux
func (s *linuxService) Run() error { func (s *linuxService) Run() error {
s.logger.Info("starting service") startTime := time.Now()
s.logger.Info("starting service", "start_time", startTime.Format(time.RFC3339))
// Notify systemd that we are starting // Notify systemd that we are starting
daemon.SdNotify(false, "STATUS=Starting HATray...\n") daemon.SdNotify(false, "STATUS=starting\n")
// Setup signal handling for systemd // Setup signal handling for systemd
sigs := make(chan os.Signal, 1) sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
// Setup watchdog to systemd
var watchdog *time.Ticker
if watchdogUSec, err := daemon.SdWatchdogEnabled(false); err == nil && watchdogUSec > 0 {
watchdog = time.NewTicker(watchdogUSec / 2)
}
defer func() {
if watchdog != nil {
watchdog.Stop()
}
}()
// Setup heartbeat to systemd // Setup heartbeat to systemd
ticker := time.NewTicker(30 * time.Second) heartbeat := time.NewTicker(2 * time.Second)
defer ticker.Stop() defer heartbeat.Stop()
// Start the service (backgrounded so that the service can still respond to systemd signals, the app layer is still designed for concurrency) // Start the service (backgrounded so that the service can still respond to systemd signals, the app layer is still designed for concurrency)
go func() { go func() {
@@ -56,21 +69,23 @@ func (s *linuxService) Run() error {
} }
// Notify systemd that we are ready (and running) // Notify systemd that we are ready (and running)
daemon.SdNotify(false, "READY=1") daemon.SdNotify(false, daemon.SdNotifyReady)
daemon.SdNotify(false, "STATUS=HATray running\n") daemon.SdNotify(false, fmt.Sprintf("STATUS=running for %s\n", time.Since(startTime).String()))
}() }()
for { for {
select { select {
case <-ticker.C: // This is only called if the service is configured with watchdog
daemon.SdNotify(false, "WATCHDOG=1") case <-watchdog.C:
s.logger.Debug("heartbeat") // TODO: add more detailed status information here daemon.SdNotify(false, daemon.SdNotifyWatchdog)
case <-heartbeat.C:
daemon.SdNotify(false, fmt.Sprintf("STATUS=running for %s\n", time.Since(startTime).String()))
case sig := <-sigs: case sig := <-sigs:
s.logger.Info("signal received", "signal", sig) s.logger.Info("signal received", "signal", sig)
switch sig { switch sig {
case syscall.SIGINT, syscall.SIGTERM: case syscall.SIGINT, syscall.SIGTERM:
daemon.SdNotify(false, "STOPPING=1") daemon.SdNotify(false, daemon.SdNotifyStopping)
s.logger.Info("stopping service") s.logger.Info("stopping service")
if err := s.app.Pause(); err != nil { if err := s.app.Pause(); err != nil {
@@ -80,13 +95,15 @@ func (s *linuxService) Run() error {
return nil // exit the service return nil // exit the service
case syscall.SIGHUP: case syscall.SIGHUP:
s.logger.Info("reloading service") s.logger.Info("reloading service")
daemon.SdNotify(false, "RELOADING=1") daemon.SdNotify(false, daemon.SdNotifyReloading)
if err := s.app.Reload(); err != nil { if err := s.app.Reload(); err != nil {
s.logger.Error("failed to reload app layer", "error", err) s.logger.Error("failed to reload app layer", "error", err)
} }
daemon.SdNotify(false, "READY=1") daemon.SdNotify(false, daemon.SdNotifyReady)
default:
s.logger.Warn("unhandled signal", "signal", sig)
} }
} }
} }