notifications: better msg content, pruning for past events

cmd/sylve/main.go

@@ -188,6 +188,7 @@ func main() {
 	}
 
 	go db.StartQueue(qCtx)
+	db.StartPruneWorker(qCtx, d)
 
 	if startAdvancedStartupWorkers {
 		logger.L.Info().Msg("Starting background watchers and queues")

internal/db/notification_retention.go

@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Copyright (c) 2025 The FreeBSD Foundation.
+//
+// This software was developed by Hayzam Sherif <hayzam@alchemilla.io>
+// of Alchemilla Ventures Pvt. Ltd. <hello@alchemilla.io>,
+// under sponsorship from the FreeBSD Foundation.
+
+package db
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/alchemillahq/sylve/internal/db/models"
+	notifier "github.com/alchemillahq/sylve/internal/notifications"
+	"gorm.io/gorm"
+)
+
+const (
+	NotificationDismissedRetentionDays   = 30
+	NotificationSuppressionRetentionDays = 90
+)
+
+func EnforceNotificationRetention(db *gorm.DB, now time.Time) error {
+	if db == nil {
+		return fmt.Errorf("db_not_initialized")
+	}
+
+	if db.Migrator().HasTable(&models.Notification{}) {
+		dismissedCutoff := now.Add(-NotificationDismissedRetentionDays * 24 * time.Hour)
+		if err := db.
+			Where("dismissed_at IS NOT NULL").
+			Where("dismissed_at < ?", dismissedCutoff).
+			Delete(&models.Notification{}).
+			Error; err != nil {
+			return fmt.Errorf("failed_to_prune_expired_notifications: %w", err)
+		}
+	}
+
+	if db.Migrator().HasTable(&models.NotificationSuppression{}) {
+		suppressionCutoff := now.Add(-NotificationSuppressionRetentionDays * 24 * time.Hour)
+		if err := db.
+			Where("created_at < ?", suppressionCutoff).
+			Delete(&models.NotificationSuppression{}).
+			Error; err != nil {
+			return fmt.Errorf("failed_to_prune_expired_notification_suppressions: %w", err)
+		}
+
+		if err := db.
+			Where("kind LIKE ?", notifier.ZFSPoolStateKindPrefix+"%").
+			Delete(&models.NotificationSuppression{}).
+			Error; err != nil {
+			return fmt.Errorf("failed_to_prune_zfs_notification_suppressions: %w", err)
+		}
+	}
+
+	return nil
+}

internal/db/notification_retention_test.go

@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Copyright (c) 2025 The FreeBSD Foundation.
+//
+// This software was developed by Hayzam Sherif <hayzam@alchemilla.io>
+// of Alchemilla Ventures Pvt. Ltd. <hello@alchemilla.io>,
+// under sponsorship from the FreeBSD Foundation.
+
+package db
+
+import (
+	"testing"
+	"time"
+
+	"github.com/alchemillahq/sylve/internal/db/models"
+	notifier "github.com/alchemillahq/sylve/internal/notifications"
+	"github.com/alchemillahq/sylve/internal/testutil"
+)
+
+func TestEnforceNotificationRetentionPrunesOldDismissedNotifications(t *testing.T) {
+	db := testutil.NewSQLiteTestDB(t, &models.Notification{})
+
+	now := time.Date(2026, time.April, 24, 12, 0, 0, 0, time.UTC)
+	oldDismissedAt := now.Add(-(NotificationDismissedRetentionDays*24*time.Hour + time.Hour))
+	freshDismissedAt := now.Add(-24 * time.Hour)
+	activeOccurredAt := now.Add(-48 * time.Hour)
+
+	oldDismissed := models.Notification{
+		Kind:            "test.kind",
+		Title:           "old dismissed",
+		Body:            "old",
+		Severity:        models.NotificationSeverityWarning,
+		Source:          "test",
+		Fingerprint:     "old-dismissed",
+		Metadata:        map[string]string{},
+		OccurrenceCount: 1,
+		FirstOccurredAt: oldDismissedAt,
+		LastOccurredAt:  oldDismissedAt,
+		DismissedAt:     &oldDismissedAt,
+		CreatedAt:       oldDismissedAt,
+		UpdatedAt:       oldDismissedAt,
+	}
+	freshDismissed := models.Notification{
+		Kind:            "test.kind",
+		Title:           "fresh dismissed",
+		Body:            "fresh",
+		Severity:        models.NotificationSeverityWarning,
+		Source:          "test",
+		Fingerprint:     "fresh-dismissed",
+		Metadata:        map[string]string{},
+		OccurrenceCount: 1,
+		FirstOccurredAt: freshDismissedAt,
+		LastOccurredAt:  freshDismissedAt,
+		DismissedAt:     &freshDismissedAt,
+		CreatedAt:       freshDismissedAt,
+		UpdatedAt:       freshDismissedAt,
+	}
+	active := models.Notification{
+		Kind:            "test.kind",
+		Title:           "active",
+		Body:            "active",
+		Severity:        models.NotificationSeverityInfo,
+		Source:          "test",
+		Fingerprint:     "active",
+		Metadata:        map[string]string{},
+		OccurrenceCount: 1,
+		FirstOccurredAt: activeOccurredAt,
+		LastOccurredAt:  activeOccurredAt,
+		DismissedAt:     nil,
+		CreatedAt:       activeOccurredAt,
+		UpdatedAt:       activeOccurredAt,
+	}
+
+	if err := db.Create(&[]models.Notification{oldDismissed, freshDismissed, active}).Error; err != nil {
+		t.Fatalf("failed_to_seed_notifications: %v", err)
+	}
+
+	if err := EnforceNotificationRetention(db, now); err != nil {
+		t.Fatalf("enforce_notification_retention_failed: %v", err)
+	}
+
+	var kept []models.Notification
+	if err := db.Order("fingerprint ASC").Find(&kept).Error; err != nil {
+		t.Fatalf("failed_to_fetch_notifications: %v", err)
+	}
+
+	if len(kept) != 2 {
+		t.Fatalf("expected_2_notifications_remaining_got_%d", len(kept))
+	}
+	if kept[0].Fingerprint != "active" || kept[1].Fingerprint != "fresh-dismissed" {
+		t.Fatalf("unexpected_remaining_notifications: %s, %s", kept[0].Fingerprint, kept[1].Fingerprint)
+	}
+}
+
+func TestEnforceNotificationRetentionPrunesSuppressionsByAgeAndZFSPrefix(t *testing.T) {
+	db := testutil.NewSQLiteTestDB(t, &models.NotificationSuppression{})
+
+	now := time.Date(2026, time.April, 24, 12, 0, 0, 0, time.UTC)
+	oldCreatedAt := now.Add(-(NotificationSuppressionRetentionDays*24*time.Hour + time.Hour))
+	freshCreatedAt := now.Add(-2 * time.Hour)
+
+	rows := []models.NotificationSuppression{
+		{
+			Fingerprint: "storage.disks|disk-ada0-failure",
+			Kind:        "storage.disks",
+			CreatedAt:   oldCreatedAt,
+		},
+		{
+			Fingerprint: "storage.disks|disk-ada1-failure",
+			Kind:        "storage.disks",
+			CreatedAt:   freshCreatedAt,
+		},
+		{
+			Fingerprint: notifier.KindForZFSPoolState("test") + "|test|vdev0|degraded",
+			Kind:        notifier.KindForZFSPoolState("test"),
+			CreatedAt:   freshCreatedAt,
+		},
+	}
+
+	if err := db.Create(&rows).Error; err != nil {
+		t.Fatalf("failed_to_seed_suppressions: %v", err)
+	}
+
+	if err := EnforceNotificationRetention(db, now); err != nil {
+		t.Fatalf("enforce_notification_retention_failed: %v", err)
+	}
+
+	var kept []models.NotificationSuppression
+	if err := db.Order("fingerprint ASC").Find(&kept).Error; err != nil {
+		t.Fatalf("failed_to_fetch_suppressions: %v", err)
+	}
+
+	if len(kept) != 1 {
+		t.Fatalf("expected_1_suppression_remaining_got_%d", len(kept))
+	}
+	if kept[0].Kind != "storage.disks" || kept[0].Fingerprint != "storage.disks|disk-ada1-failure" {
+		t.Fatalf("unexpected_remaining_suppression: kind=%s fingerprint=%s", kept[0].Kind, kept[0].Fingerprint)
+	}
+}

internal/db/prune.go

@@ -9,6 +9,7 @@
 package db
 
 import (
+	"context"
 	"time"
 
 	clusterModels "github.com/alchemillahq/sylve/internal/db/models/cluster"
@@ -41,5 +42,34 @@ func PruneJobs(db *gorm.DB) error {
 		return err
 	}
 
+	if err := EnforceNotificationRetention(db, time.Now()); err != nil {
+		return err
+	}
+
 	return nil
 }
+
+func StartPruneWorker(ctx context.Context, db *gorm.DB) {
+	cleanup := func() {
+		if err := PruneJobs(db); err != nil {
+			logger.L.Error().Err(err).Msg("periodic_prune_jobs_failed")
+		}
+	}
+
+	go func() {
+		cleanup()
+
+		ticker := time.NewTicker(6 * time.Hour)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ctx.Done():
+				logger.L.Debug().Msg("Stopped prune worker")
+				return
+			case <-ticker.C:
+				cleanup()
+			}
+		}
+	}()
+}

internal/services/notifications/email_html.go

@@ -0,0 +1,143 @@
+// SPDX-License-Identifier: BSD-2-Clause
+//
+// Copyright (c) 2025 The FreeBSD Foundation.
+//
+// This software was developed by Hayzam Sherif <hayzam@alchemilla.io>
+// of Alchemilla Ventures Pvt. Ltd. <hello@alchemilla.io>,
+// under sponsorship from the FreeBSD Foundation.
+
+package notifications
+
+import (
+	"bytes"
+	"html/template"
+	"strings"
+	"time"
+
+	notifier "github.com/alchemillahq/sylve/internal/notifications"
+)
+
+const sylveLogoURL = "https://public-bucket.sylve.io/mail-logo.png"
+
+type emailTemplateData struct {
+	LogoURL       string
+	Title         string
+	Body          string
+	SeverityLabel string
+	SeverityColor string
+	SeverityIcon  template.HTML
+	ShowMetadata  bool
+	Pool          string
+	VdevPath      string
+	State         string
+	Source        string
+	Kind          string
+	Year          int
+}
+
+var svgCheckCircle = template.HTML(`<svg xmlns="http://www.w3.org/2000/svg" width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="#22c55e" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><polyline points="9 12 11 14 15 10"/></svg>`)
+var svgWarningTriangle = template.HTML(`<svg xmlns="http://www.w3.org/2000/svg" width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="#f59e0b" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"><path d="M10.29 3.86L1.82 18a2 2 0 0 0 1.71 3h16.94a2 2 0 0 0 1.71-3L13.71 3.86a2 2 0 0 0-3.42 0z"/><line x1="12" y1="9" x2="12" y2="13"/><line x1="12" y1="17" x2="12.01" y2="17"/></svg>`)
+var svgXCircle = template.HTML(`<svg xmlns="http://www.w3.org/2000/svg" width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="#ef4444" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><line x1="15" y1="9" x2="9" y2="15"/><line x1="9" y1="9" x2="15" y2="15"/></svg>`)
+var svgAlertCircle = template.HTML(`<svg xmlns="http://www.w3.org/2000/svg" width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="#dc2626" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><line x1="12" y1="8" x2="12" y2="12"/><line x1="12" y1="16" x2="12.01" y2="16"/></svg>`)
+
+func severityEmailStyle(severity string) (label, color string, icon template.HTML) {
+	switch strings.TrimSpace(strings.ToLower(severity)) {
+	case "warning":
+		return "Warning", "#f59e0b", svgWarningTriangle
+	case "error":
+		return "Error", "#ef4444", svgXCircle
+	case "critical":
+		return "Critical", "#dc2626", svgAlertCircle
+	default:
+		return "Info", "#22c55e", svgCheckCircle
+	}
+}
+
+func buildEmailHTML(input notifier.EventInput, now time.Time) string {
+	severityLabel, severityColor, severityIcon := severityEmailStyle(input.Severity)
+
+	pool := strings.TrimSpace(input.Metadata["pool"])
+	vdevPath := strings.TrimSpace(input.Metadata["vdev_path"])
+	state := strings.TrimSpace(input.Metadata["state"])
+	showMetadata := pool != "" || vdevPath != "" || state != ""
+
+	data := emailTemplateData{
+		LogoURL:       sylveLogoURL,
+		Title:         input.Title,
+		Body:          strings.TrimSpace(input.Body),
+		SeverityLabel: severityLabel,
+		SeverityColor: severityColor,
+		SeverityIcon:  severityIcon,
+		ShowMetadata:  showMetadata,
+		Pool:          pool,
+		VdevPath:      vdevPath,
+		State:         state,
+		Source:        strings.TrimSpace(input.Source),
+		Kind:          strings.TrimSpace(input.Kind),
+		Year:          now.Year(),
+	}
+
+	var buf bytes.Buffer
+	if err := emailTemplate.Execute(&buf, data); err != nil {
+		return "<html><body>" + template.HTMLEscapeString(input.Title) + "</body></html>"
+	}
+
+	return buf.String()
+}
+
+var emailTemplate = template.Must(template.New("email").Parse(`<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>{{.Title}}</title>
+<style>
+  body { margin:0; padding:0; font-family: Arial, sans-serif; background-color:#f4f4f4; color:#333; line-height:1.6; }
+  .wrapper { max-width:680px; margin:40px auto; background-color:#fff; }
+  .header { background-color:#161616; padding:24px 32px; text-align:center; }
+  .header img { height:72px; display:inline-block; }
+  .content { padding:32px 32px 24px; }
+  .severity-badge { display:inline-flex; align-items:center; gap:8px; padding:6px 14px; border-radius:999px; background-color:{{.SeverityColor}}1a; border:1px solid {{.SeverityColor}}66; margin-bottom:18px; }
+  .severity-badge span { font-size:13px; font-weight:600; color:{{.SeverityColor}}; letter-spacing:0.04em; text-transform:uppercase; }
+  .alert-title { font-size:22px; font-weight:700; color:#161616; margin:0 0 12px; }
+  .alert-body { font-size:15px; color:#444; margin:0 0 24px; }
+  .meta-table { width:100%; border-collapse:collapse; margin-bottom:20px; }
+  .meta-table td { padding:8px 10px; font-size:13px; border-bottom:1px solid #ebebeb; }
+  .meta-table td:first-child { font-weight:600; color:#161616; width:110px; white-space:nowrap; }
+  .meta-table td:last-child { color:#555; word-break:break-all; }
+  .meta-row-last td { border-bottom:none; }
+  .divider { border:none; border-top:1px solid #ebebeb; margin:24px 0 20px; }
+  .footer-meta { font-size:11px; color:#999; margin-bottom:4px; }
+  .footer { background-color:#161616; padding:18px 32px; text-align:center; }
+  .footer p { margin:0; font-size:12px; color:#888; }
+</style>
+</head>
+<body>
+  <div class="wrapper">
+    <div class="header">
+      <img src="{{.LogoURL}}" alt="Sylve">
+    </div>
+    <div class="content">
+      <div class="severity-badge">
+        {{.SeverityIcon}}
+        <span>{{.SeverityLabel}}</span>
+      </div>
+      <div class="alert-title">{{.Title}}</div>
+      {{if .Body}}<p class="alert-body">{{.Body}}</p>{{end}}
+      {{if .ShowMetadata}}
+      <table class="meta-table">
+        {{if .Pool}}<tr><td>Pool</td><td>{{.Pool}}</td></tr>{{end}}
+        {{if .VdevPath}}<tr><td>Device</td><td>{{.VdevPath}}</td></tr>{{end}}
+        {{if .State}}<tr class="meta-row-last"><td>State</td><td>{{.State}}</td></tr>{{end}}
+      </table>
+      {{end}}
+      <hr class="divider">
+      {{if .Source}}<p class="footer-meta">Source: {{.Source}}</p>{{end}}
+      {{if .Kind}}<p class="footer-meta">Kind: {{.Kind}}</p>{{end}}
+    </div>
+    <div class="footer">
+      <p>&copy; {{.Year}} <a href="https://sylve.io" style="color:#aaa; text-decoration:none;">Sylve</a>. All rights reserved.</p>
+    </div>
+  </div>
+</body>
+</html>`))

internal/services/notifications/service.go

@@ -245,6 +245,7 @@ func (s *Service) Emit(ctx context.Context, input notifier.EventInput) (notifier
 
 	result := notifier.EmitResult{}
 	var kindRule models.NotificationKindRule
+	canSuppress := shouldPersistSuppressionForKind(normalized.Kind)
 
 	err := s.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
 		var err error
@@ -254,19 +255,21 @@ func (s *Service) Emit(ctx context.Context, input notifier.EventInput) (notifier
 			return err
 		}
 
-		suppressionFingerprint := suppressionKey(normalized.Kind, normalized.Fingerprint)
+		if canSuppress {
+			suppressionFingerprint := suppressionKey(normalized.Kind, normalized.Fingerprint)
 
-		var suppression models.NotificationSuppression
-		err = tx.
-			Where("kind = ?", normalized.Kind).
-			Where("fingerprint = ?", suppressionFingerprint).
-			First(&suppression).Error
-		if err == nil {
-			result.Suppressed = true
-			return nil
-		}
-		if err != nil && err != gorm.ErrRecordNotFound {
-			return err
+			var suppression models.NotificationSuppression
+			err = tx.
+				Where("kind = ?", normalized.Kind).
+				Where("fingerprint = ?", suppressionFingerprint).
+				First(&suppression).Error
+			if err == nil {
+				result.Suppressed = true
+				return nil
+			}
+			if err != nil && err != gorm.ErrRecordNotFound {
+				return err
+			}
 		}
 
 		if kindRule.UIEnabled {
@@ -428,13 +431,15 @@ func (s *Service) Dismiss(ctx context.Context, id uint) error {
 			}
 		}
 
-		suppression := models.NotificationSuppression{
-			Fingerprint: suppressionKey(notif.Kind, notif.Fingerprint),
-			Kind:        notif.Kind,
-		}
+		if shouldPersistSuppressionForKind(notif.Kind) {
+			suppression := models.NotificationSuppression{
+				Fingerprint: suppressionKey(notif.Kind, notif.Fingerprint),
+				Kind:        notif.Kind,
+			}
 
-		if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&suppression).Error; err != nil {
-			return err
+			if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&suppression).Error; err != nil {
+				return err
+			}
 		}
 
 		return nil
@@ -1291,7 +1296,7 @@ func (s *Service) sendNtfy(ctx context.Context, cfg models.NotificationTransport
 	}
 
 	req.Header.Set("Title", input.Title)
-	req.Header.Set("Tags", strings.TrimSpace(input.Severity))
+	req.Header.Set("Tags", ntfyTagForSeverity(input.Severity))
 	req.Header.Set("Content-Type", "text/plain; charset=utf-8")
 	if token != "" {
 		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
@@ -1330,11 +1335,8 @@ func (s *Service) sendEmail(ctx context.Context, cfg models.NotificationTranspor
 		port = defaultSMTPPort
 	}
 
-	subject := fmt.Sprintf("[Sylve][%s] %s", strings.ToUpper(strings.TrimSpace(input.Severity)), input.Title)
-	body := strings.TrimSpace(input.Body)
-	if body == "" {
-		body = input.Title
-	}
+	subject := fmt.Sprintf("Sylve | %s", input.Title)
+	htmlBody := buildEmailHTML(input, s.now())
 
 	msg := strings.Builder{}
 	msg.WriteString("From: ")
@@ -1346,17 +1348,10 @@ func (s *Service) sendEmail(ctx context.Context, cfg models.NotificationTranspor
 	msg.WriteString("Subject: ")
 	msg.WriteString(subject)
 	msg.WriteString("\r\n")
-	msg.WriteString("Content-Type: text/plain; charset=UTF-8\r\n")
+	msg.WriteString("MIME-Version: 1.0\r\n")
+	msg.WriteString("Content-Type: text/html; charset=UTF-8\r\n")
 	msg.WriteString("\r\n")
-	msg.WriteString(body)
-	if strings.TrimSpace(input.Source) != "" {
-		msg.WriteString("\n\nSource: ")
-		msg.WriteString(strings.TrimSpace(input.Source))
-	}
-	if strings.TrimSpace(input.Kind) != "" {
-		msg.WriteString("\nKind: ")
-		msg.WriteString(strings.TrimSpace(input.Kind))
-	}
+	msg.WriteString(htmlBody)
 
 	client, conn, err := dialSMTPClient(ctx, host, port, cfg.SMTPUseTLS)
 	if err != nil {
@@ -1583,3 +1578,21 @@ func normalizePoolNames(raw []string) []string {
 func suppressionKey(kind, fingerprint string) string {
 	return strings.TrimSpace(strings.ToLower(kind)) + "|" + strings.TrimSpace(fingerprint)
 }
+
+func shouldPersistSuppressionForKind(kind string) bool {
+	kind = strings.TrimSpace(strings.ToLower(kind))
+	return !strings.HasPrefix(kind, notifier.ZFSPoolStateKindPrefix)
+}
+
+func ntfyTagForSeverity(severity string) string {
+	switch strings.TrimSpace(strings.ToLower(severity)) {
+	case "warning":
+		return "warning"
+	case "error":
+		return "x"
+	case "critical":
+		return "rotating_light"
+	default:
+		return "white_check_mark"
+	}
+}

internal/services/notifications/service_test.go

@@ -881,3 +881,91 @@ func TestEmitHonorsUIAndChannelRuleToggles(t *testing.T) {
 		t.Fatalf("expected_no_ui_notifications_stored got: %d", count)
 	}
 }
+
+func TestDismissDoesNotPersistSuppressionForZFSPoolState(t *testing.T) {
+	svc := newTestService(t)
+
+	ntfyCalls := 0
+	emailCalls := 0
+	svc.SetNtfySender(func(ctx context.Context, cfg models.NotificationTransportConfig, input notifier.EventInput, token string) error {
+		ntfyCalls++
+		return nil
+	})
+	svc.SetEmailSender(func(ctx context.Context, cfg models.NotificationTransportConfig, input notifier.EventInput, password string) error {
+		emailCalls++
+		return nil
+	})
+
+	_, err := svc.UpdateTransportConfig(context.Background(), TransportConfigUpdate{
+		Transports: []TransportConfigEntryUpdate{
+			{
+				Name:    "ntfy",
+				Type:    TransportTypeNtfy,
+				Enabled: true,
+				Ntfy: &NtfyTransportConfigUpdate{
+					BaseURL: "https://ntfy.sh",
+					Topic:   "ops",
+				},
+			},
+			{
+				Name:    "smtp",
+				Type:    TransportTypeSMTP,
+				Enabled: true,
+				Email: &EmailTransportConfigUpdate{
+					SMTPHost:   "smtp.example.com",
+					SMTPPort:   587,
+					SMTPFrom:   "ops@example.com",
+					Recipients: []string{"ops@example.com"},
+				},
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("update_transport_config_failed: %v", err)
+	}
+
+	input := notifier.EventInput{
+		Kind:        notifier.KindForZFSPoolState("test"),
+		Title:       "Pool degraded",
+		Body:        "test pool degraded",
+		Severity:    "warning",
+		Fingerprint: "test|vdev0|degraded",
+	}
+
+	first, err := svc.Emit(context.Background(), input)
+	if err != nil {
+		t.Fatalf("first_emit_failed: %v", err)
+	}
+	if first.Suppressed {
+		t.Fatalf("expected_first_emit_not_suppressed")
+	}
+
+	if err := svc.Dismiss(context.Background(), first.NotificationID); err != nil {
+		t.Fatalf("dismiss_failed: %v", err)
+	}
+
+	second, err := svc.Emit(context.Background(), input)
+	if err != nil {
+		t.Fatalf("second_emit_failed: %v", err)
+	}
+	if second.Suppressed {
+		t.Fatalf("expected_zfs_emit_not_suppressed_after_dismiss")
+	}
+
+	if ntfyCalls != 2 {
+		t.Fatalf("expected_ntfy_called_twice got: %d", ntfyCalls)
+	}
+	if emailCalls != 2 {
+		t.Fatalf("expected_email_called_twice got: %d", emailCalls)
+	}
+
+	var suppressions int64
+	if err := svc.DB.Model(&models.NotificationSuppression{}).
+		Where("kind = ?", input.Kind).
+		Count(&suppressions).Error; err != nil {
+		t.Fatalf("failed_to_count_suppressions: %v", err)
+	}
+	if suppressions != 0 {
+		t.Fatalf("expected_no_suppression_rows_for_zfs_kind got: %d", suppressions)
+	}
+}

internal/services/system/zfs_notifications.go

@@ -71,9 +71,13 @@ func (s *Service) buildPoolStateChangeNotification(ctx context.Context, ev *mode
 		return notifier.EventInput{}, false, fmt.Errorf("zfs_pool_not_found")
 	}
 
-	status, err := pool.Status(ctx)
-	if err != nil {
-		return notifier.EventInput{}, false, err
+	status, statusErr := pool.Status(ctx)
+	if statusErr != nil {
+		logger.L.Debug().
+			Err(statusErr).
+			Str("pool", poolName).
+			Msg("zfs_pool_status_unavailable_falling_back_to_pool_state")
+		status = nil
 	}
 
 	state := resolvePoolStateFromStatus(status, pool, ev.Attrs)