Add retry mechanism and failure notifications for nCore/qBittorrent setup

- Implemented a retry mechanism for external calls, allowing up to 3 attempts before failing.
- Enhanced error handling to send failure notifications when setup steps fail, including detailed error messages.
- Updated RunSummary model to include status, error step, and error message fields for better tracking of run outcomes.
- Modified database schema to store failure metadata for runs.
- Updated README.md to reflect changes in error handling and notification behavior.

internal/notify/notify.go

@@ -14,9 +14,11 @@ import (
 )
 
 const manualNeededSubject = "nCore HnR manual work"
+const failureSubject = "nCore HnR check failed"
 
 type Sender interface {
 	SendManualNeeded(results []model.ActionResult) error
+	SendFailure(step string, failure error) error
 }
 
 type NotificationNTFY struct {
@@ -29,6 +31,18 @@ func (n NotificationNTFY) SendManualNeeded(results []model.ActionResult) error {
 	if strings.TrimSpace(n.URL) == "" || !ok {
 		return nil
 	}
+	return n.sendMessage(manualNeededSubject, body)
+}
+
+func (n NotificationNTFY) SendFailure(step string, failure error) error {
+	if strings.TrimSpace(n.URL) == "" {
+		return nil
+	}
+	_, body := FailureMessage(step, failure)
+	return n.sendMessage(failureSubject, body)
+}
+
+func (n NotificationNTFY) sendMessage(subject string, body string) error {
 	client := n.HTTPClient
 	if client == nil {
 		client = &http.Client{Timeout: 15 * time.Second}
@@ -39,7 +53,7 @@ func (n NotificationNTFY) SendManualNeeded(results []model.ActionResult) error {
 		return err
 	}
 	req.Header.Set("Content-Type", "text/plain; charset=utf-8")
-	req.Header.Set("Title", manualNeededSubject)
+	req.Header.Set("Title", subject)
 
 	resp, err := client.Do(req)
 	if err != nil {
@@ -66,7 +80,18 @@ func (s NotificationSMTP) SendManualNeeded(results []model.ActionResult) error {
 	if strings.TrimSpace(s.Host) == "" || !ok {
 		return nil
 	}
+	return s.sendMessage(manualNeededSubject, body)
+}
 
+func (s NotificationSMTP) SendFailure(step string, failure error) error {
+	if strings.TrimSpace(s.Host) == "" {
+		return nil
+	}
+	_, body := FailureMessage(step, failure)
+	return s.sendMessage(failureSubject, body)
+}
+
+func (s NotificationSMTP) sendMessage(subject string, body string) error {
 	host := strings.TrimSpace(s.Host)
 	addr := net.JoinHostPort(host, strings.TrimSpace(s.Port))
 
@@ -92,7 +117,7 @@ func (s NotificationSMTP) SendManualNeeded(results []model.ActionResult) error {
 	message := strings.Builder{}
 	message.WriteString(fmt.Sprintf("From: %s\r\n", from.String()))
 	message.WriteString(fmt.Sprintf("To: %s\r\n", formatAddressList(recipients)))
-	message.WriteString(fmt.Sprintf("Subject: %s\r\n", manualNeededSubject))
+	message.WriteString(fmt.Sprintf("Subject: %s\r\n", subject))
 	message.WriteString("MIME-Version: 1.0\r\n")
 	message.WriteString("Content-Type: text/plain; charset=UTF-8\r\n")
 	message.WriteString("\r\n")
@@ -109,6 +134,17 @@ func ManualNeededMessage(results []model.ActionResult) (string, string, bool) {
 	return manualNeededSubject, body, ok
 }
 
+func FailureMessage(step string, failure error) (string, string) {
+	var body strings.Builder
+	body.WriteString("nCore HnR check failed after retries.\n")
+	body.WriteString(fmt.Sprintf("Step: %s\n", strings.TrimSpace(step)))
+	if failure != nil {
+		body.WriteString(fmt.Sprintf("Error: %s\n", failure.Error()))
+	}
+	body.WriteString("Action: review the CronJob logs. Failed nCore/qBittorrent setup steps do not clear unresolved torrent state; the next cron run will retry.\n")
+	return failureSubject, body.String()
+}
+
 func manualNeededText(results []model.ActionResult) (string, bool) {
 	var body strings.Builder
 	manualCount := 0