alfonzso/ncore-hnr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add retry mechanism and failure notifications for nCore/qBittorrent setup

Browse Source 
- Implemented a retry mechanism for external calls, allowing up to 3 attempts before failing.
- Enhanced error handling to send failure notifications when setup steps fail, including detailed error messages.
- Updated RunSummary model to include status, error step, and error message fields for better tracking of run outcomes.
- Modified database schema to store failure metadata for runs.
- Updated README.md to reflect changes in error handling and notification behavior.
This commit is contained in:
Zsolt Alföldi
2026-05-07 12:03:00 +02:00
parent 696d0227a3
commit 89835b237c
6 changed files with 268 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
internal/app/app.go
View File

@@ -16,52 +16,68 @@ import (
"ncore-hnr/internal/store"
)
const retryAttempts = 3
const retryDelay = 20 * time.Second
func Run(cfg config.Config) (model.RunSummary, error) {
startedAt := time.Now().UTC()
summary := model.RunSummary{StartedAt: startedAt, DryRun: cfg.DryRun}
summary := model.RunSummary{StartedAt: startedAt, Status: "success", DryRun: cfg.DryRun}
db, err := store.Open(cfg.DBPath)
if err != nil {
return summary, err
return failRun(cfg, nil, summary, "open database", err)
}
defer db.Close()
ncoreClient := ncore.New(newHTTPClient(cfg.HTTPTimeout), cfg.NCoreLoginURL, cfg.NCoreHitRunURL)
if err := ncoreClient.Login(cfg.NCoreUsername, cfg.NCorePassword); err != nil {
return summary, err
}
page, err := ncoreClient.FetchHitRun()
if err != nil {
return summary, err
var page model.HitRunPage
if err := withRetry("fetch nCore HnR page", func() error {
ncoreClient := ncore.New(newHTTPClient(cfg.HTTPTimeout), cfg.NCoreLoginURL, cfg.NCoreHitRunURL)
if err := ncoreClient.Login(cfg.NCoreUsername, cfg.NCorePassword); err != nil {
return fmt.Errorf("login: %w", err)
}
fetched, err := ncoreClient.FetchHitRun()
if err != nil {
return fmt.Errorf("fetch: %w", err)
}
page = fetched
return nil
}); err != nil {
return failRun(cfg, db, summary, "fetch nCore HnR page", err)
}
summary.TotalRisk = len(page.Torrents)
var qbitClient *qbit.Client
var qbitTorrents []model.QBitTorrent
if !cfg.SkipQBit && cfg.QBitURL != "" {
if err := withRetry("load qBittorrent torrents", func() error {
client := qbit.New(cfg.QBitURL, newHTTPClient(cfg.HTTPTimeout))
if err := client.Login(cfg.QBitUsername, cfg.QBitPassword); err != nil {
return fmt.Errorf("login: %w", err)
}
torrents, err := client.Torrents()
if err != nil {
return fmt.Errorf("list torrents: %w", err)
}
qbitClient = client
qbitTorrents = torrents
return nil
}); err != nil {
return failRun(cfg, db, summary, "load qBittorrent torrents", err)
}
}
activeIDs := map[int64]bool{}
for _, torrent := range page.Torrents {
activeIDs[torrent.ID] = true
}
if err := db.MarkResolved(activeIDs, startedAt); err != nil {
return summary, err
}
var qbitClient *qbit.Client
var qbitTorrents []model.QBitTorrent
if !cfg.SkipQBit && cfg.QBitURL != "" {
qbitClient = qbit.New(cfg.QBitURL, newHTTPClient(cfg.HTTPTimeout))
if err := qbitClient.Login(cfg.QBitUsername, cfg.QBitPassword); err != nil {
return summary, err
}
qbitTorrents, err = qbitClient.Torrents()
if err != nil {
return summary, err
}
return failRun(cfg, db, summary, "mark resolved torrents", err)
}
for _, torrent := range page.Torrents {
state, err := db.UpsertSeen(torrent, startedAt)
if err != nil {
return summary, err
return failRun(cfg, db, summary, "upsert torrent state", err)
}
result := model.ActionResult{
@@ -90,28 +106,32 @@ func Run(cfg config.Config) (model.RunSummary, error) {
summary.Matched++
if !cfg.DryRun {
if err := qbitClient.ForceStart(matched.Hash); err != nil {
return summary, fmt.Errorf("force-start %q: %w", torrent.Name, err)
if err := withRetry(fmt.Sprintf("force-start %q", torrent.Name), func() error {
return qbitClient.ForceStart(matched.Hash)
}); err != nil {
return failRun(cfg, db, summary, fmt.Sprintf("force-start %q", torrent.Name), err)
}
result.ForceStarted = true
summary.ForceStarted++
if err := qbitClient.Reannounce(matched.Hash); err != nil {
return summary, fmt.Errorf("reannounce %q: %w", torrent.Name, err)
if err := withRetry(fmt.Sprintf("reannounce %q", torrent.Name), func() error {
return qbitClient.Reannounce(matched.Hash)
}); err != nil {
return failRun(cfg, db, summary, fmt.Sprintf("reannounce %q", torrent.Name), err)
}
result.Reannounced = true
summary.Reannounced++
}
if err := db.RecordQBit(torrent, matched, startedAt, result.ForceStarted, result.Reannounced); err != nil {
return summary, err
return failRun(cfg, db, summary, "record qBittorrent state", err)
}
if startedAt.Sub(state.FirstSeenAt) >= cfg.AlertAfter {
result.ManualNeeded = true
summary.ManualNeeded++
if err := db.MarkManualNeeded(torrent.ID, startedAt); err != nil {
return summary, err
return failRun(cfg, db, summary, "mark manual-needed torrent", err)
}
}
@@ -119,7 +139,7 @@ func Run(cfg config.Config) (model.RunSummary, error) {
}
if err := db.InsertRun(summary); err != nil {
return summary, err
return failRun(cfg, db, summary, "record run", err)
}
if cfg.NotificationDryRun {
@@ -129,7 +149,9 @@ func Run(cfg config.Config) (model.RunSummary, error) {
notifier := newNotifier(cfg)
if notifier != nil {
if err := notifier.SendManualNeeded(summary.Results); err != nil {
if err := withRetry("send manual-needed notification", func() error {
return notifier.SendManualNeeded(summary.Results)
}); err != nil {
return summary, err
}
}
@@ -215,8 +237,10 @@ func PrintStats(snapshot model.StatsSnapshot, asJSON bool) error {
fmt.Println("Last run: none")
} else {
run := snapshot.LastRun
fmt.Printf("Last run: %s | total=%d matched=%d unmatched=%d force-started=%d reannounced=%d manual-needed=%d dry-run=%t\n",
status := defaultText(run.Status, "success")
fmt.Printf("Last run: %s | status=%s total=%d matched=%d unmatched=%d force-started=%d reannounced=%d manual-needed=%d dry-run=%t\n",
run.StartedAt,
status,
run.TotalRisk,
run.Matched,
run.Unmatched,
@@ -225,6 +249,9 @@ func PrintStats(snapshot model.StatsSnapshot, asJSON bool) error {
run.ManualNeeded,
run.DryRun,
)
if status == "failed" {
fmt.Printf("Last error: %s | %s\n", defaultText(run.ErrorStep, "-"), defaultText(run.ErrorMessage, "-"))
}
}
if len(snapshot.ManualNeeded) > 0 {
@@ -319,3 +346,58 @@ func printNotificationPreview(notificationType string, results []model.ActionRes
fmt.Fprintf(os.Stderr, "notification dry-run: type=%s\n", notificationType)
fmt.Fprintf(os.Stderr, "Subject: %s\n\n%s", subject, body)
}
func withRetry(step string, fn func() error) error {
var lastErr error
for attempt := 1; attempt <= retryAttempts; attempt++ {
if err := fn(); err != nil {
lastErr = err
fmt.Fprintf(os.Stderr, "%s failed (attempt %d/%d): %v\n", step, attempt, retryAttempts, err)
if attempt < retryAttempts {
time.Sleep(retryDelay)
}
continue
}
return nil
}
return fmt.Errorf("failed after %d attempts: %w", retryAttempts, lastErr)
}
func failRun(cfg config.Config, db *store.Store, summary model.RunSummary, step string, err error) (model.RunSummary, error) {
summary.Status = "failed"
summary.ErrorStep = step
summary.ErrorMessage = err.Error()
fmt.Fprintf(os.Stderr, "run failed at %s: %v\n", step, err)
if db != nil {
if insertErr := db.InsertRun(summary); insertErr != nil {
fmt.Fprintf(os.Stderr, "record failed run: %v\n", insertErr)
}
}
if notifyErr := sendFailureNotification(cfg, step, err); notifyErr != nil {
return summary, fmt.Errorf("%s: %w; failure notification failed: %v", step, err, notifyErr)
}
return summary, fmt.Errorf("%s: %w", step, err)
}
func sendFailureNotification(cfg config.Config, step string, err error) error {
if cfg.NotificationDryRun {
subject, body := notify.FailureMessage(step, err)
notificationType := cfg.NotificationType
if notificationType == "" {
notificationType = "unconfigured"
}
fmt.Fprintf(os.Stderr, "failure notification dry-run: type=%s\n", notificationType)
fmt.Fprintf(os.Stderr, "Subject: %s\n\n%s", subject, body)
return nil
}
notifier := newNotifier(cfg)
if notifier == nil {
return nil
}
return withRetry("send failure notification", func() error {
return notifier.SendFailure(step, err)
})
}