Continue tuning log messages in scrape

This commit is contained in:
2024-01-31 18:56:22 -06:00
parent a9603ba562
commit a2ccef6d48
2 changed files with 4 additions and 5 deletions

View File

@@ -285,7 +285,6 @@ func main() {
log.Err(err).Stack().Msg("Periodic Scrape Failed")
}
// Wait 5 minutes
time.Sleep(3 * time.Minute)
}
}()

View File

@@ -45,12 +45,13 @@ func Scrape() error {
}
}
subjects, err := GetExpiredSubjects()
expiredSubjects, err := GetExpiredSubjects()
if err != nil {
return fmt.Errorf("failed to get scrapable majors: %w", err)
}
for _, subject := range subjects {
log.Info().Strs("majors", expiredSubjects).Msg("Scraping majors")
for _, subject := range expiredSubjects {
err := ScrapeMajor(subject)
if err != nil {
return fmt.Errorf("failed to scrape major %s: %w", subject, err)
@@ -104,7 +105,6 @@ func IsSubjectExpired(subject string, term string) (bool, error) {
// ScrapeMajor is the scraping invocation for a specific major.
// This function does not check whether scraping is required at this time, it is assumed that the caller has already done so.
func ScrapeMajor(subject string) error {
log.Debug().Str("subject", subject).Msg("Scraping Major")
offset := 0
totalClassCount := 0
@@ -149,7 +149,7 @@ func ScrapeMajor(subject string) error {
continue
} else {
// Log the number of classes scraped
log.Info().Str("subject", subject).Int("total", totalClassCount).Msg("Major Scraped")
log.Info().Str("subject", subject).Int("total", totalClassCount).Msgf("Subject %s Scraped", subject)
break
}
}