xevion/banner
1
0
Fork 0
mirror of https://github.com/Xevion/banner.git synced 2025-12-07 07:14:25 -06:00
Code Issues Packages Projects Releases Wiki Activity

Continue tuning log messages in scrape

Browse Source
This commit is contained in:
Xevion
2024-01-31 18:56:22 -06:00
parent a9603ba562
commit a2ccef6d48
2 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
main.go
View File

@@ -285,7 +285,6 @@ func main() {
log.Err(err).Stack().Msg("Periodic Scrape Failed") log.Err(err).Stack().Msg("Periodic Scrape Failed")
} }
// Wait 5 minutes
time.Sleep(3 * time.Minute) time.Sleep(3 * time.Minute)
} }
}() }()

8
scrape.go
View File

@@ -45,12 +45,13 @@ func Scrape() error {
} }
} }
subjects, err := GetExpiredSubjects() expiredSubjects, err := GetExpiredSubjects()
if err != nil { if err != nil {
return fmt.Errorf("failed to get scrapable majors: %w", err) return fmt.Errorf("failed to get scrapable majors: %w", err)
} }
for _, subject := range subjects { log.Info().Strs("majors", expiredSubjects).Msg("Scraping majors")
for _, subject := range expiredSubjects {
err := ScrapeMajor(subject) err := ScrapeMajor(subject)
if err != nil { if err != nil {
return fmt.Errorf("failed to scrape major %s: %w", subject, err) return fmt.Errorf("failed to scrape major %s: %w", subject, err)
@@ -104,7 +105,6 @@ func IsSubjectExpired(subject string, term string) (bool, error) {
// ScrapeMajor is the scraping invocation for a specific major. // ScrapeMajor is the scraping invocation for a specific major.
// This function does not check whether scraping is required at this time, it is assumed that the caller has already done so. // This function does not check whether scraping is required at this time, it is assumed that the caller has already done so.
func ScrapeMajor(subject string) error { func ScrapeMajor(subject string) error {
log.Debug().Str("subject", subject).Msg("Scraping Major")
offset := 0 offset := 0
totalClassCount := 0 totalClassCount := 0
@@ -149,7 +149,7 @@ func ScrapeMajor(subject string) error {
continue continue
} else { } else {
// Log the number of classes scraped // Log the number of classes scraped
log.Info().Str("subject", subject).Int("total", totalClassCount).Msg("Major Scraped") log.Info().Str("subject", subject).Int("total", totalClassCount).Msgf("Subject %s Scraped", subject)
break break
} }
} }