Improve CalculateExpiry with Slope-based duration calculation

- Remove ICS NewCalendar() func
- TODO IsViewOnlyTerm
This commit is contained in:
2024-02-01 15:15:16 -06:00
parent 9885c202ae
commit b9ff35b711
2 changed files with 52 additions and 28 deletions

View File

@@ -144,16 +144,17 @@ func ScrapeMajor(subject string) error {
}
}
term := Default(time.Now()).ToString()
// Calculate the expiry time for the scrape (1 hour for every 200 classes, random +-15%) with a minimum of 1 hour
var scrapeExpiry time.Duration
if totalClassCount == 0 {
scrapeExpiry = time.Hour * 12
} else {
scrapeExpiry = CalculateExpiry(totalClassCount, lo.Contains(PriorityMajors, subject))
scrapeExpiry = CalculateExpiry(term, totalClassCount, lo.Contains(PriorityMajors, subject))
}
// Mark the major as scraped
term := Default(time.Now()).ToString()
if totalClassCount == 0 {
totalClassCount = -1
}
@@ -166,27 +167,45 @@ func ScrapeMajor(subject string) error {
}
// CalculateExpiry calculates the expiry time until the next scrape for a major.
func CalculateExpiry(count int, priority bool) time.Duration {
scrapeExpiry := time.Hour * time.Duration(count/100)
partial := scrapeExpiry.Seconds() * (rand.Float64() * 0.15) // Between 0 and 15% of the total
// term is the term for which the relevant course is occurring within.
// count is the number of courses that were scraped.
// priority is a boolean indicating whether the major is a priority major.
func CalculateExpiry(term string, count int, priority bool) time.Duration {
// Subjects with less than 50 classes have a reversed expiry (less classes, longer interval)
// 1 class => 12 hours, 49 classes => 1 hour
if count < 50 {
hours := Slope(Point{1, 12}, Point{49, 1}, float64(count)).Y
return time.Duration(hours * float64(time.Hour))
}
// Randomly add or subtract the partial (delta between -15% and 15%)
// An hour for every 100 classes
baseExpiry := time.Hour * time.Duration(count/100)
// If the subject is a priority, then the expiry is halved without variance
if priority {
return baseExpiry / 3
}
// If the term is considered "view only", then the expiry is multipled by 5
var expiry = baseExpiry
if IsViewOnlyTerm(term) {
expiry *= 5
}
// Add minor variance to the expiry
expiryVariance := baseExpiry.Seconds() * (rand.Float64() * 0.15) // Between 0 and 15% of the total
if rand.Intn(2) == 0 {
scrapeExpiry -= time.Duration(partial) * time.Second
expiry -= time.Duration(expiryVariance) * time.Second
} else {
scrapeExpiry += time.Duration(partial) * time.Second
expiry += time.Duration(expiryVariance) * time.Second
}
// Ensure the expiry is at least 1 hour with up to 15 extra minutes
if scrapeExpiry < time.Hour {
scrapeExpiry = time.Hour + time.Duration(rand.Intn(60*15))*time.Second
if expiry < time.Hour {
baseExpiry = time.Hour + time.Duration(rand.Intn(60*15))*time.Second
}
// If the subject is a priority, then the expiry is halved
if priority {
return scrapeExpiry / 3
}
return scrapeExpiry
return baseExpiry
}
// IntakeCourse stores a course in Redis.