refactor: complete refactor into cmd/ & internal/ submodules

internal/api/api.go

@@ -0,0 +1,640 @@
+package api
+
+import (
+	"banner/internal/config"
+	"banner/internal/models"
+	"banner/internal/utils"
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"time"
+
+	"github.com/redis/go-redis/v9"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/samber/lo"
+)
+
+var (
+	latestSession string = ""
+	sessionTime   time.Time
+	expiryTime    time.Duration = 25 * time.Minute
+)
+
+// ResetSessionTimer resets the session timer to the current time.
+// This is only used by the DoRequest handler when Banner API calls are detected, which would reset the session timer.
+func ResetSessionTimer() {
+	// Only reset the session time if the session is still valid
+	if time.Since(sessionTime) <= expiryTime {
+		sessionTime = time.Now()
+	}
+}
+
+// GenerateSession generates a new session ID (nonce) for use with the Banner API.
+// Don't use this function directly, use GetSession instead.
+func GenerateSession() string {
+	return utils.RandomString(5) + utils.Nonce()
+}
+
+// DoRequest performs & logs the request, logging and returning the response
+func DoRequest(req *http.Request) (*http.Response, error) {
+	headerSize := 0
+	for key, values := range req.Header {
+		for _, value := range values {
+			headerSize += len(key)
+			headerSize += len(value)
+		}
+	}
+
+	bodySize := int64(0)
+	if req.Body != nil {
+		bodySize, _ = io.Copy(io.Discard, req.Body)
+	}
+
+	size := zerolog.Dict().Int64("body", bodySize).Int("header", headerSize).Int("url", len(req.URL.String()))
+
+	log.Debug().
+		Dict("size", size).
+		Str("method", strings.TrimRight(req.Method, " ")).
+		Str("url", req.URL.String()).
+		Str("query", req.URL.RawQuery).
+		Str("content-type", req.Header.Get("Content-Type")).
+		Msg("Request")
+
+	res, err := config.Client.Do(req)
+
+	if err != nil {
+		log.Err(err).Stack().Str("method", req.Method).Msg("Request Failed")
+	} else {
+		contentLengthHeader := res.Header.Get("Content-Length")
+		contentLength := int64(-1)
+
+		// If this request was a Banner API request, reset the session timer
+		if strings.HasPrefix(req.URL.Path, "StudentRegistrationSsb/ssb/classSearch/") {
+			ResetSessionTimer()
+		}
+
+		// Get the content length
+		if contentLengthHeader != "" {
+			contentLength, err = strconv.ParseInt(contentLengthHeader, 10, 64)
+			if err != nil {
+				contentLength = -1
+			}
+		}
+
+		log.Debug().Int("status", res.StatusCode).Int64("content-length", contentLength).Strs("content-type", res.Header["Content-Type"]).Msg("Response")
+	}
+	return res, err
+}
+
+var terms []BannerTerm
+var lastTermUpdate time.Time
+
+// TryReloadTerms attempts to reload the terms if they are not loaded or the last update was more than 24 hours ago
+func TryReloadTerms() error {
+	if len(terms) > 0 && time.Since(lastTermUpdate) < 24*time.Hour {
+		return nil
+	}
+
+	// Load the terms
+	var err error
+	terms, err = GetTerms("", 1, 100)
+	if err != nil {
+		return fmt.Errorf("failed to load terms: %w", err)
+	}
+
+	lastTermUpdate = time.Now()
+	return nil
+}
+
+// IsTermArchived checks if the given term is archived
+// TODO: Add error, switch missing term logic to error
+func IsTermArchived(term string) bool {
+	// Ensure the terms are loaded
+	err := TryReloadTerms()
+	if err != nil {
+		log.Err(err).Stack().Msg("Failed to reload terms")
+		return true
+	}
+
+	// Check if the term is in the list of terms
+	bannerTerm, exists := lo.Find(terms, func(t BannerTerm) bool {
+		return t.Code == term
+	})
+
+	if !exists {
+		log.Warn().Str("term", term).Msg("Term does not exist")
+		return true
+	}
+
+	return bannerTerm.Archived()
+}
+
+// GetSession retrieves the current session ID if it's still valid.
+// If the session ID is invalid or has expired, a new one is generated and returned.
+// SessionIDs are valid for 30 minutes, but we'll be conservative and regenerate every 25 minutes.
+func GetSession() string {
+	// Check if a reset is required
+	if latestSession == "" || time.Since(sessionTime) >= expiryTime {
+		// Generate a new session identifier
+		latestSession = GenerateSession()
+
+		// Select the current term
+		term := utils.Default(time.Now()).ToString()
+		log.Info().Str("term", term).Str("sessionID", latestSession).Msg("Setting selected term")
+		err := SelectTerm(term, latestSession)
+		if err != nil {
+			log.Fatal().Stack().Err(err).Msg("Failed to select term while generating session ID")
+		}
+
+		sessionTime = time.Now()
+	}
+
+	return latestSession
+}
+
+type Pair struct {
+	Code        string `json:"code"`
+	Description string `json:"description"`
+}
+
+type BannerTerm Pair
+type Instructor Pair
+
+// Archived returns true if the term is in it's archival state (view only)
+func (term BannerTerm) Archived() bool {
+	return strings.Contains(term.Description, "View Only")
+}
+
+// GetTerms retrieves and parses the term information for a given search term.
+// Page number must be at least 1.
+func GetTerms(search string, page int, max int) ([]BannerTerm, error) {
+	// Ensure offset is valid
+	if page <= 0 {
+		return nil, errors.New("offset must be greater than 0")
+	}
+
+	req := utils.BuildRequest("GET", "/classSearch/getTerms", map[string]string{
+		"searchTerm": search,
+		// Page vs Offset is not a mistake here, the API uses "offset" as the page number
+		"offset": strconv.Itoa(page),
+		"max":    strconv.Itoa(max),
+		"_":      utils.Nonce(),
+	})
+
+	if page <= 0 {
+		return nil, errors.New("Offset must be greater than 0")
+	}
+
+	res, err := DoRequest(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get terms: %w", err)
+	}
+
+	// Assert that the response is JSON
+	if contentType := res.Header.Get("Content-Type"); !strings.Contains(contentType, models.JsonContentType) {
+		return nil, &utils.UnexpectedContentTypeError{
+			Expected: models.JsonContentType,
+			Actual:   contentType,
+		}
+	}
+
+	// print the response body
+	defer res.Body.Close()
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	terms := make([]BannerTerm, 0, 10)
+	json.Unmarshal(body, &terms)
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse terms: %w", err)
+	}
+
+	return terms, nil
+}
+
+// SelectTerm selects the given term in the Banner system.
+// This function completes the initial term selection process, which is required before any other API calls can be made with the session ID.
+func SelectTerm(term string, sessionId string) error {
+	form := url.Values{
+		"term":            {term},
+		"studyPath":       {""},
+		"studyPathText":   {""},
+		"startDatepicker": {""},
+		"endDatepicker":   {""},
+		"uniqueSessionId": {sessionId},
+	}
+
+	params := map[string]string{
+		"mode": "search",
+	}
+
+	req := utils.BuildRequestWithBody("POST", "/term/search", params, bytes.NewBufferString(form.Encode()))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	res, err := DoRequest(req)
+	if err != nil {
+		return fmt.Errorf("failed to select term: %w", err)
+	}
+
+	// Assert that the response is JSON
+	if !utils.ContentTypeMatch(res, "application/json") {
+		return fmt.Errorf("response was not JSON: %w", res.Header.Get("Content-Type"))
+	}
+
+	// Acquire fwdUrl
+	defer res.Body.Close()
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	var redirectResponse struct {
+		FwdUrl string `json:"fwdUrl"`
+	}
+	json.Unmarshal(body, &redirectResponse)
+
+	// Make a GET request to the fwdUrl
+	req = utils.BuildRequest("GET", redirectResponse.FwdUrl, nil)
+	res, err = DoRequest(req)
+	if err != nil {
+		return fmt.Errorf("failed to follow redirect: %w", err)
+	}
+
+	// Assert that the response is OK (200)
+	if res.StatusCode != 200 {
+		return fmt.Errorf("redirect response was not 200: %w", res.StatusCode)
+	}
+
+	return nil
+}
+
+// GetPartOfTerms retrieves and parses the part of term information for a given term.
+// Ensure that the offset is greater than 0.
+func GetPartOfTerms(search string, term int, offset int, max int) ([]BannerTerm, error) {
+	// Ensure offset is valid
+	if offset <= 0 {
+		return nil, errors.New("offset must be greater than 0")
+	}
+
+	req := utils.BuildRequest("GET", "/classSearch/get_partOfTerm", map[string]string{
+		"searchTerm":      search,
+		"term":            strconv.Itoa(term),
+		"offset":          strconv.Itoa(offset),
+		"max":             strconv.Itoa(max),
+		"uniqueSessionId": GetSession(),
+		"_":               utils.Nonce(),
+	})
+
+	res, err := DoRequest(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get part of terms: %w", err)
+	}
+
+	// Assert that the response is JSON
+	if !utils.ContentTypeMatch(res, "application/json") {
+		log.Panic().Stack().Str("content-type", res.Header.Get("Content-Type")).Msg("Response was not JSON")
+	}
+
+	defer res.Body.Close()
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	terms := make([]BannerTerm, 0, 10)
+	err = json.Unmarshal(body, &terms)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse part of terms: %w", err)
+	}
+
+	return terms, nil
+}
+
+// GetInstructors retrieves and parses the instructor information for a given search term.
+// In my opinion, it is unclear what providing the term does, as the results should be the same regardless of the term.
+// This function is included for completeness, but probably isn't useful.
+// Ensure that the offset is greater than 0.
+func GetInstructors(search string, term string, offset int, max int) ([]Instructor, error) {
+	// Ensure offset is valid
+	if offset <= 0 {
+		return nil, errors.New("offset must be greater than 0")
+	}
+
+	req := utils.BuildRequest("GET", "/classSearch/get_instructor", map[string]string{
+		"searchTerm":      search,
+		"term":            term,
+		"offset":          strconv.Itoa(offset),
+		"max":             strconv.Itoa(max),
+		"uniqueSessionId": GetSession(),
+		"_":               utils.Nonce(),
+	})
+
+	res, err := DoRequest(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get instructors: %w", err)
+	}
+
+	// Assert that the response is JSON
+	if !utils.ContentTypeMatch(res, "application/json") {
+		log.Fatal().Stack().Str("content-type", res.Header.Get("Content-Type")).Msg("Response was not JSON")
+	}
+
+	defer res.Body.Close()
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	instructors := make([]Instructor, 0, 10)
+	err = json.Unmarshal(body, &instructors)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse instructors: %w", err)
+	}
+
+	return instructors, nil
+}
+
+// TODO: Finish this struct & function
+// ClassDetails represents
+type ClassDetails struct {
+}
+
+func GetCourseDetails(term int, crn int) *ClassDetails {
+	body, err := json.Marshal(map[string]string{
+		"term":                  strconv.Itoa(term),
+		"courseReferenceNumber": strconv.Itoa(crn),
+		"first":                 "first", // TODO: What is this?
+	})
+	if err != nil {
+		log.Fatal().Stack().Err(err).Msg("Failed to marshal body")
+	}
+	req := utils.BuildRequestWithBody("GET", "/searchResults/getClassDetails", nil, bytes.NewBuffer(body))
+
+	res, err := DoRequest(req)
+	if err != nil {
+		return nil
+	}
+
+	// Assert that the response is JSON
+	if !utils.ContentTypeMatch(res, "application/json") {
+		log.Fatal().Stack().Str("content-type", res.Header.Get("Content-Type")).Msg("Response was not JSON")
+	}
+
+	return &ClassDetails{}
+}
+
+// Search invokes a search on the Banner system with the given query and returns the results.
+func Search(query *Query, sort string, sortDescending bool) (*models.SearchResult, error) {
+	ResetDataForm()
+
+	params := query.Paramify()
+
+	params["txt_term"] = "202510" // TODO: Make this automatic but dynamically specifiable
+	params["uniqueSessionId"] = GetSession()
+	params["sortColumn"] = sort
+	params["sortDirection"] = "asc"
+
+	// These dates are not available for usage anywhere in the UI, but are included in every query
+	params["startDatepicker"] = ""
+	params["endDatepicker"] = ""
+
+	req := utils.BuildRequest("GET", "/searchResults/searchResults", params)
+
+	res, err := DoRequest(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to search: %w", err)
+	}
+
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("search failed with status code: %d", res.StatusCode)
+	}
+
+	// Assert that the response is JSON
+	if !utils.ContentTypeMatch(res, "application/json") {
+		// for server 500 errors, parse for the error with '#dialog-message > div.message'
+		log.Error().Stack().Str("content-type", res.Header.Get("Content-Type")).Msg("Response was not JSON")
+	}
+
+	defer res.Body.Close()
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	var result models.SearchResult
+	err = json.Unmarshal(body, &result)
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse search results: %w", err)
+	}
+
+	return &result, nil
+}
+
+// GetSubjects retrieves and parses the subject information for a given search term.
+// The results of this response shouldn't change much, but technically could as new majors are developed, or old ones are removed.
+// Ensure that the offset is greater than 0.
+func GetSubjects(search string, term string, offset int, max int) ([]Pair, error) {
+	// Ensure offset is valid
+	if offset <= 0 {
+		return nil, errors.New("offset must be greater than 0")
+	}
+
+	req := utils.BuildRequest("GET", "/classSearch/get_subject", map[string]string{
+		"searchTerm":      search,
+		"term":            term,
+		"offset":          strconv.Itoa(offset),
+		"max":             strconv.Itoa(max),
+		"uniqueSessionId": GetSession(),
+		"_":               utils.Nonce(),
+	})
+
+	res, err := DoRequest(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get subjects: %w", err)
+	}
+
+	// Assert that the response is JSON
+	if !utils.ContentTypeMatch(res, "application/json") {
+		log.Fatal().Stack().Str("content-type", res.Header.Get("Content-Type")).Msg("Response was not JSON")
+	}
+
+	defer res.Body.Close()
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	subjects := make([]Pair, 0, 10)
+	err = json.Unmarshal(body, &subjects)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse subjects: %w", err)
+	}
+
+	return subjects, nil
+}
+
+// GetCampuses retrieves and parses the campus information for a given search term.
+// In my opinion, it is unclear what providing the term does, as the results should be the same regardless of the term.
+// This function is included for completeness, but probably isn't useful.
+// Ensure that the offset is greater than 0.
+func GetCampuses(search string, term int, offset int, max int) ([]Pair, error) {
+	// Ensure offset is valid
+	if offset <= 0 {
+		return nil, errors.New("offset must be greater than 0")
+	}
+
+	req := utils.BuildRequest("GET", "/classSearch/get_campus", map[string]string{
+		"searchTerm":      search,
+		"term":            strconv.Itoa(term),
+		"offset":          strconv.Itoa(offset),
+		"max":             strconv.Itoa(max),
+		"uniqueSessionId": GetSession(),
+		"_":               utils.Nonce(),
+	})
+
+	res, err := DoRequest(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get campuses: %w", err)
+	}
+
+	// Assert that the response is JSON
+	if !utils.ContentTypeMatch(res, "application/json") {
+		log.Fatal().Stack().Str("content-type", res.Header.Get("Content-Type")).Msg("Response was not JSON")
+	}
+
+	defer res.Body.Close()
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	campuses := make([]Pair, 0, 10)
+	err = json.Unmarshal(body, &campuses)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse campuses: %w", err)
+	}
+
+	return campuses, nil
+}
+
+// GetInstructionalMethods retrieves and parses the instructional method information for a given search term.
+// In my opinion, it is unclear what providing the term does, as the results should be the same regardless of the term.
+// This function is included for completeness, but probably isn't useful.
+// Ensure that the offset is greater than 0.
+func GetInstructionalMethods(search string, term string, offset int, max int) ([]Pair, error) {
+	// Ensure offset is valid
+	if offset <= 0 {
+		return nil, errors.New("offset must be greater than 0")
+	}
+
+	req := utils.BuildRequest("GET", "/classSearch/get_instructionalMethod", map[string]string{
+		"searchTerm":      search,
+		"term":            term,
+		"offset":          strconv.Itoa(offset),
+		"max":             strconv.Itoa(max),
+		"uniqueSessionId": GetSession(),
+		"_":               utils.Nonce(),
+	})
+
+	res, err := DoRequest(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get instructional methods: %w", err)
+	}
+
+	// Assert that the response is JSON
+	if !utils.ContentTypeMatch(res, "application/json") {
+		log.Fatal().Stack().Str("content-type", res.Header.Get("Content-Type")).Msg("Response was not JSON")
+	}
+
+	defer res.Body.Close()
+	body, _ := io.ReadAll(res.Body)
+
+	methods := make([]Pair, 0, 10)
+	err = json.Unmarshal(body, &methods)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse instructional methods: %w", err)
+	}
+
+	return methods, nil
+}
+
+// GetCourseMeetingTime retrieves the meeting time information for a course based on the given term and course reference number (CRN).
+// It makes an HTTP GET request to the appropriate API endpoint and parses the response to extract the meeting time data.
+// The function returns a MeetingTimeResponse struct containing the extracted information.
+func GetCourseMeetingTime(term int, crn int) ([]models.MeetingTimeResponse, error) {
+	req := utils.BuildRequest("GET", "/searchResults/getFacultyMeetingTimes", map[string]string{
+		"term":                  strconv.Itoa(term),
+		"courseReferenceNumber": strconv.Itoa(crn),
+	})
+
+	res, err := DoRequest(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get meeting time: %w", err)
+	}
+
+	// Assert that the response is JSON
+	if !utils.ContentTypeMatch(res, "application/json") {
+		log.Fatal().Stack().Str("content-type", res.Header.Get("Content-Type")).Msg("Response was not JSON")
+	}
+
+	// Read the response body into JSON
+	defer res.Body.Close()
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	// Parse the JSON into a MeetingTimeResponse struct
+	var meetingTime struct {
+		Inner []models.MeetingTimeResponse `json:"fmt"`
+	}
+	err = json.Unmarshal(body, &meetingTime)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse meeting time: %w", err)
+	}
+
+	return meetingTime.Inner, nil
+}
+
+// ResetDataForm makes a POST request that needs to be made upon before new search requests can be made.
+func ResetDataForm() {
+	req := utils.BuildRequest("POST", "/classSearch/resetDataForm", nil)
+	_, err := DoRequest(req)
+	if err != nil {
+		log.Fatal().Stack().Err(err).Msg("Failed to reset data form")
+	}
+}
+
+// GetCourse retrieves the course information.
+// This course does not retrieve directly from the API, but rather uses scraped data stored in Redis.
+func GetCourse(crn string) (*models.Course, error) {
+	// Retrieve raw data
+	result, err := config.KV.Get(config.Ctx, fmt.Sprintf("class:%s", crn)).Result()
+	if err != nil {
+		if err == redis.Nil {
+			return nil, fmt.Errorf("course not found: %w", err)
+		}
+		return nil, fmt.Errorf("failed to get course: %w", err)
+	}
+
+	// Unmarshal the raw data
+	var course models.Course
+	err = json.Unmarshal([]byte(result), &course)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal course: %w", err)
+	}
+
+	return &course, nil
+}

internal/api/scrape.go

@@ -0,0 +1,222 @@
+package api
+
+import (
+	"banner/internal/config"
+	"banner/internal/models"
+	"banner/internal/utils"
+	"fmt"
+	"math/rand"
+	"time"
+
+	"github.com/rs/zerolog/log"
+	"github.com/samber/lo"
+)
+
+const (
+	MaxPageSize = 500
+)
+
+var (
+	// PriorityMajors is a list of majors that are considered to be high priority for scraping. This list is used to determine which majors to scrape first/most often.
+	PriorityMajors = []string{"CS", "CPE", "MAT", "EE", "IS"}
+	// AncillaryMajors is a list of majors that are considered to be low priority for scraping. This list will not contain any majors that are in PriorityMajors.
+	AncillaryMajors []string
+	// AllMajors is a list of all majors that are available in the Banner system.
+	AllMajors []string
+)
+
+// Scrape is the general scraping invocation (best called within/as a goroutine) that should be called regularly to initiate scraping of the Banner system.
+func Scrape() error {
+	// Populate AllMajors if it is empty
+	if len(AncillaryMajors) == 0 {
+		term := utils.Default(time.Now()).ToString()
+		subjects, err := GetSubjects("", term, 1, 99)
+		if err != nil {
+			return fmt.Errorf("failed to get subjects: %w", err)
+		}
+
+		// Ensure subjects were found
+		if len(subjects) == 0 {
+			return fmt.Errorf("no subjects found")
+		}
+
+		// Extract major code name
+		for _, subject := range subjects {
+			// Add to AncillaryMajors if not in PriorityMajors
+			if !lo.Contains(PriorityMajors, subject.Code) {
+				AncillaryMajors = append(AncillaryMajors, subject.Code)
+			}
+		}
+
+		AllMajors = lo.Flatten([][]string{PriorityMajors, AncillaryMajors})
+	}
+
+	expiredSubjects, err := GetExpiredSubjects()
+	if err != nil {
+		return fmt.Errorf("failed to get scrapable majors: %w", err)
+	}
+
+	log.Info().Strs("majors", expiredSubjects).Msg("Scraping majors")
+	for _, subject := range expiredSubjects {
+		err := ScrapeMajor(subject)
+		if err != nil {
+			return fmt.Errorf("failed to scrape major %s: %w", subject, err)
+		}
+	}
+
+	return nil
+}
+
+// GetExpiredSubjects returns a list of subjects that are expired and should be scraped.
+func GetExpiredSubjects() ([]string, error) {
+	term := utils.Default(time.Now()).ToString()
+	subjects := make([]string, 0)
+
+	// Get all subjects
+	values, err := config.KV.MGet(config.Ctx, lo.Map(AllMajors, func(major string, _ int) string {
+		return fmt.Sprintf("scraped:%s:%s", major, term)
+	})...).Result()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get all subjects: %w", err)
+	}
+
+	// Extract expired subjects
+	for i, value := range values {
+		subject := AllMajors[i]
+
+		// If the value is nil or "0", then the subject is expired
+		if value == nil || value == "0" {
+			subjects = append(subjects, subject)
+		}
+	}
+
+	log.Debug().Strs("majors", subjects).Msg("Expired Subjects")
+
+	return subjects, nil
+}
+
+// ScrapeMajor is the scraping invocation for a specific major.
+// This function does not check whether scraping is required at this time, it is assumed that the caller has already done so.
+func ScrapeMajor(subject string) error {
+	offset := 0
+	totalClassCount := 0
+
+	for {
+		// Build & execute the query
+		query := NewQuery().Offset(offset).MaxResults(MaxPageSize * 2).Subject(subject)
+		result, err := Search(query, "subjectDescription", false)
+		if err != nil {
+			return fmt.Errorf("search failed: %w (%s)", err, query.String())
+		}
+
+		// Isn't it bullshit that they decided not to leave an actual 'reason' field for the failure?
+		if !result.Success {
+			return fmt.Errorf("result marked unsuccessful when searching for classes (%s)", query.String())
+		}
+
+		classCount := len(result.Data)
+		totalClassCount += classCount
+		log.Debug().Str("subject", subject).Int("count", classCount).Int("offset", offset).Msg("Placing classes in Redis")
+
+		// Process each class and store it in Redis
+		for _, course := range result.Data {
+			// Store class in Redis
+			err := IntakeCourse(course)
+			if err != nil {
+				log.Error().Err(err).Msg("failed to store class in Redis")
+			}
+		}
+
+		// Increment and continue if the results are full
+		if classCount >= MaxPageSize {
+			// This is unlikely to happen, but log it just in case
+			if classCount > MaxPageSize {
+				log.Warn().Int("page", offset).Int("count", classCount).Msg("Results exceed MaxPageSize")
+			}
+
+			offset += MaxPageSize
+
+			// TODO: Replace sleep with smarter rate limiting
+			log.Debug().Str("subject", subject).Int("nextOffset", offset).Msg("Sleeping before next page")
+			time.Sleep(time.Second * 3)
+			continue
+		} else {
+			// Log the number of classes scraped
+			log.Info().Str("subject", subject).Int("total", totalClassCount).Msgf("Subject %s Scraped", subject)
+			break
+		}
+	}
+
+	term := utils.Default(time.Now()).ToString()
+
+	// Calculate the expiry time for the scrape (1 hour for every 200 classes, random +-15%) with a minimum of 1 hour
+	var scrapeExpiry time.Duration
+	if totalClassCount == 0 {
+		scrapeExpiry = time.Hour * 12
+	} else {
+		scrapeExpiry = CalculateExpiry(term, totalClassCount, lo.Contains(PriorityMajors, subject))
+	}
+
+	// Mark the major as scraped
+	if totalClassCount == 0 {
+		totalClassCount = -1
+	}
+	err := config.KV.Set(config.Ctx, fmt.Sprintf("scraped:%s:%s", subject, term), totalClassCount, scrapeExpiry).Err()
+	if err != nil {
+		log.Error().Err(err).Msg("failed to mark major as scraped")
+	}
+
+	return nil
+}
+
+// CalculateExpiry calculates the expiry time until the next scrape for a major.
+// term is the term for which the relevant course is occurring within.
+// count is the number of courses that were scraped.
+// priority is a boolean indicating whether the major is a priority major.
+func CalculateExpiry(term string, count int, priority bool) time.Duration {
+	// An hour for every 100 classes
+	baseExpiry := time.Hour * time.Duration(count/100)
+
+	// Subjects with less than 50 classes have a reversed expiry (less classes, longer interval)
+	// 1 class => 12 hours, 49 classes => 1 hour
+	if count < 50 {
+		hours := utils.Slope(utils.Point{1, 12}, utils.Point{49, 1}, float64(count)).Y
+		baseExpiry = time.Duration(hours * float64(time.Hour))
+	}
+
+	// If the subject is a priority, then the expiry is halved without variance
+	if priority {
+		return baseExpiry / 3
+	}
+
+	// If the term is considered "view only" or "archived", then the expiry is multiplied by 5
+	var expiry = baseExpiry
+	if IsTermArchived(term) {
+		expiry *= 5
+	}
+
+	// Add minor variance to the expiry
+	expiryVariance := baseExpiry.Seconds() * (rand.Float64() * 0.15) // Between 0 and 15% of the total
+	if rand.Intn(2) == 0 {
+		expiry -= time.Duration(expiryVariance) * time.Second
+	} else {
+		expiry += time.Duration(expiryVariance) * time.Second
+	}
+
+	// Ensure the expiry is at least 1 hour with up to 15 extra minutes
+	if expiry < time.Hour {
+		baseExpiry = time.Hour + time.Duration(rand.Intn(60*15))*time.Second
+	}
+
+	return baseExpiry
+}
+
+// IntakeCourse stores a course in Redis.
+// This function is mostly a stub for now, but will be used to handle change identification, notifications, and SQLite upserts in the future.
+func IntakeCourse(course models.Course) error {
+	err := config.KV.Set(config.Ctx, fmt.Sprintf("class:%s", course.CourseReferenceNumber), course, 0).Err()
+	if err != nil {
+		return fmt.Errorf("failed to store class in Redis: %w", err)
+	}
+	return nil
+}

internal/api/search.go

@@ -0,0 +1,336 @@
+package api
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/samber/lo"
+)
+
+const (
+	paramSubject           = "txt_subject"
+	paramTitle             = "txt_courseTitle"
+	paramKeywords          = "txt_keywordlike"
+	paramOpenOnly          = "chk_open_only"
+	paramTermPart          = "txt_partOfTerm"
+	paramCampus            = "txt_campus"
+	paramAttributes        = "txt_attribute"
+	paramInstructor        = "txt_instructor"
+	paramStartTimeHour     = "select_start_hour"
+	paramStartTimeMinute   = "select_start_min"
+	paramStartTimeMeridiem = "select_start_ampm"
+	paramEndTimeHour       = "select_end_hour"
+	paramEndTimeMinute     = "select_end_min"
+	paramEndTimeMeridiem   = "select_end_ampm"
+	paramMinCredits        = "txt_credithourlow"
+	paramMaxCredits        = "txt_credithourhigh"
+	paramCourseNumberLow   = "txt_course_number_range"
+	paramCourseNumberHigh  = "txt_course_number_range_to"
+	paramOffset            = "pageOffset"
+	paramMaxResults        = "pageMaxSize"
+)
+
+type Query struct {
+	subject             *string
+	title               *string
+	keywords            *[]string
+	openOnly            *bool
+	termPart            *[]string // e.g. [1, B6, 8, J]
+	campus              *[]string // e.g. [9, 1DT, 1LR]
+	instructionalMethod *[]string // e.g. [HB]
+	attributes          *[]string // e.g. [060, 010]
+	instructor          *[]uint64 // e.g. [27957, 27961]
+	startTime           *time.Duration
+	endTime             *time.Duration
+	minCredits          *int
+	maxCredits          *int
+	offset              int
+	maxResults          int
+	courseNumberRange   *Range
+}
+
+func NewQuery() *Query {
+	return &Query{maxResults: 8, offset: 0}
+}
+
+// Subject sets the subject for the query
+func (q *Query) Subject(subject string) *Query {
+	q.subject = &subject
+	return q
+}
+
+// Title sets the title for the query
+func (q *Query) Title(title string) *Query {
+	q.title = &title
+	return q
+}
+
+// Keywords sets the keywords for the query
+func (q *Query) Keywords(keywords []string) *Query {
+	q.keywords = &keywords
+	return q
+}
+
+// Keyword adds a keyword to the query
+func (q *Query) Keyword(keyword string) *Query {
+	if q.keywords == nil {
+		q.keywords = &[]string{keyword}
+	} else {
+		*q.keywords = append(*q.keywords, keyword)
+	}
+	return q
+}
+
+// OpenOnly sets the open only flag for the query
+func (q *Query) OpenOnly(openOnly bool) *Query {
+	q.openOnly = &openOnly
+	return q
+}
+
+// TermPart sets the term part for the query
+func (q *Query) TermPart(termPart []string) *Query {
+	q.termPart = &termPart
+	return q
+}
+
+func (q *Query) Campus(campus []string) *Query {
+	q.campus = &campus
+	return q
+}
+
+func (q *Query) InstructionalMethod(instructionalMethod []string) *Query {
+	q.instructionalMethod = &instructionalMethod
+	return q
+}
+
+func (q *Query) Attributes(attributes []string) *Query {
+	q.attributes = &attributes
+	return q
+}
+
+func (q *Query) Instructor(instructor []uint64) *Query {
+	q.instructor = &instructor
+	return q
+}
+
+func (q *Query) StartTime(startTime time.Duration) *Query {
+	q.startTime = &startTime
+	return q
+}
+
+func (q *Query) EndTime(endTime time.Duration) *Query {
+	q.endTime = &endTime
+	return q
+}
+
+func (q *Query) Credits(low int, high int) *Query {
+	q.minCredits = &low
+	q.maxCredits = &high
+	return q
+}
+
+func (q *Query) MinCredits(value int) *Query {
+	q.minCredits = &value
+	return q
+}
+
+func (q *Query) MaxCredits(value int) *Query {
+	q.maxCredits = &value
+	return q
+}
+
+func (q *Query) CourseNumbers(low int, high int) *Query {
+	q.courseNumberRange = &Range{low, high}
+	return q
+}
+
+// Offset sets the offset for the query, allowing for pagination
+func (q *Query) Offset(offset int) *Query {
+	q.offset = offset
+	return q
+}
+
+// MaxResults sets the maximum number of results for the query
+func (q *Query) MaxResults(maxResults int) *Query {
+	q.maxResults = maxResults
+	return q
+}
+
+type Range struct {
+	Low  int
+	High int
+}
+
+// FormatTimeParameter formats a time.Duration into a tuple of strings
+// This is mostly a private helper to keep the parameter formatting for both the start and end time consistent together
+func FormatTimeParameter(d time.Duration) (string, string, string) {
+	hourParameter, minuteParameter, meridiemParameter := "", "", ""
+
+	hours := int64(d.Hours())
+	minutes := int64(d.Minutes()) % 60
+
+	minuteParameter = strconv.FormatInt(minutes, 10)
+
+	if hours >= 12 {
+		hourParameter = "PM"
+
+		// Exceptional case: 12PM = 12, 1PM = 1, 2PM = 2
+		if hours >= 13 {
+			hourParameter = strconv.FormatInt(hours-12, 10) // 13 - 12 = 1, 14 - 12 = 2
+		} else {
+			hourParameter = strconv.FormatInt(hours, 10)
+		}
+	} else {
+		meridiemParameter = "AM"
+		hourParameter = strconv.FormatInt(hours, 10)
+	}
+
+	return hourParameter, minuteParameter, meridiemParameter
+}
+
+// Paramify converts a Query into a map of parameters that can be used in a POST request
+// This function assumes each query key only appears once.
+func (q *Query) Paramify() map[string]string {
+	params := map[string]string{}
+
+	if q.subject != nil {
+		params[paramSubject] = *q.subject
+	}
+
+	if q.title != nil {
+		// Whitespace can prevent valid queries from succeeding
+		params[paramTitle] = strings.TrimSpace(*q.title)
+	}
+
+	if q.keywords != nil {
+		params[paramKeywords] = strings.Join(*q.keywords, " ")
+	}
+
+	if q.openOnly != nil {
+		params[paramOpenOnly] = "true"
+	}
+
+	if q.termPart != nil {
+		params[paramTermPart] = strings.Join(*q.termPart, ",")
+	}
+
+	if q.campus != nil {
+		params[paramCampus] = strings.Join(*q.campus, ",")
+	}
+
+	if q.attributes != nil {
+		params[paramAttributes] = strings.Join(*q.attributes, ",")
+	}
+
+	if q.instructor != nil {
+		params[paramInstructor] = strings.Join(lo.Map(*q.instructor, func(i uint64, _ int) string {
+			return strconv.FormatUint(i, 10)
+		}), ",")
+	}
+
+	if q.startTime != nil {
+		hour, minute, meridiem := FormatTimeParameter(*q.startTime)
+		params[paramStartTimeHour] = hour
+		params[paramStartTimeMinute] = minute
+		params[paramStartTimeMeridiem] = meridiem
+	}
+
+	if q.endTime != nil {
+		hour, minute, meridiem := FormatTimeParameter(*q.endTime)
+		params[paramEndTimeHour] = hour
+		params[paramEndTimeMinute] = minute
+		params[paramEndTimeMeridiem] = meridiem
+	}
+
+	if q.minCredits != nil {
+		params[paramMinCredits] = strconv.Itoa(*q.minCredits)
+	}
+
+	if q.maxCredits != nil {
+		params[paramMaxCredits] = strconv.Itoa(*q.maxCredits)
+	}
+
+	if q.courseNumberRange != nil {
+		params[paramCourseNumberLow] = strconv.Itoa(q.courseNumberRange.Low)
+		params[paramCourseNumberHigh] = strconv.Itoa(q.courseNumberRange.High)
+	}
+
+	params[paramOffset] = strconv.Itoa(q.offset)
+	params[paramMaxResults] = strconv.Itoa(q.maxResults)
+
+	return params
+}
+
+// String returns a string representation of the query, ideal for debugging & logging.
+func (q *Query) String() string {
+	var sb strings.Builder
+
+	if q.subject != nil {
+		fmt.Fprintf(&sb, "subject=%s, ", *q.subject)
+	}
+
+	if q.title != nil {
+		// Whitespace can prevent valid queries from succeeding
+		fmt.Fprintf(&sb, "title=%s, ", strings.TrimSpace(*q.title))
+	}
+
+	if q.keywords != nil {
+		fmt.Fprintf(&sb, "keywords=%s, ", strings.Join(*q.keywords, " "))
+	}
+
+	if q.openOnly != nil {
+		fmt.Fprintf(&sb, "openOnly=%t, ", *q.openOnly)
+	}
+
+	if q.termPart != nil {
+		fmt.Fprintf(&sb, "termPart=%s, ", strings.Join(*q.termPart, ","))
+	}
+
+	if q.campus != nil {
+		fmt.Fprintf(&sb, "campus=%s, ", strings.Join(*q.campus, ","))
+	}
+
+	if q.attributes != nil {
+		fmt.Fprintf(&sb, "attributes=%s, ", strings.Join(*q.attributes, ","))
+	}
+
+	if q.instructor != nil {
+		fmt.Fprintf(&sb, "instructor=%s, ", strings.Join(lo.Map(*q.instructor, func(i uint64, _ int) string {
+			return strconv.FormatUint(i, 10)
+		}), ","))
+	}
+
+	if q.startTime != nil {
+		hour, minute, meridiem := FormatTimeParameter(*q.startTime)
+		fmt.Fprintf(&sb, "startTime=%s:%s%s, ", hour, minute, meridiem)
+	}
+
+	if q.endTime != nil {
+		hour, minute, meridiem := FormatTimeParameter(*q.endTime)
+		fmt.Fprintf(&sb, "endTime=%s:%s%s, ", hour, minute, meridiem)
+	}
+
+	if q.minCredits != nil {
+		fmt.Fprintf(&sb, "minCredits=%d, ", *q.minCredits)
+	}
+
+	if q.maxCredits != nil {
+		fmt.Fprintf(&sb, "maxCredits=%d, ", *q.maxCredits)
+	}
+
+	if q.courseNumberRange != nil {
+		fmt.Fprintf(&sb, "courseNumberRange=%d-%d, ", q.courseNumberRange.Low, q.courseNumberRange.High)
+	}
+
+	fmt.Fprintf(&sb, "offset=%d, ", q.offset)
+	fmt.Fprintf(&sb, "maxResults=%d", q.maxResults)
+
+	return sb.String()
+}
+
+// Dict returns a map representation of the query, ideal for debugging & logging.
+// This dict is represented with zerolog's Event type.
+// func (q *Query) Dict() *zerolog.Event {
+// }

internal/api/session.go

@@ -0,0 +1,54 @@
+package api
+
+import (
+	"banner/internal/config"
+	"banner/internal/utils"
+	"net/url"
+
+	log "github.com/rs/zerolog/log"
+)
+
+func Setup() {
+	// Makes the initial requests that sets up the session cookies for the rest of the application
+	log.Info().Msg("Setting up session...")
+
+	request_queue := []string{
+		"/registration/registration",
+		"/selfServiceMenu/data",
+	}
+
+	for _, path := range request_queue {
+		req := utils.BuildRequest("GET", path, nil)
+		DoRequest(req)
+	}
+
+	// Validate that cookies were set
+	baseUrlParsed, err := url.Parse(config.BaseURL)
+	if err != nil {
+		log.Fatal().Stack().Str("baseURL", config.BaseURL).Err(err).Msg("Failed to parse baseURL")
+	}
+
+	current_cookies := config.Client.Jar.Cookies(baseUrlParsed)
+	required_cookies := map[string]bool{
+		"JSESSIONID": false,
+		"SSB_COOKIE": false,
+	}
+
+	for _, cookie := range current_cookies {
+		_, present := required_cookies[cookie.Name]
+		// Check if this cookie is required
+		if present {
+			required_cookies[cookie.Name] = true
+		}
+	}
+
+	// Check if all required cookies were set
+	for cookieName, cookie_set := range required_cookies {
+		if !cookie_set {
+			log.Warn().Str("cookieName", cookieName).Msg("Required cookie not set")
+		}
+	}
+	log.Debug().Msg("All required cookies set, session setup complete")
+
+	// TODO: Validate that the session allows access to termSelection
+}