scla-unsubscribe/directory.go

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/PuerkitoBio/goquery"
	badger "github.com/dgraph-io/badger/v4"
	"github.com/pkg/errors"
	"github.com/rs/zerolog/log"
	"github.com/samber/lo"
)

func Login(username string, password string) error {
	// Setup initial redirected request
	directoryPageUrl, _ := url.Parse("https://www.utsa.edu/directory/Directory?action=Index")
	request, _ := http.NewRequest("GET", directoryPageUrl.String(), nil)
	ApplyUtsaHeaders(request)
	response, err := DoRequestNoRead(request)
	if err != nil {
		return errors.Wrap(err, "error sending initial request")
	}

	// Verify that we were redirected to the login page
	if response.StatusCode != 302 {
		return fmt.Errorf("bad request (no initial redirect)")
	} else {
		log.Debug().Str("location", response.Header.Get("Location")).Msg("Initial Page Redirected")
	}

	// Setup URL for request
	loginPageUrl, _ := url.Parse("https://www.utsa.edu/directory/Account/Login")
	query := loginPageUrl.Query()
	query.Set("ReturnUrl", "/directory/AdvancedSearch")
	loginPageUrl.RawQuery = query.Encode()

	// Build request
	request, _ = http.NewRequest("GET", loginPageUrl.String(), nil)
	ApplyUtsaHeaders(request)

	// Send request
	response, err = DoRequestNoRead(request)
	if err != nil {
		log.Fatal().Err(err).Msg("Error sending login page request")
	}
	doc, err := goquery.NewDocumentFromReader(response.Body)
	defer response.Body.Close()
	if err != nil {
		log.Fatal().Err(err).Msg("Error parsing response body")
	}

	// Get token
	token, _ := doc.Find("input[name='__RequestVerificationToken']").Attr("value")
	log.Debug().Str("token", token).Msg("Token Captured")

	// Build the login request
	form := url.Values{
		"__RequestVerificationToken": {token},
		"myUTSAID":                   {username},
		"passphrase":                 {password},
		"log-me-in":                  {"Log+In"},
	}
	request, _ = http.NewRequest("POST", "https://www.utsa.edu/directory/", strings.NewReader(form.Encode()))
	ApplyUtsaHeaders(request)
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	// Send the login request
	response, err = DoRequestNoRead(request)

	if err != nil {
		log.Fatal().Err(err).Msg("Error sending login request")
	}

	if response.StatusCode != 200 {
		switch response.StatusCode {
		case 302: // ignore

		case 500:
			return fmt.Errorf("bad request (check cookies)")
		default:
			return fmt.Errorf("unknown error")
		}
	}

	// Check for Set-Cookie of ".ADAuthCookie"
	newCookies := response.Header.Values("Set-Cookie")
	authCookie, found := lo.Find(newCookies, func(cookie string) bool {
		return strings.Contains(cookie, ".ADAuthCookie")
	})

	if !found {
		return fmt.Errorf("login failed: could not find auth cookie")
	} else {
		log.Info().Str("authCookie", authCookie).Msg("Auth Cookie Found")
	}

	// Check if redirected to directory page
	if response.Header.Get("Location") != "" {
		log.Debug().Str("location", response.Header.Get("Location")).Msg("Redirected")
	} else {
		return fmt.Errorf("login failed: no redirect")
	}

	// Request the redirect page
	redirectUrl := fmt.Sprintf("%s%s", "https://www.utsa.edu", response.Header.Get("Location"))
	request, _ = http.NewRequest("GET", redirectUrl, nil)
	ApplyUtsaHeaders(request)
	response, err = DoRequestNoRead(request)
	if err != nil {
		return errors.Wrap(err, "error sending redirect request")
	} else if response.StatusCode != 200 {
		return fmt.Errorf("non-200 status after login attempt")
	}

	// Parse the response body
	doc, err = goquery.NewDocumentFromReader(response.Body)
	if err != nil {
		return errors.Wrap(err, "error parsing response body")
	}

	// Look for field validation errors (untested)
	validationErrors := doc.Find("span.field-validation-error")
	if validationErrors.Length() > 0 {
		event := log.Debug().Int("validationErrors", validationErrors.Length())
		validationErrors.Each(func(i int, s *goquery.Selection) {
			event.Str(fmt.Sprintf("err_%d", i+1), s.Text())
		})
		return fmt.Errorf("validation error: %s", validationErrors.First().Text())
	}

	// Look for the 'Log Off' link
	logOffFound := false
	doc.Find("a.dropdown-item").Each(func(i int, s *goquery.Selection) {
		if !logOffFound && strings.Contains(s.Text(), "Log Off") {
			log.Debug().Int("index", i).Msg("Log Off Element Found")
			logOffFound = true
		}
	})

	if !logOffFound {
		return fmt.Errorf("login failed: could not find log off element")
	}

	return nil
}

func CheckLoggedIn() (bool, error) {
	// Check if required cookie exists
	utsaUrl, _ := url.Parse("https://www.utsa.edu")
	cookies := client.Jar.Cookies(utsaUrl)
	_, authCookieFound := lo.Find(cookies, func(cookie *http.Cookie) bool {
		return cookie.Name == ".ADAuthCookie"
	})

	if !authCookieFound {
		log.Debug().Int("count", len(cookies)).Msg("ActiveDirectory Auth Cookie Not Found")
		return false, nil
	}

	// Send a authenticated-only request
	directoryPageUrl, _ := url.Parse("https://www.utsa.edu/directory/AdvancedSearch")
	request, _ := http.NewRequest("GET", directoryPageUrl.String(), nil)
	ApplyUtsaHeaders(request)
	response, err := DoRequestNoRead(request)
	if err != nil {
		return false, errors.Wrap(err, "could not send redirect check request")
	}

	// If it's not a 302
	if response.StatusCode != 302 {
		// No planning for non-200 responses (this will blow up one day, probably a 400 or 500)
		if response.StatusCode != 200 {
			log.Fatal().Int("code", response.StatusCode).Msg("Unexpected Login Check Response Code")
		}

		// Parse the response document
		doc, err := goquery.NewDocumentFromReader(response.Body)
		if err != nil {
			return false, errors.Wrap(err, "error parsing response body")
		}

		// Try to find the log out button
		logOffFound := false
		doc.Find("a.dropdown-item").Each(func(i int, s *goquery.Selection) {
			if !logOffFound && strings.Contains(s.Text(), "Log Off") {
				log.Debug().Int("index", i).Msg("Log Off Element Found")
				logOffFound = true
			}
		})
		return true, nil
	}

	return false, nil
}

func GetFullDirectory() ([]Entry, error) {
	entries := make([]Entry, 0, 500)
	for letter := 'A'; letter <= 'Z'; letter++ {
		letterEntries, err := GetDirectoryCached(letter)
		if err != nil {
			return nil, errors.Wrap(err, "failed to get directory")
		}

		entries = append(entries, letterEntries...)
	}

	return entries, nil
}

func GetDirectoryCached(letter rune) ([]Entry, error) {
	key := fmt.Sprintf("directory:%s", string(letter))

	// Check if cached
	var entries []Entry
	err := db.View(func(txn *badger.Txn) error {
		directoryItem, err := txn.Get([]byte(key))

		// Check if key was found
		if err == badger.ErrKeyNotFound {
			log.Warn().Str("key", key).Msg("Directory Cache Not Found")
			return nil
		} else if err != nil {
			return errors.Wrap(err, "failed to get directory cache")
		}

		// Try to read the value
		entries = make([]Entry, 0, 500)
		return directoryItem.Value(func(val []byte) error {
			err := json.Unmarshal(val, &entries)
			return errors.Wrap(err, "failed to unmarshal directory entries")
		})
	})

	if err != nil {
		log.Error().Err(err).Msg("Failed to load from cache")
	}

	// If cached, return it
	if entries != nil {
		return entries, nil
	}

	// If not cached, get it
	entries, err = GetDirectory(letter)
	if err != nil {
		return nil, errors.Wrap(err, "failed to get directory")
	}

	// Cache it
	err = db.Update(func(txn *badger.Txn) error {
		// Marshal cookies
		marshalledEntries, err := json.Marshal(entries)
		if err != nil {
			return errors.Wrap(err, "failed to marshal directory entries")
		}

		// create transaction
		log.Debug().Str("letter", string(letter)).Str("key", key).Msg("Saving to Directory Cache")
		return txn.Set([]byte(key), []byte(marshalledEntries))
	})

	if err != nil {
		log.Error().Err(err).Msg("Failed to save to cache")
	}

	return entries, nil
}

func GetDirectory(letter rune) ([]Entry, error) {
	// Build the request
	directoryPageUrl, _ := url.Parse("https://www.utsa.edu/directory/SearchByLastName")
	query := directoryPageUrl.Query()
	query.Set("abc", string(letter))
	directoryPageUrl.RawQuery = query.Encode()

	// Send the request
	request, _ := http.NewRequest("GET", directoryPageUrl.String(), nil)
	ApplyUtsaHeaders(request)
	response, err := DoRequestNoRead(request)
	if err != nil {
		return nil, fmt.Errorf("error sending directory request")
	}

	// Parse the response
	doc, err := goquery.NewDocumentFromReader(response.Body)
	if err != nil {
		return nil, fmt.Errorf("error parsing response body")
	}

	// Acquire selector
	rows := doc.Find("table#peopleTable > tbody > tr")
	entries := make([]Entry, 0, rows.Length())
	log.Debug().Int("count", rows.Length()).Msg("Rows Found")

	// Check number of rows
	if rows.Length() < 1 {
		return nil, fmt.Errorf("no rows found in directory")
	} else if rows.Length() <= 20 {
		log.Warn().Int("count", rows.Length()).Msg("Low number of rows found")
	}

	// Iterate over rows
	rows.Each(func(i int, s *goquery.Selection) {
		entry := Entry{}
		nameElement := s.Find("a.fullName")

		// Process the HREF URL into an actual ID
		personPath, exists := nameElement.Attr("href")
		valueIndex := strings.Index(personPath, "abc=")
		if !exists || valueIndex == -1 {
			log.Warn().Str("href", personPath).Msg("Could not find ID in HREF")
			return
		}
		unescapedId, err := url.QueryUnescape(personPath[valueIndex+4:])
		if err != nil {
			log.Warn().Str("href", personPath).Msg("Could not unescape ID")
			return
		}
		entry.Id = unescapedId

		entry.Name = strings.TrimSpace(nameElement.Text())

		entry.JobTitle = strings.TrimSpace(s.Find("span.jobtitle").Text())
		entry.Department = strings.TrimSpace(s.Find("span.dept").Text())
		entry.College = strings.TrimSpace(s.Find("span.college").Text())
		entry.Phone = strings.TrimSpace(s.Find("span.phone").Text())

		entries = append(entries, entry)
	})

	return entries, nil
}

func GetFullEntryCached(id string) (*FullEntry, bool, error) {
	key := fmt.Sprintf("entry:%s", id)

	// Check if cached
	var entry FullEntry
	var cached bool
	err := db.View(func(txn *badger.Txn) error {
		entryItem, err := txn.Get([]byte(key))

		// Check if key was found
		if err == badger.ErrKeyNotFound {
			log.Debug().Str("key", key).Msg("Entry Cache Not Found")
			return nil
		} else if err != nil {
			return errors.Wrap(err, "failed to get entry cache")
		}

		// Try to read the value
		return entryItem.Value(func(val []byte) error {
			err := json.Unmarshal(val, &entry)
			cached = true
			return errors.Wrap(err, "failed to unmarshal entry")
		})
	})

	if err != nil {
		log.Error().Err(err).Msg("Failed to load from cache")
	}

	// If cached, return it
	if cached {
		return &entry, true, nil
	}

	// If not cached, get it
	entryPtr, err := GetFullEntry(id)
	if err != nil {
		return nil, false, err
	}

	// Cache it
	err = db.Update(func(txn *badger.Txn) error {
		// Marshal cookies
		marshalledEntry, err := json.Marshal(*entryPtr)
		if err != nil {
			return errors.Wrap(err, "failed to marshal entry")
		}

		// create transaction
		log.Debug().Str("id", id).Str("key", key).Msg("Saving to Entry Cache")
		return txn.Set([]byte(key), []byte(marshalledEntry))
	})

	if err != nil {
		log.Error().Err(err).Msg("Failed to save to cache")
	}

	return entryPtr, false, nil
}

func GetFullEntry(id string) (*FullEntry, error) {
	// Build the request
	directoryPageUrl, _ := url.Parse("https://www.utsa.edu/directory/Person_Detail")
	query := directoryPageUrl.Query()
	query.Set("abc", id)
	directoryPageUrl.RawQuery = query.Encode()

	// Send the request
	request, _ := http.NewRequest("GET", directoryPageUrl.String(), nil)
	ApplyUtsaHeaders(request)
	response, err := DoRequestNoRead(request)
	if err != nil {
		return nil, fmt.Errorf("error sending directory request")
	}

	// Parse the response
	doc, err := goquery.NewDocumentFromReader(response.Body)
	if err != nil {
		return nil, fmt.Errorf("error parsing response body")
	}

	// Move all rows into a map
	rows := make(map[string]string)
	rowElements := doc.Find("table.detail > tbody > tr")
	log.Debug().Int("count", rowElements.Length()).Msg("Rows Found")

	// Check number of rows
	if rowElements.Length() < 1 {
		return nil, fmt.Errorf("no rows found")
	}

	// Iterate over rows
	rowElements.Each(func(i int, s *goquery.Selection) {
		// left hand column
		rowTitle := NormalizeTitle(strings.TrimRight(
			strings.TrimSpace(s.Find("th > strong").Text()), ":",
		))

		// right hand column, add to map
		rows[rowTitle] = strings.TrimSpace(s.Find("td").Text())
	})

	// Build the entry from the map
	entry := FullEntry{}

	entry.Classification = rows["classification"]
	delete(rows, "classification")

	entry.College = rows["college"]
	delete(rows, "college")

	entry.Major = rows["major"]
	delete(rows, "major")

	entry.Email = rows["email"]
	delete(rows, "email")

	entry.Title = rows["title"]
	delete(rows, "title")

	entry.Department = rows["department"]
	delete(rows, "department")

	entry.MailingAddress = rows["mailing-address"]
	delete(rows, "mailing-address")

	entry.Building = rows["building"]
	delete(rows, "building")

	entry.Phone = rows["phone"]
	delete(rows, "phone")

	entry.Other = rows

	// Multiple names found, collect and log
	nameElement := doc.Find("body > #main span.nameBold > strong")
	if nameElement.Length() > 1 {
		var names []string
		nameElement.Each(func(i int, s *goquery.Selection) {
			names = append(names, strings.TrimSpace(s.Text()))
		})
		log.Warn().Int("count", nameElement.Length()).Interface("names", names).Msg("Multiple Names Found")

		// Use the longest name
		entry.Name = lo.MaxBy(names, func(name string, max string) bool {
			return len(name) > len(max)
		})
	} else {
		entry.Name = strings.TrimSpace(nameElement.Text())
	}

	return &entry, nil
}