diff --git a/directory.go b/directory.go index 7f5e5d7..9aaac03 100644 --- a/directory.go +++ b/directory.go @@ -334,6 +334,66 @@ func GetDirectory(letter rune) ([]Entry, error) { return entries, nil } +func GetFullEntryCached(id string) (*FullEntry, error) { + key := fmt.Sprintf("entry:%s", id) + + // Check if cached + var entry FullEntry + var cached bool + err := db.View(func(txn *badger.Txn) error { + entryItem, err := txn.Get([]byte(key)) + + // Check if key was found + if err == badger.ErrKeyNotFound { + log.Warn().Str("key", key).Msg("Entry Cache Not Found") + return nil + } else if err != nil { + return errors.Wrap(err, "failed to get entry cache") + } + + // Try to read the value + return entryItem.Value(func(val []byte) error { + err := json.Unmarshal(val, &entry) + cached = true + return errors.Wrap(err, "failed to unmarshal entry") + }) + }) + + if err != nil { + log.Error().Err(err).Msg("Failed to load from cache") + } + + // If cached, return it + if cached { + return &entry, nil + } + + // If not cached, get it + entryPtr, err := GetFullEntry(id) + if err != nil { + return nil, err + } + + // Cache it + err = db.Update(func(txn *badger.Txn) error { + // Marshal cookies + marshalledEntry, err := json.Marshal(*entryPtr) + if err != nil { + return errors.Wrap(err, "failed to marshal entry") + } + + // create transaction + log.Debug().Str("id", id).Str("key", key).Msg("Saving to Entry Cache") + return txn.Set([]byte(key), []byte(marshalledEntry)) + }) + + if err != nil { + log.Error().Err(err).Msg("Failed to save to cache") + } + + return entryPtr, nil +} + func GetFullEntry(id string) (*FullEntry, error) { // Build the request directoryPageUrl, _ := url.Parse("https://www.utsa.edu/directory/Person_Detail") diff --git a/main.go b/main.go index 2a3df6b..97cbb40 100644 --- a/main.go +++ b/main.go @@ -136,6 +136,8 @@ func main() { log.Info().Msg("Login Not Required") } + SaveCookies() + // Get the directory for letter := 'A'; letter <= 'Z'; letter++ { go func(letter rune) { @@ -156,12 +158,13 @@ func main() { for entry := range incompleteEntries { log.Debug().Str("name", entry.Name).Msg("Processing Entry") - fullEntry, err := GetFullEntry(entry.Id) + fullEntry, err := GetFullEntryCached(entry.Id) if err != nil { log.Fatal().Err(err).Msg("Failed to get full entry") } log.Debug().Str("name", fullEntry.Name).Str("email", fullEntry.Email).Msg("Entry Processed") + entries <- fullEntry.Email } }()