diff --git a/main.go b/main.go index c5b66cb..0c182f6 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,12 @@ import ( var ( client *http.Client db *badger.DB + + // A channel that will be used to buffer incomplete entries that need to be queried properly + incompleteEntries = make(chan Entry) + + // A channel that will be used to buffer emails that need to be unsubscribed + entries = make(chan string) ) func init() { @@ -131,27 +137,31 @@ func main() { } // Get the directory - go func() { - directory, err := GetFullDirectory() - if err != nil { - log.Fatal().Err(err).Msg("Failed to get directory") - } - log.Info().Int("count", len(directory)).Msg("Directory Loaded") + for letter := 'A'; letter <= 'Z'; letter++ { + go func(letter rune) { + letterEntries, err := GetDirectoryCached(string(letter)) + if err != nil { + log.Fatal().Err(err).Msg("Failed to get directory") + } - // Place each entry in the channel - for _, entry := range directory { - incompleteEntries <- entry - } - }() + // Process each entry + for _, entry := range letterEntries { + incompleteEntries <- entry + } + }(letter) + } // Process each incomplete entry - lim := GetLimiter("test") go func() { for entry := range incompleteEntries { - // Wait for a token - Wait(lim, nil) log.Debug().Str("name", entry.Name).Msg("Processing Entry") - entries <- entry.College + + fullEntry, err := GetFullEntry(entry.Id) + if err != nil { + log.Fatal().Err(err).Msg("Failed to get full entry") + } + + log.Debug().Str("name", fullEntry.Name).Str("email", fullEntry.Email).Msg("Entry Processed") } }() diff --git a/pipeline.go b/pipeline.go deleted file mode 100644 index df399d6..0000000 --- a/pipeline.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -// A channel that will be used to buffer incomplete entries that need to be queried properly -var incompleteEntries = make(chan Entry) - -// A channel that will be used to buffer emails that need to be unsubscribed -var entries = make(chan string)