Launch multiple directory routines, GetFullEntry usage

main.go

@@ -17,6 +17,12 @@ import (
 var (
 	client *http.Client
 	db     *badger.DB
+
+	// A channel that will be used to buffer incomplete entries that need to be queried properly
+	incompleteEntries = make(chan Entry)
+
+	// A channel that will be used to buffer emails that need to be unsubscribed
+	entries = make(chan string)
 )
 
 func init() {
@@ -131,27 +137,31 @@ func main() {
 	}
 
 	// Get the directory
-	go func() {
+	for letter := 'A'; letter <= 'Z'; letter++ {
-		directory, err := GetFullDirectory()
+		go func(letter rune) {
-		if err != nil {
+			letterEntries, err := GetDirectoryCached(string(letter))
-			log.Fatal().Err(err).Msg("Failed to get directory")
+			if err != nil {
-		}
+				log.Fatal().Err(err).Msg("Failed to get directory")
-		log.Info().Int("count", len(directory)).Msg("Directory Loaded")
+			}
 
-		// Place each entry in the channel
+			// Process each entry
-		for _, entry := range directory {
+			for _, entry := range letterEntries {
-			incompleteEntries <- entry
+				incompleteEntries <- entry
-		}
+			}
-	}()
+		}(letter)
+	}
 
 	// Process each incomplete entry
-	lim := GetLimiter("test")
 	go func() {
 		for entry := range incompleteEntries {
-			// Wait for a token
-			Wait(lim, nil)
 			log.Debug().Str("name", entry.Name).Msg("Processing Entry")
-			entries <- entry.College
+
+			fullEntry, err := GetFullEntry(entry.Id)
+			if err != nil {
+				log.Fatal().Err(err).Msg("Failed to get full entry")
+			}
+
+			log.Debug().Str("name", fullEntry.Name).Str("email", fullEntry.Email).Msg("Entry Processed")
 		}
 	}()
 

pipeline.go

@@ -1,7 +0,0 @@
-package main
-
-// A channel that will be used to buffer incomplete entries that need to be queried properly
-var incompleteEntries = make(chan Entry)
-
-// A channel that will be used to buffer emails that need to be unsubscribed
-var entries = make(chan string)