init

_fetch.py

@@ -0,0 +1,32 @@
+import requests, json, pprint
+
+class Fetch():
+    def __init__(self, total=999):
+        self.total = total
+        self.retrieve()
+
+    def retrieve(self, total=None):
+        if total==None: total=self.total
+        url = "https://mee6.xyz/api/plugins/levels/leaderboard/166630061217153024?limit={}&page={}"
+        dataset = []
+        pages = total // 999 # Process n amount of pages
+        print(pages)
+        last = total % 999 # Then process the last n amount of pages in addition
+        print(last)
+        for i in range(pages):
+            resp = requests.get(url.format(999, i))
+            dataset.append(resp.json()['players'])
+        if last > 0:
+            resp = requests.get(url.format(last, pages+1))
+            dataset.append(resp.json()['players'])
+        self.data = dataset
+
+    def get(self):
+        return self.data
+
+pp = pprint.PrettyPrinter()    
+fetcher = Fetch(1005)
+dataset = fetcher.get()
+
+print(len(dataset[]))
+# pp.pprint(dataset[0]['players'])

pre-api/_parser.py

@@ -0,0 +1,60 @@
+from selenium import webdriver
+from bs4 import BeautifulSoup
+import time, pprint
+
+class Leaderboard():
+    def __init__(self, pages=1):
+        self.update(pages)
+        self.pp = pprint.PrettyPrinter()
+
+    def parseEnding(self, n):
+        if n.endswith("k"):
+            n = float(n[:-1])
+            n*=1000
+        elif n.endswith("m"):
+            n = float(n[:-1])
+            n*=1000000
+        if int(n) == n:
+            n = int(n)
+        return n
+
+    def get(self):
+        return self.users
+
+    # Get the user listing off the ranks page and update the leaderboard data
+    def update(self, pages=1):
+        self.pages = pages
+        self._get()
+
+    def _get(self):
+        url = "https://mee6.xyz/leaderboard/166630061217153024"
+        browser = webdriver.Chrome()
+        browser.get(url)
+        for page in range(self.pages):
+            browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+            time.sleep(1)
+        html = browser.page_source
+        browser.quit()
+        soup = BeautifulSoup(html, 'lxml')
+        # Processing Web Page
+        table_body = soup.find_all('div', attrs={'class':'leaderboardPlayer'})
+        users = {}
+        for entry in table_body:
+            # User Info section
+            userInfo = entry.find('div', attrs={'class':'leaderboardPlayerLeft'})
+            rank = userInfo.find('div', attrs={'class':'leaderboardRank'}).string
+            icon = userInfo.find('div', attrs={'class':'leaderboardPlayerIcon'}).find('img')['src']
+            username = userInfo.find('div', attrs={'class':'leaderboardPlayerUsername'}).string
+
+            # Stats section
+            userStats = entry.find('div', attrs={'class':'leaderboardPlayerStats'}).find_all('div')
+            messages = userStats[2].string
+            experience = userStats[5].string
+            percent = userStats[6]['class'][1][1:]
+            level = userStats[9].string
+            messages, experience = self.parseEnding(messages), self.parseEnding(experience)
+            users[username] = {'rank':rank, 'icon':icon, 'messages':messages, 'percent': percent, 'experience':experience, 'level':level}
+        self.users = users
+
+    def print(self):
+        self.pp.pprint(self.users)

pre-api/example.html

@@ -0,0 +1,27 @@
+<div class="leaderboardPlayer">
+    <div class="leaderboardPlayerLeft">
+        <div class="leaderboardRank leaderboardRankFirst">1</div>
+        <div class="leaderboardPlayerIcon"><img src="https://cdn.discordapp.com/avatars/122019892093386753/a_db6eb2a7e6efd35dbfb384b18a96a8de.png"></div>
+        <div class="leaderboardPlayerUsername">InsomniaKitten</div>
+    </div>
+    <div class="leaderboardPlayerStats">
+        <div class="leaderboardPlayerStatBlock">
+            <div class="leaderboardPlayerStatName">MESSAGES</div>
+            <div class="leaderboardPlayerStatValue">63.2k</div>
+        </div>
+        <div class="leaderboardPlayerStatBlock">
+            <div class="leaderboardPlayerStatName">EXPERIENCE</div>
+            <div class="leaderboardPlayerStatValue">1.3m</div>
+        </div>
+        <div class="leaderboardPlayerStat p73">
+            <div class="leaderboardPlayerStatText">
+                <div class="leaderboardPlayerStatName">LEVEL</div>
+                <div class="leaderboardPlayerStatValue">86</div>
+            </div>
+            <div class="leaderboardPlayerStatBarContainer">
+                <div class="leaderboardPlayerStatBar"></div>
+                <div class="leaderboardPlayerStatFill"></div>
+            </div>
+        </div>
+    </div>
+</div>

pre-api/level_plot.py

@@ -0,0 +1,46 @@
+from matplotlib import pyplot as plt
+import pprint
+
+def levelXP(n):
+    return (5 * (n ** 2)) + (50 * n) + (100)
+
+def additiveXP(n):
+    return sum([levelXP(x) for x in range(0, n)])
+
+x_values = [x for x in range(0,100)]
+y_values = [additiveXP(y) for y in range(0, 100)]
+
+pp = pprint.PrettyPrinter()
+pp.pprint(x_values)
+pp.pprint(y_values)
+
+def notation(n):
+    n = int(n)
+    qu = 10**18;q = 10**15;t = 10**12;b = 10**9;m = 10**6;k = 10**3
+    if n < k:
+        return str(n)
+    elif n >= qu:
+        n = round(n / qu, 1)
+        return str(n) + " qu"
+    elif n >= q:
+        n = round(n / q, 1)
+        return str(n) + " q"
+    elif n >= t:
+        n = round(n / t, 1)
+        return str(n) + " t"
+    elif n >= b:
+        n = round(n / b, 1)
+        return str(n) + " b"
+    elif n >= m:
+        n = round(n / m, 1)
+        return str(n) + " m"
+    elif n >= k:
+        n = round(n / k, 1)
+        return str(n) + " k"
+
+with plt.xkcd():
+    plt.plot(x_values, y_values)
+    for i, txt in enumerate(y_values):
+        if i % 10 == 0:
+            plt.annotate(notation(txt), (x_values[i], y_values[i]))
+    plt.show()

pre-api/scraper.py

@@ -0,0 +1,7 @@
+from _parser import *
+
+Board = Leaderboard(120)
+import json, os, sys
+with open(os.path.join(sys.path[0], 'data.json'), 'w+') as data:
+    json.dump(Board.get(), data)
+Board.print()