diff --git a/app/__init__.py b/app/__init__.py
index f599e65..602b676 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -1,12 +1,13 @@
# Main Flask and Flask configs
from flask import Flask
-from config import Config
-# Flask Extensions
-from flask_sqlalchemy import SQLAlchemy
-from flask_migrate import Migrate
-from flask_login import LoginManager
from flask_limiter import Limiter
from flask_limiter.util import get_remote_address
+from flask_login import LoginManager
+from flask_migrate import Migrate
+# Flask Extensions
+from flask_sqlalchemy import SQLAlchemy
+
+from config import Config
# App & App config setup
app = Flask(__name__)
@@ -21,4 +22,4 @@ limiter = Limiter(app, key_func=get_remote_address, default_limits=["10 per seco
from app import models
from app import routes, simple_routes, hidden, dashboard
-from app import ftbhot, custom, spotify, panzer, sound
\ No newline at end of file
+from app import ftbhot, custom, spotify, panzer, sound
diff --git a/app/custom.py b/app/custom.py
index 3e6f928..6fb7870 100644
--- a/app/custom.py
+++ b/app/custom.py
@@ -1,6 +1,8 @@
+from functools import wraps
+
from flask import abort
from flask_login import current_user
-from functools import wraps
+
def require_role(roles=["User"]):
def wrap(func):
@@ -10,5 +12,7 @@ def require_role(roles=["User"]):
if current_user.has_roles(roles):
return func(*args, **kwargs)
return abort(401)
+
return decorated_view
- return wrap
\ No newline at end of file
+
+ return wrap
diff --git a/app/dashboard.py b/app/dashboard.py
index 2d6c245..6e6a470 100644
--- a/app/dashboard.py
+++ b/app/dashboard.py
@@ -1,15 +1,17 @@
-from app import app, db, login
-from app.forms import ProfileSettingsForm, ProfilePictureForm
-from app.models import User, Search
+from flask import render_template, request, jsonify
+from flask_login import login_required
+
+from app import app
from app.custom import require_role
-from flask import render_template, redirect, url_for, request, jsonify
-from flask_login import current_user, login_required
+from app.forms import ProfileSettingsForm, ProfilePictureForm
+
@app.route('/dashboard')
@login_required
def dashboard():
return render_template('/dashboard/dashboard.html')
+
@app.route('/dashboard/profile_settings', methods=['GET'])
@login_required
def profile_settings():
@@ -17,25 +19,28 @@ def profile_settings():
ppform = ProfilePictureForm()
return render_template('/dashboard/profile_settings.html', psform=psform, ppform=ppform)
+
@app.route('/dashboard/profile_settings/submit', methods=['POST'])
@login_required
def profile_settings_submit():
form = ProfileSettingsForm()
if form.validate_on_submit():
data = {
- 'show_email' : form.show_email.data or None,
- 'profile_picture_file' : request.files
+ 'show_email': form.show_email.data or None,
+ 'profile_picture_file': request.files
}
return jsonify(data=data)
return '{}'
+
@app.route('/dashboard/constants')
@login_required
@require_role(roles=['Admin'])
def constants():
return render_template('/dashboard/constants.html')
+
@app.route('/dashboard/rbac')
@login_required
def rbac():
- return render_template('/dashboard/rbac.html')
\ No newline at end of file
+ return render_template('/dashboard/rbac.html')
diff --git a/app/forms.py b/app/forms.py
index f165a93..228f5a1 100644
--- a/app/forms.py
+++ b/app/forms.py
@@ -1,14 +1,17 @@
from flask_wtf import FlaskForm
from wtforms import StringField, PasswordField, BooleanField, SubmitField, RadioField, FileField
from wtforms.validators import ValidationError, DataRequired, EqualTo, Email, URL
+
from app.models import User
+
class LoginForm(FlaskForm):
username = StringField('Username', validators=[DataRequired()])
password = PasswordField('Password', validators=[DataRequired()])
remember_me = BooleanField('Remember Me')
submit = SubmitField('Sign in')
+
class RegistrationForm(FlaskForm):
username = StringField('Username', validators=[DataRequired()])
email = StringField('Email', validators=[DataRequired(), Email()])
@@ -20,17 +23,21 @@ class RegistrationForm(FlaskForm):
user = User.query.filter_by(username=username.data).first()
if user is not None:
raise ValidationError('That username is not available.')
-
+
def validate_email(self, email):
user = User.query.filter_by(email=email.data).first()
if user is not None:
raise ValidationError('That email address is not available.')
+
class ProfileSettingsForm(FlaskForm):
- show_email = RadioField('Show Email', default='registered', choices=[('public', 'Public'), ('registered', 'Registered Users Only'), ('hidden', 'Hidden')])
+ show_email = RadioField('Show Email', default='registered',
+ choices=[('public', 'Public'), ('registered', 'Registered Users Only'),
+ ('hidden', 'Hidden')])
submit = SubmitField('Save Profile Settings')
+
class ProfilePictureForm(FlaskForm):
profile_picture_file = FileField('Upload Profile Picture')
profile_picture_url = StringField('Use URL for Profile Picture', validators=[URL()])
- submit = SubmitField('Submit Profile Picture')
\ No newline at end of file
+ submit = SubmitField('Submit Profile Picture')
diff --git a/app/ftbhot.py b/app/ftbhot.py
index 30f9351..d71312e 100644
--- a/app/ftbhot.py
+++ b/app/ftbhot.py
@@ -1,22 +1,27 @@
-from app import app
import flask
+from app import app
+
+
@app.route('/ftbhot/about')
@app.route('/ftbhot/about/')
def ftbhot_about():
return flask.render_template('/ftbhot/about.html')
+
@app.route('/ftbhot/auth')
@app.route('/ftbhot/auth/')
def ftbhot_auth():
return 'WIP'
+
@app.route('/ftbhot')
@app.route('/ftbhot/')
def ftbhot():
return flask.render_template('/ftbhot/embed.html')
+
@app.route('/ftbhot/json')
@app.route('/ftbhot/json/')
def ftbhot_embed():
- return flask.render_template('/ftbhot/current.json')
\ No newline at end of file
+ return flask.render_template('/ftbhot/current.json')
diff --git a/app/hidden.py b/app/hidden.py
index b5cb0af..43b023e 100644
--- a/app/hidden.py
+++ b/app/hidden.py
@@ -1,13 +1,16 @@
-from app import app, db, login
-from app.models import Search
-from app.custom import require_role
-from flask_login import current_user, login_user, logout_user, login_required
-from flask import request, render_template
-import requests
-import xmltodict
import base64
import json
+import requests
+import xmltodict
+from flask import request, render_template
+from flask_login import current_user, login_required
+
+from app import app, db
+from app.custom import require_role
+from app.models import Search
+
+
@app.route('/hidden/history')
@login_required
@require_role(roles=['Hidden', 'Admin'])
@@ -21,6 +24,7 @@ def hidden_history():
def hidden_help():
return render_template('hidden_help.html')
+
# Parses strings to test for "boolean-ness"
def boolparse(string, default=False):
trues = ['true', '1']
@@ -30,6 +34,7 @@ def boolparse(string, default=False):
return True
return False
+
@app.route('/hidden/')
@login_required
@require_role(roles=['Hidden'])
@@ -39,7 +44,8 @@ def hidden():
try:
page = int(request.args.get('page') or 1)
except (TypeError, ValueError):
- return '\"page\" parameter must be Integer.
Invalid \"page\" parameter: \"{}\"'.format(request.args.get('page'))
+ return '\"page\" parameter must be Integer.
Invalid \"page\" parameter: \"{}\"'.format(
+ request.args.get('page'))
# Handled within building
try:
count = int(request.args.get('count') or 50)
@@ -50,7 +56,7 @@ def hidden():
showfull = boolparse(request.args.get('showfull'))
showtags = boolparse(request.args.get('showtags'))
# Request, Parse & Build Data
- data = build_data(tags, page-1, count, base64, showfull)
+ data = build_data(tags, page - 1, count, base64, showfull)
# Handling for limiters
if base64:
if showfull:
@@ -60,14 +66,18 @@ def hidden():
search = Search(user_id=current_user.id, exact_url=str(request.url), query_args=json.dumps(request.args.to_dict()))
db.session.add(search)
db.session.commit()
- return render_template('hidden.html', title='Gelbooru Browser', data=data, tags=tags, page=page, count=count, base64=base64, showfull=showfull, showtags=showtags)
+ return render_template('hidden.html', title='Gelbooru Browser', data=data, tags=tags, page=page, count=count,
+ base64=base64, showfull=showfull, showtags=showtags)
+
def base64ify(url):
return base64.b64encode(requests.get(url).content).decode()
+
gelbooru_api_url = "https://gelbooru.com/index.php?page=dapi&s=post&q=index&tags={}&pid={}&limit={}"
gelbooru_view_url = "https://gelbooru.com/index.php?page=post&s=view&id={}"
+
def build_data(tags, page, count, base64, showfull):
# URL Building & Request
temp = gelbooru_api_url.format(tags, page, count)
@@ -75,7 +85,7 @@ def build_data(tags, page, count, base64, showfull):
# XML Parsing & Data Building
parse = xmltodict.parse(response)
build = []
-
+
try:
parse['posts']['post']
except KeyError:
@@ -83,13 +93,13 @@ def build_data(tags, page, count, base64, showfull):
for index, element in enumerate(parse['posts']['post'][:count]):
temp = {
- 'index' : str(index + 1),
- 'real_url' : element['@file_url'],
- 'sample_url' : element['@preview_url'],
- # strips tags, ensures no empty tags (may be unnecessary)
- 'tags' : list(filter(lambda tag : tag != '', [tag.strip() for tag in element['@tags'].split(' ')])),
- 'view' : gelbooru_view_url.format(element['@id'])
- }
+ 'index': str(index + 1),
+ 'real_url': element['@file_url'],
+ 'sample_url': element['@preview_url'],
+ # strips tags, ensures no empty tags (may be unnecessary)
+ 'tags': list(filter(lambda tag: tag != '', [tag.strip() for tag in element['@tags'].split(' ')])),
+ 'view': gelbooru_view_url.format(element['@id'])
+ }
if base64:
if not showfull:
temp['base64'] = base64ify(temp['sample_url'])
@@ -97,4 +107,4 @@ def build_data(tags, page, count, base64, showfull):
temp['base64'] = base64ify(temp['real_url'])
build.append(temp)
- return build
\ No newline at end of file
+ return build
diff --git a/app/models.py b/app/models.py
index a40b7e9..580d771 100644
--- a/app/models.py
+++ b/app/models.py
@@ -1,9 +1,11 @@
-from flask import abort
-from flask_login import UserMixin
from datetime import datetime
-from app import db, login
+
+from flask_login import UserMixin
from werkzeug.security import generate_password_hash, check_password_hash
+from app import db, login
+
+
# Just a note, my role system is really quite terrible, but I've implemented as good as a system as I can for a simple RBAC without Hierarchy.
# Once could create a complex system, but it would be better to properly work with SQLAlchemy to create proper permissions, hierarchy, parent/child etc. rather than to work with simple strings.
# One should look into perhaps Pickled Python objects if they were interested in simplifying interactions while opening a lot more data storage.
@@ -20,7 +22,7 @@ class User(UserMixin, db.Model):
about_me = db.Column(db.String(320))
last_seen = db.Column(db.DateTime, default=datetime.utcnow)
show_email = db.Column(db.Boolean, default=False)
-
+
def set_password(self, password):
self.password_hash = generate_password_hash(password)
@@ -28,7 +30,7 @@ class User(UserMixin, db.Model):
if self.password_hash is None:
raise "{} has no password_hash set!".format(self.__repr__())
return check_password_hash(self.password_hash, password)
-
+
# Retains order while making sure that there are no duplicate role values and they are capitalized
def post_role_processing(self):
user_roles = self.get_roles()
@@ -51,7 +53,7 @@ class User(UserMixin, db.Model):
raise e
success = False
return success
-
+
def get_roles(self):
return self.uroles.split(' ')
@@ -70,7 +72,7 @@ class User(UserMixin, db.Model):
self.uroles = user_roles
if postprocess:
self.post_role_processing()
-
+
def has_role(self, role):
return self.has_roles([role])
@@ -92,6 +94,7 @@ class User(UserMixin, db.Model):
def __repr__(self):
return ''.format(self.username)
+
class Search(db.Model):
id = db.Column(db.Integer, primary_key=True)
exact_url = db.Column(db.String(160))
@@ -102,6 +105,7 @@ class Search(db.Model):
def __repr__(self):
return ''.format(User.query.filter_by(id=self.user_id).first().username, self.timestamp)
+
class Post(db.Model):
id = db.Column(db.Integer, primary_key=True)
body = db.Column(db.String(140))
@@ -111,6 +115,7 @@ class Post(db.Model):
def __repr__(self):
return ''.format(self.body)
+
@login.user_loader
def load_user(id):
- return User.query.get(int(id))
\ No newline at end of file
+ return User.query.get(int(id))
diff --git a/app/panzer.py b/app/panzer.py
index 7f74468..c9f88f5 100644
--- a/app/panzer.py
+++ b/app/panzer.py
@@ -1,8 +1,11 @@
-from app import app
-from textwrap import wrap
-from PIL import Image, ImageDraw, ImageFont
from io import BytesIO
+from textwrap import wrap
+
import flask
+from PIL import Image, ImageDraw, ImageFont
+
+from app import app
+
@app.route('/panzer/')
@app.route('/panzer')
@@ -14,6 +17,7 @@ def panzer(string='bionicles are cooler than sex'):
image = create_panzer(string)
return serve_pil_image(image)
+
def create_panzer(string):
img = Image.open("./app/static/panzer.jpeg")
draw = ImageDraw.Draw(img)
@@ -27,8 +31,9 @@ def create_panzer(string):
draw.text((topleft[0], topleft[1] + (y * 33)), text, font=font2)
return img
+
def serve_pil_image(pil_img):
img_io = BytesIO()
pil_img.save(img_io, 'JPEG', quality=50)
img_io.seek(0)
- return flask.send_file(img_io, mimetype='image/jpeg')
\ No newline at end of file
+ return flask.send_file(img_io, mimetype='image/jpeg')
diff --git a/app/routes.py b/app/routes.py
index c5a4834..dbb23d9 100644
--- a/app/routes.py
+++ b/app/routes.py
@@ -1,35 +1,37 @@
-from app import app, db, login
-from app.models import User, Search
-from app.forms import LoginForm, RegistrationForm
-from app.custom import require_role
-from werkzeug.urls import url_parse
-from flask import render_template, redirect, url_for, flash, request, jsonify, abort, send_file
-from flask_login import current_user, login_user, logout_user, login_required
-from multiprocessing import Value
-import flask
-import requests
-import xmltodict
-import random
-import string
-import faker
import json
import pprint
-import os
-import sys
+import random
+import string
+
+import faker
+import requests
+from flask import render_template, redirect, url_for, flash, request, jsonify
+from flask_login import current_user, login_user, logout_user, login_required
+from werkzeug.urls import url_parse
+
+from app import app, db
+from app.custom import require_role
+from app.forms import LoginForm, RegistrationForm
+from app.models import User
print = pprint.PrettyPrinter().pprint
fake = faker.Faker()
-strgen = lambda length, charset=string.ascii_letters, weights=None : ''.join(random.choices(list(charset), k=length, weights=weights))
+strgen = lambda length, charset=string.ascii_letters, weights=None: ''.join(
+ random.choices(list(charset), k=length, weights=weights))
+
@app.route('/', subdomain='api')
def api_index():
return "api"
+
@app.route('/time/')
def time():
value = request.args.get('value')
if not value:
- return '
'.join(['[int] value', '[int list] lengths', '[string list] strings', '[boolean] reverse', '[string] pluralappend', '[boolean] synonym'])
+ return '
'.join(
+ ['[int] value', '[int list] lengths', '[string list] strings', '[boolean] reverse', '[string] pluralappend',
+ '[boolean] synonym'])
value = int(value)
lengths = request.args.get('lengths')
if lengths: lengths = lengths.split(',')
@@ -40,9 +42,13 @@ def time():
if lengths: lengths = list(map(int, lengths))
reverse = request.args.get('reverse')
if reverse: reverse = bool(reverse)
- return timeformat(value=value, lengths=lengths or [60, 60, 24, 365], strings=strings or ['second', 'minute', 'hour', 'day', 'year'], reverse=True if reverse is None else reverse)
+ return timeformat(value=value, lengths=lengths or [60, 60, 24, 365],
+ strings=strings or ['second', 'minute', 'hour', 'day', 'year'],
+ reverse=True if reverse is None else reverse)
-def timeformat(value, lengths=[60, 60, 24, 365], strings=['second', 'minute', 'hour', 'day', 'year'], reverse=True, pluralappend='s', synonym=False):
+
+def timeformat(value, lengths=[60, 60, 24, 365], strings=['second', 'minute', 'hour', 'day', 'year'], reverse=True,
+ pluralappend='s', synonym=False):
converted = [value]
for index, length in enumerate(lengths):
temp = converted[-1] // length
@@ -51,18 +57,20 @@ def timeformat(value, lengths=[60, 60, 24, 365], strings=['second', 'minute', 'h
if temp != 0:
converted.append(temp)
else:
- break
+ break
strings = strings[:len(converted)]
- build = ['{} {}'.format(value, strings[i] + pluralappend if value > 1 or value == 0 else strings[i]) for i, value in enumerate(converted)][::-1]
+ build = ['{} {}'.format(value, strings[i] + pluralappend if value > 1 or value == 0 else strings[i]) for i, value in
+ enumerate(converted)][::-1]
build = ', '.join(build)
return build
+
@app.route('/avatar/')
@app.route('/avatar//')
@app.route('/avatar/')
def getAvatar(id=''):
# Constants
- headers = {'Authorization' : f'Bot {app.config["DISCORD_TOKEN"]}'}
+ headers = {'Authorization': f'Bot {app.config["DISCORD_TOKEN"]}'}
api = "https://discordapp.com/api/v6/users/{}"
cdn = "https://cdn.discordapp.com/avatars/{}/{}.png"
# Get User Data which contains Avatar Hash
@@ -73,23 +81,25 @@ def getAvatar(id=''):
url = cdn.format(id, user['avatar'])
return "
".format(url)
+
@app.route('/userinfo/')
@login_required
@require_role(roles=['Admin'])
def user_info():
prepare = {
- 'id' : current_user.get_id(),
- 'email' : current_user.email,
- 'username' : current_user.username,
- 'password_hash' : current_user.password_hash,
- 'is_active' : current_user.is_active,
- 'is_anonymous' : current_user.is_anonymous,
- 'is_authenticated' : current_user.is_authenticated,
- 'metadata' : current_user.metadata.info,
- 'uroles' : current_user.get_roles()
+ 'id': current_user.get_id(),
+ 'email': current_user.email,
+ 'username': current_user.username,
+ 'password_hash': current_user.password_hash,
+ 'is_active': current_user.is_active,
+ 'is_anonymous': current_user.is_anonymous,
+ 'is_authenticated': current_user.is_authenticated,
+ 'metadata': current_user.metadata.info,
+ 'uroles': current_user.get_roles()
}
return jsonify(prepare)
+
@app.route('/')
def index():
jobs = [
@@ -101,6 +111,7 @@ def index():
]
return render_template('index.html', job=random.choice(jobs))
+
@app.route('/register/', methods=['GET', 'POST'])
def register():
if current_user.is_authenticated:
@@ -115,6 +126,7 @@ def register():
return redirect(url_for('login'))
return render_template('register.html', title='Register', form=form, hideRegister=True)
+
@app.route('/login/', methods=['GET', 'POST'])
def login():
if current_user.is_authenticated:
@@ -132,7 +144,8 @@ def login():
return redirect(next_page)
return render_template('login.html', title='Login', form=form, hideLogin=True)
+
@app.route('/logout/')
def logout():
logout_user()
- return redirect(url_for('index'))
\ No newline at end of file
+ return redirect(url_for('index'))
diff --git a/app/simple_routes.py b/app/simple_routes.py
index 7634e4b..b08e36b 100644
--- a/app/simple_routes.py
+++ b/app/simple_routes.py
@@ -1,27 +1,35 @@
-from app import app
-from flask import send_from_directory, redirect, url_for, render_template
-import mistune
import os
+import mistune
+from flask import send_from_directory, redirect, url_for, render_template
+
+from app import app
+
markdown = mistune.Markdown()
+
@app.route('/keybase.txt')
def keybase():
return app.send_static_file('keybase.txt')
+
@app.route('/modpacks')
def modpacks():
return markdown(open(os.path.join(app.root_path, 'static', 'MODPACKS.MD'), 'r').read())
+
@app.route('/favicon.ico')
def favicon():
- return send_from_directory(os.path.join(app.root_path, 'static'), 'favicon.ico', mimetype='image/vnd.microsoft.icon')
+ return send_from_directory(os.path.join(app.root_path, 'static'), 'favicon.ico',
+ mimetype='image/vnd.microsoft.icon')
+
@app.errorhandler(401)
def unauthorized(e):
return redirect(url_for('login'))
+
@app.errorhandler(404)
def page_not_found(e):
# note that we set the 404 status explicitly
- return render_template('error.html', code=404, message='Content not found...'), 404
\ No newline at end of file
+ return render_template('error.html', code=404, message='Content not found...'), 404
diff --git a/app/sound.py b/app/sound.py
index 44fb831..652bb23 100644
--- a/app/sound.py
+++ b/app/sound.py
@@ -1,20 +1,16 @@
-from app import app, db, limiter
-from app.sound_models import YouTubeAudio, SoundcloudAudio, CouldNotDecode, CouldNotDownload, CouldNotProcess
-from flask import Response, send_file, redirect, url_for, render_template, request, jsonify
+from flask import Response, send_file, request, jsonify
from flask_login import current_user
-from multiprocessing import Value
-from mutagen.mp3 import MP3
-import os
-import re
-import json
-import subprocess
+
+from app import app, db, limiter
+from app.sound_models import YouTubeAudio, CouldNotDecode, CouldNotDownload, CouldNotProcess
# Selection of Lambdas for creating new responses
# Not sure if Responses change based on Request Context, but it doesn't hurt.
-getBadRequest = lambda : Response('Bad request', status=400, mimetype='text/plain')
-getNotImplemented = lambda : Response('Not implemented', status=501, mimetype='text/plain')
-getInvalidID = lambda : Response('Invalid ID', status=400, mimetype='text/plain')
-getNotDownloaded = lambda : Response('Media not yet downloaded', status=400, mimetype='text/plain')
+getBadRequest = lambda: Response('Bad request', status=400, mimetype='text/plain')
+getNotImplemented = lambda: Response('Not implemented', status=501, mimetype='text/plain')
+getInvalidID = lambda: Response('Invalid ID', status=400, mimetype='text/plain')
+getNotDownloaded = lambda: Response('Media not yet downloaded', status=400, mimetype='text/plain')
+
# Retrieves the YouTubeAudio object relevant to the mediaid if available. If not, it facilitates the creation and writing of one.
# Also helps with access times.
@@ -22,22 +18,24 @@ def get_youtube(mediaid):
audio = YouTubeAudio.query.get(mediaid)
if audio is not None:
audio.access()
- return audio # sets the access time to now
+ return audio # sets the access time to now
else:
- audio = YouTubeAudio(id=mediaid)
- audio.fill_metadata()
- audio.download()
- # Commit and save new audio object into the database
- db.session.add(audio)
- db.session.commit()
- return audio
+ audio = YouTubeAudio(id=mediaid)
+ audio.fill_metadata()
+ audio.download()
+ # Commit and save new audio object into the database
+ db.session.add(audio)
+ db.session.commit()
+ return audio
+
basic_responses = {
- CouldNotDecode : 'Could not decode process response.',
- CouldNotDownload : 'Could not download video.',
- CouldNotProcess : 'Could not process.'
+ CouldNotDecode: 'Could not decode process response.',
+ CouldNotDownload: 'Could not download video.',
+ CouldNotProcess: 'Could not process.'
}
+
# A simple function among the routes to determine what should be returned.
# Not particularly sure how request context is passed, but it seems that either it passed or can access current_user's authenitcation/role's properly, so no problem.
# Shows error in full context IF authenticated + admin, otherwise basic error description, OTHERWISE a basic error message.
@@ -48,7 +46,8 @@ def errorCheck(e):
raise e
if current_user.is_authenticated and current_user.has_role('Admin'): response = str(e) + '\n' + response
return Response(response, status=200, mimetype='text/plain')
-
+
+
# Under the request context, it grabs the same args needed to decide whether the stream has been downloaded previously
# It applies rate limiting differently based on service, and whether the stream has been accessed previously
def downloadLimiter():
@@ -60,9 +59,10 @@ def downloadLimiter():
else:
return '10/minute'
+
# Streams back the specified media back to the client
@app.route('/stream//')
-@limiter.limit(downloadLimiter, lambda : 'global', error_message='429 Too Many Requests')
+@limiter.limit(downloadLimiter, lambda: 'global', error_message='429 Too Many Requests')
def stream(service, mediaid):
if service == 'youtube':
if YouTubeAudio.isValid(mediaid):
@@ -80,6 +80,7 @@ def stream(service, mediaid):
else:
return getBadRequest()
+
# Returns the duration of a specific media
@app.route('/duration//')
def duration(service, mediaid):
@@ -93,6 +94,7 @@ def duration(service, mediaid):
else:
return getBadRequest()
+
# Returns a detailed JSON export of a specific database entry.
# Will not create a new database entry where one didn't exist before.
@app.route('/status//')
@@ -113,6 +115,7 @@ def status(service, mediaid):
else:
return getBadRequest()
+
@app.route('/list/')
def list(service):
if service == 'youtube':
@@ -125,6 +128,7 @@ def list(service):
else:
return getBadRequest()
+
@app.route('/all/')
def all(service):
if service == 'youtube':
@@ -136,4 +140,4 @@ def all(service):
elif service == 'spotify':
return getNotImplemented()
else:
- return getBadRequest()
\ No newline at end of file
+ return getBadRequest()
diff --git a/app/sound_models.py b/app/sound_models.py
index fe4870c..5d885f3 100644
--- a/app/sound_models.py
+++ b/app/sound_models.py
@@ -1,9 +1,11 @@
-from datetime import datetime
-from app import db
-import subprocess
import json
import os
import re
+import subprocess
+from datetime import datetime
+
+from app import db
+
# Returned when a erroring status code is returned. May end up hitting false positives, where the file was still produced properly
# yet a erroring status code was returned. May be a good measure to always disconnect when a error code is found.
@@ -11,27 +13,31 @@ import re
class CouldNotProcess(Exception):
pass
+
# Shouldn't happen in most cases. When a file isn't found, yet the status code for the process returned positive.
class CouldNotDownload(Exception):
pass
+
# When a JSON returning command returns undecodable JSON
# This shouldn't occur and will only be available when a unforseen error occurs where JSON cannot be read,
# yet a non-erroring status code was returned!
class CouldNotDecode(Exception):
pass
+
# A Database Object describing a Audio File originating from YouTube
# Stores basic information like Title/Uploader/URL etc. as well as holds methods useful
# for manipulating, deleting, downloading, updating, and accessing the relevant information or file.
class YouTubeAudio(db.Model):
- id = db.Column(db.String(11), primary_key=True) # 11 char id, presumed to stay the same for the long haul. Should be able to change to 12 chars.
- url = db.Column(db.String(64)) # 43 -> 64
- title = db.Column(db.String(128)) # 120 > 128
- creator = db.Column(db.String(128)) # Seems to be Uploader set, so be careful with this
- uploader = db.Column(db.String(32)) # 20 -> 32
- filename = db.Column(db.String(156)) # 128 + 11 + 1 -> 156
- duration = db.Column(db.Integer)
+ id = db.Column(db.String(11),
+ primary_key=True) # 11 char id, presumed to stay the same for the long haul. Should be able to change to 12 chars.
+ url = db.Column(db.String(64)) # 43 -> 64
+ title = db.Column(db.String(128)) # 120 > 128
+ creator = db.Column(db.String(128)) # Seems to be Uploader set, so be careful with this
+ uploader = db.Column(db.String(32)) # 20 -> 32
+ filename = db.Column(db.String(156)) # 128 + 11 + 1 -> 156
+ duration = db.Column(db.Integer)
access_count = db.Column(db.Integer, default=0)
download_timestamp = db.Column(db.DateTime, index=True, default=datetime.utcnow)
last_access_timestamp = db.Column(db.DateTime, index=True, default=datetime.utcnow)
@@ -61,20 +67,23 @@ class YouTubeAudio(db.Model):
self.filename = self.id + '.mp3'
command = f'youtube-dl -4 -x --audio-format mp3 --restrict-filenames --dump-json {self.id}'
process = subprocess.Popen(command.split(' '),
- encoding='utf-8', stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ encoding='utf-8', stdout=subprocess.PIPE, stderr=subprocess.PIPE)
data = process.communicate()
if process.returncode != 0:
- raise CouldNotProcess(f'Command: {command}\n{data[1]}Exit Code: {process.returncode}') # process ends with a newline, not needed between
+ raise CouldNotProcess(
+ f'Command: {command}\n{data[1]}Exit Code: {process.returncode}') # process ends with a newline, not needed between
try:
data = json.loads(data[0])
except json.JSONDecodeError:
- raise CouldNotDecode(data) # We'll return the process data, figure out what to do with it higher up in stack (return/diagnose etc.)
+ raise CouldNotDecode(
+ data) # We'll return the process data, figure out what to do with it higher up in stack (return/diagnose etc.)
print(f'JSON acquired for {self.id}, beginning to fill.')
self.duration = data['duration']
- self.url = data['webpage_url'] # Could be created, but we'll just infer from JSON response
+ self.url = data['webpage_url'] # Could be created, but we'll just infer from JSON response
self.creator = data['creator'] or data['uploader']
self.uploader = data['uploader'] or data['creator']
- self.title = data['title'] or data['alt_title'] # Do not trust alt-title ; it is volatile and uploader set, e.x. https://i.imgur.com/Tgff4rI.png
+ self.title = data['title'] or data[
+ 'alt_title'] # Do not trust alt-title ; it is volatile and uploader set, e.x. https://i.imgur.com/Tgff4rI.png
print(f'Metadata filled for {self.id}')
db.session.commit()
@@ -83,26 +92,27 @@ class YouTubeAudio(db.Model):
print(f'Attempting download of {self.id}')
command = f'youtube-dl -x -4 --restrict-filenames --audio-quality 64K --audio-format mp3 -o ./app/sounds/youtube/%(id)s.%(ext)s {self.id}'
process = subprocess.Popen(command.split(' '), encoding='utf-8', stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- data = process.communicate() # Not the data for the mp3, just the output. We have to separate this in order to 'wait' for the process to complete fully.
+ data = process.communicate() # Not the data for the mp3, just the output. We have to separate this in order to 'wait' for the process to complete fully.
print('Checking process return code...')
if process.returncode != 0:
raise CouldNotProcess(f'Command: {command}\n{data[1] or data[0]}Exit Code: {process.returncode}')
print('Checking for expected file...')
if not os.path.exists(self.getPath()):
raise CouldNotDownload(data[1] or data[0])
- print(f'Download attempt for {self.id} finished successfully.')
+ print(f'Download attempt for {self.id} finished successfully.')
+
+ # Validates whether the specified ID could be a valid YouTube video ID
- # Validates whether the specified ID could be a valid YouTube video ID
@staticmethod
def isValid(id):
return re.match(r'^[A-Za-z0-9_-]{11}$', id) is not None
# Returns a JSON serialization of the database entry
def toJSON(self, noConvert=False):
- data = {'id' : self.id, 'url' : self.url, 'title' : self.title, 'creator' : self.creator,
- 'uploader' : self.uploader, 'filename' : self.filename, 'duration' : self.duration,
- 'access_count' : self.access_count, 'download_timestamp' : self.download_timestamp.isoformat(),
- 'last_access_timestamp' : self.last_access_timestamp.isoformat()}
+ data = {'id': self.id, 'url': self.url, 'title': self.title, 'creator': self.creator,
+ 'uploader': self.uploader, 'filename': self.filename, 'duration': self.duration,
+ 'access_count': self.access_count, 'download_timestamp': self.download_timestamp.isoformat(),
+ 'last_access_timestamp': self.last_access_timestamp.isoformat()}
return data if noConvert else json.dumps(data)
def delete(self):
@@ -114,10 +124,11 @@ class YouTubeAudio(db.Model):
db.session.delete(self)
db.session.commit()
+
class SoundcloudAudio(db.Model):
- id = db.Column(db.Integer, primary_key=True) # hidden API-accessible only ID
+ id = db.Column(db.Integer, primary_key=True) # hidden API-accessible only ID
url = db.Column(db.String(256))
title = db.Column(db.String(128))
creator = db.Column(db.String(64))
filename = db.Column(db.String(156))
- duration = db.Column(db.Integer)
\ No newline at end of file
+ duration = db.Column(db.Integer)
diff --git a/app/spotify.py b/app/spotify.py
index 4aa2d88..a7cbd80 100644
--- a/app/spotify.py
+++ b/app/spotify.py
@@ -1,22 +1,23 @@
+import json
+import os
+import time
+
+from flask import send_file
+
from app import app
from config import Config
-from flask import send_from_directory, redirect, url_for, render_template, send_file
-import json
-import subprocess
-import time
-import os
from .spotify_explicit import main
-
path = os.path.join('app/spotify_explicit/recent.json')
+
def check_and_update():
try:
with open(path) as file:
file = json.load(file)
except (FileNotFoundError, json.JSONDecodeError):
- file = {'last_generated' : -1}
-
+ file = {'last_generated': -1}
+
if file['last_generated'] == -1:
return True
else:
@@ -28,12 +29,13 @@ def check_and_update():
ideal = file['last_generated'] + Config.SPOTIFY_CACHE_TIME
# print(f'Waiting another {int(ideal - time.time())} seconds')
return False
-
+
+
@app.route('/spotify/')
def spotify():
if check_and_update():
print('Graph out of date - running update command')
with open(path, 'w+') as file:
- file = json.dump({'last_generated' : int(time.time())}, file)
+ file = json.dump({'last_generated': int(time.time())}, file)
main.main()
- return send_file('spotify_explicit/export/export.png')
\ No newline at end of file
+ return send_file('spotify_explicit/export/export.png')
diff --git a/app/spotify_explicit/auth.py b/app/spotify_explicit/auth.py
index 994ef6c..ca7a6a1 100644
--- a/app/spotify_explicit/auth.py
+++ b/app/spotify_explicit/auth.py
@@ -1,4 +1,7 @@
-import logging, sys, os, json
+import json
+import logging
+import os
+import sys
# Path to API Credentials file
PATH = os.path.join(sys.path[0], 'auth.json')
@@ -9,11 +12,11 @@ if not os.path.exists(PATH):
# Dump a pretty-printed dictionary with default values
json.dump(
{
- 'USERNAME' : 'Your Username Here',
- 'CLIENT_ID' : 'Your Client ID Here',
- 'CLIENT_SECRET' : 'Your Client Secret Here',
- 'REDIRECT_URI' : 'Your Redirect URI Callback Here',
- 'SCOPE' : ['Your Scopes Here']
+ 'USERNAME': 'Your Username Here',
+ 'CLIENT_ID': 'Your Client ID Here',
+ 'CLIENT_SECRET': 'Your Client Secret Here',
+ 'REDIRECT_URI': 'Your Redirect URI Callback Here',
+ 'SCOPE': ['Your Scopes Here']
},
file,
indent=3
@@ -31,4 +34,4 @@ USERNAME = FILE['USERNAME']
CLIENT_ID = FILE['CLIENT_ID']
CLIENT_SECRET = FILE['CLIENT_SECRET']
REDIRECT_URI = FILE['REDIRECT_URI']
-SCOPE = ' '.join(FILE['SCOPE'])
\ No newline at end of file
+SCOPE = ' '.join(FILE['SCOPE'])
diff --git a/app/spotify_explicit/main.py b/app/spotify_explicit/main.py
index 741df1b..9bfe1b5 100644
--- a/app/spotify_explicit/main.py
+++ b/app/spotify_explicit/main.py
@@ -1,11 +1,12 @@
+import json
+import logging
import os
import sys
import time
-import json
+
from . import auth
-from . import pull
from . import process
-import logging
+from . import pull
def main():
@@ -14,6 +15,7 @@ def main():
refresh()
process.main()
+
# Refreshes tracks from files if the token from Spotipy has expired,
# thus keeping us up to date in most cases while keeping rate limits
def refresh():
@@ -28,5 +30,6 @@ def refresh():
else:
pull.main()
+
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/app/spotify_explicit/process.py b/app/spotify_explicit/process.py
index 47c277e..e408c3c 100644
--- a/app/spotify_explicit/process.py
+++ b/app/spotify_explicit/process.py
@@ -1,13 +1,11 @@
-import os
-import sys
import json
import logging
-import datetime
-import collections
-import numpy as np
+import os
+
import dateutil.parser
-import PIL.Image as Image
import matplotlib.pyplot as plt
+import numpy as np
+
# Gets all files in tracks folder, returns them in parsed JSON
def get_files():
@@ -20,6 +18,7 @@ def get_files():
)
return files
+
# Simple function to combine a bunch of items from different files
def combine_files(files):
items = []
@@ -27,6 +26,7 @@ def combine_files(files):
items.extend(file['items'])
return items
+
# Prints the data in a interesting format
def print_data(data):
for i, item in enumerate(data):
@@ -36,6 +36,7 @@ def print_data(data):
artists = ' & '.join(artist['name'] for artist in item['track']['artists'])
print('[{}] {} "{}" by {}'.format(date, explicit, track_name, artists))
+
def process_data(data):
# Process the data by Month/Year, then by Clean/Explicit
scores = {}
@@ -44,7 +45,7 @@ def process_data(data):
if date not in scores.keys():
scores[date] = [0, 0]
scores[date][1 if item['track']['explicit'] else 0] += 1
-
+
# Create simplified arrays for each piece of data
months = list(scores.keys())[::-1]
clean, explicit = [], []
@@ -59,17 +60,17 @@ def process_data(data):
ind = np.arange(n)
width = 0.55
# Resizer figuresize to be 2.0 wider
- plt.figure(figsize=(10.0, 6.0))
+ plt.figure(figsize=(10.0, 6.0))
# Stacked Bars
p1 = plt.bar(ind, explicit, width)
- p2 = plt.bar(ind, clean, width, bottom=explicit) # bottom= just has the bar sit on top of the explicit
+ p2 = plt.bar(ind, clean, width, bottom=explicit) # bottom= just has the bar sit on top of the explicit
# Plot labeling
plt.title('Song Count by Clean/Explicit')
plt.ylabel('Song Count')
plt.xlabel('Month')
- plt.xticks(ind, months, rotation=270) # Rotation 90 will have the
+ plt.xticks(ind, months, rotation=270) # Rotation 90 will have the
plt.legend((p1[0], p2[0]), ('Explicit', 'Clean'))
- fig = plt.gcf() # Magic to save to image and then show
+ fig = plt.gcf() # Magic to save to image and then show
# Save the figure, overwriting anything in your way
logging.info('Saving the figure to the \'export\' folder')
@@ -86,7 +87,7 @@ def process_data(data):
dpi=100,
quality=95
)
-
+
# Finally show the figure to
logging.info('Showing plot to User')
# plt.show()
@@ -95,6 +96,7 @@ def process_data(data):
# logging.info('Copying the plot data to clipboard')
# copy(months, clean, explicit)
+
# Simple method for exporting data to a table like format
# Will paste into Excel very easily
def copy(months, clean, explicit):
@@ -104,6 +106,7 @@ def copy(months, clean, explicit):
f'{item[0]}\t{item[1]}\t{item[2]}' for item in zip(months, clean, explicit)
]))
+
def main():
# logging.basicConfig(level=logging.INFO)
logging.info("Reading track files")
@@ -111,7 +114,7 @@ def main():
logging.info(f"Read and parse {len(files)} track files")
logging.info("Combining into single track file for ease of access")
data = combine_files(files)
- data.sort(key=lambda item : dateutil.parser.parse(item['added_at']).timestamp(), reverse=True)
+ data.sort(key=lambda item: dateutil.parser.parse(item['added_at']).timestamp(), reverse=True)
logging.info(f'File combined with {len(data)} items')
logging.info('Processing file...')
- process_data(data)
\ No newline at end of file
+ process_data(data)
diff --git a/app/spotify_explicit/pull.py b/app/spotify_explicit/pull.py
index b1d6a57..2a82776 100644
--- a/app/spotify_explicit/pull.py
+++ b/app/spotify_explicit/pull.py
@@ -1,13 +1,14 @@
-import os
-import sys
-from . import auth
import json
-import shutil
-import pprint
-import spotipy
import logging
-from hurry.filesize import size, alternative
+import os
+import shutil
+
+import spotipy
import spotipy.util as util
+from hurry.filesize import size, alternative
+
+from . import auth
+
def main():
# Get Authorization
@@ -27,8 +28,8 @@ def main():
tracks_folder = os.path.join(os.path.dirname(__file__), 'tracks')
logging.warning('Clearing all files in tracks folder for new files')
if os.path.exists(tracks_folder):
- shutil.rmtree(tracks_folder) # Delete folder and all contents (old track files)
- os.makedirs(tracks_folder) # Recreate the folder just deleted
+ shutil.rmtree(tracks_folder) # Delete folder and all contents (old track files)
+ os.makedirs(tracks_folder) # Recreate the folder just deleted
logging.info('Cleared folder, ready to download new track files')
curoffset, curlimit = 0, 50
@@ -62,4 +63,4 @@ def main():
))
break
# Continuing, so increment offset
- curoffset += curlimit
\ No newline at end of file
+ curoffset += curlimit