Add media processing command with ImageMagick thumbnailing & subprocess

This commit is contained in:
Xevion
2022-05-21 12:02:38 -05:00
parent 2c058f7840
commit 06d7880bab
91 changed files with 214 additions and 1 deletions

View File

@@ -0,0 +1 @@
{"name": "aj", "summary": "", "actor": "", "appearances": 31}

View File

@@ -0,0 +1 @@
{"name": "Andy Bernard", "summary": "Andy joined the sales staff at Dunder Mifflin after spending time at the Stamford Branch before it was closed due to downsizing. Andy did later become manager of Dunder Mifflin for a time, but for most of the series he acts as a salesman. His early appearances displayed him as a one-upping suckup employee with a tendency to loose one's temper. Andy is also unusually talented when it comes to singing, often boasting of his education at Cornell University in a capella group.", "actor": "Ed Helms", "appearances": 3963}

View File

@@ -0,0 +1 @@
{"name": "Angela Martin", "summary": "Angela is one of the lead accountants in The Office who has a tremendous affection for cats and tends to be a bit judgmental. Her no nonsense approach to how things should be done in the office appeals to Dwight.", "actor": "Angela Kinsey", "appearances": 1665}

View File

@@ -0,0 +1 @@
{"name": "ben-franklin", "summary": "", "actor": "", "appearances": 24}

View File

@@ -0,0 +1 @@
{"name": "ben-nugent", "summary": "", "actor": "", "appearances": 4}

View File

@@ -0,0 +1 @@
{"name": "bert", "summary": "", "actor": "", "appearances": 23}

View File

@@ -0,0 +1 @@
{"name": "Billy Merchant", "summary": "", "actor": "", "appearances": 9}

View File

@@ -0,0 +1 @@
{"name": "Billy", "summary": "", "actor": "", "appearances": 10}

View File

@@ -0,0 +1 @@
{"name": "Bob V ance", "summary": "", "actor": "", "appearances": 20}

View File

@@ -0,0 +1 @@
{"name": "brian", "summary": "", "actor": "", "appearances": 53}

View File

@@ -0,0 +1 @@
{"name": "broccoli-rob", "summary": "", "actor": "", "appearances": 11}

View File

@@ -0,0 +1 @@
{"name": "Captain Jack", "summary": "", "actor": "", "appearances": 35}

View File

@@ -0,0 +1 @@
{"name": "Carla Fern", "summary": "", "actor": "", "appearances": 15}

View File

@@ -0,0 +1 @@
{"name": "Carrol Stills", "summary": "", "actor": "", "appearances": 106}

View File

@@ -0,0 +1 @@
{"name": "Cathy Simms", "summary": "", "actor": "Lindsey Broad", "appearances": 50}

View File

@@ -0,0 +1 @@
{"name": "Cece", "summary": "Jim and Pam's first child", "actor": "Mia Cavolic, then Bailey and Sienna Strull", "appearances": 12}

View File

@@ -0,0 +1 @@
{"name": "charles", "summary": "", "actor": "", "appearances": 153}

View File

@@ -0,0 +1 @@
{"name": "cindy", "summary": "", "actor": "", "appearances": 15}

View File

@@ -0,0 +1 @@
{"name": "Clark Green", "summary": "", "actor": "Clark Duke", "appearances": 0}

View File

@@ -0,0 +1 @@
{"name": "concierge-marie", "summary": "", "actor": "", "appearances": 10}

View File

@@ -0,0 +1 @@
{"name": "Creed Bratton", "summary": "Creed is ostensibly in charge of quality control for Dunder Mifflin paper. Creed is a highly unusual among the employees of Dunder Mifflin, showing a questionable history and affiliation with a complete lack of morals.", "actor": "Creed Bratton", "appearances": 452}

View File

@@ -0,0 +1 @@
{"name": "Darryl Philbin", "summary": "", "actor": "Craig Robinson", "appearances": 1281}

View File

@@ -0,0 +1 @@
{"name": "David Brent", "summary": "", "actor": "Ricky Gervais", "appearances": 2}

View File

@@ -0,0 +1 @@
{"name": "David Wallace", "summary": "", "actor": "Andy Buckley", "appearances": 470}

View File

@@ -0,0 +1 @@
{"name": "DeAngelo", "summary": "", "actor": "", "appearances": 240}

View File

@@ -0,0 +1 @@
{"name": "Devon White", "summary": "", "actor": "", "appearances": 9}

View File

@@ -0,0 +1 @@
{"name": "Donna", "summary": "", "actor": "", "appearances": 81}

View File

@@ -0,0 +1 @@
{"name": "Dwight Schrute", "summary": "Dwight is unofficially the assistant (to the) regional manager at Dunder Mifflin. Dwight is extremely passionate about paper and is constantly trying to impress Michael. Dwight is also the leading salesman in the Scranton office.", "actor": "Rainn Wilson", "appearances": 7387}

View File

@@ -0,0 +1 @@
{"name": "Erin Hannon", "summary": "Erin Hannon replaces Pam Beasley (Halpert) as the receptionist at Dunder Mifflin Scranton. Erin is cheerful and loves being a receptionist. She is typically positive and supportive of Michael which she demonstrates in the \"Nepotism\" scene. Erin could be described as the employee who is the least likely to quit her job in search of a better opportunity because she is very content at Dunder Mifflin.", "actor": "Ellie Kemper", "appearances": 1477}

View File

@@ -0,0 +1 @@
{"name": "ernesto", "summary": "", "actor": "", "appearances": 4}

View File

@@ -0,0 +1 @@
{"name": "esther", "summary": "", "actor": "", "appearances": 21}

View File

@@ -0,0 +1 @@
{"name": "fannie", "summary": "", "actor": "", "appearances": 19}

View File

@@ -0,0 +1 @@
{"name": "frank", "summary": "", "actor": "", "appearances": 8}

View File

@@ -0,0 +1 @@
{"name": "Gabe Lewis", "summary": "After Dunder Mifflin was taken over by Sabre Corporation, Gabe was assigned by the corporate headquarters to report on the activities at Scranton.", "actor": "Zach Woods", "appearances": 431}

View File

@@ -0,0 +1 @@
{"name": "Gil", "summary": "Oscar Martinez's boyfriend in the early seasons of the office. The two travel ", "actor": "Tom W. Chick", "appearances": 6}

View File

@@ -0,0 +1 @@
{"name": "Hank", "summary": "The security guard.", "actor": "", "appearances": 51}

View File

@@ -0,0 +1 @@
{"name": "Helene Beesly", "summary": "", "actor": "", "appearances": 87}

View File

@@ -0,0 +1 @@
{"name": "here-comes-treble", "summary": "", "actor": "", "appearances": 14}

View File

@@ -0,0 +1 @@
{"name": "Hide", "summary": "", "actor": "", "appearances": 12}

View File

@@ -0,0 +1 @@
{"name": "Holly Flax", "summary": "", "actor": "Amy Ryan", "appearances": 606}

View File

@@ -0,0 +1 @@
{"name": "irene", "summary": "", "actor": "", "appearances": 23}

View File

@@ -0,0 +1 @@
{"name": "Jada Philbin", "summary": "The daughter of Darryl and Justine (divorced).", "actor": "", "appearances": 22}

View File

@@ -0,0 +1 @@
{"name": "Jake Palmer", "summary": "Meredith's son.", "actor": "Spencer Daniels", "appearances": 32}

View File

@@ -0,0 +1 @@
{"name": "Jan Levinson", "summary": "", "actor": "Melora Hardin", "appearances": 913}

View File

@@ -0,0 +1 @@
{"name": "Jim Halpert", "summary": "Jim is also in sales and is considered to be one of the \"heroes\" of the show because of his laid back attitude and ability to get along with people. During the course of the series, Jim marries Pam Beasley and subsequently becomes quite motivated to succeed as a paper salesman. There are many instances in The Office where Jim spends very little time attempting to increase his paper sales instead spending considerable time trying to win the affections of Pam. Jim also spends an inordinate amount of time playing tricks on Dwight or entertaining his co-workers which he primarily does to entertain Pam.", "actor": "John Krakinski", "appearances": 6664}

View File

@@ -0,0 +1 @@
{"name": "Jo Bennett", "summary": "", "actor": "Kathy Bates", "appearances": 186}

View File

@@ -0,0 +1 @@
{"name": "Josh Porter", "summary": "The Stamford Branch's manager who ends up leveraging his pending promotion to Dunder Mifflin Northeast to win a job offer at Staples.", "actor": "Charles Esten", "appearances": 65}

View File

@@ -0,0 +1 @@
{"name": "Karen Filippelli", "summary": "", "actor": "Rashida Jones", "appearances": 340}

View File

@@ -0,0 +1 @@
{"name": "Katy", "summary": "", "actor": "", "appearances": 99}

View File

@@ -0,0 +1 @@
{"name": "Kelly Kapoor", "summary": "Kelly Kapoor is in charge of customer relations at Dunder Mifflin Scranton. She is a very competent employee but she is extremely talkative and a bit of a gossip. She enjoys talking about trivial matters such as celebrity gossip and could certainly be accused of being obsessed about getting married (particularly to her on again off again boyfriend Ryan).", "actor": "Mindy Kaling", "appearances": 936}

View File

@@ -0,0 +1 @@
{"name": "Kevin Malone", "summary": "Kevin is the third accountant in the office and he is less adept at accounting than one might expect. Kevin was given his job by Michael despite being under qualified and that helps to explain why he occasionally becomes confused about fairly simple concepts.", "actor": "Brian Baumgartner", "appearances": 1686}

View File

@@ -0,0 +1 @@
{"name": "Lonny", "summary": "", "actor": "", "appearances": 22}

View File

@@ -0,0 +1 @@
{"name": "Madge", "summary": "", "actor": "", "appearances": 7}

View File

@@ -0,0 +1 @@
{"name": "martin", "summary": "", "actor": "", "appearances": 18}

View File

@@ -0,0 +1 @@
{"name": "melvina", "summary": "", "actor": "", "appearances": 5}

View File

@@ -0,0 +1 @@
{"name": "Meredith Palmer", "summary": "Meredith is in \"supplier relations\" for Dunder Mifflin although she is seldom shown working on the show. Meredith spends most of her time playing solitaire on her computer. ", "actor": "Kate Flannery", "appearances": 647}

View File

@@ -0,0 +1 @@
{"name": "Michael Scott", "summary": "Michael is the regional manager of Dunder Mifflin Paper Company in Scranton, Pennsylvania. Michael thinks of himself as an extremely capable boss who can handle any problem efficiently. In reality, Scott is ill-equipped to handle most of the problems that arise at Dunder Mifflin. Michael loves to host conference room meetings where typically very little is accomplished but he gets to be the center of attention. ", "actor": "Steve Carell", "appearances": 11768}

View File

@@ -0,0 +1 @@
{"name": "mose", "summary": "", "actor": "", "appearances": 33}

View File

@@ -0,0 +1 @@
{"name": "Mr. Brown", "summary": "", "actor": "", "appearances": 52}

View File

@@ -0,0 +1 @@
{"name": "mr-figaro", "summary": "", "actor": "", "appearances": 4}

View File

@@ -0,0 +1 @@
{"name": "Nate", "summary": "", "actor": "", "appearances": 55}

View File

@@ -0,0 +1 @@
{"name": "Nellie Bertram", "summary": "", "actor": "Catherine T", "appearances": 531}

View File

@@ -0,0 +1 @@
{"name": "nick", "summary": "", "actor": "", "appearances": 31}

View File

@@ -0,0 +1 @@
{"name": "Oscar Gutierrez", "summary": "Oscar is the other lead accountant in the office. Oscar and Angela frequently disagree about what is acceptable office behavior. ", "actor": "Oscar Nu\u00c3\u00b1ez", "appearances": 1478}

View File

@@ -0,0 +1 @@
{"name": "Pam Beasley", "summary": "Pam is the secretary at Dunder Mifflin and tries to keep Michael somewhat focused on work and being productive. Pam is unaware of Jim's romantic interest in her during the early seasons of the show.", "actor": "Jenna Fischer", "appearances": 5271}

View File

@@ -0,0 +1 @@
{"name": "Pete Miller", "summary": "", "actor": "Jake Lacy", "appearances": 0}

View File

@@ -0,0 +1 @@
{"name": "phillip", "summary": "", "actor": "", "appearances": 5}

View File

@@ -0,0 +1 @@
{"name": "Phyllis Lapin", "summary": "Phyllis is also a Dunder Mifflin salesperson. While Phyllis is typically very laid back and quiet she is very protective of her position in the company and her sales commissions.", "actor": "Phyllis Smith", "appearances": 1059}

View File

@@ -0,0 +1 @@
{"name": "randy", "summary": "", "actor": "", "appearances": 2}

View File

@@ -0,0 +1 @@
{"name": "ravi", "summary": "", "actor": "", "appearances": 9}

View File

@@ -0,0 +1 @@
{"name": "robert-dunder", "summary": "", "actor": "", "appearances": 11}

View File

@@ -0,0 +1 @@
{"name": "Robert California", "summary": "", "actor": "", "appearances": 445}

View File

@@ -0,0 +1 @@
{"name": "rolando", "summary": "", "actor": "", "appearances": 4}

View File

@@ -0,0 +1 @@
{"name": "Rolf", "summary": "One of Dwight's friends.", "actor": "", "appearances": 29}

View File

@@ -0,0 +1 @@
{"name": "Roy Anderson", "summary": "", "actor": "David Denman", "appearances": 252}

View File

@@ -0,0 +1 @@
{"name": "Ryan Howard", "summary": "Ryan Howard is a temporary employee in season one but eventually gets promoted to be Michael\u00e2\u20ac\u2122s supervisor at Dunder Mifflin.", "actor": "B.J. Novak", "appearances": 1351}

View File

@@ -0,0 +1 @@
{"name": "Senator Lipton", "summary": "", "actor": "", "appearances": 47}

View File

@@ -0,0 +1 @@
{"name": "sensei-billy", "summary": "", "actor": "", "appearances": 11}

View File

@@ -0,0 +1 @@
{"name": "Sherri", "summary": "Jan's secretary", "actor": "", "appearances": 11}

View File

@@ -0,0 +1 @@
{"name": "Stanley Hudson", "summary": "Stanley is a fairly successful salesman who resents spending time in an office where he typically is not productive. Stanley tolerates Michael's long and typically pointless conference room meetings by spending his time completing crossword puzzles. ", "actor": "Leslie David Baker", "appearances": 751}

View File

@@ -0,0 +1 @@
{"name": "Sweeney Todd", "summary": "", "actor": "", "appearances": 14}

View File

@@ -0,0 +1 @@
{"name": "Sylvia", "summary": "", "actor": "", "appearances": 15}

View File

@@ -0,0 +1 @@
{"name": "Toby Flenderson", "summary": "Toby is the head of human resources and he and Michael do not get along well. Toby frequently tries (with little success) to prevent Michael from violating corporate policy. As a result, Michael develops a strong grudge against Toby.", "actor": "Paul Lieberstein", "appearances": 924}

View File

@@ -0,0 +1 @@
{"name": "Todd Packer", "summary": "", "actor": "David Koechner", "appearances": 167}

View File

@@ -0,0 +1 @@
{"name": "trevor", "summary": "", "actor": "", "appearances": 31}

View File

@@ -0,0 +1 @@
{"name": "uncle-al", "summary": "", "actor": "", "appearances": 7}

View File

@@ -0,0 +1 @@
{"name": "Val Johnson", "summary": "Hired after the old warehouse crew quits, Val is the warehouse foreman and has a relationship with Darryl after she breaks up with her initial boyfriend, Brandon.", "actor": "Ameenah Kaplan", "appearances": 73}

View File

@@ -0,0 +1 @@
{"name": "vikram", "summary": "", "actor": "", "appearances": 21}

View File

@@ -0,0 +1 @@
{"name": "Walter Bernard Jr.", "summary": "Andrew Bernard's younger brother.", "actor": "Josh Groban", "appearances": 11}

View File

@@ -0,0 +1 @@
{"name": "Walter Bernard Sr.", "summary": "Andrew Bernard's father.", "actor": "Stephen Collins, ?", "appearances": 25}

View File

@@ -4,6 +4,7 @@ import logging
import os
import re
import shutil
import subprocess
from collections import Counter, OrderedDict
from typing import Any, Dict, List, Optional, Tuple, Union
@@ -785,7 +786,7 @@ def app(path: str, make_dir: bool) -> None:
json.dump(character_data, character_file)
# Ensure character folder exists before writing files
if not os.path.lexists(character_folder):
if not os.path.exists(character_folder):
os.makedirs(character_folder)
for id, data in character_data.items():
@@ -794,5 +795,127 @@ def app(path: str, make_dir: bool) -> None:
json.dump(data, file)
@build.command('media')
@click.option('--suppress/--no-suppress', default=True, help='Disable stdout suppression for image magick commandline output.')
@click.option('--copy/--no-copy', default=True, help='Complete the copying stage.')
@click.option('--thumbnail/--no-thumbnail', default=True, help='Complete the thumbnailing stage.')
@click.argument('path', type=click.Path(file_okay=False))
def media(path: str, suppress: bool, copy: bool, thumbnail: bool) -> None:
def get_fullsize_args(input_path: str, output_path: str, geometry: str) -> List[str]:
return ['magick',
input_path,
'-gravity', 'Center',
'-crop', '1:1+0+0',
'+repage',
'-quality', '95',
'-interlace', 'none',
'-colorspace', 'sRGB',
'-strip',
output_path]
def get_thumbnailing_args(input_path: str, output_path: str, geometry: str) -> List[str]:
return [
'magick',
input_path,
'-gravity', 'Center',
'-crop', '1:1+0+0',
'+repage',
'-filter', 'Triangle',
'-define', 'filter:support=2',
'-thumbnail', geometry,
'-unsharp', '0.25x0.25+8+0.065',
'-dither', 'None',
'-posterize', '136',
'-quality', '82',
'-define', 'jpeg:fancy-upsampling=off',
'-define', 'png:compression-filter=5',
'-define', 'png:compression-level=9',
'-define', 'png:compression-strategy=1',
'-define', 'png:exclude-chunk=all',
'-interlace', 'none',
'-colorspace', 'sRGB',
'-strip',
output_path,
]
if not (copy or thumbnail):
logger.error('Both copy and thumbnail stages are disabled. Quitting early.')
return
copy_operations: List[Tuple[str, str]] = []
thumbnail_operations: List[Tuple[str, str]] = []
# /img/episode/03/04/full.jpeg
all_episodes: List[Tuple[int, int]] = [(season + 1, episode + 1) for season in range(9) for episode in range(EPISODE_COUNTS[season])]
progress = Progress(SpinnerColumn('dots10'), *Progress.get_default_columns(), MofNCompleteColumn(),
TimeElapsedColumn())
with progress:
for season, episode in progress.track(all_episodes, description='Finding images'):
# Find what images are available, select the one with the lowest integer
episode_dir = os.path.join(IMG_EPISODES_DIR, f'{season:02}', f'{episode:02}')
if not os.path.exists(episode_dir):
os.makedirs(episode_dir)
images_available = os.listdir(episode_dir)
images_available.sort(key=lambda x: int(x.split('.')[0]))
input_path: str = os.path.join(episode_dir, images_available[0])
output_dir: str = os.path.abspath(os.path.join(path, f'{season:02}', f'{episode:02}'))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
output_copy_path: str = os.path.join(output_dir, 'full.jpeg')
output_thumb_path: str = os.path.join(output_dir, 'thumbnail.jpeg')
if copy: copy_operations.append((input_path, output_copy_path))
if thumbnail: thumbnail_operations.append((input_path, output_thumb_path))
logger.debug(f'Starting {len(copy_operations) + len(thumbnail_operations)} copy/thumbnailing operations.')
sp_kwargs = {'capture_output': True, 'text': True} if suppress else {}
if copy:
with progress:
logger.debug('Beginning "smart copying"...')
for input, output in progress.track(copy_operations, description='Smart copying...'):
sp_args = get_fullsize_args(input, output, '1440')
try:
completed = subprocess.run(sp_args, **sp_kwargs, check=True)
except subprocess.CalledProcessError as e:
logger.error('Failed to process copy operation.', exc_info=e)
logger.error(f'Input: "{input}"')
logger.error(f'Output: "{output}"')
logger.error(f'Args: "{" ".join(sp_args)}"')
logger.error(f'Stdout: "{e.stdout.rstrip()}"')
logger.error(f'Stderr: "{e.stderr.rstrip()}"')
if thumbnail:
with progress:
logger.debug('Beginning "thumbnailing"...')
for input, output in progress.track(thumbnail_operations, description='Thumbnailing...'):
sp_args = get_thumbnailing_args(input, output, '156')
try:
completed = subprocess.run(sp_args, **sp_kwargs, check=True)
except subprocess.CalledProcessError as e:
logger.error('Failed to process thumbnail operation.', exc_info=e)
logger.error(f'Input: "{input}"')
logger.error(f'Output: "{output}"')
logger.error(f'Args: "{" ".join(sp_args)}"')
logger.error(f'Stdout: "{e.stdout.rstrip()}"')
logger.error(f'Stderr: "{e.stderr.rstrip()}"')
with progress:
output_paths = [y for x, y in copy_operations]
output_paths.extend([y for x, y in thumbnail_operations])
file_sizes = []
for path in progress.track(output_paths, description='Acquiring sizes...'):
file_sizes.append(os.stat(path).st_size)
if __name__ == '__main__':
cli()