diff --git a/phototag/__init__.py b/phototag/__init__.py index c5f89fa..727899f 100644 --- a/phototag/__init__.py +++ b/phototag/__init__.py @@ -11,8 +11,9 @@ import os from rich.logging import RichHandler from . import config - # noinspection PyArgumentList +from .exceptions import EmptyConfigurationValueError, InvalidConfigurationError + logging.basicConfig( format='[bold deep_pink2]%(threadName)s[/bold deep_pink2] %(message)s', level=logging.ERROR, @@ -31,11 +32,21 @@ INPUT_PATH = ROOT SCRIPT_ROOT = os.path.dirname(os.path.realpath(__file__)) TEMP_PATH = os.path.join(ROOT, "temp") OUTPUT_PATH = os.path.join(ROOT, "output") +CONFIG_PATH = os.path.join(SCRIPT_ROOT, "config") logger.info("Path constants built successfully...") # Environment Variables -os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.join(SCRIPT_ROOT, "config", - config.config["google"]["credentials"]) +try: + if not config.config["google"]["credentials"]: + raise EmptyConfigurationValueError( + "Please use the configuration command to add a Google API authorization file." + ) +except (ValueError, AttributeError): + raise InvalidConfigurationError( + "The configuration file appears to be damaged. Please fix, delete or replace it with a valid configuration." + ) +else: + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.join(CONFIG_PATH, config.config["google"]["credentials"]) # Extension Constants RAW_EXTS = ["3fr", "ari", "arw", "bay", "braw", "crw", "cr2", "cr3", "cap", "data", "dcs", "dcr", "dng", "drf", "eip", diff --git a/phototag/app.py b/phototag/app.py index 153954f..6681732 100644 --- a/phototag/app.py +++ b/phototag/app.py @@ -38,11 +38,8 @@ def run(): os.makedirs(TEMP_PATH) try: - with Progress( - "[progress.description]{task.description}", - BarColumn(bar_width=None), - "[progress.percentage]{task.percentage:>3.0f}%", - ) as progress: + with Progress("[progress.description]{task.description}", BarColumn(bar_width=None), + "[progress.percentage]{task.percentage:>3.0f}%") as progress: mp = MasterFileProcessor(select, 10, convert_to_bytes("1780 KB"), True, client=client, progress=progress) logger.info('MasterFileProcessor created.') mp.load() diff --git a/phototag/cli.py b/phototag/cli.py index c90fa07..3b6ad46 100644 --- a/phototag/cli.py +++ b/phototag/cli.py @@ -6,18 +6,18 @@ The file responsible for providing commandline functionality to the user. import logging import os -import re import shutil from typing import Tuple -from glob import glob -from rich.traceback import install + import click +from google.cloud import vision +from rich.progress import Progress, BarColumn -from . import config, INPUT_PATH +from . import config, TEMP_PATH from .exceptions import InvalidSelectionError -from .helpers import get_extension, valid_extension +from .helpers import select_files, convert_to_bytes +from .process import MasterFileProcessor -# install() logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -33,44 +33,64 @@ def cli(): @click.option('-a', '--all', is_flag=True, help='Add all files in the current directory to be tagged.') @click.option('-r', '--regex', help='Use RegEx to match files in the current directory') @click.option('-g', '--glob', 'glob_pattern', help='Use Glob (UNIX-style file pattern matching) to match files.') -def run(files: Tuple[str], all: bool = False, regex: str = None, glob_pattern: str = None): +@click.option('--max-threads', type=int, help='The maximum number of threads that can be running at any point') +@click.option('--max-buffer-size', 'max_buffer', + help='Keep the total size of the files in memory at or below this point') +@click.option('--forget', is_flag=True, help='Don\'t utilize labels received from the Vision API previously.') +@click.option('--overwrite', is_flag=True, help='Instead of adding tags, clear and overwrite them') +@click.option('-d', '--dry-run', is_flag=True, help='Dry-run mode: Don\'t actually write to or modify files.') +@click.option('-t', '--test', is_flag=True, + help='Don\'t actually query the Vision API, just generate fake tags for testing purposes.') +def run(files: Tuple[str], all: bool = False, regex: str = None, glob_pattern: str = None, max_threads: int = None, + max_buffer: str = None, forget: bool = False, overwrite: bool = False, dry_run: bool = False, + test: bool = False): """ Run tagging on FILES. Files can also be selected using --all, --regex and --glob. + --max-threads, --max-buffer-size and --forget will inherit their settings from the global config. """ - files = list(files) - - # Just add all files in current working directory - if all: - files.extend(os.listdir(INPUT_PATH)) + try: + files = select_files(list(files), regex, glob_pattern) + except InvalidSelectionError: + logger.exception(InvalidSelectionError.__doc__, exc_info=False) else: - # RegEx option pattern matching - if regex: - files.extend( - filter(lambda filename: re.match(re.compile(regex), filename) is not None, os.listdir(INPUT_PATH)) - ) + client = vision.ImageAnnotatorClient() + try: + # Create the 'temp' directory + if not os.path.exists(TEMP_PATH): + logger.info("Creating temporary processing directory") + os.makedirs(TEMP_PATH) - # Glob option pattern matching - if glob_pattern: - files.extend(glob(glob_pattern)) + with Progress("[progress.description]{task.description}", BarColumn(bar_width=None), + "{task.completed}/{task.total} [progress.percentage]{task.percentage:>3.0f}%") as progress: + mp = MasterFileProcessor(files, 10, convert_to_bytes("2 MB"), True, client=client, progress=progress) + mp.load() + logger.info('Finished loading/starting initial threads.') + mp.join() + logger.info('Finished joining threads, now quitting.') + except Exception as error: + logger.exception(str(error)) + finally: + os.rmdir(TEMP_PATH) + logger.info("Temporary directory removed.") - # Format file selection into relative paths, filter down to 'valid' image files - files = list(dict.fromkeys(os.path.relpath(file) for file in files)) - select = list(filter(lambda filename: valid_extension(get_extension(filename)), files)) - if len(select) == 0: - if len(files) == 0: - raise InvalidSelectionError('No files selected.') - else: - raise InvalidSelectionError('No valid images selected.') - else: - logger.debug(f'Found {len(select)} valid images out of {len(files)} files selected.') +@cli.command('collect') +@click.argument('files', nargs=-1, type=click.Path(exists=True)) +@click.argument('output', type=click.File(mode="w"), required=False) +@click.option('--level', default=0.25) +@click.option('-a', '--all', is_flag=True, help='Add all files in the current directory to be tagged.') +@click.option('-r', '--regex', help='Use RegEx to match files in the current directory') +@click.option('-g', '--glob', 'glob_pattern', help='Use Glob (UNIX-style file pattern matching) to match files.') +def collect(files: Tuple[str], output=None, all: bool = False, regex: str = None, glob: str = None): + """ + Collects tags from selected images for compiling the average tags of an album. - print(files) - # from .app import run - # - # run() + Input is selected with FILES or using --all, --regex and --glob. + """ + files = select_files(list(files), regex, glob) + pass @cli.command('auth') diff --git a/phototag/exceptions.py b/phototag/exceptions.py index ab1f1af..273c6fa 100644 --- a/phototag/exceptions.py +++ b/phototag/exceptions.py @@ -23,6 +23,11 @@ class InvalidConfigurationError(UserError): pass +class EmptyConfigurationValueError(InvalidConfigurationError): + """The configuration did not include values required to run the application.""" + pass + + class NoSidecarFileError(PhototagException): """ The application is confused as a sidecar file was not found where it was expected. diff --git a/phototag/helpers.py b/phototag/helpers.py index 6bd8ad3..c662386 100644 --- a/phototag/helpers.py +++ b/phototag/helpers.py @@ -3,16 +3,22 @@ helpers.py Simple helper functions and constants separated from the primary application functionality. """ +import logging import os import random import re import string +from glob import glob +from typing import List, Optional -from phototag import LOSSY_EXTS, RAW_EXTS -from phototag.exceptions import PhototagException +from phototag import LOSSY_EXTS, RAW_EXTS, INPUT_PATH +from phototag.exceptions import PhototagException, InvalidSelectionError ALL_EXTENSIONS = RAW_EXTS + LOSSY_EXTS +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + byte_magnitudes = { "B": 1024 ** 0, "KB": 1024 ** 1, @@ -75,3 +81,43 @@ def convert_to_bytes(size_string: str) -> int: """ match = re.match(r"(\d+)\s*(\w{1,2})", size_string) return int(match.group(1)) * byte_magnitudes[match.group(2)] + + +def select_files(files: List[str], regex: Optional[str], glob_pattern: Optional[str]) -> List[str]: + """ + Helper function for selecting files in the CWD (or subdirectories, via Glob) and filtering them. + Combines direct file argument selections, RegEx filters and Glob patterns together. + + :param files: Specific files chosen by the user. + :param regex: A full RegEx matching pattern + :param glob_pattern: A Glob pattern + :return: A list of files relative to the CWD + """ + # Just add all files in current working directory + if all: + files.extend(os.listdir(INPUT_PATH)) + else: + # RegEx option pattern matching + if regex: + files.extend( + filter(lambda filename: re.match(re.compile(regex), filename) is not None, os.listdir(INPUT_PATH)) + ) + + # Glob option pattern matching + if glob_pattern: + files.extend(glob(glob_pattern)) + + # Format file selection into relative paths, filter down to 'valid' image files + files = list(dict.fromkeys(os.path.relpath(file) for file in files)) + select = list(filter(lambda filename: valid_extension(get_extension(filename)), files)) + + if len(select) == 0: + logger.debug(f'{len(files)} files found, 0 images found.') + if len(files) == 0: + raise InvalidSelectionError('No files selected.') + else: + raise InvalidSelectionError('No valid images selected.') + else: + logger.info(f'Found {len(select)} valid images out of {len(files)} files selected.') + + return files