From fab1241a79b3cf0d239de075841724e842af32b7 Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 21:05:17 -0500 Subject: [PATCH 01/16] basic refactoring of extension/path/xmp management --- package/__init__.py | 22 ++++++++ package/app.py | 126 +++++++------------------------------------- package/process.py | 105 ++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 108 deletions(-) create mode 100644 package/__init__.py create mode 100644 package/process.py diff --git a/package/__init__.py b/package/__init__.py new file mode 100644 index 0000000..269e8f1 --- /dev/null +++ b/package/__init__.py @@ -0,0 +1,22 @@ +import os +import sys + +# Path Constants +ROOT = sys"path", [0] +PROCESSING_PATH = os"path", "join", (ROOT, 'package', 'processing') +INPUT_PATH = os"path", "join", (PROCESSING_PATH, 'input') +TEMP_PATH = os"path", "join", (PROCESSING_PATH, 'temp') +OUTPUT_PATH = os"path", "join", (PROCESSING_PATH, 'output') + +# Extension Constants +RAW_EXTS = [ + "3fr", "ari", "arw", "bay", "braw", "crw", + "cr2", "cr3", "cap", "data", "dcs", "dcr", + "dng", "drf", "eip", "erf", "fff", "gpr", + "iiq", "k25", "kdc", "mdc", "mef", "mos", + "mrw", "nef", "nrw", "obm", "orf", "pef", + "ptx", "pxn", "r3d", "raf", "raw", "rwl", + "rw2", "rwz", "sr2", "srf", "srw", "tif", + "x3f", +] +LOSSY_EXTS = ["JPEG", "JPG", "PNG"] \ No newline at end of file diff --git a/package/app.py b/package/app.py index d3e3381..db2fa44 100644 --- a/package/app.py +++ b/package/app.py @@ -1,13 +1,10 @@ -import io, sys, os, time, rawpy, imageio, progressbar, shutil, iptcinfo3 -from google.cloud.vision import types -from google.cloud import vision -from package import xmp -from PIL import Image +import io, sys, os, time, rawpy, imageio, progressbar, shutil -# The name of the image file to annotate -input_path = os.path.join(sys.path[0], 'package', 'processing', 'input') -temp_path = os.path.join(sys.path[0], 'package', 'processing', 'temp') -output_path = os.path.join(sys.path[0], 'package', 'processing', 'output') +from .xmp import XMPParser +from google.cloud import vision + +from .process import FileProcessor +from . import INPUT_PATH, TEMP_PATH, OUTPUT_PATH, PROCESSING_PATH # Process a single file in these steps: # 1) Create a temporary file @@ -15,115 +12,28 @@ output_path = os.path.join(sys.path[0], 'package', 'processing', 'output') # 3) Read XMP, then write new tags to it # 4) Delete temporary file, move NEF/JPEG and XMP -def process_file(file_name, xmp_name=None): - global client - - # Remove the temporary file - def _cleanup(): - if os.path.exists(temp_file_path): - # Deletes the temporary file - os.remove(temp_file_path) - - # Get the size of the file. Is concerned with filesize type. 1024KiB -> 1MiB - def _size(file_path): - size, type = os.path.getsize(file_path) / 1024, 'KiB' - if size >= 1024: size /= 1024; type = 'MiB' - return round(size, 2), type - - # Optimizes a file using JPEG thumbnailing and compression. - def _optimize(file_path, size=(512, 512), quality=85, copy=None): - image = Image.open(file_path) - image.thumbnail(size, resample=Image.ANTIALIAS) - if copy: - image.save(copy, format='jpeg', optimize=True, quality=quality) - else: - image.save(file_path, format='jpeg', optimize=True, quality=quality) - - base, ext = os.path.splitext(file_name) - temp_file_path = os.path.join(temp_path, base + '.jpeg') - - try: - if xmp_name: - # Process the file into a JPEG - rgb = rawpy.imread(os.path.join(input_path, file_name)) - imageio.imsave(temp_file_path, rgb.postprocess()) - rgb.close() - - # Information on file sizes - print("Raw Size: {} {}".format(*_size(os.path.join(input_path, file_name))), end=' | ') - print("Resave Size: {} {}".format(*_size(temp_file_path)), end=' | ') - pre = os.path.getsize(temp_file_path) - _optimize(temp_file_path) - post = os.path.getsize(temp_file_path) - print("Optimized Size: {} {} ({}% savings)".format(*_size(temp_file_path), round((1.0 - (post / pre)) * 100), 2) ) - else: - pre = os.path.getsize(os.path.join(input_path, file_name)) - _optimize(os.path.join(input_path, file_name), copy=temp_file_path) - post = os.path.getsize(temp_file_path) - print("Optimized Size: {} {} ({}% savings)".format(*_size(temp_file_path), round((1.0 - (post / pre)) * 100), 2) ) - - # Open the image, read as bytes, convert to types Image - image = Image.open(temp_file_path) - bytesIO = io.BytesIO() - image.save(bytesIO, format='jpeg') - image.close() - image = vision.types.Image(content=bytesIO.getvalue()) - - # Performs label detection on the image file - response = client.label_detection(image=image) - labels = [label.description for label in response.label_annotations] - print('\tLabels: {}'.format(', '.join(labels))) - - # XMP sidecar file specified, write to it using XML module - if xmp_name: - print('\tWriting {} tags to output XMP...'.format(len(labels))) - parser = xmp.XMPParser(os.path.join(input_path, xmp_name)) - parser.add_keywords(labels) - # Save the new XMP file - parser.save(os.path.join(output_path, xmp_name)) - # Remove the old XMP file - os.remove(os.path.join(input_path, xmp_name)) - # No XMP file is specified, using IPTC tagging - else: - print('\tWriting {} tags to output {}'.format(len(labels), ext[1:].upper())) - info = iptcinfo3.IPTCInfo(os.path.join(input_path, file_name)) - info['keywords'].extend(labels) - info.save() - # Remove the weird ghsot file created by this iptc read/writer. - os.remove(os.path.join(input_path, file_name + '~')) - - # Copy dry-run - # shutil.copy2(os.path.join(input_path, file_name), os.path.join(output_path, file_name)) - os.rename(os.path.join(input_path, file_name), os.path.join(output_path, file_name)) - except: - _cleanup() - raise - _cleanup() - # Driver code for the package -def run(): - global client - +def run(client): # Ensure that 'input' and 'output' directories are created - if not os.path.exists(input_path): + if not os.path.exists(INPUT_PATH): print('Input directory did not exist, creating and quitting.') - os.makedirs(input_path) + os.makedirs(INPUT_PATH) return - if not os.path.exists(output_path): + if not os.path.exists(OUTPUT_PATH): print('Output directory did not exist. Creating...') - os.makedirs(output_path) + os.makedirs(OUTPUT_PATH) # Clients client = vision.ImageAnnotatorClient() # Find files we want to process based on if they have a corresponding .XMP - files = os.listdir(input_path) + files = os.listdir(INPUT_PATH) select = [file for file in files if os.path.splitext(file)[1] != '.xmp'] # Create the 'temp' directory print(f'Initializing file processing for {len(select)} files...') - os.makedirs(temp_path) + os.makedirs(TEMP_PATH) try: # Process files @@ -157,15 +67,15 @@ def run(): # Process individual file else: print('Processing file {}, \'{}\''.format(index + 1, xmps[0]), end=' | ') - process_file(file_name=file, xmp_name=xmps[0]) - elif ext in ['.JPEG', '.JPG', '.PNG']: + file = FileProcessor(file, xmps[0]) + elif ext in BASIC_EXTENSIONS: print('Processing file {}, \'{}\''.format(index + 1, file), end=' | ') - process_file(file_name=file) + file = FileProcessor(file, xmps[0]) except: - os.rmdir(temp_path) + os.rmdir(TEMP_PATH) raise # Remove the directory, we are done here print('Cleaning up temporary directory...') - os.rmdir(temp_path) \ No newline at end of file + os.rmdir(TEMP_PATH) \ No newline at end of file diff --git a/package/process.py b/package/process.py new file mode 100644 index 0000000..7831bf2 --- /dev/null +++ b/package/process.py @@ -0,0 +1,105 @@ +import os +import sys +import rawpy +import imageio +import io +import iptcinfo3 +from PIL import Image +from google.cloud.vision import types +from google.cloud import vision + +from . import TEMP_PATH, INPUT_PATH, OUTPUT_PATH +from .xmp import XMPParser + +class FileProcessor(object): + def __init__(self, file_name, xmp_name=None): + self.file_name, self.xmp_name = file_name, xmp_name + self.base, self.ext = os.path.splitext(self.file_name) + self.temp_file_path = os.path.join(TEMP_PATH, self.base + '.jpeg') + + def rawOptimize(self): + rgb = rawpy.imread(os.path.join(INPUT_PATH, self.file_name)) + imageio.imsave(temp_file_path, rgb.postprocess()) + rgb.close() + + # Information on file sizes + print("Raw Size: {} {}".format(*_size(os.path.join(INPUT_PATH, self.file_name))), end=' | ') + print("Resave Size: {} {}".format(*_size(temp_file_path)), end=' | ') + pre = os.path.getsize(temp_file_path) + _optimize(temp_file_path) + post = os.path.getsize(temp_file_path) + print("Optimized Size: {} {} ({}% savings)".format(*_size(temp_file_path), round((1.0 - (post / pre)) * 100), 2) ) + + def basicOptimize(self): + pre = os.path.getsize(os.path.join(INPUT_PATH, self.file_name)) + _optimize(os.path.join(INPUT_PATH, self.file_name), copy=temp_file_path) + post = os.path.getsize(temp_file_path) + print("Optimized Size: {} {} ({}% savings)".format(*_size(temp_file_path), round((1.0 - (post / pre)) * 100), 2) ) + + + def run(self, client): + try: + if self.xmp_name: + # Process the file into a JPEG + self.rawOptimize() + else: + self.basicOptimize() + + # Open the image, read as bytes, convert to types Image + image = Image.open(temp_file_path) + bytesIO = io.BytesIO() + image.save(bytesIO, format='jpeg') + image.close() + image = vision.types.Image(content=bytesIO.getvalue()) + + # Performs label detection on the image file + response = client.label_detection(image=image) + labels = [label.description for label in response.label_annotations] + print('\tLabels: {}'.format(', '.join(labels))) + + # XMP sidecar file specified, write to it using XML module + if self.xmp_name: + print('\tWriting {} tags to output XMP...'.format(len(labels))) + parser = XMPParser(os.path.join(INPUT_PATH, self.xmp_name)) + parser.add_keywords(labels) + # Save the new XMP file + parser.save(os.path.join(OUTPUT_PATH, self.xmp_name)) + # Remove the old XMP file + os.remove(os.path.join(INPUT_PATH, self.xmp_name)) + # No XMP file is specified, using IPTC tagging + else: + print('\tWriting {} tags to output {}'.format(len(labels), ext[1:].upper())) + info = iptcinfo3.IPTCInfo(os.path.join(INPUT_PATH, self.file_name)) + info['keywords'].extend(labels) + info.save() + # Remove the weird ghsot file created by this iptc read/writer. + os.remove(os.path.join(INPUT_PATH, self.file_name + '~')) + + # Copy dry-run + # shutil.copy2(os.path.join(INPUT_PATH, self.file_name), os.path.join(OUTPUT_PATH, self.file_name)) + os.rename(os.path.join(INPUT_PATH, self.file_name), os.path.join(OUTPUT_PATH, self.file_name)) + except: + self._cleanup() + raise + self._cleanup() + + # Remove the temporary file + def _cleanup(self): + if os.path.exists(self.temp_file_path): + # Deletes the temporary file + os.remove(self.temp_file_path) + + # Get the size of the file. Is concerned with filesize type. 1024KiB -> 1MiB + def _size(self, file_path): + size, type = os.path.getsize(file_path) / 1024, 'KiB' + if size >= 1024: size /= 1024; type = 'MiB' + return round(size, 2), type + + # Optimizes a file using JPEG thumbnailing and compression. + def _optimize(self, file_path, size=(512, 512), quality=85, copy=None): + image = Image.open(file_path) + image.thumbnail(size, resample=Image.ANTIALIAS) + if copy: + image.save(copy, format='jpeg', optimize=True, quality=quality) + else: + image.save(file_path, format='jpeg', optimize=True, quality=quality) \ No newline at end of file From 9165e675be66f90a27feef46ac37e22a56fe4a55 Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 21:07:05 -0500 Subject: [PATCH 02/16] fix failure of a formatting job --- package/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/package/__init__.py b/package/__init__.py index 269e8f1..9252d31 100644 --- a/package/__init__.py +++ b/package/__init__.py @@ -2,11 +2,11 @@ import os import sys # Path Constants -ROOT = sys"path", [0] -PROCESSING_PATH = os"path", "join", (ROOT, 'package', 'processing') -INPUT_PATH = os"path", "join", (PROCESSING_PATH, 'input') -TEMP_PATH = os"path", "join", (PROCESSING_PATH, 'temp') -OUTPUT_PATH = os"path", "join", (PROCESSING_PATH, 'output') +ROOT = sys.path, [0] +PROCESSING_PATH = os.path.join(ROOT, 'package', 'processing') +INPUT_PATH = os.path.join(PROCESSING_PATH, 'input') +TEMP_PATH = os.path.join(PROCESSING_PATH, 'temp') +OUTPUT_PATH = os.path.join(PROCESSING_PATH, 'output') # Extension Constants RAW_EXTS = [ From 40d200dd74140ecd551b3c78aef49dffb2f0277d Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 21:17:38 -0500 Subject: [PATCH 03/16] further class refactoring i have really bad commit names rn lol --- package/__init__.py | 4 ++-- package/app.py | 16 +++++++++------- package/process.py | 31 +++++++++++++++---------------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/package/__init__.py b/package/__init__.py index 9252d31..1d1bce0 100644 --- a/package/__init__.py +++ b/package/__init__.py @@ -2,7 +2,7 @@ import os import sys # Path Constants -ROOT = sys.path, [0] +ROOT = sys.path[0] PROCESSING_PATH = os.path.join(ROOT, 'package', 'processing') INPUT_PATH = os.path.join(PROCESSING_PATH, 'input') TEMP_PATH = os.path.join(PROCESSING_PATH, 'temp') @@ -19,4 +19,4 @@ RAW_EXTS = [ "rw2", "rwz", "sr2", "srf", "srw", "tif", "x3f", ] -LOSSY_EXTS = ["JPEG", "JPG", "PNG"] \ No newline at end of file +LOSSY_EXTS = ["jpeg", "jpg", "png"] \ No newline at end of file diff --git a/package/app.py b/package/app.py index db2fa44..4df864f 100644 --- a/package/app.py +++ b/package/app.py @@ -5,6 +5,7 @@ from google.cloud import vision from .process import FileProcessor from . import INPUT_PATH, TEMP_PATH, OUTPUT_PATH, PROCESSING_PATH +from . import RAW_EXTS, LOSSY_EXTS # Process a single file in these steps: # 1) Create a temporary file @@ -13,7 +14,7 @@ from . import INPUT_PATH, TEMP_PATH, OUTPUT_PATH, PROCESSING_PATH # 4) Delete temporary file, move NEF/JPEG and XMP # Driver code for the package -def run(client): +def run(): # Ensure that 'input' and 'output' directories are created if not os.path.exists(INPUT_PATH): print('Input directory did not exist, creating and quitting.') @@ -24,7 +25,7 @@ def run(client): print('Output directory did not exist. Creating...') os.makedirs(OUTPUT_PATH) - # Clients + # Client client = vision.ImageAnnotatorClient() # Find files we want to process based on if they have a corresponding .XMP @@ -39,9 +40,9 @@ def run(client): # Process files for index, file in progressbar.progressbar(list(enumerate(select)), redirect_stdout=True, term_width=110): name, ext = os.path.splitext(file) - ext = ext.upper() + ext = ext.lower().strip('.') # Raw files contain their metadata in an XMP file usually - if ext in ['.NEF', '.CR2']: + if ext in RAW_EXTS: # Get all possible files identicals = [possible for possible in files if possible.startswith(os.path.splitext(file)[0]) @@ -68,10 +69,11 @@ def run(client): else: print('Processing file {}, \'{}\''.format(index + 1, xmps[0]), end=' | ') file = FileProcessor(file, xmps[0]) - elif ext in BASIC_EXTENSIONS: + file.run(client) + elif ext in LOSSY_EXTS: print('Processing file {}, \'{}\''.format(index + 1, file), end=' | ') - file = FileProcessor(file, xmps[0]) - + file = FileProcessor(file) + file.run(client) except: os.rmdir(TEMP_PATH) raise diff --git a/package/process.py b/package/process.py index 7831bf2..65ac4b9 100644 --- a/package/process.py +++ b/package/process.py @@ -17,6 +17,15 @@ class FileProcessor(object): self.base, self.ext = os.path.splitext(self.file_name) self.temp_file_path = os.path.join(TEMP_PATH, self.base + '.jpeg') + # Optimizes a file using JPEG thumbnailing and compression. + def _optimize(self, file_path, size=(512, 512), quality=85, copy=None): + image = Image.open(file_path) + image.thumbnail(size, resample=Image.ANTIALIAS) + if copy: + image.save(copy, format='jpeg', optimize=True, quality=quality) + else: + image.save(file_path, format='jpeg', optimize=True, quality=quality) + def rawOptimize(self): rgb = rawpy.imread(os.path.join(INPUT_PATH, self.file_name)) imageio.imsave(temp_file_path, rgb.postprocess()) @@ -26,13 +35,13 @@ class FileProcessor(object): print("Raw Size: {} {}".format(*_size(os.path.join(INPUT_PATH, self.file_name))), end=' | ') print("Resave Size: {} {}".format(*_size(temp_file_path)), end=' | ') pre = os.path.getsize(temp_file_path) - _optimize(temp_file_path) + self._optimize(temp_file_path) post = os.path.getsize(temp_file_path) print("Optimized Size: {} {} ({}% savings)".format(*_size(temp_file_path), round((1.0 - (post / pre)) * 100), 2) ) def basicOptimize(self): pre = os.path.getsize(os.path.join(INPUT_PATH, self.file_name)) - _optimize(os.path.join(INPUT_PATH, self.file_name), copy=temp_file_path) + self._optimize(os.path.join(INPUT_PATH, self.file_name), copy=temp_file_path) post = os.path.getsize(temp_file_path) print("Optimized Size: {} {} ({}% savings)".format(*_size(temp_file_path), round((1.0 - (post / pre)) * 100), 2) ) @@ -68,16 +77,15 @@ class FileProcessor(object): os.remove(os.path.join(INPUT_PATH, self.xmp_name)) # No XMP file is specified, using IPTC tagging else: - print('\tWriting {} tags to output {}'.format(len(labels), ext[1:].upper())) + print('\tWriting {} tags to output {}'.format(len(labels), self.ext[1:].upper())) info = iptcinfo3.IPTCInfo(os.path.join(INPUT_PATH, self.file_name)) info['keywords'].extend(labels) info.save() # Remove the weird ghsot file created by this iptc read/writer. os.remove(os.path.join(INPUT_PATH, self.file_name + '~')) - # Copy dry-run - # shutil.copy2(os.path.join(INPUT_PATH, self.file_name), os.path.join(OUTPUT_PATH, self.file_name)) - os.rename(os.path.join(INPUT_PATH, self.file_name), os.path.join(OUTPUT_PATH, self.file_name)) + shutil.copy2(os.path.join(INPUT_PATH, self.file_name), os.path.join(OUTPUT_PATH, self.file_name)) + # os.rename(os.path.join(INPUT_PATH, self.file_name), os.path.join(OUTPUT_PATH, self.file_name)) except: self._cleanup() raise @@ -93,13 +101,4 @@ class FileProcessor(object): def _size(self, file_path): size, type = os.path.getsize(file_path) / 1024, 'KiB' if size >= 1024: size /= 1024; type = 'MiB' - return round(size, 2), type - - # Optimizes a file using JPEG thumbnailing and compression. - def _optimize(self, file_path, size=(512, 512), quality=85, copy=None): - image = Image.open(file_path) - image.thumbnail(size, resample=Image.ANTIALIAS) - if copy: - image.save(copy, format='jpeg', optimize=True, quality=quality) - else: - image.save(file_path, format='jpeg', optimize=True, quality=quality) \ No newline at end of file + return round(size, 2), type \ No newline at end of file From ade2f7c01872b21f709f1759f0ce9f3fbfd97f06 Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 21:28:10 -0500 Subject: [PATCH 04/16] main file black format --- main.py | 8 +++++--- package/app.py | 2 +- package/process.py | 22 +++++++++++----------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/main.py b/main.py index 9fd911a..b138446 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,9 @@ -import sys, os +import sys +import os from package import app -os.environ["GOOGLE_APPLICATION_CREDENTIALS"]=os.path.join(sys.path[0], 'package', 'key', 'photo_tagging_service.json') +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.join( + sys.path[0], 'package', 'key', 'photo_tagging_service.json') if __name__ == "__main__": - sys.exit(app.run()) \ No newline at end of file + sys.exit(app.run()) diff --git a/package/app.py b/package/app.py index 4df864f..df30670 100644 --- a/package/app.py +++ b/package/app.py @@ -51,7 +51,7 @@ def run(): # Alert the user that there are duplicates in the directory and ask whether or not to continue if len(identicals) > 0: - print('Identical files were found in the directory, continue?') + print('Identical files were the directory, continue?') print(',\n\t'.join(identicals)) xmps = [possible for possible in files diff --git a/package/process.py b/package/process.py index 65ac4b9..1a68683 100644 --- a/package/process.py +++ b/package/process.py @@ -28,22 +28,22 @@ class FileProcessor(object): def rawOptimize(self): rgb = rawpy.imread(os.path.join(INPUT_PATH, self.file_name)) - imageio.imsave(temp_file_path, rgb.postprocess()) + imageio.imsave(self.temp_file_path, rgb.postprocess()) rgb.close() # Information on file sizes - print("Raw Size: {} {}".format(*_size(os.path.join(INPUT_PATH, self.file_name))), end=' | ') - print("Resave Size: {} {}".format(*_size(temp_file_path)), end=' | ') - pre = os.path.getsize(temp_file_path) - self._optimize(temp_file_path) - post = os.path.getsize(temp_file_path) - print("Optimized Size: {} {} ({}% savings)".format(*_size(temp_file_path), round((1.0 - (post / pre)) * 100), 2) ) + print("Raw Size: {} {}".format(*self._size(os.path.join(INPUT_PATH, self.file_name))), end=' | ') + print("Resave Size: {} {}".format(*self._size(self.temp_file_path)), end=' | ') + pre = os.path.getsize(self.temp_file_path) + self._optimize(self.temp_file_path) + post = os.path.getsize(self.temp_file_path) + print("Optimized Size: {} {} ({}% savings)".format(*self._size(self.temp_file_path), round((1.0 - (post / pre)) * 100), 2) ) def basicOptimize(self): pre = os.path.getsize(os.path.join(INPUT_PATH, self.file_name)) - self._optimize(os.path.join(INPUT_PATH, self.file_name), copy=temp_file_path) - post = os.path.getsize(temp_file_path) - print("Optimized Size: {} {} ({}% savings)".format(*_size(temp_file_path), round((1.0 - (post / pre)) * 100), 2) ) + self._optimize(os.path.join(INPUT_PATH, self.file_name), copy=self.temp_file_path) + post = os.path.getsize(self.temp_file_path) + print("Optimized Size: {} {} ({}% savings)".format(*self._size(self.temp_file_path), round((1.0 - (post / pre)) * 100), 2) ) def run(self, client): @@ -55,7 +55,7 @@ class FileProcessor(object): self.basicOptimize() # Open the image, read as bytes, convert to types Image - image = Image.open(temp_file_path) + image = Image.open(self.temp_file_path) bytesIO = io.BytesIO() image.save(bytesIO, format='jpeg') image.close() From 96685e33d3976623731d93803f43f95ca5836404 Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 22:22:38 -0500 Subject: [PATCH 05/16] logging additions and removal of archaic .XMP sidecar file "detection" --- main.py | 2 ++ package/__init__.py | 4 ++++ package/app.py | 52 ++++++++++++--------------------------------- package/process.py | 5 ----- 4 files changed, 19 insertions(+), 44 deletions(-) diff --git a/main.py b/main.py index b138446..b4b20a2 100644 --- a/main.py +++ b/main.py @@ -1,9 +1,11 @@ import sys import os +import logging from package import app os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.join( sys.path[0], 'package', 'key', 'photo_tagging_service.json') if __name__ == "__main__": + logging.info('Executing package...') sys.exit(app.run()) diff --git a/package/__init__.py b/package/__init__.py index 1d1bce0..c43ec7e 100644 --- a/package/__init__.py +++ b/package/__init__.py @@ -1,5 +1,8 @@ import os import sys +import logging + +logging.basicConfig(level=logging.INFO) # Path Constants ROOT = sys.path[0] @@ -7,6 +10,7 @@ PROCESSING_PATH = os.path.join(ROOT, 'package', 'processing') INPUT_PATH = os.path.join(PROCESSING_PATH, 'input') TEMP_PATH = os.path.join(PROCESSING_PATH, 'temp') OUTPUT_PATH = os.path.join(PROCESSING_PATH, 'output') +logging.info('Path Constants Built.') # Extension Constants RAW_EXTS = [ diff --git a/package/app.py b/package/app.py index df30670..e77d7ec 100644 --- a/package/app.py +++ b/package/app.py @@ -1,8 +1,16 @@ -import io, sys, os, time, rawpy, imageio, progressbar, shutil +import io +import sys +import os +import time +import rawpy +import imageio +import progressbar +import shutil +import logging -from .xmp import XMPParser from google.cloud import vision +from .xmp import XMPParser from .process import FileProcessor from . import INPUT_PATH, TEMP_PATH, OUTPUT_PATH, PROCESSING_PATH from . import RAW_EXTS, LOSSY_EXTS @@ -15,16 +23,6 @@ from . import RAW_EXTS, LOSSY_EXTS # Driver code for the package def run(): - # Ensure that 'input' and 'output' directories are created - if not os.path.exists(INPUT_PATH): - print('Input directory did not exist, creating and quitting.') - os.makedirs(INPUT_PATH) - return - - if not os.path.exists(OUTPUT_PATH): - print('Output directory did not exist. Creating...') - os.makedirs(OUTPUT_PATH) - # Client client = vision.ImageAnnotatorClient() @@ -43,33 +41,9 @@ def run(): ext = ext.lower().strip('.') # Raw files contain their metadata in an XMP file usually if ext in RAW_EXTS: - # Get all possible files - identicals = [possible for possible in files - if possible.startswith(os.path.splitext(file)[0]) - and not possible.endswith(os.path.splitext(file)[1]) - and not possible.upper().endswith('.XMP')] - - # Alert the user that there are duplicates in the directory and ask whether or not to continue - if len(identicals) > 0: - print('Identical files were the directory, continue?') - print(',\n\t'.join(identicals)) - - xmps = [possible for possible in files - if possible.startswith(os.path.splitext(file)[0]) - and possible.upper().endswith('.XMP')] - - # Skip and warn if more than 1 possible files, user error - if len(xmps) > 1: - print('More than 1 possible XMP metadata file for \'{}\'...'.format(file)) - print(',\n'.join(['\t{}'.format(possible) for possible in xmps])) - # Zero possible files, user error, likely - elif len(xmps) <= 0: - print('No matching XMP metadata file for \'{}\'. skipping...'.format(file)) - # Process individual file - else: - print('Processing file {}, \'{}\''.format(index + 1, xmps[0]), end=' | ') - file = FileProcessor(file, xmps[0]) - file.run(client) + print('Processing file {}, \'{}\''.format(index + 1, xmps[0]), end=' | ') + file = FileProcessor(file, xmps[0]) + file.run(client) elif ext in LOSSY_EXTS: print('Processing file {}, \'{}\''.format(index + 1, file), end=' | ') file = FileProcessor(file) diff --git a/package/process.py b/package/process.py index 1a68683..6cdc2e9 100644 --- a/package/process.py +++ b/package/process.py @@ -32,19 +32,14 @@ class FileProcessor(object): rgb.close() # Information on file sizes - print("Raw Size: {} {}".format(*self._size(os.path.join(INPUT_PATH, self.file_name))), end=' | ') - print("Resave Size: {} {}".format(*self._size(self.temp_file_path)), end=' | ') pre = os.path.getsize(self.temp_file_path) self._optimize(self.temp_file_path) post = os.path.getsize(self.temp_file_path) - print("Optimized Size: {} {} ({}% savings)".format(*self._size(self.temp_file_path), round((1.0 - (post / pre)) * 100), 2) ) def basicOptimize(self): pre = os.path.getsize(os.path.join(INPUT_PATH, self.file_name)) self._optimize(os.path.join(INPUT_PATH, self.file_name), copy=self.temp_file_path) post = os.path.getsize(self.temp_file_path) - print("Optimized Size: {} {} ({}% savings)".format(*self._size(self.temp_file_path), round((1.0 - (post / pre)) * 100), 2) ) - def run(self, client): try: From 33f4ffa760d5561c32fe6657f70a8750a47bbbd0 Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 22:31:03 -0500 Subject: [PATCH 06/16] removal of large amounts of archaic file processing and printing in favor of simplicity --- package/__main__.py | 13 +++++++++++++ package/app.py | 15 +++++---------- 2 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 package/__main__.py diff --git a/package/__main__.py b/package/__main__.py new file mode 100644 index 0000000..0cf84d7 --- /dev/null +++ b/package/__main__.py @@ -0,0 +1,13 @@ +import os +import logging + +from . import INPUT_PATH, OUTPUT_PATH + +# Ensure that 'input' and 'output' directories are created +if not os.path.exists(INPUT_PATH): + logging.fatal('Input directory did not exist, creating and quitting.') + os.makedirs(INPUT_PATH) + +if not os.path.exists(OUTPUT_PATH): + logging.info('Output directory did not exist. Creating...') + os.makedirs(OUTPUT_PATH) \ No newline at end of file diff --git a/package/app.py b/package/app.py index e77d7ec..994a052 100644 --- a/package/app.py +++ b/package/app.py @@ -27,25 +27,20 @@ def run(): client = vision.ImageAnnotatorClient() # Find files we want to process based on if they have a corresponding .XMP + logging.info('Locating processable files...') files = os.listdir(INPUT_PATH) select = [file for file in files if os.path.splitext(file)[1] != '.xmp'] # Create the 'temp' directory - print(f'Initializing file processing for {len(select)} files...') + logging.info(f'Found {len(files)} valid files, beginning processing...') os.makedirs(TEMP_PATH) try: # Process files for index, file in progressbar.progressbar(list(enumerate(select)), redirect_stdout=True, term_width=110): - name, ext = os.path.splitext(file) - ext = ext.lower().strip('.') - # Raw files contain their metadata in an XMP file usually - if ext in RAW_EXTS: - print('Processing file {}, \'{}\''.format(index + 1, xmps[0]), end=' | ') - file = FileProcessor(file, xmps[0]) - file.run(client) - elif ext in LOSSY_EXTS: - print('Processing file {}, \'{}\''.format(index + 1, file), end=' | ') + _, ext = os.path.splitext(file) + if ext in LOSSY_EXTS or ext in RAW_EXTS: + logging.info(f"Processing file '{file}'...") file = FileProcessor(file) file.run(client) except: From 9d56c5ebfc8834f70fc0024c8eea4b07e645eff3 Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 22:37:51 -0500 Subject: [PATCH 07/16] cleanup/refactor of FileProcessor class further --- package/process.py | 47 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/package/process.py b/package/process.py index 6cdc2e9..eb783ed 100644 --- a/package/process.py +++ b/package/process.py @@ -4,6 +4,7 @@ import rawpy import imageio import io import iptcinfo3 +import logging from PIL import Image from google.cloud.vision import types from google.cloud import vision @@ -12,11 +13,16 @@ from . import TEMP_PATH, INPUT_PATH, OUTPUT_PATH from .xmp import XMPParser class FileProcessor(object): - def __init__(self, file_name, xmp_name=None): - self.file_name, self.xmp_name = file_name, xmp_name - self.base, self.ext = os.path.splitext(self.file_name) + def __init__(self, file_name): + self.file_name = file_name + self.base, self.ext = os.path.splitext(self.file_name) # fileNAME and fileEXTENSIOn + # Path to temporary file that will be optimized for upload to Google self.temp_file_path = os.path.join(TEMP_PATH, self.base + '.jpeg') + @property + def hasXMP(self): + return self.ext.lower() == 'xmp' + # Optimizes a file using JPEG thumbnailing and compression. def _optimize(self, file_path, size=(512, 512), quality=85, copy=None): image = Image.open(file_path) @@ -26,28 +32,20 @@ class FileProcessor(object): else: image.save(file_path, format='jpeg', optimize=True, quality=quality) - def rawOptimize(self): - rgb = rawpy.imread(os.path.join(INPUT_PATH, self.file_name)) - imageio.imsave(self.temp_file_path, rgb.postprocess()) - rgb.close() - - # Information on file sizes - pre = os.path.getsize(self.temp_file_path) - self._optimize(self.temp_file_path) - post = os.path.getsize(self.temp_file_path) - - def basicOptimize(self): - pre = os.path.getsize(os.path.join(INPUT_PATH, self.file_name)) - self._optimize(os.path.join(INPUT_PATH, self.file_name), copy=self.temp_file_path) - post = os.path.getsize(self.temp_file_path) + def optimize(self): + if self.hasXMP: + # Long runn + rgb = rawpy.imread(os.path.join(INPUT_PATH, self.file_name)) + imageio.imsave(self.temp_file_path, rgb.postprocess()) + rgb.close() + self._optimize(self.temp_file_path) + else: + self._optimize(os.path.join(INPUT_PATH, self.file_name), copy=self.temp_file_path) def run(self, client): try: - if self.xmp_name: - # Process the file into a JPEG - self.rawOptimize() - else: - self.basicOptimize() + if self.hasXMP: + self.optimize() # Open the image, read as bytes, convert to types Image image = Image.open(self.temp_file_path) @@ -79,17 +77,16 @@ class FileProcessor(object): # Remove the weird ghsot file created by this iptc read/writer. os.remove(os.path.join(INPUT_PATH, self.file_name + '~')) # Copy dry-run - shutil.copy2(os.path.join(INPUT_PATH, self.file_name), os.path.join(OUTPUT_PATH, self.file_name)) + # shutil.copy2(os.path.join(INPUT_PATH, self.file_name), os.path.join(OUTPUT_PATH, self.file_name)) # os.rename(os.path.join(INPUT_PATH, self.file_name), os.path.join(OUTPUT_PATH, self.file_name)) except: self._cleanup() raise self._cleanup() - # Remove the temporary file + # Remove the temporary file (if it exists) def _cleanup(self): if os.path.exists(self.temp_file_path): - # Deletes the temporary file os.remove(self.temp_file_path) # Get the size of the file. Is concerned with filesize type. 1024KiB -> 1MiB From a6bfd8daefaab93c5fa535db5c1862ad8fce0c93 Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 22:44:29 -0500 Subject: [PATCH 08/16] added in type hinting, removed unnecessary size calc methods and added bits of logging --- package/app.py | 3 ++- package/process.py | 18 ++++++------------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/package/app.py b/package/app.py index 994a052..26ebbaf 100644 --- a/package/app.py +++ b/package/app.py @@ -44,9 +44,10 @@ def run(): file = FileProcessor(file) file.run(client) except: + logging.warning('Removing temporary directory before raising exception.') os.rmdir(TEMP_PATH) raise # Remove the directory, we are done here - print('Cleaning up temporary directory...') + logging.info('Removing temporary directory.') os.rmdir(TEMP_PATH) \ No newline at end of file diff --git a/package/process.py b/package/process.py index eb783ed..44c70f5 100644 --- a/package/process.py +++ b/package/process.py @@ -13,7 +13,7 @@ from . import TEMP_PATH, INPUT_PATH, OUTPUT_PATH from .xmp import XMPParser class FileProcessor(object): - def __init__(self, file_name): + def __init__(self, file_name : str): self.file_name = file_name self.base, self.ext = os.path.splitext(self.file_name) # fileNAME and fileEXTENSIOn # Path to temporary file that will be optimized for upload to Google @@ -24,13 +24,13 @@ class FileProcessor(object): return self.ext.lower() == 'xmp' # Optimizes a file using JPEG thumbnailing and compression. - def _optimize(self, file_path, size=(512, 512), quality=85, copy=None): - image = Image.open(file_path) + def _optimize(self, file : str, size : tuple =(512, 512), quality=85, copy=None): + image = Image.open(file) image.thumbnail(size, resample=Image.ANTIALIAS) if copy: image.save(copy, format='jpeg', optimize=True, quality=quality) else: - image.save(file_path, format='jpeg', optimize=True, quality=quality) + image.save(file, format='jpeg', optimize=True, quality=quality) def optimize(self): if self.hasXMP: @@ -42,7 +42,7 @@ class FileProcessor(object): else: self._optimize(os.path.join(INPUT_PATH, self.file_name), copy=self.temp_file_path) - def run(self, client): + def run(self, client : vision.ImageAnnotatorClient): try: if self.hasXMP: self.optimize() @@ -87,10 +87,4 @@ class FileProcessor(object): # Remove the temporary file (if it exists) def _cleanup(self): if os.path.exists(self.temp_file_path): - os.remove(self.temp_file_path) - - # Get the size of the file. Is concerned with filesize type. 1024KiB -> 1MiB - def _size(self, file_path): - size, type = os.path.getsize(file_path) / 1024, 'KiB' - if size >= 1024: size /= 1024; type = 'MiB' - return round(size, 2), type \ No newline at end of file + os.remove(self.temp_file_path) \ No newline at end of file From 30bb1352167f2a4e452bf4e9a17af4ae692783aa Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 22:47:03 -0500 Subject: [PATCH 09/16] formatting, additional logging --- package/process.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/package/process.py b/package/process.py index 44c70f5..fd96af4 100644 --- a/package/process.py +++ b/package/process.py @@ -61,16 +61,17 @@ class FileProcessor(object): # XMP sidecar file specified, write to it using XML module if self.xmp_name: - print('\tWriting {} tags to output XMP...'.format(len(labels))) + logging.info('\tWriting {} tags to output XMP.'.format(len(labels))) parser = XMPParser(os.path.join(INPUT_PATH, self.xmp_name)) parser.add_keywords(labels) # Save the new XMP file + loggin.debug('Saving to new XMP file.') parser.save(os.path.join(OUTPUT_PATH, self.xmp_name)) - # Remove the old XMP file + logging.debug('Removing old XMP file.') os.remove(os.path.join(INPUT_PATH, self.xmp_name)) # No XMP file is specified, using IPTC tagging else: - print('\tWriting {} tags to output {}'.format(len(labels), self.ext[1:].upper())) + logging.info('\tWriting {} tags to image IPTC'.format(len(labels))) info = iptcinfo3.IPTCInfo(os.path.join(INPUT_PATH, self.file_name)) info['keywords'].extend(labels) info.save() From f267a43a26aebc291a2df65aa9831390edf9d510 Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 23:03:31 -0500 Subject: [PATCH 10/16] exception handling -> printing, xmp identification and handling --- package/app.py | 18 +++++++++++------- package/process.py | 43 ++++++++++++++++++++++++------------------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/package/app.py b/package/app.py index 26ebbaf..043194a 100644 --- a/package/app.py +++ b/package/app.py @@ -21,33 +21,37 @@ from . import RAW_EXTS, LOSSY_EXTS # 3) Read XMP, then write new tags to it # 4) Delete temporary file, move NEF/JPEG and XMP -# Driver code for the package + + def run(): - # Client client = vision.ImageAnnotatorClient() # Find files we want to process based on if they have a corresponding .XMP logging.info('Locating processable files...') files = os.listdir(INPUT_PATH) select = [file for file in files if os.path.splitext(file)[1] != '.xmp'] + logging.info(f'Found {len(files)} valid files') # Create the 'temp' directory - logging.info(f'Found {len(files)} valid files, beginning processing...') + logging.info('Creating temporary processing directory') os.makedirs(TEMP_PATH) try: # Process files for index, file in progressbar.progressbar(list(enumerate(select)), redirect_stdout=True, term_width=110): _, ext = os.path.splitext(file) + ext = ext[1:] if ext in LOSSY_EXTS or ext in RAW_EXTS: - logging.info(f"Processing file '{file}'...") file = FileProcessor(file) + logging.info(f"Processing file '{file}'...") file.run(client) - except: - logging.warning('Removing temporary directory before raising exception.') + except Exception as error: + logging.error(str(error)) + logging.warning( + 'Removing temporary directory before raising exception.') os.rmdir(TEMP_PATH) raise # Remove the directory, we are done here logging.info('Removing temporary directory.') - os.rmdir(TEMP_PATH) \ No newline at end of file + os.rmdir(TEMP_PATH) diff --git a/package/process.py b/package/process.py index fd96af4..599d164 100644 --- a/package/process.py +++ b/package/process.py @@ -12,19 +12,20 @@ from google.cloud import vision from . import TEMP_PATH, INPUT_PATH, OUTPUT_PATH from .xmp import XMPParser + class FileProcessor(object): - def __init__(self, file_name : str): + def __init__(self, file_name: str): self.file_name = file_name - self.base, self.ext = os.path.splitext(self.file_name) # fileNAME and fileEXTENSIOn + self.base, self.ext = os.path.splitext( + self.file_name) # fileNAME and fileEXTENSIOn # Path to temporary file that will be optimized for upload to Google self.temp_file_path = os.path.join(TEMP_PATH, self.base + '.jpeg') - - @property - def hasXMP(self): - return self.ext.lower() == 'xmp' - + # Decide whether a XMP file is available + self.xmp = os.path.join(INPUT_PATH, base + '.xmp') + self.xmp = self.xmp if os.path.exists(self.xmp) else None + # Optimizes a file using JPEG thumbnailing and compression. - def _optimize(self, file : str, size : tuple =(512, 512), quality=85, copy=None): + def _optimize(self, file: str, size: tuple = (512, 512), quality : int = 85, copy : str = None): image = Image.open(file) image.thumbnail(size, resample=Image.ANTIALIAS) if copy: @@ -40,12 +41,13 @@ class FileProcessor(object): rgb.close() self._optimize(self.temp_file_path) else: - self._optimize(os.path.join(INPUT_PATH, self.file_name), copy=self.temp_file_path) + self._optimize(os.path.join( + INPUT_PATH, self.file_name), copy=self.temp_file_path) - def run(self, client : vision.ImageAnnotatorClient): + def run(self, client: vision.ImageAnnotatorClient): try: - if self.hasXMP: - self.optimize() + + self.optimize() # Open the image, read as bytes, convert to types Image image = Image.open(self.temp_file_path) @@ -53,15 +55,16 @@ class FileProcessor(object): image.save(bytesIO, format='jpeg') image.close() image = vision.types.Image(content=bytesIO.getvalue()) - + # Performs label detection on the image file response = client.label_detection(image=image) labels = [label.description for label in response.label_annotations] print('\tLabels: {}'.format(', '.join(labels))) - + # XMP sidecar file specified, write to it using XML module - if self.xmp_name: - logging.info('\tWriting {} tags to output XMP.'.format(len(labels))) + if self.hasXMP: + logging.info( + '\tWriting {} tags to output XMP.'.format(len(labels))) parser = XMPParser(os.path.join(INPUT_PATH, self.xmp_name)) parser.add_keywords(labels) # Save the new XMP file @@ -71,8 +74,10 @@ class FileProcessor(object): os.remove(os.path.join(INPUT_PATH, self.xmp_name)) # No XMP file is specified, using IPTC tagging else: - logging.info('\tWriting {} tags to image IPTC'.format(len(labels))) - info = iptcinfo3.IPTCInfo(os.path.join(INPUT_PATH, self.file_name)) + logging.info( + '\tWriting {} tags to image IPTC'.format(len(labels))) + info = iptcinfo3.IPTCInfo( + os.path.join(INPUT_PATH, self.file_name)) info['keywords'].extend(labels) info.save() # Remove the weird ghsot file created by this iptc read/writer. @@ -88,4 +93,4 @@ class FileProcessor(object): # Remove the temporary file (if it exists) def _cleanup(self): if os.path.exists(self.temp_file_path): - os.remove(self.temp_file_path) \ No newline at end of file + os.remove(self.temp_file_path) From 3d7179be34eed838cf9ecb6c927b58425abccd68 Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 23:12:50 -0500 Subject: [PATCH 11/16] fixed xmp detection and cleaned up log messages --- main.py | 4 ++-- package/__init__.py | 6 +++++- package/app.py | 4 ++-- package/process.py | 44 +++++++++++++++++++++++--------------------- 4 files changed, 32 insertions(+), 26 deletions(-) diff --git a/main.py b/main.py index b4b20a2..a1b2012 100644 --- a/main.py +++ b/main.py @@ -1,11 +1,11 @@ import sys import os import logging -from package import app +from package import log, app os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.join( sys.path[0], 'package', 'key', 'photo_tagging_service.json') if __name__ == "__main__": - logging.info('Executing package...') + log.info('Executing package...') sys.exit(app.run()) diff --git a/package/__init__.py b/package/__init__.py index c43ec7e..0bc0c82 100644 --- a/package/__init__.py +++ b/package/__init__.py @@ -1,8 +1,12 @@ import os import sys import logging +import progressbar +# Logging and Progressbar work +progressbar.streams.wrap_stderr() logging.basicConfig(level=logging.INFO) +log = logging.getLogger(__name__) # Path Constants ROOT = sys.path[0] @@ -10,7 +14,7 @@ PROCESSING_PATH = os.path.join(ROOT, 'package', 'processing') INPUT_PATH = os.path.join(PROCESSING_PATH, 'input') TEMP_PATH = os.path.join(PROCESSING_PATH, 'temp') OUTPUT_PATH = os.path.join(PROCESSING_PATH, 'output') -logging.info('Path Constants Built.') +log.info('Path Constants Built.') # Extension Constants RAW_EXTS = [ diff --git a/package/app.py b/package/app.py index 043194a..63572f8 100644 --- a/package/app.py +++ b/package/app.py @@ -42,9 +42,9 @@ def run(): _, ext = os.path.splitext(file) ext = ext[1:] if ext in LOSSY_EXTS or ext in RAW_EXTS: - file = FileProcessor(file) + process = FileProcessor(file) logging.info(f"Processing file '{file}'...") - file.run(client) + process.run(client) except Exception as error: logging.error(str(error)) logging.warning( diff --git a/package/process.py b/package/process.py index 599d164..746bd5c 100644 --- a/package/process.py +++ b/package/process.py @@ -9,21 +9,26 @@ from PIL import Image from google.cloud.vision import types from google.cloud import vision -from . import TEMP_PATH, INPUT_PATH, OUTPUT_PATH +from . import TEMP_PATH, INPUT_PATH, OUTPUT_PATH, RAW_EXTS, LOSSY_EXTS from .xmp import XMPParser class FileProcessor(object): def __init__(self, file_name: str): self.file_name = file_name - self.base, self.ext = os.path.splitext( - self.file_name) # fileNAME and fileEXTENSIOn + self.base, self.ext = os.path.splitext(self.file_name) + self.ext = self.ext[1:] # Path to temporary file that will be optimized for upload to Google self.temp_file_path = os.path.join(TEMP_PATH, self.base + '.jpeg') # Decide whether a XMP file is available - self.xmp = os.path.join(INPUT_PATH, base + '.xmp') - self.xmp = self.xmp if os.path.exists(self.xmp) else None - + self.xmp = None + if self.ext.lower() in RAW_EXTS: + self.xmp = self.base + '.xmp' + self.input_xmp = os.path.join(INPUT_PATH, self.xmp) + self.output_xmp = os.path.join(OUTPUT_PATH, self.xmp) + if not os.path.exists(self.input_xmp): + raise Exception('Sidecar file for \'{}\' does not exist.'.format(self.xmp)) + # Optimizes a file using JPEG thumbnailing and compression. def _optimize(self, file: str, size: tuple = (512, 512), quality : int = 85, copy : str = None): image = Image.open(file) @@ -34,7 +39,7 @@ class FileProcessor(object): image.save(file, format='jpeg', optimize=True, quality=quality) def optimize(self): - if self.hasXMP: + if self.xmp: # Long runn rgb = rawpy.imread(os.path.join(INPUT_PATH, self.file_name)) imageio.imsave(self.temp_file_path, rgb.postprocess()) @@ -46,7 +51,6 @@ class FileProcessor(object): def run(self, client: vision.ImageAnnotatorClient): try: - self.optimize() # Open the image, read as bytes, convert to types Image @@ -59,32 +63,30 @@ class FileProcessor(object): # Performs label detection on the image file response = client.label_detection(image=image) labels = [label.description for label in response.label_annotations] - print('\tLabels: {}'.format(', '.join(labels))) + logging.info('Keywords Identified: {}'.format(', '.join(labels))) # XMP sidecar file specified, write to it using XML module - if self.hasXMP: - logging.info( - '\tWriting {} tags to output XMP.'.format(len(labels))) - parser = XMPParser(os.path.join(INPUT_PATH, self.xmp_name)) + if self.xmp: + logging.info('Writing {} tags to output XMP.'.format(len(labels))) + parser = XMPParser(self.input_xmp) parser.add_keywords(labels) # Save the new XMP file - loggin.debug('Saving to new XMP file.') - parser.save(os.path.join(OUTPUT_PATH, self.xmp_name)) + logging.debug('Saving to new XMP file.') + parser.save(self.output_xmp) logging.debug('Removing old XMP file.') - os.remove(os.path.join(INPUT_PATH, self.xmp_name)) + os.remove(self.input_xmp) # No XMP file is specified, using IPTC tagging else: - logging.info( - '\tWriting {} tags to image IPTC'.format(len(labels))) - info = iptcinfo3.IPTCInfo( - os.path.join(INPUT_PATH, self.file_name)) + logging.info('Writing {} tags to image IPTC'.format(len(labels))) + info = iptcinfo3.IPTCInfo(os.path.join(INPUT_PATH, self.file_name)) info['keywords'].extend(labels) info.save() # Remove the weird ghsot file created by this iptc read/writer. os.remove(os.path.join(INPUT_PATH, self.file_name + '~')) + # Copy dry-run # shutil.copy2(os.path.join(INPUT_PATH, self.file_name), os.path.join(OUTPUT_PATH, self.file_name)) - # os.rename(os.path.join(INPUT_PATH, self.file_name), os.path.join(OUTPUT_PATH, self.file_name)) + os.rename(os.path.join(INPUT_PATH, self.file_name), os.path.join(OUTPUT_PATH, self.file_name)) except: self._cleanup() raise From 7e3cb12513ec625acec8db7ff94aab755e79f559 Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 23:36:15 -0500 Subject: [PATCH 12/16] change to named loggers --- main.py | 4 +++- package/__init__.py | 7 ++++++- package/app.py | 27 +++++++++++---------------- package/process.py | 11 ++++++----- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/main.py b/main.py index a1b2012..37b2f9a 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,9 @@ import sys import os import logging -from package import log, app +from package import app + +log = logging.getLogger('main') os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.join( sys.path[0], 'package', 'key', 'photo_tagging_service.json') diff --git a/package/__init__.py b/package/__init__.py index 0bc0c82..815214b 100644 --- a/package/__init__.py +++ b/package/__init__.py @@ -6,11 +6,16 @@ import progressbar # Logging and Progressbar work progressbar.streams.wrap_stderr() logging.basicConfig(level=logging.INFO) -log = logging.getLogger(__name__) +log = logging.getLogger('init') +log.info('Progressbar/Logging ready.') + # Path Constants +# ROOT = '' ROOT = sys.path[0] +# PROCESSING_PATH = ROOT PROCESSING_PATH = os.path.join(ROOT, 'package', 'processing') +# INPUT_PATH = PROCESSING_PATH INPUT_PATH = os.path.join(PROCESSING_PATH, 'input') TEMP_PATH = os.path.join(PROCESSING_PATH, 'temp') OUTPUT_PATH = os.path.join(PROCESSING_PATH, 'output') diff --git a/package/app.py b/package/app.py index 63572f8..1c14635 100644 --- a/package/app.py +++ b/package/app.py @@ -15,43 +15,38 @@ from .process import FileProcessor from . import INPUT_PATH, TEMP_PATH, OUTPUT_PATH, PROCESSING_PATH from . import RAW_EXTS, LOSSY_EXTS -# Process a single file in these steps: -# 1) Create a temporary file -# 2) Send it to GoogleAPI -# 3) Read XMP, then write new tags to it -# 4) Delete temporary file, move NEF/JPEG and XMP - - +log = logging.getLogger('app') def run(): client = vision.ImageAnnotatorClient() # Find files we want to process based on if they have a corresponding .XMP - logging.info('Locating processable files...') + log.info('Locating processable files...') files = os.listdir(INPUT_PATH) select = [file for file in files if os.path.splitext(file)[1] != '.xmp'] - logging.info(f'Found {len(files)} valid files') + log.info(f'Found {len(files)} valid files') # Create the 'temp' directory - logging.info('Creating temporary processing directory') + log.info('Creating temporary processing directory') os.makedirs(TEMP_PATH) - + os.makedirs(OUTPUT_PATH) + try: # Process files for index, file in progressbar.progressbar(list(enumerate(select)), redirect_stdout=True, term_width=110): _, ext = os.path.splitext(file) - ext = ext[1:] + ext = ext[1:].lower() if ext in LOSSY_EXTS or ext in RAW_EXTS: process = FileProcessor(file) - logging.info(f"Processing file '{file}'...") + log.info(f"Processing file '{file}'...") process.run(client) except Exception as error: - logging.error(str(error)) - logging.warning( + log.error(str(error)) + log.warning( 'Removing temporary directory before raising exception.') os.rmdir(TEMP_PATH) raise # Remove the directory, we are done here - logging.info('Removing temporary directory.') + log.info('Removing temporary directory.') os.rmdir(TEMP_PATH) diff --git a/package/process.py b/package/process.py index 746bd5c..b1a0bc2 100644 --- a/package/process.py +++ b/package/process.py @@ -12,6 +12,7 @@ from google.cloud import vision from . import TEMP_PATH, INPUT_PATH, OUTPUT_PATH, RAW_EXTS, LOSSY_EXTS from .xmp import XMPParser +log = logging.getLogger('process') class FileProcessor(object): def __init__(self, file_name: str): @@ -63,21 +64,21 @@ class FileProcessor(object): # Performs label detection on the image file response = client.label_detection(image=image) labels = [label.description for label in response.label_annotations] - logging.info('Keywords Identified: {}'.format(', '.join(labels))) + log.info('Keywords Identified: {}'.format(', '.join(labels))) # XMP sidecar file specified, write to it using XML module if self.xmp: - logging.info('Writing {} tags to output XMP.'.format(len(labels))) + log.info('Writing {} tags to output XMP.'.format(len(labels))) parser = XMPParser(self.input_xmp) parser.add_keywords(labels) # Save the new XMP file - logging.debug('Saving to new XMP file.') + log.debug('Saving to new XMP file.') parser.save(self.output_xmp) - logging.debug('Removing old XMP file.') + log.debug('Removing old XMP file.') os.remove(self.input_xmp) # No XMP file is specified, using IPTC tagging else: - logging.info('Writing {} tags to image IPTC'.format(len(labels))) + log.info('Writing {} tags to image IPTC'.format(len(labels))) info = iptcinfo3.IPTCInfo(os.path.join(INPUT_PATH, self.file_name)) info['keywords'].extend(labels) info.save() From 9c1c05c7d01a10bba3700bfc48b9dcfebf78b8b3 Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 1 Nov 2019 23:45:06 -0500 Subject: [PATCH 13/16] migration towards command line only based utility --- package/__init__.py | 12 ++++-------- package/app.py | 15 +++++++++------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/package/__init__.py b/package/__init__.py index 815214b..37cb1e2 100644 --- a/package/__init__.py +++ b/package/__init__.py @@ -11,14 +11,10 @@ log.info('Progressbar/Logging ready.') # Path Constants -# ROOT = '' -ROOT = sys.path[0] -# PROCESSING_PATH = ROOT -PROCESSING_PATH = os.path.join(ROOT, 'package', 'processing') -# INPUT_PATH = PROCESSING_PATH -INPUT_PATH = os.path.join(PROCESSING_PATH, 'input') -TEMP_PATH = os.path.join(PROCESSING_PATH, 'temp') -OUTPUT_PATH = os.path.join(PROCESSING_PATH, 'output') +ROOT = os.path.dirname(os.path.realpath(__file__)) +INPUT_PATH = ROOT +TEMP_PATH = os.path.join(ROOT, 'temp') +OUTPUT_PATH = os.path.join(ROOT, 'output') log.info('Path Constants Built.') # Extension Constants diff --git a/package/app.py b/package/app.py index 1c14635..cb0216e 100644 --- a/package/app.py +++ b/package/app.py @@ -12,7 +12,7 @@ from google.cloud import vision from .xmp import XMPParser from .process import FileProcessor -from . import INPUT_PATH, TEMP_PATH, OUTPUT_PATH, PROCESSING_PATH +from . import INPUT_PATH, TEMP_PATH, OUTPUT_PATH from . import RAW_EXTS, LOSSY_EXTS log = logging.getLogger('app') @@ -23,13 +23,16 @@ def run(): # Find files we want to process based on if they have a corresponding .XMP log.info('Locating processable files...') files = os.listdir(INPUT_PATH) - select = [file for file in files if os.path.splitext(file)[1] != '.xmp'] - log.info(f'Found {len(files)} valid files') + select = [file for file in files if os.path.splitext(file)[1][1:].lower() in (RAW_EXTS + LOSSY_EXTS)] + log.info(f'Found {len(select)} valid files') # Create the 'temp' directory - log.info('Creating temporary processing directory') - os.makedirs(TEMP_PATH) - os.makedirs(OUTPUT_PATH) + if not os.path.exists(TEMP_PATH): + log.info('Creating temporary processing directory') + os.makedirs(TEMP_PATH) + if not os.path.exists(OUTPUT_PATH): + log.info('Creating output processing directory') + os.makedirs(OUTPUT_PATH) try: # Process files From 01e4d7c4f4b5c7f3e16c822aa2c6bc32d2334cca Mon Sep 17 00:00:00 2001 From: Xevion Date: Sat, 2 Nov 2019 00:05:49 -0500 Subject: [PATCH 14/16] prep for click --- main.py => phototag.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename main.py => phototag.py (100%) diff --git a/main.py b/phototag.py similarity index 100% rename from main.py rename to phototag.py From 5865819aa0c80c3b1581d071554d2d2914951d2e Mon Sep 17 00:00:00 2001 From: Xevion Date: Sat, 2 Nov 2019 01:58:02 -0500 Subject: [PATCH 15/16] add setup.py for automatic installation --- setup.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c32bfa4 --- /dev/null +++ b/setup.py @@ -0,0 +1,37 @@ +import sys +import os +import io +from setuptools import find_packages, setup + +DEPENDENCIES = [] +EXCLUDE_FROM_PACKAGES = [] +CURDIR = sys.path[0] + +with open(os.path.join(CURDIR, 'README.md')) as file: + README = file.read() + +setup( + name="phototag", + version="1.0.0", + author="Xevion", + author_email="xevion@xevion.dev", + description="", + long_description=README, + long_description_content_type="text/markdown", + url="https://github.com/xevion/photo-tagging", + packages=find_packages(exclude=EXCLUDE_FROM_PACKAGES), + include_package_data=True, + keywords=[], + scripts=[], + entry_points={"console_scripts": ["phototag=phototag.main:main"]}, + zip_safe=False, + install_requires=DEPENDENCIES, + python_requires=">=3.6", + # license and classifier list: + # https://pypi.org/pypi?%3Aaction=list_classifiers + license="License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + classifiers=[ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + ], +) \ No newline at end of file From 19a52c2a5510395a41ab39e27dada7abdee311d1 Mon Sep 17 00:00:00 2001 From: Xevion Date: Sat, 2 Nov 2019 02:16:04 -0500 Subject: [PATCH 16/16] setup.py update for click cli proper --- phototag.py | 9 ++++++++- setup.py | 7 +++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/phototag.py b/phototag.py index 37b2f9a..4812820 100644 --- a/phototag.py +++ b/phototag.py @@ -1,6 +1,7 @@ import sys import os import logging +import click from package import app log = logging.getLogger('main') @@ -8,6 +9,12 @@ log = logging.getLogger('main') os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.join( sys.path[0], 'package', 'key', 'photo_tagging_service.json') -if __name__ == "__main__": + +@click.command() +def cli(): log.info('Executing package...') sys.exit(app.run()) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index c32bfa4..66d763b 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import os import io from setuptools import find_packages, setup -DEPENDENCIES = [] +DEPENDENCIES = ['Click'] EXCLUDE_FROM_PACKAGES = [] CURDIR = sys.path[0] @@ -23,7 +23,10 @@ setup( include_package_data=True, keywords=[], scripts=[], - entry_points={"console_scripts": ["phototag=phototag.main:main"]}, + entry_points=''' + [console_scripts] + phototag=phototag.phototag:cli + ''', zip_safe=False, install_requires=DEPENDENCIES, python_requires=">=3.6",