the-office/app/models.py

import requests
import json
import os

from bs4 import BeautifulSoup
from app import db, login

episodes = [
    5,
    6,
    22,
    23,
    14,
    26,
    24,
    24,
    24,
    23,
]  # Episode counts. Index 0 is for Webisodes.
quotePattern = r"([\w\s\.\',-\[\]\d&\"#]+):(.+)"
with open(os.path.join('app', 'static', 'titles.json'), 'r', encoding="utf-8") as file:
    titles = json.load(file)

class Season(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    episodes = db.relationship("Episode", backref="season", lazy="dynamic")

    def __init__(self, **kwargs):
        assert 0 <= kwargs.get("id") <= 9, "Season ID must be 0-9 inclusive"
        super(Season, self).__init__(**kwargs)

    def build(self, rebuild=False):
        """runs build operations on every Episode under this season"""
        print(f"Running build() on Season {self.id}")
        for episode in range(1, episodes[self.id] + 1):
            ep = Episode.query.filter_by(season_id=self.id, number=episode).first()
            if ep is None:
                # Add the episode, then build
                print(f"Creating new Episode, Season {self.id}, Episode {episode}")
                ep = Episode(season_id=self.id, number=episode)
                db.session.add(ep)
                # I'm commiting early, which is a bit taboo, but I'm more worried about what the Episode object will need while building.
                db.session.commit()
                ep.build()
            else:
                if rebuild:
                    print(f"Rebuilding Season {self.id}, Episode {episode}")
                    ep.build()
                pass

    def download(self, force=False):
        episodes = Episode.query.filter_by(season_id=self.id).all()
        for ep in episodes:
            ep.rebuild()

    @staticmethod
    def create_all(build=True):
        """
        creates new Season objects and runs build() on them"""
        for i in range(1, 10):
            if Season.query.get(i) is None:
                s = Season(id=i)
                db.session.add(s)
                if build:
                    s.build()
        db.session.commit()

    @staticmethod
    def rebuild_all():
        """
        Runs .build() on all Season objects in database
        """
        for season in Season.query.all():
            season.rebuild()

    @property
    def characters(self, sort):
        """
        returns a List of all characters in this Season, built off the Episode's .characters() method
        """
        pass


class Episode(db.Model):
    """
    represents a Episode with underlying Sections (representing a specific cutscene in the show)
    also has some other attributes useful for identify the episode and displaying, as well as countless methods
    aimed at providing easy to access information using the database collection
    """

    id = db.Column(
        db.Integer, primary_key=True
    )  # arbitrary ID, should NOT be relied on to determine episode number or correlating season
    number = db.Column(db.Integer)  # episode number
    title = db.Column(db.String(32))
    season_id = db.Column(
        db.Integer, db.ForeignKey("season.id")
    )  # correlating season number
    built = db.Column(db.Boolean, default=False)
    sections = db.relationship(
        "Section", backref="episode", lazy="dynamic"
    )  # sections of quotes under this episode

    @property
    def link(self):
        return f"http://officequotes.net/no{self.season_id}-{str(self.number).zfill(2)}.php"

    @property
    def HTMLpath(self):
        return os.path.join("app", "data", "raw", f"{self.season_id}-{self.number}.html")

    @property
    def HTMLdata(self):
        return open(self.HTMLpath, "r", encoding="utf-8").read()

    @property
    def JSONpath(self):
        """
        returns the path for the JSON file with data for this episode
        @return: a path
        """
        return os.path.join("app", "data", "preprocess", f"{self.season_id}-{self.number}.json")

    @property
    def JSONdata(self):
        """
        Returns the raw JSON data for this episode
        """
        return open(self.JSONpath, "r", encoding="utf-8").read()

    @property
    def downloaded(self):
        """
        Checks whether the raw episode script data has been downloaded.
        @return: boolean stating the existence (and thus likely properly downloaded) of raw data
        """
        return os.path.exists(self.HTMLpath)

    def download(self, force=False):
        """downloads data"""
        if not self.downloaded or force:
            print(f"Downloading e{self.number}/s{self.season_id} from {self.link}")
            data = requests.get(self.link).text
            open(self.HTMLpath, "w+", encoding="utf-8").write(data)

    def preprocess(self):
        """
        Runs pre-processing on this Episode, which creates and automatically builds a JSON file full of the data
        required to create a Episode properly, right before the Developer edits a episode and then enters it into the
        database as a full fledged 'processed' episode.
        """
        print(f'Pre-processing data for {self}')
        print(f'Rebuilding s{self.season_id} e{self.number}')
        self.download()

        soup = BeautifulSoup(self.data, "html.parser")
        sections = soup.find_all(attrs={"class": "quote"})
        deleted = 0

        root = []

        for section in sections:
            isNewpeat = False
            isDeleted = "deleted scene" in section.text
            if isDeleted:
                deleted += 1

            quotes = []
            for quote in section.find_all("b"):
                if "Newpeat" in quote.string:
                    quote = quote.next_sibling
                    isNewpeat = True
                # if quote is None or quote.next_sibling is None:
                #     print("Quote is None or next sibling is None")
                #     continue
                quotes.append(quote.string + quote.next_sibling.string)

            if len(quotes) == 0:
                print(f"Section found with Zero quotes. Newpeat: {isNewpeat} Deleted: {isDeleted}")
                if not (isNewpeat or isDeleted):
                    continue

            sectionData = {'isNewpeat' : isNewpeat, 'isDeleted' : isDeleted, 'quotes' : quotes}
            root.append(sectionData)

        with open(self.JSONpath, 'w+', encoding='utf-8') as file:
            json.dump(root, file)



    def build(self):
        """
        Downloads, Processes, and Automatically creates Sections and Quotes
        """

        self.built = True
        self.title = titles[self.season_id][self.number - 1]
        print(self.title)
        db.session.commit()

    def rebuild(self):
        """
        Clears all sections f
        """
        print(f'Rebuilding s{self.season_id} e{self.number}')
        self.clear()
        self.build()

    def clear(self):
        """delete all sections relevant to this episode in order to reprocess"""
        sections = Section.query.filter_by(episode_id=self.id).all()
        if len(sections > 0):
            print(f"Clearing {len(sections)} Sections of Ep {self.number} Season {self.season_id}")
            for section in sections:
                section.clear(commit=False, delete=True)
            self.built = False
            db.session.commit()
        else:
            print('No sections for this episode (s{self.season_id}/e{self.number}) could be found.')

    @staticmethod
    def clear_all():
        """
        Runs clear() on every episode in the database
        """
        print('Clearing all episodes in database.')
        for episode in Episode.query.all():
            episode.clear()

    def __repr__(self):
        sections = len(Section.query.filter_by(episode_id=self.id).all())
        return f"Episode(id={self.id} s={self.season_id} ep={self.number} sects=[{sections}...])"


class Section(db.Model):
    """represents a Section of Quotes, a specific scene with relevant dialog"""

    id = db.Column(db.Integer, primary_key=True)
    episode_id = db.Column(db.Integer, db.ForeignKey("episode.id"))
    deleted = db.Column(db.Integer, default=-1)
    newpeat = db.Column(db.Boolean, default=False)
    quotes = db.relationship("Quote", backref="section", lazy="dynamic")

    def build(self, quotes, commit=False, reset=False):
        """
        Given an List of unformatted script quotes, automatically creates Quotes assigned to this Section
        """
        for i, quote in enumerate(quotes):
            if quote.lower().startswith("deleted scene"):
                raise Exception(
                    f'Deleted Scene Quote passed to Section Builder: "{quote}"'
                )
            # match = re.match(quotePattern, quote)
            # assert match != None, f"Quote '{quote}' could not be processed."
            # q = Quote(section=self, speaker=match[1].strip(), text=match[2].strip())
            mark = quote.find(":")
            q = Quote(
                section=self,
                speaker=quote[:mark],
                text=quote[mark + 1 :],
                section_index=i,
            )
            db.session.add(q)
        if commit:
            db.session.commit()

    def clear(self, doprint=False, commit=True, delete=False):
        """
        Delete all quotes relevant to this section.
        """
        quotes = Quote.query.filter_by(section_id=self.id).all()
        if doprint:
            print(f"Clearing {len(quotes)} quotes from Section ID {self.id}")
        for quote in quotes:
            db.session.delete(quote)
        if delete:
            db.session.delete(self)
        if commit:
            db.session.commit()

    def __repr__(self):
        season = Episode.query.get(self.episode_id).id
        quotes = len(Quote.query.filter_by(section_id=self.id).all())
        return f"Section(id={self.id} S-EP={season}/{self.episode_id} quotes=[{quotes}...])"


class Quote(db.Model):
    """represents a specific quote by a specific speaker"""

    id = db.Column(db.Integer, primary_key=True)
    section_id = db.Column(
        db.Integer, db.ForeignKey("section.id")
    )  # The section this quote belongs to.
    speaker = db.Column(db.String(32))  # The name of a character
    text = db.Column(
        db.String(512)
    )  # The content of the Quote. Usually a sentence, sometimes more.
    section_index = db.Column(db.Integer)  # The index of this quote in the section

    def __repr__(self):
        return f"Quote(speaker='{self.speaker}' text='{self.text[:50]}{'...' if len(self.text) > 51 else ''}')"