Setup GitLab contribution fetching & fix instance logger level, add dateutil package

Pipfile

@@ -7,6 +7,7 @@ name = "pypi"
 rich = "*"
 requests = "*"
 python-decouple = "*"
+python-dateutil = "*"
 
 [dev-packages]
 

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "00c679bbeb1ce7bb2e0deb8c98894525960340fcf4a428b3920ae1ce7c2bcd37"
+            "sha256": "a7f90696a0d9986fc93f4cfd376ba8cc1bddbc5d6effa7e2a82b7a24b6439061"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -51,6 +51,14 @@
             ],
             "version": "==2.12.0"
         },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
+                "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
+            ],
+            "index": "pypi",
+            "version": "==2.8.2"
+        },
         "python-decouple": {
             "hashes": [
                 "sha256:2838cdf77a5cf127d7e8b339ce14c25bceb3af3e674e039d4901ba16359968c7",
@@ -75,6 +83,13 @@
             "index": "pypi",
             "version": "==12.4.4"
         },
+        "six": {
+            "hashes": [
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+            ],
+            "version": "==1.16.0"
+        },
         "typing-extensions": {
             "hashes": [
                 "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708",

sources.py

@@ -1,9 +1,10 @@
 import abc
-from datetime import date, datetime
+import logging
-from pprint import pprint
+from datetime import date
 from typing import Any, Callable, Dict, List, Optional
 
 import requests
+from dateutil import parser
 from decouple import config
 
 from models import Commit
@@ -33,7 +34,9 @@ class CommitSource(abc.ABC):
 
     def getLogger(self, name: Optional[str] = None) -> logging.Logger:
         """Returns a new instance of a Logger"""
-        return logging.getLogger(name or self.name.lower())
+        logger = logging.getLogger(name or self.name.lower())
+        logger.setLevel(logging.INFO)
+        return logger
 
 
 class Gitlab(CommitSource):
@@ -49,22 +52,32 @@ class Gitlab(CommitSource):
         self.__api_key: str = config(self.API_KEY_CONSTANT)
         self.__username: str = config(self.USERNAME_CONSTANT)
 
-    @property
+    def fetch(self, check_seen_function: Callable) -> List[Commit]:
-    def url(self) -> str:
+        """Automatically fetch all commits from the database."""
-        """Returns the request URL from which events will be sourced."""
-        return f"https://gitlab.com/api/v4/users/{self.__username}/events"
 
-    @property
+        page: int = 1
-    def headers(self) -> Dict[str, str]:
+        continue_fetching: bool = True
-        """Returns the required headers for authoring API requests."""
+        results: List[Commit] = []
-        return {
-            'PRIVATE-TOKEN': self.__api_key
-        }
 
-    def fetch(self, check_function: Callable) -> List[Commit]:
+        self.logger.info('Beginning fetching process for GitLab source.')
-        self.events(page=3)
 
-        return []
+        while continue_fetching:
+            continue_fetching = False
+
+            # Check all events in the list
+            for event in self.events(page=page, per_page=50):
+                if not check_seen_function(event['id']):
+                    continue_fetching = True
+
+                    results.append(Commit(
+                            id=event['id'],
+                            name='Private Contribution',
+                            timestamp=parser.isoparse(event['created_at'])
+                    ))
+
+            page += 1
+
+        return results
 
     def events(self, action: Optional[str] = None, target_type: Optional[str] = None, before: Optional[date] = None,
                after: Optional[date] = None, sort: Optional[str] = None, page: Optional[int] = None,
@@ -76,10 +89,25 @@ class Gitlab(CommitSource):
         if after: params['after'] = after.isoformat()
 
         params = {k: v for k, v in params.items() if v is not None}
-        response = requests.get(self.url, params=params, headers=self.headers)
+        request = requests.Request('GET', self.url, params=params, headers=self.headers)
-
+        prepped = self.session.prepare_request(request)
-        pprint(params)
+        response = self.session.send(prepped)
-
-        pprint(response.json())
 
         return response.json()
+
+    @property
+    def source_type(self) -> str:
+        """Provides the source type for this class in the database."""
+        return 'gitlab'
+
+    @property
+    def url(self) -> str:
+        """Returns the request URL from which events will be sourced."""
+        return f"https://gitlab.com/api/v4/users/{self.__username}/events"
+
+    @property
+    def headers(self) -> Dict[str, str]:
+        """Returns the required headers for authoring API requests."""
+        return {
+            'PRIVATE-TOKEN': self.__api_key
+        }