automatic update cron implementation, vercel cron, type guards

package.json

@@ -14,6 +14,7 @@
     "@headlessui/react": "^2.2.0",
     "@kodingdotninja/use-tailwind-breakpoint": "^1.0.0",
     "@next/eslint-plugin-next": "^15.1.1",
+    "@octokit/core": "^6.1.2",
     "@plaiceholder/next": "^3.0.0",
     "@tailwindcss/typography": "^0.5.8",
     "@tanstack/react-query": "^4.16.0",

pnpm-lock.yaml

@@ -20,6 +20,9 @@ importers:
       '@next/eslint-plugin-next':
         specifier: ^15.1.1
         version: 15.1.1
+      '@octokit/core':
+        specifier: ^6.1.2
+        version: 6.1.2
       '@plaiceholder/next':
         specifier: ^3.0.0
         version: 3.0.0(next@15.1.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(sass@1.56.2))(plaiceholder@3.0.0(sharp@0.32.1))(sharp@0.32.1)
@@ -433,6 +436,36 @@ packages:
     resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==}
     engines: {node: '>=12.4.0'}
 
+  '@octokit/auth-token@5.1.1':
+    resolution: {integrity: sha512-rh3G3wDO8J9wSjfI436JUKzHIxq8NaiL0tVeB2aXmG6p/9859aUOAjA9pmSPNGGZxfwmaJ9ozOJImuNVJdpvbA==}
+    engines: {node: '>= 18'}
+
+  '@octokit/core@6.1.2':
+    resolution: {integrity: sha512-hEb7Ma4cGJGEUNOAVmyfdB/3WirWMg5hDuNFVejGEDFqupeOysLc2sG6HJxY2etBp5YQu5Wtxwi020jS9xlUwg==}
+    engines: {node: '>= 18'}
+
+  '@octokit/endpoint@10.1.2':
+    resolution: {integrity: sha512-XybpFv9Ms4hX5OCHMZqyODYqGTZ3H6K6Vva+M9LR7ib/xr1y1ZnlChYv9H680y77Vd/i/k+thXApeRASBQkzhA==}
+    engines: {node: '>= 18'}
+
+  '@octokit/graphql@8.1.2':
+    resolution: {integrity: sha512-bdlj/CJVjpaz06NBpfHhp4kGJaRZfz7AzC+6EwUImRtrwIw8dIgJ63Xg0OzV9pRn3rIzrt5c2sa++BL0JJ8GLw==}
+    engines: {node: '>= 18'}
+
+  '@octokit/openapi-types@22.2.0':
+    resolution: {integrity: sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==}
+
+  '@octokit/request-error@6.1.6':
+    resolution: {integrity: sha512-pqnVKYo/at0NuOjinrgcQYpEbv4snvP3bKMRqHaD9kIsk9u1LCpb2smHZi8/qJfgeNqLo5hNW4Z7FezNdEo0xg==}
+    engines: {node: '>= 18'}
+
+  '@octokit/request@9.1.4':
+    resolution: {integrity: sha512-tMbOwGm6wDII6vygP3wUVqFTw3Aoo0FnVQyhihh8vVq12uO3P+vQZeo2CKMpWtPSogpACD0yyZAlVlQnjW71DA==}
+    engines: {node: '>= 18'}
+
+  '@octokit/types@13.6.2':
+    resolution: {integrity: sha512-WpbZfZUcZU77DrSW4wbsSgTPfKcp286q3ItaIgvSbBpZJlu6mnYXAkjZz6LVZPXkEvLIM8McanyZejKTYUHipA==}
+
   '@pkgjs/parseargs@0.11.0':
     resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
     engines: {node: '>=14'}
@@ -758,6 +791,9 @@ packages:
   base64-js@1.5.1:
     resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
 
+  before-after-hook@3.0.2:
+    resolution: {integrity: sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==}
+
   binary-extensions@2.2.0:
     resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==}
     engines: {node: '>=8'}
@@ -1146,6 +1182,9 @@ packages:
   extend@3.0.2:
     resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==}
 
+  fast-content-type-parse@2.0.0:
+    resolution: {integrity: sha512-fCqg/6Sps8tqk8p+kqyKqYfOF0VjPNYrqpLiqNl0RBKmD80B080AJWVV6EkSkscjToNExcXg1+Mfzftrx6+iSA==}
+
   fast-deep-equal@3.1.3:
     resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
 
@@ -2338,6 +2377,9 @@ packages:
   unist-util-visit@5.0.0:
     resolution: {integrity: sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==}
 
+  universal-user-agent@7.0.2:
+    resolution: {integrity: sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==}
+
   update-browserslist-db@1.0.10:
     resolution: {integrity: sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==}
     hasBin: true
@@ -2667,6 +2709,47 @@ snapshots:
 
   '@nolyfill/is-core-module@1.0.39': {}
 
+  '@octokit/auth-token@5.1.1': {}
+
+  '@octokit/core@6.1.2':
+    dependencies:
+      '@octokit/auth-token': 5.1.1
+      '@octokit/graphql': 8.1.2
+      '@octokit/request': 9.1.4
+      '@octokit/request-error': 6.1.6
+      '@octokit/types': 13.6.2
+      before-after-hook: 3.0.2
+      universal-user-agent: 7.0.2
+
+  '@octokit/endpoint@10.1.2':
+    dependencies:
+      '@octokit/types': 13.6.2
+      universal-user-agent: 7.0.2
+
+  '@octokit/graphql@8.1.2':
+    dependencies:
+      '@octokit/request': 9.1.4
+      '@octokit/types': 13.6.2
+      universal-user-agent: 7.0.2
+
+  '@octokit/openapi-types@22.2.0': {}
+
+  '@octokit/request-error@6.1.6':
+    dependencies:
+      '@octokit/types': 13.6.2
+
+  '@octokit/request@9.1.4':
+    dependencies:
+      '@octokit/endpoint': 10.1.2
+      '@octokit/request-error': 6.1.6
+      '@octokit/types': 13.6.2
+      fast-content-type-parse: 2.0.0
+      universal-user-agent: 7.0.2
+
+  '@octokit/types@13.6.2':
+    dependencies:
+      '@octokit/openapi-types': 22.2.0
+
   '@pkgjs/parseargs@0.11.0':
     optional: true
 
@@ -3034,6 +3117,8 @@ snapshots:
 
   base64-js@1.5.1: {}
 
+  before-after-hook@3.0.2: {}
+
   binary-extensions@2.2.0: {}
 
   bl@4.1.0:
@@ -3395,7 +3480,7 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  eslint-module-utils@2.12.0(@typescript-eslint/parser@8.18.1(eslint@9.17.0(jiti@1.21.7))(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0)(eslint@9.17.0(jiti@1.21.7)))(eslint@9.17.0(jiti@1.21.7)):
+  eslint-module-utils@2.12.0(@typescript-eslint/parser@8.18.1(eslint@9.17.0(jiti@1.21.7))(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0)(eslint@9.17.0(jiti@1.21.7)):
     dependencies:
       debug: 3.2.7
     optionalDependencies:
@@ -3417,7 +3502,7 @@ snapshots:
       doctrine: 2.1.0
       eslint: 9.17.0(jiti@1.21.7)
       eslint-import-resolver-node: 0.3.9
-      eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.18.1(eslint@9.17.0(jiti@1.21.7))(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0)(eslint@9.17.0(jiti@1.21.7)))(eslint@9.17.0(jiti@1.21.7))
+      eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.18.1(eslint@9.17.0(jiti@1.21.7))(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0)(eslint@9.17.0(jiti@1.21.7))
       hasown: 2.0.2
       is-core-module: 2.16.0
       is-glob: 4.0.3
@@ -3554,6 +3639,8 @@ snapshots:
 
   extend@3.0.2: {}
 
+  fast-content-type-parse@2.0.0: {}
+
   fast-deep-equal@3.1.3: {}
 
   fast-glob@3.3.1:
@@ -4998,6 +5085,8 @@ snapshots:
       unist-util-is: 6.0.0
       unist-util-visit-parents: 6.0.1
 
+  universal-user-agent@7.0.2: {}
+
   update-browserslist-db@1.0.10(browserslist@4.21.4):
     dependencies:
       browserslist: 4.21.4

src/pages/api/cron/updated.ts

@@ -0,0 +1,315 @@
+/**
+ * This is a cron job handler for acquiring the latest 'updated' data for the site's projects.
+ *
+ * 1) Fetch the list of all projects including their link URLs.
+ * 2) Filter the list only for projects with 'autocheck_update' enabled and any 'github.com' link.
+ * 3) For each project, query the GitHub API for the latest commit date on all branches.
+ * 4) If the latest commit date is newer than the project's 'last_updated' date, update the project's 'last_updated' date.
+ * 5) If any project's 'last_updated' date was updated, revalidate the index and/or project page.
+ * 6) Report the results of this cron job invocation.
+ *
+ * - This cron job runs at least once a day, at most once an hour.
+ * - This cron job is completely asynchronous but respects GitHub API rate limits.
+ * - This cron job requires authentication with the Directus API.
+ * - This cron job requires authentication with the GitHub API (mostly for rate limits).
+ */
+import directus, { ProjectLink } from "@/utils/directus";
+import { readItems, updateItem } from "@directus/sdk";
+import { NextApiRequest, NextApiResponse } from "next";
+import { Octokit } from "@octokit/core";
+import { isFulfilled, isRejected } from "@/utils/types";
+
+const octokit = new Octokit({
+  auth: process.env.GITHUB_API_TOKEN,
+  request: {
+    fetch: (url: string | URL, options: RequestInit) => {
+      console.log(`${options.method} ${url}`);
+      return fetch(url, options);
+    },
+  },
+});
+
+type ProjectResult = {
+  id: string;
+  previousUpdated: Date | null;
+  latestUpdated: Date | null;
+};
+
+function getRepository(url: string): [string, string] | null {
+  const pattern = /github.com\/([^/]+)\/([^/]+)/;
+  const match = pattern.exec(url);
+
+  if (match === null) return null;
+  return [match[1]!, match[2]!];
+}
+
+async function handleProject({
+  id: project_id,
+  urls,
+  date_updated: previousUpdated,
+}: {
+  id: string;
+  urls: string[];
+  date_updated: Date | null;
+}): Promise<ProjectResult> {
+  // Extract the branches from each URL
+  const allBranches = await Promise.all(
+    urls.map(async (url) => {
+      const details = getRepository(url);
+      if (!details) {
+        return [];
+      }
+
+      // TODO: Handle deduplication of repository targets
+      const [owner, repo] = details;
+      const branches = await octokit.request(
+        "GET /repos/{owner}/{repo}/branches",
+        {
+          owner,
+          repo,
+          headers: {
+            "X-GitHub-Api-Version": "2022-11-28",
+          },
+        },
+      );
+
+      return branches.data.map((branch) => ({
+        branch: branch.name,
+        owner: owner,
+        repo: repo,
+      }));
+    }),
+  );
+
+  // Get the latest commit date for each branch (flattened)
+  const latestCommits = allBranches
+    .flat()
+    .map(async ({ owner, repo, branch }) => {
+      const commits = await octokit.request(
+        "GET /repos/{owner}/{repo}/commits",
+        {
+          owner,
+          repo,
+          sha: branch,
+          per_page: 1,
+          headers: {
+            "X-GitHub-Api-Version": "2022-11-28",
+          },
+        },
+      );
+      const latestCommit = commits.data[0];
+
+      // Commits not returned
+      if (latestCommit == null) {
+        console.warn({
+          target: `${owner}/${repo}@${branch}`,
+          message: "No commits available",
+        });
+        return null;
+      }
+
+      // Handle missing commit data in unpredictable cases
+      if (latestCommit.commit.author == null) {
+        console.warn({
+          target: `${owner}/${repo}@${branch}`,
+          sha: latestCommit.sha,
+          commit: latestCommit.commit.message,
+          url: latestCommit.html_url,
+          message: "No author available",
+        });
+        return null;
+      } else if (latestCommit.commit.author.date == null) {
+        console.warn({
+          target: `${owner}/${repo}@${branch}`,
+          sha: latestCommit.sha,
+          commit: latestCommit.commit.message,
+          url: latestCommit.html_url,
+          message: "No date available",
+        });
+        return null;
+      }
+
+      return new Date(latestCommit.commit.author.date);
+    });
+
+  const results = await Promise.allSettled(latestCommits);
+
+  // Handle the promises that failed
+  results.filter(isRejected).forEach((result) => {
+    // TODO: Add more context to the error message
+    console.error("Failed to fetch latest commit date", result.reason);
+  });
+
+  // Find the latest commit date
+  const latestUpdated = results
+    .filter(isFulfilled)
+    .map((v) => v.value)
+    .filter((v) => v != null)
+    .reduce((previous: Date | null, current: Date) => {
+      if (previous == null) return current;
+      return current > previous ? current : previous;
+    }, null);
+
+  if (latestUpdated == null) {
+    console.error("Unable to acquire the latest commit date for project");
+    return {
+      id: project_id,
+      previousUpdated,
+      latestUpdated: null,
+    };
+  }
+
+  // Ensure it's a reasonable date
+  if (latestUpdated != null && latestUpdated < new Date("2015-01-01")) {
+    console.error("Invalid commit date acquired", latestUpdated);
+    return {
+      id: project_id,
+      previousUpdated,
+      latestUpdated: null,
+    };
+  }
+
+  const result = { id: project_id, previousUpdated, latestUpdated: null };
+
+  // Update the project's 'last_updated' date if the latest commit date is newer
+  if (previousUpdated == null || latestUpdated > previousUpdated) {
+    await directus.request(
+      updateItem("project", project_id, {
+        date_updated: latestUpdated,
+      }),
+    );
+
+    // 'latestUpdated' is not null ONLY if the project was actually updated
+    return {
+      ...result,
+      latestUpdated,
+    };
+  }
+
+  return result;
+}
+
+export default async function handler(
+  req: NextApiRequest,
+  res: NextApiResponse,
+) {
+  // Check for the required environment variables
+  const { CRON_SECRET, GITHUB_API_TOKEN, DIRECTUS_API_TOKEN } = process.env;
+  if (!CRON_SECRET || !GITHUB_API_TOKEN || !DIRECTUS_API_TOKEN) {
+    res.status(500).json({ error: "Missing environment variables" });
+  }
+
+  // Ensure the cron request is authenticated
+  if (process.env.NODE_ENV !== "development") {
+    const authHeader = req.headers["authorization"];
+    if (authHeader !== `Bearer ${CRON_SECRET}`) {
+      return new Response("Unauthorized", {
+        status: 401,
+      });
+    }
+  }
+
+  let request_count = 0;
+  octokit.hook.before("request", async () => {
+    request_count++;
+  });
+
+  try {
+    // Fetch the list of all projects including their link URLs.
+    const projects = await directus.request(
+      readItems("project", {
+        fields: [
+          "id",
+          "name",
+          "autocheckUpdated",
+          "date_updated",
+          { links: ["url"] },
+        ],
+      }),
+    );
+
+    // Filter the list only for projects with 'autocheck_update' enabled and any 'github.com' link.
+    const eligibleProjects = projects
+      .map((project) => {
+        // Skip projects that don't have autocheckUpdated enabled.
+        if (!project.autocheckUpdated) return null;
+
+        // Acquire the URL from the link, then filter out any non-GitHub URLs.
+        const urls = project
+          .links!.map((link) => {
+            return (<ProjectLink>link).url;
+          })
+          .filter((url) => url.includes("github.com"));
+
+        // Skip projects that don't have any GitHub URLs.
+        if (urls.length === 0) return null;
+
+        // Return the project's most important data for further processing.
+        return {
+          id: project.id,
+          name: project.name,
+          date_updated: project.date_updated,
+          urls,
+        };
+      })
+      // null values are still included in the array, so filter them out.
+      .filter((project) => project !== null);
+
+    // Log the date_updated for each project
+    eligibleProjects.forEach((project) => {
+      console.log({
+        name: project.name,
+        date_updated: project.date_updated,
+      });
+    });
+
+    // For each project, query the GitHub API for the latest commit date on all branches.
+    const projectPromises = eligibleProjects.map((project) =>
+      handleProject({
+        id: project.id,
+        urls: project.urls,
+        date_updated:
+          project.date_updated != null ? new Date(project.date_updated) : null,
+      }),
+    );
+
+    // Wait for all project promises to resolve
+    const results = await Promise.allSettled(projectPromises);
+
+    // If more than 10% of the requests failed, return an error status code
+    const isFailed = results.filter(isRejected).length > results.length * 0.1;
+
+    type Response = {
+      request_count: number;
+      errors: { project_name: string; reason: string }[];
+      ignored: string[];
+      changed: { project_name: string; previous: Date | null; latest: Date }[];
+    };
+
+    const fulfilled = results.filter(isFulfilled);
+
+    const response: Response = {
+      request_count,
+      errors: results.filter(isRejected).map((r) => ({
+        // TODO: Fix this project name
+        project_name: "unknown",
+        reason: r.reason,
+      })),
+      ignored: fulfilled
+        .filter((r) => r.value.latestUpdated == null)
+        .map((r) => r.value.id),
+      changed: fulfilled
+        .filter((r) => r.value.latestUpdated != null)
+        .map((r) => ({
+          project_name: r.value.id,
+          previous: r.value.previousUpdated,
+          latest: r.value.latestUpdated!,
+        })),
+    };
+
+    res.status(!isFailed ? 200 : 500).json(response);
+  } catch (error) {
+    res.status(500).json({ error });
+    return;
+  }
+}

src/utils/directus.ts

@@ -71,7 +71,14 @@ export interface Metadata {
   resumeFilename: string;
 }
 
-const directus = createDirectus<Schema>("https://api.xevion.dev")
+const directus = createDirectus<Schema>("https://api.xevion.dev", {
+  globals: {
+    fetch: (input, init) => {
+      console.log(`${init.method?.toUpperCase()} ${input}`);
+      return fetch(input, init);
+    },
+  },
+})
   .with(staticToken(env.DIRECTUS_API_TOKEN))
   .with(rest());
 

src/utils/types.ts

@@ -2,15 +2,13 @@ import type { IconType } from "react-icons";
 import { AiFillGithub, AiOutlineLink } from "react-icons/ai";
 import { RxOpenInNewWindow } from "react-icons/rx";
 
-export type Project = {
+// Promise.allSettled type guards
-  title: string;
+export const isFulfilled = <T>(
-  banner: string;
+  p: PromiseSettledResult<T>,
-  bannerSettings?: { quality: number };
+): p is PromiseFulfilledResult<T> => p.status === "fulfilled";
-  longDescription: string;
+export const isRejected = <T>(
-  shortDescription: string;
+  p: PromiseSettledResult<T>,
-  links?: LinkIcon[];
+): p is PromiseRejectedResult => p.status === "rejected";
-  location: string;
-};
 
 export const LinkIcons: Record<string, IconType> = {
   github: AiFillGithub,

vercel.json

@@ -0,0 +1,8 @@
+{
+  "crons": [
+    {
+      "path": "/api/cron/updated",
+      "schedule": "7 */2 * * *"
+    }
+  ]
+}