diff --git a/package.json b/package.json index cbc260a..fb3b231 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "@headlessui/react": "^2.2.0", "@kodingdotninja/use-tailwind-breakpoint": "^1.0.0", "@next/eslint-plugin-next": "^15.1.1", + "@octokit/core": "^6.1.2", "@plaiceholder/next": "^3.0.0", "@tailwindcss/typography": "^0.5.8", "@tanstack/react-query": "^4.16.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eddaad0..321fd41 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@next/eslint-plugin-next': specifier: ^15.1.1 version: 15.1.1 + '@octokit/core': + specifier: ^6.1.2 + version: 6.1.2 '@plaiceholder/next': specifier: ^3.0.0 version: 3.0.0(next@15.1.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0)(sass@1.56.2))(plaiceholder@3.0.0(sharp@0.32.1))(sharp@0.32.1) @@ -433,6 +436,36 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} + '@octokit/auth-token@5.1.1': + resolution: {integrity: sha512-rh3G3wDO8J9wSjfI436JUKzHIxq8NaiL0tVeB2aXmG6p/9859aUOAjA9pmSPNGGZxfwmaJ9ozOJImuNVJdpvbA==} + engines: {node: '>= 18'} + + '@octokit/core@6.1.2': + resolution: {integrity: sha512-hEb7Ma4cGJGEUNOAVmyfdB/3WirWMg5hDuNFVejGEDFqupeOysLc2sG6HJxY2etBp5YQu5Wtxwi020jS9xlUwg==} + engines: {node: '>= 18'} + + '@octokit/endpoint@10.1.2': + resolution: {integrity: sha512-XybpFv9Ms4hX5OCHMZqyODYqGTZ3H6K6Vva+M9LR7ib/xr1y1ZnlChYv9H680y77Vd/i/k+thXApeRASBQkzhA==} + engines: {node: '>= 18'} + + '@octokit/graphql@8.1.2': + resolution: {integrity: sha512-bdlj/CJVjpaz06NBpfHhp4kGJaRZfz7AzC+6EwUImRtrwIw8dIgJ63Xg0OzV9pRn3rIzrt5c2sa++BL0JJ8GLw==} + engines: {node: '>= 18'} + + '@octokit/openapi-types@22.2.0': + resolution: {integrity: sha512-QBhVjcUa9W7Wwhm6DBFu6ZZ+1/t/oYxqc2tp81Pi41YNuJinbFRx8B133qVOrAaBbF7D/m0Et6f9/pZt9Rc+tg==} + + '@octokit/request-error@6.1.6': + resolution: {integrity: sha512-pqnVKYo/at0NuOjinrgcQYpEbv4snvP3bKMRqHaD9kIsk9u1LCpb2smHZi8/qJfgeNqLo5hNW4Z7FezNdEo0xg==} + engines: {node: '>= 18'} + + '@octokit/request@9.1.4': + resolution: {integrity: sha512-tMbOwGm6wDII6vygP3wUVqFTw3Aoo0FnVQyhihh8vVq12uO3P+vQZeo2CKMpWtPSogpACD0yyZAlVlQnjW71DA==} + engines: {node: '>= 18'} + + '@octokit/types@13.6.2': + resolution: {integrity: sha512-WpbZfZUcZU77DrSW4wbsSgTPfKcp286q3ItaIgvSbBpZJlu6mnYXAkjZz6LVZPXkEvLIM8McanyZejKTYUHipA==} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -758,6 +791,9 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + before-after-hook@3.0.2: + resolution: {integrity: sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==} + binary-extensions@2.2.0: resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} engines: {node: '>=8'} @@ -1146,6 +1182,9 @@ packages: extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + fast-content-type-parse@2.0.0: + resolution: {integrity: sha512-fCqg/6Sps8tqk8p+kqyKqYfOF0VjPNYrqpLiqNl0RBKmD80B080AJWVV6EkSkscjToNExcXg1+Mfzftrx6+iSA==} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -2338,6 +2377,9 @@ packages: unist-util-visit@5.0.0: resolution: {integrity: sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==} + universal-user-agent@7.0.2: + resolution: {integrity: sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==} + update-browserslist-db@1.0.10: resolution: {integrity: sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==} hasBin: true @@ -2667,6 +2709,47 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} + '@octokit/auth-token@5.1.1': {} + + '@octokit/core@6.1.2': + dependencies: + '@octokit/auth-token': 5.1.1 + '@octokit/graphql': 8.1.2 + '@octokit/request': 9.1.4 + '@octokit/request-error': 6.1.6 + '@octokit/types': 13.6.2 + before-after-hook: 3.0.2 + universal-user-agent: 7.0.2 + + '@octokit/endpoint@10.1.2': + dependencies: + '@octokit/types': 13.6.2 + universal-user-agent: 7.0.2 + + '@octokit/graphql@8.1.2': + dependencies: + '@octokit/request': 9.1.4 + '@octokit/types': 13.6.2 + universal-user-agent: 7.0.2 + + '@octokit/openapi-types@22.2.0': {} + + '@octokit/request-error@6.1.6': + dependencies: + '@octokit/types': 13.6.2 + + '@octokit/request@9.1.4': + dependencies: + '@octokit/endpoint': 10.1.2 + '@octokit/request-error': 6.1.6 + '@octokit/types': 13.6.2 + fast-content-type-parse: 2.0.0 + universal-user-agent: 7.0.2 + + '@octokit/types@13.6.2': + dependencies: + '@octokit/openapi-types': 22.2.0 + '@pkgjs/parseargs@0.11.0': optional: true @@ -3034,6 +3117,8 @@ snapshots: base64-js@1.5.1: {} + before-after-hook@3.0.2: {} + binary-extensions@2.2.0: {} bl@4.1.0: @@ -3395,7 +3480,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.0(@typescript-eslint/parser@8.18.1(eslint@9.17.0(jiti@1.21.7))(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0)(eslint@9.17.0(jiti@1.21.7)))(eslint@9.17.0(jiti@1.21.7)): + eslint-module-utils@2.12.0(@typescript-eslint/parser@8.18.1(eslint@9.17.0(jiti@1.21.7))(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0)(eslint@9.17.0(jiti@1.21.7)): dependencies: debug: 3.2.7 optionalDependencies: @@ -3417,7 +3502,7 @@ snapshots: doctrine: 2.1.0 eslint: 9.17.0(jiti@1.21.7) eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.18.1(eslint@9.17.0(jiti@1.21.7))(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0)(eslint@9.17.0(jiti@1.21.7)))(eslint@9.17.0(jiti@1.21.7)) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.18.1(eslint@9.17.0(jiti@1.21.7))(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0)(eslint@9.17.0(jiti@1.21.7)) hasown: 2.0.2 is-core-module: 2.16.0 is-glob: 4.0.3 @@ -3554,6 +3639,8 @@ snapshots: extend@3.0.2: {} + fast-content-type-parse@2.0.0: {} + fast-deep-equal@3.1.3: {} fast-glob@3.3.1: @@ -4998,6 +5085,8 @@ snapshots: unist-util-is: 6.0.0 unist-util-visit-parents: 6.0.1 + universal-user-agent@7.0.2: {} + update-browserslist-db@1.0.10(browserslist@4.21.4): dependencies: browserslist: 4.21.4 diff --git a/src/pages/api/cron/updated.ts b/src/pages/api/cron/updated.ts new file mode 100644 index 0000000..0da9568 --- /dev/null +++ b/src/pages/api/cron/updated.ts @@ -0,0 +1,315 @@ +/** + * This is a cron job handler for acquiring the latest 'updated' data for the site's projects. + * + * 1) Fetch the list of all projects including their link URLs. + * 2) Filter the list only for projects with 'autocheck_update' enabled and any 'github.com' link. + * 3) For each project, query the GitHub API for the latest commit date on all branches. + * 4) If the latest commit date is newer than the project's 'last_updated' date, update the project's 'last_updated' date. + * 5) If any project's 'last_updated' date was updated, revalidate the index and/or project page. + * 6) Report the results of this cron job invocation. + * + * - This cron job runs at least once a day, at most once an hour. + * - This cron job is completely asynchronous but respects GitHub API rate limits. + * - This cron job requires authentication with the Directus API. + * - This cron job requires authentication with the GitHub API (mostly for rate limits). + */ +import directus, { ProjectLink } from "@/utils/directus"; +import { readItems, updateItem } from "@directus/sdk"; +import { NextApiRequest, NextApiResponse } from "next"; +import { Octokit } from "@octokit/core"; +import { isFulfilled, isRejected } from "@/utils/types"; + +const octokit = new Octokit({ + auth: process.env.GITHUB_API_TOKEN, + request: { + fetch: (url: string | URL, options: RequestInit) => { + console.log(`${options.method} ${url}`); + return fetch(url, options); + }, + }, +}); + +type ProjectResult = { + id: string; + previousUpdated: Date | null; + latestUpdated: Date | null; +}; + +function getRepository(url: string): [string, string] | null { + const pattern = /github.com\/([^/]+)\/([^/]+)/; + const match = pattern.exec(url); + + if (match === null) return null; + return [match[1]!, match[2]!]; +} + +async function handleProject({ + id: project_id, + urls, + date_updated: previousUpdated, +}: { + id: string; + urls: string[]; + date_updated: Date | null; +}): Promise { + // Extract the branches from each URL + const allBranches = await Promise.all( + urls.map(async (url) => { + const details = getRepository(url); + if (!details) { + return []; + } + + // TODO: Handle deduplication of repository targets + const [owner, repo] = details; + const branches = await octokit.request( + "GET /repos/{owner}/{repo}/branches", + { + owner, + repo, + headers: { + "X-GitHub-Api-Version": "2022-11-28", + }, + }, + ); + + return branches.data.map((branch) => ({ + branch: branch.name, + owner: owner, + repo: repo, + })); + }), + ); + + // Get the latest commit date for each branch (flattened) + const latestCommits = allBranches + .flat() + .map(async ({ owner, repo, branch }) => { + const commits = await octokit.request( + "GET /repos/{owner}/{repo}/commits", + { + owner, + repo, + sha: branch, + per_page: 1, + headers: { + "X-GitHub-Api-Version": "2022-11-28", + }, + }, + ); + const latestCommit = commits.data[0]; + + // Commits not returned + if (latestCommit == null) { + console.warn({ + target: `${owner}/${repo}@${branch}`, + message: "No commits available", + }); + return null; + } + + // Handle missing commit data in unpredictable cases + if (latestCommit.commit.author == null) { + console.warn({ + target: `${owner}/${repo}@${branch}`, + sha: latestCommit.sha, + commit: latestCommit.commit.message, + url: latestCommit.html_url, + message: "No author available", + }); + return null; + } else if (latestCommit.commit.author.date == null) { + console.warn({ + target: `${owner}/${repo}@${branch}`, + sha: latestCommit.sha, + commit: latestCommit.commit.message, + url: latestCommit.html_url, + message: "No date available", + }); + return null; + } + + return new Date(latestCommit.commit.author.date); + }); + + const results = await Promise.allSettled(latestCommits); + + // Handle the promises that failed + results.filter(isRejected).forEach((result) => { + // TODO: Add more context to the error message + console.error("Failed to fetch latest commit date", result.reason); + }); + + // Find the latest commit date + const latestUpdated = results + .filter(isFulfilled) + .map((v) => v.value) + .filter((v) => v != null) + .reduce((previous: Date | null, current: Date) => { + if (previous == null) return current; + return current > previous ? current : previous; + }, null); + + if (latestUpdated == null) { + console.error("Unable to acquire the latest commit date for project"); + return { + id: project_id, + previousUpdated, + latestUpdated: null, + }; + } + + // Ensure it's a reasonable date + if (latestUpdated != null && latestUpdated < new Date("2015-01-01")) { + console.error("Invalid commit date acquired", latestUpdated); + return { + id: project_id, + previousUpdated, + latestUpdated: null, + }; + } + + const result = { id: project_id, previousUpdated, latestUpdated: null }; + + // Update the project's 'last_updated' date if the latest commit date is newer + if (previousUpdated == null || latestUpdated > previousUpdated) { + await directus.request( + updateItem("project", project_id, { + date_updated: latestUpdated, + }), + ); + + // 'latestUpdated' is not null ONLY if the project was actually updated + return { + ...result, + latestUpdated, + }; + } + + return result; +} + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse, +) { + // Check for the required environment variables + const { CRON_SECRET, GITHUB_API_TOKEN, DIRECTUS_API_TOKEN } = process.env; + if (!CRON_SECRET || !GITHUB_API_TOKEN || !DIRECTUS_API_TOKEN) { + res.status(500).json({ error: "Missing environment variables" }); + } + + // Ensure the cron request is authenticated + if (process.env.NODE_ENV !== "development") { + const authHeader = req.headers["authorization"]; + if (authHeader !== `Bearer ${CRON_SECRET}`) { + return new Response("Unauthorized", { + status: 401, + }); + } + } + + let request_count = 0; + octokit.hook.before("request", async () => { + request_count++; + }); + + try { + // Fetch the list of all projects including their link URLs. + const projects = await directus.request( + readItems("project", { + fields: [ + "id", + "name", + "autocheckUpdated", + "date_updated", + { links: ["url"] }, + ], + }), + ); + + // Filter the list only for projects with 'autocheck_update' enabled and any 'github.com' link. + const eligibleProjects = projects + .map((project) => { + // Skip projects that don't have autocheckUpdated enabled. + if (!project.autocheckUpdated) return null; + + // Acquire the URL from the link, then filter out any non-GitHub URLs. + const urls = project + .links!.map((link) => { + return (link).url; + }) + .filter((url) => url.includes("github.com")); + + // Skip projects that don't have any GitHub URLs. + if (urls.length === 0) return null; + + // Return the project's most important data for further processing. + return { + id: project.id, + name: project.name, + date_updated: project.date_updated, + urls, + }; + }) + // null values are still included in the array, so filter them out. + .filter((project) => project !== null); + + // Log the date_updated for each project + eligibleProjects.forEach((project) => { + console.log({ + name: project.name, + date_updated: project.date_updated, + }); + }); + + // For each project, query the GitHub API for the latest commit date on all branches. + const projectPromises = eligibleProjects.map((project) => + handleProject({ + id: project.id, + urls: project.urls, + date_updated: + project.date_updated != null ? new Date(project.date_updated) : null, + }), + ); + + // Wait for all project promises to resolve + const results = await Promise.allSettled(projectPromises); + + // If more than 10% of the requests failed, return an error status code + const isFailed = results.filter(isRejected).length > results.length * 0.1; + + type Response = { + request_count: number; + errors: { project_name: string; reason: string }[]; + ignored: string[]; + changed: { project_name: string; previous: Date | null; latest: Date }[]; + }; + + const fulfilled = results.filter(isFulfilled); + + const response: Response = { + request_count, + errors: results.filter(isRejected).map((r) => ({ + // TODO: Fix this project name + project_name: "unknown", + reason: r.reason, + })), + ignored: fulfilled + .filter((r) => r.value.latestUpdated == null) + .map((r) => r.value.id), + changed: fulfilled + .filter((r) => r.value.latestUpdated != null) + .map((r) => ({ + project_name: r.value.id, + previous: r.value.previousUpdated, + latest: r.value.latestUpdated!, + })), + }; + + res.status(!isFailed ? 200 : 500).json(response); + } catch (error) { + res.status(500).json({ error }); + return; + } +} diff --git a/src/utils/directus.ts b/src/utils/directus.ts index 83ef6e2..dfff833 100644 --- a/src/utils/directus.ts +++ b/src/utils/directus.ts @@ -71,7 +71,14 @@ export interface Metadata { resumeFilename: string; } -const directus = createDirectus("https://api.xevion.dev") +const directus = createDirectus("https://api.xevion.dev", { + globals: { + fetch: (input, init) => { + console.log(`${init.method?.toUpperCase()} ${input}`); + return fetch(input, init); + }, + }, +}) .with(staticToken(env.DIRECTUS_API_TOKEN)) .with(rest()); diff --git a/src/utils/types.ts b/src/utils/types.ts index 18f36e2..120ae9e 100644 --- a/src/utils/types.ts +++ b/src/utils/types.ts @@ -2,15 +2,13 @@ import type { IconType } from "react-icons"; import { AiFillGithub, AiOutlineLink } from "react-icons/ai"; import { RxOpenInNewWindow } from "react-icons/rx"; -export type Project = { - title: string; - banner: string; - bannerSettings?: { quality: number }; - longDescription: string; - shortDescription: string; - links?: LinkIcon[]; - location: string; -}; +// Promise.allSettled type guards +export const isFulfilled = ( + p: PromiseSettledResult, +): p is PromiseFulfilledResult => p.status === "fulfilled"; +export const isRejected = ( + p: PromiseSettledResult, +): p is PromiseRejectedResult => p.status === "rejected"; export const LinkIcons: Record = { github: AiFillGithub, diff --git a/vercel.json b/vercel.json new file mode 100644 index 0000000..0eef132 --- /dev/null +++ b/vercel.json @@ -0,0 +1,8 @@ +{ + "crons": [ + { + "path": "/api/cron/updated", + "schedule": "7 */2 * * *" + } + ] +}