feat: add GitHub activity sync background job

Adds automatic syncing of repository activity for projects with github_repo set. Background task fetches latest activity from GitHub API (issues, PRs, default branch commits) and updates last_github_activity timestamp. Configurable sync interval (default: 15 minutes), requires GITHUB_TOKEN env var.
This commit is contained in:
2026-01-13 22:54:48 -06:00
parent f79c7711f0
commit c7dbd77b72
7 changed files with 792 additions and 76 deletions
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE projects SET last_github_activity = $2 WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Timestamptz"
]
},
"nullable": []
},
"hash": "82118a82eb002e4155689f37c50b7a286d2381d9a195f1665979d3a338bedc01"
}
@@ -0,0 +1,92 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n id,\n slug,\n name,\n short_description,\n description,\n status as \"status: ProjectStatus\",\n github_repo,\n demo_url,\n last_github_activity,\n created_at,\n updated_at\n FROM projects\n WHERE github_repo IS NOT NULL\n ORDER BY updated_at DESC\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "slug",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "name",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "short_description",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "status: ProjectStatus",
"type_info": {
"Custom": {
"name": "project_status",
"kind": {
"Enum": [
"active",
"maintained",
"archived",
"hidden"
]
}
}
}
},
{
"ordinal": 6,
"name": "github_repo",
"type_info": "Text"
},
{
"ordinal": 7,
"name": "demo_url",
"type_info": "Text"
},
{
"ordinal": 8,
"name": "last_github_activity",
"type_info": "Timestamptz"
},
{
"ordinal": 9,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 10,
"name": "updated_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
false,
false,
false,
true,
true,
true,
false,
false
]
},
"hash": "8302d5621d743bd3e5e2a029e30ad1e017e1f2ef1f9cb09aa924436b0c6b8c22"
}
+166 -76
View File
@@ -5,63 +5,158 @@ pub async fn run(pool: &PgPool) -> Result<(), Box<dyn std::error::Error>> {
println!("Seeding database..."); println!("Seeding database...");
// Clear existing data (tags will cascade delete project_tags and tag_cooccurrence) // Clear existing data (tags will cascade delete project_tags and tag_cooccurrence)
sqlx::query("DELETE FROM social_links")
.execute(pool)
.await?;
sqlx::query("DELETE FROM tags").execute(pool).await?; sqlx::query("DELETE FROM tags").execute(pool).await?;
sqlx::query("DELETE FROM projects").execute(pool).await?; sqlx::query("DELETE FROM projects").execute(pool).await?;
// Seed projects with diverse data // Seed site identity
sqlx::query(
r#"
INSERT INTO site_identity (id, display_name, occupation, bio, site_title)
VALUES (1, $1, $2, $3, $4)
ON CONFLICT (id) DO UPDATE SET
display_name = EXCLUDED.display_name,
occupation = EXCLUDED.occupation,
bio = EXCLUDED.bio,
site_title = EXCLUDED.site_title
"#,
)
.bind("Ryan Walters")
.bind("Full-Stack Software Engineer")
.bind("A fanatical software engineer with expertise and passion for sound, scalable and high-performance applications. I'm always working on something new.\nSometimes innovative — sometimes crazy.")
.bind("Xevion.dev")
.execute(pool)
.await?;
println!(" Seeded site identity");
// Seed social links
let social_links = vec![
(
"github",
"GitHub",
"https://github.com/Xevion",
"simple-icons:github",
1,
),
(
"linkedin",
"LinkedIn",
"https://linkedin.com/in/ryancwalters",
"simple-icons:linkedin",
2,
),
("discord", "Discord", ".xevion", "simple-icons:discord", 3),
(
"email",
"Email",
"xevion@xevion.dev",
"material-symbols:mail-rounded",
4,
),
];
for (platform, label, value, icon, order) in &social_links {
sqlx::query(
r#"
INSERT INTO social_links (platform, label, value, icon, visible, display_order)
VALUES ($1, $2, $3, $4, true, $5)
"#,
)
.bind(platform)
.bind(label)
.bind(value)
.bind(icon)
.bind(order)
.execute(pool)
.await?;
}
println!(" Seeded {} social links", social_links.len());
// Seed projects matching production data
let projects = vec![ let projects = vec![
( (
"xevion-dev", "xevion-dev",
"xevion.dev", "xevion.dev",
"Personal portfolio and project showcase", "A dual-process portfolio website built with Rust and SvelteKit",
"Personal portfolio site with fuzzy tag discovery and ISR caching", "A modern portfolio website featuring a dual-process architecture with Rust (Axum) handling API serving, reverse proxying, and static asset embedding, while Bun runs SvelteKit for SSR rendering. Includes a full admin interface for content management, PostgreSQL for persistence, and Cloudflare R2 for media storage. Features ISR caching, session-based authentication, dynamic OG image generation with Satori, and a CLI for remote content management.",
"active", "active",
Some("Xevion/xevion.dev"), Some("xevion/xevion.dev"),
Some("https://xevion.dev"),
),
(
"rdap",
"rdap",
"Modern RDAP query client for domain and IP lookups",
"A web-based RDAP (Registration Data Access Protocol) client for querying domain and IP registration data. Built with Next.js and statically hosted for fast global access. Features comprehensive schema validation, JSContact/vCard parsing for contact information, support for all RDAP object types, and a clean UI built with Radix components. Replaces traditional WHOIS lookups with the modern, structured RDAP standard that provides machine-readable responses.",
"active",
Some("Xevion/rdap"),
Some("https://rdap.xevion.dev"),
),
(
"byte-me",
"byte-me",
"Cross-platform media bitrate visualizer built with Tauri",
"A desktop application for visualizing media file bitrates over time, built with Tauri and Rust. Parses video containers to extract frame-level bitrate data and renders interactive graphs showing encoding quality distribution. Helps content creators and video engineers identify bitrate spikes, understand encoder behavior, and optimize their encoding settings for streaming or distribution.",
"active",
Some("Xevion/byte-me"),
None, None,
), ),
( (
"contest", "pac-man",
"Contest", "pac-man",
"Competitive programming archive", "Classic Pac-Man arcade game clone, playable in the browser",
"Archive and analysis platform for competitive programming problems", "A faithful recreation of the classic Pac-Man arcade game, built entirely in Rust using SDL2 for graphics, audio, and input handling. Compiled to WebAssembly via Emscripten for browser play without plugins. Features authentic ghost AI behaviors, original maze layout, power pellet mechanics, and retro pixel aesthetics. Demonstrates Rust's capability for game development and seamless WASM compilation.",
"active", "active",
Some("Xevion/contest"), Some("Xevion/Pac-Man"),
Some("https://contest.xevion.dev"), Some("https://pacman.xevion.dev"),
), ),
( (
"reforge", "rebinded",
"Reforge", "Rebinded",
"Rocket League replay parser", "Cross-platform key remapping daemon with per-app context",
"Rust library for parsing and manipulating Replay files from Rocket League", "A system-level key remapping daemon supporting both Windows and Linux with unified configuration. Features per-application context awareness, allowing different key mappings based on the active window or application. Includes stateful debouncing to prevent key repeat issues, supports complex multi-key sequences and chords, and runs as a lightweight background service. Ideal for power users who need consistent keyboard customization across operating systems.",
"maintained", "active",
Some("Xevion/reforge"), Some("Xevion/rebinded"),
None, None,
), ),
( (
"algorithms", "the-office-quotes",
"Algorithms", "The Office Quotes",
"Algorithm implementations in Python", "Search and browse quotes from The Office TV series",
"Collection of algorithm implementations and data structures in Python", "A serverless single-page application for browsing and searching quotes from The Office TV show. Built with Vue.js and Vue Router, styled with Bootstrap 4. Features Algolia-powered instant search with typo tolerance, Firebase hosting for serverless deployment, and a Python data pipeline for processing episode transcripts. Includes scene context, character filtering, and episode navigation for finding that perfect Dwight or Michael quote.",
"archived",
Some("Xevion/algorithms"),
None,
),
(
"wordplay",
"WordPlay",
"Real-time multiplayer word game",
"Interactive word game with real-time multiplayer using WebSockets",
"maintained",
Some("Xevion/wordplay"),
Some("https://wordplay.example.com"),
),
(
"dotfiles",
"Dotfiles",
"Development environment configs",
"Personal configuration files and development environment setup scripts",
"active", "active",
Some("Xevion/dotfiles"), Some("Xevion/the-office"),
Some("https://the-office.xevion.dev"),
),
(
"grain",
"grain",
"SVG-based film grain noise and radial gradient effects",
"A visual experiment demonstrating dynamically scaled SVG-based film grain noise combined with stacked radial gradients. Built with TypeScript and Preact, deployed as a lightweight static site. Showcases techniques for creating organic, film-like textures using pure SVG filters without raster images, with real-time scaling that maintains quality at any resolution.",
"active",
Some("Xevion/grain"),
Some("https://grain.xevion.dev"),
),
(
"dynamic-preauth",
"dynamic-preauth",
"Server-side executable pre-authentication proof of concept",
"A proof of concept demonstrating server-side executable modification for pre-authentication. Built with Rust using the Salvo web framework for the backend and Astro for the frontend. Explores techniques for embedding unique authentication tokens directly into downloadable executables at request time, with real-time WebSocket communication for status updates. Demonstrates binary patching, cryptographic signing, and secure token embedding for software distribution scenarios.",
"active",
Some("Xevion/dynamic-preauth"),
Some("https://dynamic-preauth.xevion.dev"),
),
(
"rustdoc-mcp",
"rustdoc-mcp",
"MCP server providing AI assistants access to Rust documentation",
"A Model Context Protocol (MCP) server that provides AI assistants with direct access to Rust crate documentation. Enables LLMs to query rustdoc-generated documentation, search for types, traits, and functions, and retrieve detailed API information for any published Rust crate. Integrates with Claude, GPT, and other MCP-compatible AI tools to provide accurate, up-to-date Rust API references without hallucination.",
"active",
Some("Xevion/rustdoc-mcp"),
None, None,
), ),
]; ];
@@ -88,35 +183,39 @@ pub async fn run(pool: &PgPool) -> Result<(), Box<dyn std::error::Error>> {
println!(" Seeded {} projects", project_count); println!(" Seeded {} projects", project_count);
// Seed tags // Seed tags matching production data
let tags = vec![ let tags = vec![
("rust", "Rust", "simple-icons:rust", "CE422B"), ("astro", "Astro", "simple-icons:astro", "FF5D01"),
("cli", "CLI", "lucide:terminal", "22C55E"),
("game", "Game", "lucide:gamepad-2", "EF4444"),
("mcp", "MCP", "lucide:plug", "8B5CF6"),
("nextjs", "Nextjs", "simple-icons:nextdotjs", "000000"),
("preact", "Preact", "simple-icons:preact", "673AB8"),
("python", "Python", "simple-icons:python", "3776AB"), ("python", "Python", "simple-icons:python", "3776AB"),
("react", "React", "simple-icons:react", "61DAFB"),
("rust", "Rust", "simple-icons:rust", "DEA584"),
("security", "Security", "lucide:shield", "F59E0B"),
("sveltekit", "SvelteKit", "simple-icons:svelte", "FF3E00"),
("tauri", "Tauri", "simple-icons:tauri", "24C8DB"),
( (
"typescript", "typescript",
"TypeScript", "TypeScript",
"simple-icons:typescript", "simple-icons:typescript",
"3178C6", "3178C6",
), ),
("vue", "Vue", "simple-icons:vuedotjs", "4FC08D"),
( (
"javascript", "webassembly",
"JavaScript", "WebAssembly",
"simple-icons:javascript", "simple-icons:webassembly",
"EAB308", "654FF0",
), ),
("web", "Web", "lucide:globe", "2563EB"),
("cli", "CLI", "lucide:terminal", "64748B"),
("library", "Library", "lucide:package", "8B5CF6"),
("game", "Game", "lucide:gamepad-2", "EC4899"),
( (
"data-structures", "web-development",
"Data Structures", "Web Development",
"lucide:database", "lucide:globe",
"10B981", "3B82F6",
), ),
("algorithms", "Algorithms", "lucide:cpu", "F59E0B"),
("multiplayer", "Multiplayer", "lucide:users", "06B6D4"),
("config", "Config", "lucide:settings", "6366F1"),
]; ];
let mut tag_ids = std::collections::HashMap::new(); let mut tag_ids = std::collections::HashMap::new();
@@ -141,29 +240,20 @@ pub async fn run(pool: &PgPool) -> Result<(), Box<dyn std::error::Error>> {
println!(" Seeded {} tags", tag_ids.len()); println!(" Seeded {} tags", tag_ids.len());
// Associate tags with projects // Associate tags with projects (matching production)
let project_tag_associations = vec![ let project_tag_associations = vec![
// xevion-dev ("xevion-dev", vec!["cli", "rust", "sveltekit", "typescript"]),
("xevion-dev", vec!["rust", "web", "typescript"]), ("rdap", vec!["nextjs", "react", "typescript"]),
// Contest ("byte-me", vec!["rust", "tauri", "typescript"]),
("pac-man", vec!["game", "rust", "webassembly"]),
("rebinded", vec!["cli", "rust"]),
("the-office-quotes", vec!["python", "vue"]),
("grain", vec!["preact", "typescript"]),
( (
"contest", "dynamic-preauth",
vec!["python", "web", "algorithms", "data-structures"], vec!["astro", "rust", "security", "typescript"],
), ),
// Reforge ("rustdoc-mcp", vec!["cli", "mcp", "rust"]),
("reforge", vec!["rust", "library", "game"]),
// Algorithms
(
"algorithms",
vec!["python", "algorithms", "data-structures"],
),
// WordPlay
(
"wordplay",
vec!["typescript", "javascript", "web", "game", "multiplayer"],
),
// Dotfiles
("dotfiles", vec!["config", "cli"]),
]; ];
let mut association_count = 0; let mut association_count = 0;
+60
View File
@@ -6,6 +6,7 @@ use tower_http::{cors::CorsLayer, limit::RequestBodyLimitLayer};
use crate::cache::{IsrCache, IsrCacheConfig}; use crate::cache::{IsrCache, IsrCacheConfig};
use crate::config::ListenAddr; use crate::config::ListenAddr;
use crate::github;
use crate::middleware::RequestIdLayer; use crate::middleware::RequestIdLayer;
use crate::state::AppState; use crate::state::AppState;
use crate::tarpit::{TarpitConfig, TarpitState}; use crate::tarpit::{TarpitConfig, TarpitState};
@@ -150,6 +151,65 @@ pub async fn run(
} }
}); });
// Spawn GitHub activity sync background task (if GITHUB_TOKEN is set)
tokio::spawn({
let pool = pool.clone();
async move {
// Check if GitHub client is available (GITHUB_TOKEN set)
if github::GitHubClient::get().await.is_none() {
return;
}
let interval_secs: u64 = std::env::var("GITHUB_SYNC_INTERVAL_SEC")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(900); // Default: 15 minutes
// Brief delay to let server finish initializing, then run initial sync
tokio::time::sleep(Duration::from_secs(2)).await;
tracing::info!(
interval_sec = interval_secs,
"Running initial GitHub activity sync"
);
match github::sync_github_activity(&pool).await {
Ok(stats) => {
tracing::info!(
synced = stats.synced,
skipped = stats.skipped,
errors = stats.errors,
"Initial GitHub activity sync completed"
);
}
Err(e) => {
tracing::error!(error = %e, "Initial GitHub sync failed");
}
}
// Regular interval sync
let mut interval = tokio::time::interval(Duration::from_secs(interval_secs));
interval.tick().await; // Skip the first tick (already ran initial sync)
loop {
interval.tick().await;
match github::sync_github_activity(&pool).await {
Ok(stats) => {
tracing::info!(
synced = stats.synced,
skipped = stats.skipped,
errors = stats.errors,
"GitHub activity sync completed"
);
}
Err(e) => {
tracing::error!(error = %e, "GitHub sync failed");
}
}
}
}
});
// Apply middleware to router // Apply middleware to router
fn apply_middleware( fn apply_middleware(
router: axum::Router<Arc<AppState>>, router: axum::Router<Arc<AppState>>,
+42
View File
@@ -421,3 +421,45 @@ pub async fn get_admin_stats(pool: &PgPool) -> Result<AdminStats, sqlx::Error> {
total_tags: tag_count.count, total_tags: tag_count.count,
}) })
} }
/// Get all projects that have a github_repo set (for GitHub sync)
pub async fn get_projects_with_github_repo(pool: &PgPool) -> Result<Vec<DbProject>, sqlx::Error> {
query_as!(
DbProject,
r#"
SELECT
id,
slug,
name,
short_description,
description,
status as "status: ProjectStatus",
github_repo,
demo_url,
last_github_activity,
created_at,
updated_at
FROM projects
WHERE github_repo IS NOT NULL
ORDER BY updated_at DESC
"#
)
.fetch_all(pool)
.await
}
/// Update the last_github_activity timestamp for a project
pub async fn update_last_github_activity(
pool: &PgPool,
id: Uuid,
activity_time: OffsetDateTime,
) -> Result<(), sqlx::Error> {
query!(
"UPDATE projects SET last_github_activity = $2 WHERE id = $1",
id,
activity_time
)
.execute(pool)
.await?;
Ok(())
}
+416
View File
@@ -0,0 +1,416 @@
//! GitHub API client for syncing repository activity.
//!
//! Fetches the latest activity from GitHub for projects that have `github_repo` set.
//! Only considers:
//! - Project-wide activity: Issues, PRs
//! - Main branch activity: Commits/pushes to the default branch only
use dashmap::DashMap;
use reqwest::header::{ACCEPT, AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT};
use serde::Deserialize;
use sqlx::PgPool;
use std::sync::Arc;
use time::OffsetDateTime;
use tokio::sync::OnceCell;
static GITHUB_CLIENT: OnceCell<Option<Arc<GitHubClient>>> = OnceCell::const_new();
/// Statistics from a sync run
#[derive(Debug, Default)]
pub struct SyncStats {
pub synced: u32,
pub skipped: u32,
pub errors: u32,
}
/// GitHub API client with caching for default branches
pub struct GitHubClient {
client: reqwest::Client,
/// Cache of "owner/repo" -> default_branch
branch_cache: DashMap<String, String>,
}
// GitHub API response types
#[derive(Debug, Deserialize)]
struct RepoInfo {
default_branch: String,
}
#[derive(Debug, Deserialize)]
struct ActivityEvent {
/// The activity endpoint uses `timestamp`, not `created_at`
timestamp: String,
}
#[derive(Debug, Deserialize)]
struct IssueEvent {
created_at: String,
}
/// Errors that can occur during GitHub API operations
#[derive(Debug)]
pub enum GitHubError {
/// HTTP request failed
Request(reqwest::Error),
/// Repository not found (404)
NotFound(String),
/// Rate limited (429)
RateLimited,
/// Failed to parse timestamp
ParseTime(String),
/// Other API error
Api(u16, String),
}
impl std::fmt::Display for GitHubError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
GitHubError::Request(e) => write!(f, "HTTP request failed: {e}"),
GitHubError::NotFound(repo) => write!(f, "Repository not found: {repo}"),
GitHubError::RateLimited => write!(f, "GitHub API rate limit exceeded"),
GitHubError::ParseTime(s) => write!(f, "Failed to parse timestamp: {s}"),
GitHubError::Api(status, msg) => write!(f, "GitHub API error ({status}): {msg}"),
}
}
}
impl std::error::Error for GitHubError {}
impl GitHubClient {
/// Create a new GitHub client if GITHUB_TOKEN is set.
fn new() -> Option<Self> {
let token = std::env::var("GITHUB_TOKEN").ok()?;
if token.is_empty() {
return None;
}
let mut headers = HeaderMap::new();
headers.insert(
ACCEPT,
HeaderValue::from_static("application/vnd.github+json"),
);
headers.insert(
"X-GitHub-Api-Version",
HeaderValue::from_static("2022-11-28"),
);
headers.insert(
AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {token}")).ok()?,
);
headers.insert(USER_AGENT, HeaderValue::from_static("xevion-dev/1.0"));
let client = reqwest::Client::builder()
.default_headers(headers)
.build()
.ok()?;
Some(Self {
client,
branch_cache: DashMap::new(),
})
}
/// Get the shared GitHub client instance.
/// Returns None if GITHUB_TOKEN is not set, logging a warning once.
pub async fn get() -> Option<Arc<Self>> {
GITHUB_CLIENT
.get_or_init(|| async {
match GitHubClient::new() {
Some(client) => {
tracing::info!("GitHub sync client initialized");
Some(Arc::new(client))
}
None => {
tracing::warn!(
"GitHub sync disabled: GITHUB_TOKEN not set. \
Set GITHUB_TOKEN to enable automatic activity sync."
);
None
}
}
})
.await
.clone()
}
/// Fetch repository info (primarily for default_branch).
async fn get_repo_info(&self, owner: &str, repo: &str) -> Result<RepoInfo, GitHubError> {
let url = format!("https://api.github.com/repos/{owner}/{repo}");
let response = self
.client
.get(&url)
.send()
.await
.map_err(GitHubError::Request)?;
let status = response.status();
if status == reqwest::StatusCode::NOT_FOUND {
return Err(GitHubError::NotFound(format!("{owner}/{repo}")));
}
if status == reqwest::StatusCode::TOO_MANY_REQUESTS {
return Err(GitHubError::RateLimited);
}
if !status.is_success() {
let body = response.text().await.unwrap_or_default();
return Err(GitHubError::Api(status.as_u16(), body));
}
response
.json::<RepoInfo>()
.await
.map_err(GitHubError::Request)
}
/// Get the default branch for a repo, using cache if available.
async fn get_default_branch(&self, owner: &str, repo: &str) -> Result<String, GitHubError> {
let cache_key = format!("{owner}/{repo}");
// Check cache first
if let Some(branch) = self.branch_cache.get(&cache_key) {
return Ok(branch.clone());
}
// Fetch from API
let info = self.get_repo_info(owner, repo).await?;
self.branch_cache
.insert(cache_key, info.default_branch.clone());
Ok(info.default_branch)
}
/// Fetch the latest activity on the default branch.
/// Returns the timestamp of the most recent push, if any.
async fn get_latest_branch_activity(
&self,
owner: &str,
repo: &str,
branch: &str,
) -> Result<Option<OffsetDateTime>, GitHubError> {
let url =
format!("https://api.github.com/repos/{owner}/{repo}/activity?ref={branch}&per_page=1");
let response = self
.client
.get(&url)
.send()
.await
.map_err(GitHubError::Request)?;
let status = response.status();
if status == reqwest::StatusCode::NOT_FOUND {
return Err(GitHubError::NotFound(format!("{owner}/{repo}")));
}
if status == reqwest::StatusCode::TOO_MANY_REQUESTS {
return Err(GitHubError::RateLimited);
}
if !status.is_success() {
let body = response.text().await.unwrap_or_default();
return Err(GitHubError::Api(status.as_u16(), body));
}
let events: Vec<ActivityEvent> = response.json().await.map_err(GitHubError::Request)?;
if let Some(event) = events.first() {
let timestamp = OffsetDateTime::parse(
&event.timestamp,
&time::format_description::well_known::Rfc3339,
)
.map_err(|_| GitHubError::ParseTime(event.timestamp.clone()))?;
Ok(Some(timestamp))
} else {
Ok(None)
}
}
/// Fetch the latest issue/PR event.
/// Returns the timestamp of the most recent event, if any.
async fn get_latest_issue_event(
&self,
owner: &str,
repo: &str,
) -> Result<Option<OffsetDateTime>, GitHubError> {
let url = format!("https://api.github.com/repos/{owner}/{repo}/issues/events?per_page=1");
let response = self
.client
.get(&url)
.send()
.await
.map_err(GitHubError::Request)?;
let status = response.status();
if status == reqwest::StatusCode::NOT_FOUND {
return Err(GitHubError::NotFound(format!("{owner}/{repo}")));
}
if status == reqwest::StatusCode::TOO_MANY_REQUESTS {
return Err(GitHubError::RateLimited);
}
if !status.is_success() {
let body = response.text().await.unwrap_or_default();
return Err(GitHubError::Api(status.as_u16(), body));
}
let events: Vec<IssueEvent> = response.json().await.map_err(GitHubError::Request)?;
if let Some(event) = events.first() {
let timestamp = OffsetDateTime::parse(
&event.created_at,
&time::format_description::well_known::Rfc3339,
)
.map_err(|_| GitHubError::ParseTime(event.created_at.clone()))?;
Ok(Some(timestamp))
} else {
Ok(None)
}
}
/// Fetch the latest activity for a repository.
/// Considers both branch activity and issue/PR events, returning the most recent.
pub async fn get_latest_activity(
&self,
owner: &str,
repo: &str,
) -> Result<Option<OffsetDateTime>, GitHubError> {
// Get default branch (cached)
let branch = self.get_default_branch(owner, repo).await?;
// Fetch both activity sources in parallel
let (branch_activity, issue_activity) = tokio::join!(
self.get_latest_branch_activity(owner, repo, &branch),
self.get_latest_issue_event(owner, repo)
);
// Take the most recent timestamp from either source
let branch_time = branch_activity?;
let issue_time = issue_activity?;
Ok(match (branch_time, issue_time) {
(Some(a), Some(b)) => Some(a.max(b)),
(Some(a), None) => Some(a),
(None, Some(b)) => Some(b),
(None, None) => None,
})
}
}
/// Parse a "owner/repo" string into (owner, repo) tuple.
fn parse_github_repo(github_repo: &str) -> Option<(&str, &str)> {
let parts: Vec<&str> = github_repo.split('/').collect();
if parts.len() == 2 && !parts[0].is_empty() && !parts[1].is_empty() {
Some((parts[0], parts[1]))
} else {
None
}
}
/// Sync GitHub activity for all projects that have github_repo set.
/// Updates the last_github_activity field with the most recent activity timestamp.
pub async fn sync_github_activity(pool: &PgPool) -> Result<SyncStats, Box<dyn std::error::Error>> {
let client = GitHubClient::get()
.await
.ok_or("GitHub client not initialized")?;
let projects = crate::db::projects::get_projects_with_github_repo(pool).await?;
let mut stats = SyncStats::default();
tracing::debug!(count = projects.len(), "Starting GitHub activity sync");
for project in projects {
let github_repo = match &project.github_repo {
Some(repo) => repo,
None => continue,
};
let (owner, repo) = match parse_github_repo(github_repo) {
Some(parsed) => parsed,
None => {
tracing::warn!(
project_id = %project.id,
github_repo = %github_repo,
"Invalid github_repo format, expected 'owner/repo'"
);
stats.skipped += 1;
continue;
}
};
match client.get_latest_activity(owner, repo).await {
Ok(Some(activity_time)) => {
// Only update if newer than current value
let should_update = project
.last_github_activity
.map_or(true, |current| activity_time > current);
if should_update {
if let Err(e) = crate::db::projects::update_last_github_activity(
pool,
project.id,
activity_time,
)
.await
{
tracing::error!(
project_id = %project.id,
error = %e,
"Failed to update last_github_activity"
);
stats.errors += 1;
} else {
tracing::debug!(
project_id = %project.id,
github_repo = %github_repo,
activity_time = %activity_time,
"Updated last_github_activity"
);
stats.synced += 1;
}
} else {
stats.skipped += 1;
}
}
Ok(None) => {
tracing::debug!(
project_id = %project.id,
github_repo = %github_repo,
"No activity found for repository"
);
stats.skipped += 1;
}
Err(GitHubError::NotFound(repo)) => {
tracing::warn!(
project_id = %project.id,
github_repo = %repo,
"Repository not found or inaccessible, skipping"
);
stats.skipped += 1;
}
Err(GitHubError::RateLimited) => {
tracing::warn!("GitHub API rate limit hit, stopping sync early");
stats.errors += 1;
break;
}
Err(e) => {
tracing::error!(
project_id = %project.id,
github_repo = %github_repo,
error = %e,
"Failed to fetch GitHub activity"
);
stats.errors += 1;
}
}
}
Ok(stats)
}
+1
View File
@@ -8,6 +8,7 @@ mod cli;
mod config; mod config;
mod db; mod db;
mod formatter; mod formatter;
mod github;
mod handlers; mod handlers;
mod health; mod health;
mod http; mod http;