From 139e4aa63544020685e0005ab20babbb9f2b9257 Mon Sep 17 00:00:00 2001 From: Xevion Date: Fri, 29 Aug 2025 13:26:53 -0500 Subject: [PATCH] feat: translate over to sqlx, remove diesel --- Cargo.lock | 96 +------------------- Cargo.toml | 3 - migrations/20250829175305_initial_schema.sql | 56 ++++++++++++ src/data/models.rs | 70 ++------------ src/data/schema.rs | 69 -------------- 5 files changed, 66 insertions(+), 228 deletions(-) create mode 100644 migrations/20250829175305_initial_schema.sql delete mode 100644 src/data/schema.rs diff --git a/Cargo.lock b/Cargo.lock index 3cbce81..ef60e73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -183,9 +183,6 @@ dependencies = [ "chrono", "chrono-tz", "compile-time", - "diesel", - "diesel-derive-enum", - "diesel_derives", "dotenvy", "figment", "fundu", @@ -617,57 +614,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "diesel" -version = "2.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229850a212cd9b84d4f0290ad9d294afc0ae70fccaa8949dbe8b43ffafa1e20c" -dependencies = [ - "bitflags 2.9.3", - "byteorder", - "chrono", - "diesel_derives", - "itoa", - "pq-sys", - "r2d2", - "serde_json", - "uuid", -] - -[[package]] -name = "diesel-derive-enum" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81c5131a2895ef64741dad1d483f358c2a229a3a2d1b256778cdc5e146db64d4" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "diesel_derives" -version = "2.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b96984c469425cb577bf6f17121ecb3e4fe1e81de5d8f780dd372802858d756" -dependencies = [ - "diesel_table_macro_syntax", - "dsl_auto_type", - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "diesel_table_macro_syntax" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" -dependencies = [ - "syn 2.0.106", -] - [[package]] name = "digest" version = "0.10.7" @@ -706,20 +652,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "dsl_auto_type" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ae9aca7527f85f26dd76483eb38533fd84bd571065da1739656ef71c5ff5b" -dependencies = [ - "darling", - "either", - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.106", -] - [[package]] name = "either" version = "1.15.0" @@ -1125,12 +1057,6 @@ dependencies = [ "hashbrown 0.15.5", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -2061,16 +1987,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "pq-sys" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfd6cf44cca8f9624bc19df234fc4112873432f5fda1caff174527846d026fa9" -dependencies = [ - "libc", - "vcpkg", -] - [[package]] name = "proc-macro2" version = "1.0.101" @@ -2966,7 +2882,7 @@ checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" dependencies = [ "dotenvy", "either", - "heck 0.5.0", + "heck", "hex", "once_cell", "proc-macro2", @@ -3758,16 +3674,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "uuid" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "uwl" version = "0.6.0" diff --git a/Cargo.toml b/Cargo.toml index 581d2bf..3c091ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,9 +11,6 @@ bitflags = { version = "2.9.3", features = ["serde"] } chrono = { version = "0.4", features = ["serde"] } chrono-tz = "0.10.4" compile-time = "0.2.0" -diesel = { version = "2.2.12", features = ["chrono", "postgres", "r2d2", "uuid", "serde_json"] } -diesel-derive-enum = { version = "2.1.0", features = ["postgres"] } -diesel_derives = "2.2.7" dotenvy = "0.15.7" figment = { version = "0.10.19", features = ["toml", "env"] } fundu = "2.0.1" diff --git a/migrations/20250829175305_initial_schema.sql b/migrations/20250829175305_initial_schema.sql new file mode 100644 index 0000000..897ce0b --- /dev/null +++ b/migrations/20250829175305_initial_schema.sql @@ -0,0 +1,56 @@ +-- Drop all old tables +DROP TABLE IF EXISTS scrape_jobs; +DROP TABLE IF EXISTS course_metrics; +DROP TABLE IF EXISTS course_audits; +DROP TABLE IF EXISTS courses; + +-- Enums for scrape_jobs +CREATE TYPE scrape_priority AS ENUM ('Low', 'Medium', 'High', 'Critical'); +CREATE TYPE target_type AS ENUM ('Subject', 'CourseRange', 'CrnList', 'SingleCrn'); + +-- Main course data table +CREATE TABLE courses ( + id SERIAL PRIMARY KEY, + crn VARCHAR NOT NULL, + subject VARCHAR NOT NULL, + course_number VARCHAR NOT NULL, + title VARCHAR NOT NULL, + term_code VARCHAR NOT NULL, + enrollment INTEGER NOT NULL, + max_enrollment INTEGER NOT NULL, + wait_count INTEGER NOT NULL, + wait_capacity INTEGER NOT NULL, + last_scraped_at TIMESTAMPTZ NOT NULL, + UNIQUE(crn, term_code) +); + +-- Time-series data for course enrollment +CREATE TABLE course_metrics ( + id SERIAL PRIMARY KEY, + course_id INTEGER NOT NULL REFERENCES courses(id) ON DELETE CASCADE, + timestamp TIMESTAMPTZ NOT NULL, + enrollment INTEGER NOT NULL, + wait_count INTEGER NOT NULL, + seats_available INTEGER NOT NULL +); + +-- Audit trail for changes to course data +CREATE TABLE course_audits ( + id SERIAL PRIMARY KEY, + course_id INTEGER NOT NULL REFERENCES courses(id) ON DELETE CASCADE, + timestamp TIMESTAMPTZ NOT NULL, + field_changed VARCHAR NOT NULL, + old_value TEXT NOT NULL, + new_value TEXT NOT NULL +); + +-- Job queue for the scraper +CREATE TABLE scrape_jobs ( + id SERIAL PRIMARY KEY, + target_type target_type NOT NULL, + target_payload JSONB NOT NULL, + priority scrape_priority NOT NULL, + execute_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + locked_at TIMESTAMPTZ +); diff --git a/src/data/models.rs b/src/data/models.rs index 449556a..cbc3a28 100644 --- a/src/data/models.rs +++ b/src/data/models.rs @@ -1,12 +1,9 @@ -//! Diesel models for the database schema. +//! `sqlx` models for the database schema. -use crate::data::schema::{course_audits, course_metrics, courses, scrape_jobs}; use chrono::{DateTime, Utc}; -use diesel::{Insertable, Queryable, QueryableByName, Selectable}; use serde_json::Value; -#[derive(Queryable, Selectable)] -#[diesel(table_name = courses)] +#[derive(sqlx::FromRow, Debug, Clone)] pub struct Course { pub id: i32, pub crn: String, @@ -21,24 +18,7 @@ pub struct Course { pub last_scraped_at: DateTime, } -#[derive(Insertable)] -#[diesel(table_name = courses)] -pub struct NewCourse<'a> { - pub crn: &'a str, - pub subject: &'a str, - pub course_number: &'a str, - pub title: &'a str, - pub term_code: &'a str, - pub enrollment: i32, - pub max_enrollment: i32, - pub wait_count: i32, - pub wait_capacity: i32, - pub last_scraped_at: DateTime, -} - -#[derive(Queryable, Selectable)] -#[diesel(table_name = course_metrics)] -#[diesel(belongs_to(Course))] +#[derive(sqlx::FromRow, Debug, Clone)] pub struct CourseMetric { pub id: i32, pub course_id: i32, @@ -48,19 +28,7 @@ pub struct CourseMetric { pub seats_available: i32, } -#[derive(Insertable)] -#[diesel(table_name = course_metrics)] -pub struct NewCourseMetric { - pub course_id: i32, - pub timestamp: DateTime, - pub enrollment: i32, - pub wait_count: i32, - pub seats_available: i32, -} - -#[derive(Queryable, Selectable)] -#[diesel(table_name = course_audits)] -#[diesel(belongs_to(Course))] +#[derive(sqlx::FromRow, Debug, Clone)] pub struct CourseAudit { pub id: i32, pub course_id: i32, @@ -70,18 +38,9 @@ pub struct CourseAudit { pub new_value: String, } -#[derive(Insertable)] -#[diesel(table_name = course_audits)] -pub struct NewCourseAudit<'a> { - pub course_id: i32, - pub timestamp: DateTime, - pub field_changed: &'a str, - pub old_value: &'a str, - pub new_value: &'a str, -} - /// The priority level of a scrape job. -#[derive(diesel_derive_enum::DbEnum, Copy, Debug, Clone)] +#[derive(sqlx::Type, Copy, Debug, Clone)] +#[sqlx(type_name = "scrape_priority", rename_all = "PascalCase")] pub enum ScrapePriority { Low, Medium, @@ -90,7 +49,8 @@ pub enum ScrapePriority { } /// The type of target for a scrape job, determining how the payload is interpreted. -#[derive(diesel_derive_enum::DbEnum, Copy, Debug, Clone)] +#[derive(sqlx::Type, Copy, Debug, Clone)] +#[sqlx(type_name = "target_type", rename_all = "PascalCase")] pub enum TargetType { Subject, CourseRange, @@ -99,8 +59,7 @@ pub enum TargetType { } /// Represents a queryable job from the database. -#[derive(Debug, Clone, Queryable, QueryableByName)] -#[diesel(table_name = scrape_jobs)] +#[derive(sqlx::FromRow, Debug, Clone)] pub struct ScrapeJob { pub id: i32, pub target_type: TargetType, @@ -110,14 +69,3 @@ pub struct ScrapeJob { pub created_at: DateTime, pub locked_at: Option>, } - -/// Represents a new job to be inserted into the database. -#[derive(Debug, Clone, Insertable)] -#[diesel(table_name = scrape_jobs)] -pub struct NewScrapeJob { - pub target_type: TargetType, - #[diesel(sql_type = diesel::sql_types::Jsonb)] - pub target_payload: Value, - pub priority: ScrapePriority, - pub execute_at: DateTime, -} diff --git a/src/data/schema.rs b/src/data/schema.rs deleted file mode 100644 index ee4998a..0000000 --- a/src/data/schema.rs +++ /dev/null @@ -1,69 +0,0 @@ -pub mod sql_types { - #[derive(diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "scrape_priority"))] - pub struct ScrapePriority; - - #[derive(diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "target_type"))] - pub struct TargetType; -} - -use super::models::{ScrapePriorityMapping, TargetTypeMapping}; - -diesel::table! { - use diesel::sql_types::*; - use super::{ScrapePriorityMapping, TargetTypeMapping}; - - scrape_jobs (id) { - id -> Int4, - target_type -> TargetTypeMapping, - target_payload -> Jsonb, - priority -> ScrapePriorityMapping, - execute_at -> Timestamptz, - created_at -> Timestamptz, - locked_at -> Nullable, - } -} - -diesel::table! { - courses (id) { - id -> Int4, - crn -> Varchar, - subject -> Varchar, - course_number -> Varchar, - title -> Varchar, - term_code -> Varchar, - enrollment -> Int4, - max_enrollment -> Int4, - wait_count -> Int4, - wait_capacity -> Int4, - last_scraped_at -> Timestamptz, - } -} - -diesel::table! { - course_metrics (id) { - id -> Int4, - course_id -> Int4, - timestamp -> Timestamptz, - enrollment -> Int4, - wait_count -> Int4, - seats_available -> Int4, - } -} - -diesel::table! { - course_audits (id) { - id -> Int4, - course_id -> Int4, - timestamp -> Timestamptz, - field_changed -> Varchar, - old_value -> Text, - new_value -> Text, - } -} - -diesel::joinable!(course_metrics -> courses (course_id)); -diesel::joinable!(course_audits -> courses (course_id)); - -diesel::allow_tables_to_appear_in_same_query!(courses, course_metrics, course_audits, scrape_jobs,);