mirror of
https://github.com/Xevion/banner.git
synced 2025-12-05 23:14:20 -06:00
Compare commits
2 Commits
9da48b9985
...
23be6035ed
| Author | SHA1 | Date | |
|---|---|---|---|
| 23be6035ed | |||
| 139e4aa635 |
114
Cargo.lock
generated
114
Cargo.lock
generated
@@ -183,9 +183,8 @@ dependencies = [
|
||||
"chrono",
|
||||
"chrono-tz",
|
||||
"compile-time",
|
||||
"diesel",
|
||||
"diesel-derive-enum",
|
||||
"diesel_derives",
|
||||
"cookie",
|
||||
"dashmap 6.1.0",
|
||||
"dotenvy",
|
||||
"figment",
|
||||
"fundu",
|
||||
@@ -195,6 +194,7 @@ dependencies = [
|
||||
"redis",
|
||||
"regex",
|
||||
"reqwest 0.12.23",
|
||||
"reqwest-middleware",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serenity",
|
||||
@@ -617,57 +617,6 @@ dependencies = [
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "diesel"
|
||||
version = "2.2.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "229850a212cd9b84d4f0290ad9d294afc0ae70fccaa8949dbe8b43ffafa1e20c"
|
||||
dependencies = [
|
||||
"bitflags 2.9.3",
|
||||
"byteorder",
|
||||
"chrono",
|
||||
"diesel_derives",
|
||||
"itoa",
|
||||
"pq-sys",
|
||||
"r2d2",
|
||||
"serde_json",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "diesel-derive-enum"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81c5131a2895ef64741dad1d483f358c2a229a3a2d1b256778cdc5e146db64d4"
|
||||
dependencies = [
|
||||
"heck 0.4.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.106",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "diesel_derives"
|
||||
version = "2.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b96984c469425cb577bf6f17121ecb3e4fe1e81de5d8f780dd372802858d756"
|
||||
dependencies = [
|
||||
"diesel_table_macro_syntax",
|
||||
"dsl_auto_type",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.106",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "diesel_table_macro_syntax"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25"
|
||||
dependencies = [
|
||||
"syn 2.0.106",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
@@ -706,20 +655,6 @@ version = "0.15.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
|
||||
|
||||
[[package]]
|
||||
name = "dsl_auto_type"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "139ae9aca7527f85f26dd76483eb38533fd84bd571065da1739656ef71c5ff5b"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"either",
|
||||
"heck 0.5.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.106",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.15.0"
|
||||
@@ -1125,12 +1060,6 @@ dependencies = [
|
||||
"hashbrown 0.15.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
@@ -2061,16 +1990,6 @@ dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pq-sys"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dfd6cf44cca8f9624bc19df234fc4112873432f5fda1caff174527846d026fa9"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.101"
|
||||
@@ -2391,6 +2310,21 @@ dependencies = [
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest-middleware"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57f17d28a6e6acfe1733fe24bcd30774d13bffa4b8a22535b4c8c98423088d4e"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"http 1.3.1",
|
||||
"reqwest 0.12.23",
|
||||
"serde",
|
||||
"thiserror 1.0.69",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.17.14"
|
||||
@@ -2966,7 +2900,7 @@ checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b"
|
||||
dependencies = [
|
||||
"dotenvy",
|
||||
"either",
|
||||
"heck 0.5.0",
|
||||
"heck",
|
||||
"hex",
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
@@ -3758,16 +3692,6 @@ version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "uwl"
|
||||
version = "0.6.0"
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
name = "banner"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
default-run = "banner"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.99"
|
||||
@@ -11,9 +12,8 @@ bitflags = { version = "2.9.3", features = ["serde"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
chrono-tz = "0.10.4"
|
||||
compile-time = "0.2.0"
|
||||
diesel = { version = "2.2.12", features = ["chrono", "postgres", "r2d2", "uuid", "serde_json"] }
|
||||
diesel-derive-enum = { version = "2.1.0", features = ["postgres"] }
|
||||
diesel_derives = "2.2.7"
|
||||
cookie = "0.18.1"
|
||||
dashmap = "6.1.0"
|
||||
dotenvy = "0.15.7"
|
||||
figment = { version = "0.10.19", features = ["toml", "env"] }
|
||||
fundu = "2.0.1"
|
||||
@@ -23,6 +23,7 @@ rand = "0.9.2"
|
||||
redis = { version = "0.32.5", features = ["tokio-comp", "r2d2"] }
|
||||
regex = "1.10"
|
||||
reqwest = { version = "0.12.23", features = ["json", "cookies"] }
|
||||
reqwest-middleware = { version = "0.4.2", features = ["json"] }
|
||||
serde = { version = "1.0.219", features = ["derive"] }
|
||||
serde_json = "1.0.143"
|
||||
serenity = { version = "0.12.4", features = ["rustls_backend"] }
|
||||
|
||||
56
migrations/20250829175305_initial_schema.sql
Normal file
56
migrations/20250829175305_initial_schema.sql
Normal file
@@ -0,0 +1,56 @@
|
||||
-- Drop all old tables
|
||||
DROP TABLE IF EXISTS scrape_jobs;
|
||||
DROP TABLE IF EXISTS course_metrics;
|
||||
DROP TABLE IF EXISTS course_audits;
|
||||
DROP TABLE IF EXISTS courses;
|
||||
|
||||
-- Enums for scrape_jobs
|
||||
CREATE TYPE scrape_priority AS ENUM ('Low', 'Medium', 'High', 'Critical');
|
||||
CREATE TYPE target_type AS ENUM ('Subject', 'CourseRange', 'CrnList', 'SingleCrn');
|
||||
|
||||
-- Main course data table
|
||||
CREATE TABLE courses (
|
||||
id SERIAL PRIMARY KEY,
|
||||
crn VARCHAR NOT NULL,
|
||||
subject VARCHAR NOT NULL,
|
||||
course_number VARCHAR NOT NULL,
|
||||
title VARCHAR NOT NULL,
|
||||
term_code VARCHAR NOT NULL,
|
||||
enrollment INTEGER NOT NULL,
|
||||
max_enrollment INTEGER NOT NULL,
|
||||
wait_count INTEGER NOT NULL,
|
||||
wait_capacity INTEGER NOT NULL,
|
||||
last_scraped_at TIMESTAMPTZ NOT NULL,
|
||||
UNIQUE(crn, term_code)
|
||||
);
|
||||
|
||||
-- Time-series data for course enrollment
|
||||
CREATE TABLE course_metrics (
|
||||
id SERIAL PRIMARY KEY,
|
||||
course_id INTEGER NOT NULL REFERENCES courses(id) ON DELETE CASCADE,
|
||||
timestamp TIMESTAMPTZ NOT NULL,
|
||||
enrollment INTEGER NOT NULL,
|
||||
wait_count INTEGER NOT NULL,
|
||||
seats_available INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Audit trail for changes to course data
|
||||
CREATE TABLE course_audits (
|
||||
id SERIAL PRIMARY KEY,
|
||||
course_id INTEGER NOT NULL REFERENCES courses(id) ON DELETE CASCADE,
|
||||
timestamp TIMESTAMPTZ NOT NULL,
|
||||
field_changed VARCHAR NOT NULL,
|
||||
old_value TEXT NOT NULL,
|
||||
new_value TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Job queue for the scraper
|
||||
CREATE TABLE scrape_jobs (
|
||||
id SERIAL PRIMARY KEY,
|
||||
target_type target_type NOT NULL,
|
||||
target_payload JSONB NOT NULL,
|
||||
priority scrape_priority NOT NULL,
|
||||
execute_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
locked_at TIMESTAMPTZ
|
||||
);
|
||||
@@ -7,7 +7,7 @@ use redis::AsyncCommands;
|
||||
use redis::Client;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
pub banner_api: Arc<BannerApi>,
|
||||
pub redis: Arc<Client>,
|
||||
|
||||
@@ -1,25 +1,79 @@
|
||||
//! Main Banner API client implementation.
|
||||
|
||||
use crate::banner::{models::*, query::SearchQuery, session::SessionManager, util::user_agent};
|
||||
use anyhow::{Context, Result};
|
||||
use axum::http::HeaderValue;
|
||||
use reqwest::Client;
|
||||
use serde_json;
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
sync::{Arc, Mutex},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
use tracing::{error, info};
|
||||
use crate::banner::{
|
||||
BannerSession, SessionPool, models::*, nonce, query::SearchQuery, util::user_agent,
|
||||
};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use axum::http::{Extensions, HeaderValue};
|
||||
use cookie::Cookie;
|
||||
use dashmap::DashMap;
|
||||
use reqwest::{Client, Request, Response};
|
||||
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware, Middleware, Next};
|
||||
use serde_json;
|
||||
use tracing::{Level, Metadata, Span, debug, error, field::ValueSet, info, span};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum BannerApiError {
|
||||
#[error("Banner session is invalid or expired")]
|
||||
InvalidSession,
|
||||
#[error(transparent)]
|
||||
RequestFailed(#[from] anyhow::Error),
|
||||
}
|
||||
|
||||
/// Main Banner API client.
|
||||
#[derive(Debug)]
|
||||
pub struct BannerApi {
|
||||
sessions: SessionManager,
|
||||
http: Client,
|
||||
pub sessions: SessionPool,
|
||||
http: ClientWithMiddleware,
|
||||
base_url: String,
|
||||
}
|
||||
|
||||
pub struct TransparentMiddleware;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Middleware for TransparentMiddleware {
|
||||
async fn handle(
|
||||
&self,
|
||||
req: Request,
|
||||
extensions: &mut Extensions,
|
||||
next: Next<'_>,
|
||||
) -> std::result::Result<Response, reqwest_middleware::Error> {
|
||||
debug!(
|
||||
domain = req.url().domain(),
|
||||
"{method} {path}",
|
||||
method = req.method().to_string(),
|
||||
path = req.url().path(),
|
||||
);
|
||||
let response = next.run(req, extensions).await;
|
||||
|
||||
match &response {
|
||||
Ok(response) => {
|
||||
debug!(
|
||||
"{code} {reason} {path}",
|
||||
code = response.status().as_u16(),
|
||||
reason = response.status().canonical_reason().unwrap_or("??"),
|
||||
path = response.url().path(),
|
||||
);
|
||||
}
|
||||
Err(error) => {
|
||||
debug!("!!! {error}");
|
||||
}
|
||||
}
|
||||
|
||||
response
|
||||
}
|
||||
}
|
||||
|
||||
impl BannerApi {
|
||||
/// Creates a new Banner API client.
|
||||
pub fn new(base_url: String) -> Result<Self> {
|
||||
let http = Client::builder()
|
||||
let http = ClientBuilder::new(
|
||||
Client::builder()
|
||||
.cookie_store(true)
|
||||
.user_agent(user_agent())
|
||||
.tcp_keepalive(Some(std::time::Duration::from_secs(60 * 5)))
|
||||
@@ -27,63 +81,18 @@ impl BannerApi {
|
||||
.connect_timeout(std::time::Duration::from_secs(10))
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.build()
|
||||
.context("Failed to create HTTP client")?;
|
||||
|
||||
let session_manager = SessionManager::new(base_url.clone(), http.clone());
|
||||
.context("Failed to create HTTP client")?,
|
||||
)
|
||||
.with(TransparentMiddleware)
|
||||
.build();
|
||||
|
||||
Ok(Self {
|
||||
sessions: session_manager,
|
||||
sessions: SessionPool::new(http.clone(), base_url.clone()),
|
||||
http,
|
||||
base_url,
|
||||
})
|
||||
}
|
||||
|
||||
/// Sets up the API client by initializing session cookies.
|
||||
pub async fn setup(&self) -> Result<()> {
|
||||
info!(base_url = self.base_url, "setting up banner api client");
|
||||
let result = self.sessions.setup().await;
|
||||
match &result {
|
||||
Ok(()) => info!("banner api client setup completed successfully"),
|
||||
Err(e) => error!(error = ?e, "banner api client setup failed"),
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Retrieves a list of terms from the Banner API.
|
||||
pub async fn get_terms(
|
||||
&self,
|
||||
search: &str,
|
||||
page: i32,
|
||||
max_results: i32,
|
||||
) -> Result<Vec<BannerTerm>> {
|
||||
if page <= 0 {
|
||||
return Err(anyhow::anyhow!("Page must be greater than 0"));
|
||||
}
|
||||
|
||||
let url = format!("{}/classSearch/getTerms", self.base_url);
|
||||
let params = [
|
||||
("searchTerm", search),
|
||||
("offset", &page.to_string()),
|
||||
("max", &max_results.to_string()),
|
||||
("_", &SessionManager::nonce()),
|
||||
];
|
||||
|
||||
let response = self
|
||||
.http
|
||||
.get(&url)
|
||||
.query(¶ms)
|
||||
.send()
|
||||
.await
|
||||
.context("Failed to get terms")?;
|
||||
|
||||
let terms: Vec<BannerTerm> = response
|
||||
.json()
|
||||
.await
|
||||
.context("Failed to parse terms response")?;
|
||||
|
||||
Ok(terms)
|
||||
}
|
||||
|
||||
/// Retrieves a list of subjects from the Banner API.
|
||||
pub async fn get_subjects(
|
||||
&self,
|
||||
@@ -96,15 +105,15 @@ impl BannerApi {
|
||||
return Err(anyhow::anyhow!("Offset must be greater than 0"));
|
||||
}
|
||||
|
||||
let session_id = self.sessions.ensure_session()?;
|
||||
let session = self.sessions.acquire(term.parse()?).await?;
|
||||
let url = format!("{}/classSearch/get_subject", self.base_url);
|
||||
let params = [
|
||||
("searchTerm", search),
|
||||
("term", term),
|
||||
("offset", &offset.to_string()),
|
||||
("max", &max_results.to_string()),
|
||||
("uniqueSessionId", &session_id),
|
||||
("_", &SessionManager::nonce()),
|
||||
("uniqueSessionId", &session.id()),
|
||||
("_", &nonce()),
|
||||
];
|
||||
|
||||
let response = self
|
||||
@@ -135,15 +144,15 @@ impl BannerApi {
|
||||
return Err(anyhow::anyhow!("Offset must be greater than 0"));
|
||||
}
|
||||
|
||||
let session_id = self.sessions.ensure_session()?;
|
||||
let session = self.sessions.acquire(term.parse()?).await?;
|
||||
let url = format!("{}/classSearch/get_instructor", self.base_url);
|
||||
let params = [
|
||||
("searchTerm", search),
|
||||
("term", term),
|
||||
("offset", &offset.to_string()),
|
||||
("max", &max_results.to_string()),
|
||||
("uniqueSessionId", &session_id),
|
||||
("_", &SessionManager::nonce()),
|
||||
("uniqueSessionId", &session.id()),
|
||||
("_", &nonce()),
|
||||
];
|
||||
|
||||
let response = self
|
||||
@@ -174,15 +183,15 @@ impl BannerApi {
|
||||
return Err(anyhow::anyhow!("Offset must be greater than 0"));
|
||||
}
|
||||
|
||||
let session_id = self.sessions.ensure_session()?;
|
||||
let session = self.sessions.acquire(term.parse()?).await?;
|
||||
let url = format!("{}/classSearch/get_campus", self.base_url);
|
||||
let params = [
|
||||
("searchTerm", search),
|
||||
("term", term),
|
||||
("offset", &offset.to_string()),
|
||||
("max", &max_results.to_string()),
|
||||
("uniqueSessionId", &session_id),
|
||||
("_", &SessionManager::nonce()),
|
||||
("uniqueSessionId", &session.id()),
|
||||
("_", &nonce()),
|
||||
];
|
||||
|
||||
let response = self
|
||||
@@ -259,15 +268,15 @@ impl BannerApi {
|
||||
query: &SearchQuery,
|
||||
sort: &str,
|
||||
sort_descending: bool,
|
||||
) -> Result<SearchResult> {
|
||||
self.sessions.reset_data_form().await?;
|
||||
) -> Result<SearchResult, BannerApiError> {
|
||||
// self.sessions.reset_data_form().await?;
|
||||
|
||||
let session_id = self.sessions.ensure_session()?;
|
||||
let session = self.sessions.acquire(term.parse()?).await?;
|
||||
let mut params = query.to_params();
|
||||
|
||||
// Add additional parameters
|
||||
params.insert("txt_term".to_string(), term.to_string());
|
||||
params.insert("uniqueSessionId".to_string(), session_id);
|
||||
params.insert("uniqueSessionId".to_string(), session.id());
|
||||
params.insert("sortColumn".to_string(), sort.to_string());
|
||||
params.insert(
|
||||
"sortDirection".to_string(),
|
||||
@@ -280,37 +289,50 @@ impl BannerApi {
|
||||
let response = self
|
||||
.http
|
||||
.get(&url)
|
||||
.header("Cookie", session.cookie())
|
||||
.query(¶ms)
|
||||
.send()
|
||||
.await
|
||||
.context("Failed to search courses")?;
|
||||
|
||||
let search_result: SearchResult = response
|
||||
.json()
|
||||
let status = response.status();
|
||||
let url = response.url().clone();
|
||||
let body = response
|
||||
.text()
|
||||
.await
|
||||
.context("Failed to parse search response")?;
|
||||
.with_context(|| format!("Failed to read body (status={status})"))?;
|
||||
|
||||
let search_result: SearchResult = parse_json_with_context(&body).map_err(|e| {
|
||||
BannerApiError::RequestFailed(anyhow!(
|
||||
"Failed to parse search response (status={status}, url={url}): {e}\nBody: {body}"
|
||||
))
|
||||
})?;
|
||||
|
||||
// Check for signs of an invalid session, based on docs/Sessions.md
|
||||
if search_result.path_mode.is_none() || search_result.data.is_none() {
|
||||
return Err(BannerApiError::InvalidSession);
|
||||
}
|
||||
|
||||
if !search_result.success {
|
||||
return Err(anyhow::anyhow!(
|
||||
return Err(BannerApiError::RequestFailed(anyhow!(
|
||||
"Search marked as unsuccessful by Banner API"
|
||||
));
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(search_result)
|
||||
}
|
||||
|
||||
/// Selects a term for the current session.
|
||||
pub async fn select_term(&self, term: &str) -> Result<()> {
|
||||
self.sessions.select_term(term).await
|
||||
}
|
||||
|
||||
/// Retrieves a single course by CRN by issuing a minimal search
|
||||
pub async fn get_course_by_crn(&self, term: &str, crn: &str) -> Result<Option<Course>> {
|
||||
self.sessions.reset_data_form().await?;
|
||||
pub async fn get_course_by_crn(
|
||||
&self,
|
||||
term: &str,
|
||||
crn: &str,
|
||||
) -> Result<Option<Course>, BannerApiError> {
|
||||
// self.sessions.reset_data_form().await?;
|
||||
// Ensure session is configured for this term
|
||||
self.select_term(term).await?;
|
||||
// self.select_term(term).await?;
|
||||
|
||||
let session_id = self.sessions.ensure_session()?;
|
||||
let session = self.sessions.acquire(term.parse()?).await?;
|
||||
|
||||
let query = SearchQuery::new()
|
||||
.course_reference_number(crn)
|
||||
@@ -318,7 +340,7 @@ impl BannerApi {
|
||||
|
||||
let mut params = query.to_params();
|
||||
params.insert("txt_term".to_string(), term.to_string());
|
||||
params.insert("uniqueSessionId".to_string(), session_id);
|
||||
params.insert("uniqueSessionId".to_string(), session.id());
|
||||
params.insert("sortColumn".to_string(), "subjectDescription".to_string());
|
||||
params.insert("sortDirection".to_string(), "asc".to_string());
|
||||
params.insert("startDatepicker".to_string(), String::new());
|
||||
@@ -328,27 +350,36 @@ impl BannerApi {
|
||||
let response = self
|
||||
.http
|
||||
.get(&url)
|
||||
.header("Cookie", session.cookie())
|
||||
.query(¶ms)
|
||||
.send()
|
||||
.await
|
||||
.context("Failed to search course by CRN")?;
|
||||
|
||||
let status = response.status();
|
||||
let url = response.url().clone();
|
||||
let body = response
|
||||
.text()
|
||||
.await
|
||||
.with_context(|| format!("Failed to read body (status={status})"))?;
|
||||
|
||||
let search_result: SearchResult = parse_json_with_context(&body).map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"Failed to parse search response for CRN (status={status}, url={url}): {e}",
|
||||
)
|
||||
BannerApiError::RequestFailed(anyhow!(
|
||||
"Failed to parse search response for CRN (status={status}, url={url}): {e}"
|
||||
))
|
||||
})?;
|
||||
|
||||
// Check for signs of an invalid session, based on docs/Sessions.md
|
||||
if search_result.path_mode == Some("registration".to_string())
|
||||
&& search_result.data.is_none()
|
||||
{
|
||||
return Err(BannerApiError::InvalidSession);
|
||||
}
|
||||
|
||||
if !search_result.success {
|
||||
return Err(anyhow::anyhow!(
|
||||
return Err(BannerApiError::RequestFailed(anyhow!(
|
||||
"Search marked as unsuccessful by Banner API"
|
||||
));
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(search_result
|
||||
@@ -382,13 +413,14 @@ impl BannerApi {
|
||||
}
|
||||
}
|
||||
|
||||
/// Attempt to parse JSON and, on failure, include a contextual snippet around the error location
|
||||
/// Attempt to parse JSON and, on failure, include a contextual snippet of the
|
||||
/// line where the error occurred. This prevents dumping huge JSON bodies to logs.
|
||||
fn parse_json_with_context<T: serde::de::DeserializeOwned>(body: &str) -> Result<T> {
|
||||
match serde_json::from_str::<T>(body) {
|
||||
Ok(value) => Ok(value),
|
||||
Err(err) => {
|
||||
let (line, column) = (err.line(), err.column());
|
||||
let snippet = build_error_snippet(body, line, column, 120);
|
||||
let snippet = build_error_snippet(body, line, column, 80);
|
||||
Err(anyhow::anyhow!(
|
||||
"{err} at line {line}, column {column}\nSnippet:\n{snippet}",
|
||||
))
|
||||
@@ -396,21 +428,23 @@ fn parse_json_with_context<T: serde::de::DeserializeOwned>(body: &str) -> Result
|
||||
}
|
||||
}
|
||||
|
||||
fn build_error_snippet(body: &str, line: usize, column: usize, max_len: usize) -> String {
|
||||
fn build_error_snippet(body: &str, line: usize, column: usize, context_len: usize) -> String {
|
||||
let target_line = body.lines().nth(line.saturating_sub(1)).unwrap_or("");
|
||||
if target_line.is_empty() {
|
||||
return String::new();
|
||||
return "(empty line)".to_string();
|
||||
}
|
||||
|
||||
let start = column.saturating_sub(max_len.min(column));
|
||||
let end = (column + max_len).min(target_line.len());
|
||||
// column is 1-based, convert to 0-based for slicing
|
||||
let error_idx = column.saturating_sub(1);
|
||||
|
||||
let half_len = context_len / 2;
|
||||
let start = error_idx.saturating_sub(half_len);
|
||||
let end = (error_idx + half_len).min(target_line.len());
|
||||
|
||||
let slice = &target_line[start..end];
|
||||
let indicator_pos = error_idx - start;
|
||||
|
||||
let mut indicator = String::new();
|
||||
if column > start {
|
||||
indicator.push_str(&" ".repeat(column - start - 1));
|
||||
indicator.push('^');
|
||||
}
|
||||
let indicator = " ".repeat(indicator_pos) + "^";
|
||||
|
||||
format!("{slice}\n{indicator}")
|
||||
format!("...{slice}...\n {indicator}")
|
||||
}
|
||||
|
||||
@@ -44,8 +44,8 @@ pub struct FacultyItem {
|
||||
pub struct MeetingTime {
|
||||
pub start_date: String, // MM/DD/YYYY, e.g 08/26/2025
|
||||
pub end_date: String, // MM/DD/YYYY, e.g 08/26/2025
|
||||
pub begin_time: String, // HHMM, e.g 1000
|
||||
pub end_time: String, // HHMM, e.g 1100
|
||||
pub begin_time: Option<String>, // HHMM, e.g 1000
|
||||
pub end_time: Option<String>, // HHMM, e.g 1100
|
||||
pub category: String, // unknown meaning, e.g. 01, 02, etc
|
||||
pub class: String, // internal class name, e.g. net.hedtech.banner.general.overallMeetingTimeDecorator
|
||||
pub monday: bool, // true if the meeting time occurs on Monday
|
||||
@@ -55,13 +55,13 @@ pub struct MeetingTime {
|
||||
pub friday: bool, // true if the meeting time occurs on Friday
|
||||
pub saturday: bool, // true if the meeting time occurs on Saturday
|
||||
pub sunday: bool, // true if the meeting time occurs on Sunday
|
||||
pub room: String, // e.g. 1238
|
||||
pub room: Option<String>, // e.g. 1.238
|
||||
#[serde(deserialize_with = "deserialize_string_to_term")]
|
||||
pub term: Term, // e.g 202510
|
||||
pub building: String, // e.g NPB
|
||||
pub building_description: String, // e.g North Paseo Building
|
||||
pub campus: String, // campus code, e.g 11
|
||||
pub campus_description: String, // name of campus, e.g Main Campus
|
||||
pub building: Option<String>, // e.g NPB
|
||||
pub building_description: Option<String>, // e.g North Paseo Building
|
||||
pub campus: Option<String>, // campus code, e.g 11
|
||||
pub campus_description: Option<String>, // name of campus, e.g Main Campus
|
||||
pub course_reference_number: String, // CRN, e.g 27294
|
||||
pub credit_hour_session: f64, // e.g. 30
|
||||
pub hours_week: f64, // e.g. 30
|
||||
@@ -347,42 +347,58 @@ impl MeetingType {
|
||||
|
||||
/// Meeting location information
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MeetingLocation {
|
||||
pub campus: String,
|
||||
pub building: String,
|
||||
pub building_description: String,
|
||||
pub room: String,
|
||||
pub is_online: bool,
|
||||
pub enum MeetingLocation {
|
||||
Online,
|
||||
InPerson {
|
||||
campus: String,
|
||||
campus_description: String,
|
||||
building: String,
|
||||
building_description: String,
|
||||
room: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl MeetingLocation {
|
||||
/// Create from raw MeetingTime data
|
||||
pub fn from_meeting_time(meeting_time: &MeetingTime) -> Self {
|
||||
let is_online = meeting_time.room.is_empty();
|
||||
if meeting_time.campus.is_none()
|
||||
|| meeting_time.building.is_none()
|
||||
|| meeting_time.building_description.is_none()
|
||||
|| meeting_time.room.is_none()
|
||||
|| meeting_time.campus_description.is_none()
|
||||
|| meeting_time
|
||||
.campus_description
|
||||
.eq(&Some("Internet".to_string()))
|
||||
{
|
||||
return MeetingLocation::Online;
|
||||
}
|
||||
|
||||
MeetingLocation {
|
||||
campus: meeting_time.campus_description.clone(),
|
||||
building: meeting_time.building.clone(),
|
||||
building_description: meeting_time.building_description.clone(),
|
||||
room: meeting_time.room.clone(),
|
||||
is_online,
|
||||
MeetingLocation::InPerson {
|
||||
campus: meeting_time.campus.as_ref().unwrap().clone(),
|
||||
campus_description: meeting_time.campus_description.as_ref().unwrap().clone(),
|
||||
building: meeting_time.building.as_ref().unwrap().clone(),
|
||||
building_description: meeting_time.building_description.as_ref().unwrap().clone(),
|
||||
room: meeting_time.room.as_ref().unwrap().clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for MeetingLocation {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
if self.is_online {
|
||||
write!(f, "Online")
|
||||
} else {
|
||||
write!(
|
||||
match self {
|
||||
MeetingLocation::Online => write!(f, "Online"),
|
||||
MeetingLocation::InPerson {
|
||||
campus,
|
||||
building,
|
||||
building_description,
|
||||
room,
|
||||
..
|
||||
} => write!(
|
||||
f,
|
||||
"{campus} | {building_name} | {building_code} {room}",
|
||||
campus = self.campus,
|
||||
building_name = self.building_description,
|
||||
building_code = self.building,
|
||||
room = self.room
|
||||
)
|
||||
building_name = building_description,
|
||||
building_code = building,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -402,7 +418,11 @@ impl MeetingScheduleInfo {
|
||||
/// Create from raw MeetingTime data
|
||||
pub fn from_meeting_time(meeting_time: &MeetingTime) -> Self {
|
||||
let days = MeetingDays::from_meeting_time(meeting_time);
|
||||
let time_range = TimeRange::from_hhmm(&meeting_time.begin_time, &meeting_time.end_time);
|
||||
let time_range = match (&meeting_time.begin_time, &meeting_time.end_time) {
|
||||
(Some(begin), Some(end)) => TimeRange::from_hhmm(&begin, &end),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let date_range =
|
||||
DateRange::from_mm_dd_yyyy(&meeting_time.start_date, &meeting_time.end_date)
|
||||
.unwrap_or_else(|| {
|
||||
@@ -470,16 +490,18 @@ impl MeetingScheduleInfo {
|
||||
|
||||
/// Returns a formatted string representing the location of the meeting
|
||||
pub fn place_string(&self) -> String {
|
||||
if self.location.room.is_empty() {
|
||||
"Online".to_string()
|
||||
} else {
|
||||
format!(
|
||||
match &self.location {
|
||||
MeetingLocation::Online => "Online".to_string(),
|
||||
MeetingLocation::InPerson {
|
||||
campus,
|
||||
building,
|
||||
building_description,
|
||||
room,
|
||||
..
|
||||
} => format!(
|
||||
"{} | {} | {} {}",
|
||||
self.location.campus,
|
||||
self.location.building_description,
|
||||
self.location.building,
|
||||
self.location.room
|
||||
)
|
||||
campus, building_description, building, room
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ pub struct SearchResult {
|
||||
pub total_count: i32,
|
||||
pub page_offset: i32,
|
||||
pub page_max_size: i32,
|
||||
pub path_mode: String,
|
||||
pub search_results_config: Vec<SearchResultConfig>,
|
||||
pub path_mode: Option<String>,
|
||||
pub search_results_config: Option<Vec<SearchResultConfig>>,
|
||||
pub data: Option<Vec<Course>>,
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ const CURRENT_YEAR: u32 = compile_time::date!().year() as u32;
|
||||
const VALID_YEARS: RangeInclusive<u32> = 2007..=(CURRENT_YEAR + 10);
|
||||
|
||||
/// Represents a term in the Banner system
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
|
||||
pub struct Term {
|
||||
pub year: u32, // 2024, 2025, etc
|
||||
pub season: Season,
|
||||
@@ -29,7 +29,7 @@ pub enum TermPoint {
|
||||
}
|
||||
|
||||
/// Represents a season within a term
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
|
||||
pub enum Season {
|
||||
Fall,
|
||||
Spring,
|
||||
|
||||
@@ -1,75 +1,40 @@
|
||||
//! Session management for Banner API.
|
||||
|
||||
use crate::banner::util::user_agent;
|
||||
use anyhow::Result;
|
||||
use rand::distributions::{Alphanumeric, DistString};
|
||||
use crate::banner::BannerTerm;
|
||||
use crate::banner::models::Term;
|
||||
use anyhow::{Context, Result};
|
||||
use cookie::Cookie;
|
||||
use dashmap::DashMap;
|
||||
use rand::distr::{Alphanumeric, SampleString};
|
||||
use reqwest::Client;
|
||||
use std::sync::Mutex;
|
||||
use reqwest_middleware::ClientWithMiddleware;
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::ops::{Deref, DerefMut};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, info};
|
||||
use url::Url;
|
||||
|
||||
/// Session manager for Banner API interactions
|
||||
#[derive(Debug)]
|
||||
pub struct SessionManager {
|
||||
current_session: Mutex<Option<SessionData>>,
|
||||
base_url: String,
|
||||
client: Client,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct SessionData {
|
||||
session_id: String,
|
||||
created_at: Instant,
|
||||
}
|
||||
|
||||
impl SessionManager {
|
||||
const SESSION_EXPIRY: Duration = Duration::from_secs(25 * 60); // 25 minutes
|
||||
|
||||
/// Creates a new session manager
|
||||
pub fn new(base_url: String, client: Client) -> Self {
|
||||
Self {
|
||||
current_session: Mutex::new(None),
|
||||
base_url,
|
||||
client,
|
||||
}
|
||||
}
|
||||
|
||||
/// Ensures a valid session is available, creating one if necessary
|
||||
pub fn ensure_session(&self) -> Result<String> {
|
||||
let start_time = std::time::Instant::now();
|
||||
let mut session_guard = self.current_session.lock().unwrap();
|
||||
|
||||
if let Some(ref session) = *session_guard
|
||||
&& session.created_at.elapsed() < Self::SESSION_EXPIRY
|
||||
{
|
||||
let elapsed = start_time.elapsed();
|
||||
debug!(
|
||||
session_id = session.session_id,
|
||||
elapsed = format!("{:.2?}", elapsed),
|
||||
"reusing existing banner session"
|
||||
);
|
||||
return Ok(session.session_id.clone());
|
||||
}
|
||||
|
||||
// Generate new session
|
||||
let session_id = self.generate_session_id();
|
||||
*session_guard = Some(SessionData {
|
||||
session_id: session_id.clone(),
|
||||
created_at: Instant::now(),
|
||||
});
|
||||
|
||||
let elapsed = start_time.elapsed();
|
||||
debug!(
|
||||
session_id = session_id,
|
||||
elapsed = format!("{:.2?}", elapsed),
|
||||
"generated new banner session"
|
||||
);
|
||||
Ok(session_id)
|
||||
/// Represents an active anonymous session within the Banner API.
|
||||
/// Identified by multiple persistent cookies, as well as a client-generated "unique session ID".
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BannerSession {
|
||||
// Randomly generated
|
||||
unique_session_id: String,
|
||||
// Timestamp of creation
|
||||
created_at: Instant,
|
||||
// Timestamp of last activity
|
||||
last_activity: Option<Instant>,
|
||||
// Cookie values from initial registration page
|
||||
jsessionid: String,
|
||||
ssb_cookie: String,
|
||||
}
|
||||
|
||||
/// Generates a new session ID mimicking Banner's format
|
||||
fn generate_session_id(&self) -> String {
|
||||
let random_part = Alphanumeric.sample_string(&mut rand::thread_rng(), 5);
|
||||
fn generate_session_id() -> String {
|
||||
let random_part = Alphanumeric.sample_string(&mut rand::rng(), 5);
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
@@ -77,57 +42,267 @@ impl SessionManager {
|
||||
format!("{}{}", random_part, timestamp)
|
||||
}
|
||||
|
||||
/// Generates a timestamp-based nonce
|
||||
pub fn nonce() -> String {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
impl BannerSession {
|
||||
/// Creates a new session
|
||||
pub async fn new(unique_session_id: &str, jsessionid: &str, ssb_cookie: &str) -> Result<Self> {
|
||||
let now = Instant::now();
|
||||
|
||||
Ok(Self {
|
||||
created_at: now,
|
||||
last_activity: None,
|
||||
unique_session_id: unique_session_id.to_string(),
|
||||
jsessionid: jsessionid.to_string(),
|
||||
ssb_cookie: ssb_cookie.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the unique session ID
|
||||
pub fn id(&self) -> String {
|
||||
self.unique_session_id.clone()
|
||||
}
|
||||
|
||||
/// Updates the last activity timestamp
|
||||
pub fn touch(&mut self) {
|
||||
debug!("Session {} is being used", self.unique_session_id);
|
||||
self.last_activity = Some(Instant::now());
|
||||
}
|
||||
|
||||
/// Returns true if the session is expired
|
||||
pub fn is_expired(&self) -> bool {
|
||||
self.last_activity.unwrap_or(self.created_at).elapsed() > SESSION_EXPIRY
|
||||
}
|
||||
|
||||
/// Returns a string used to for the "Cookie" header
|
||||
pub fn cookie(&self) -> String {
|
||||
format!(
|
||||
"JSESSIONID={}; SSB_COOKIE={}",
|
||||
self.jsessionid, self.ssb_cookie
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// A smart pointer that returns a BannerSession to the pool when dropped.
|
||||
pub struct PooledSession {
|
||||
session: Option<BannerSession>,
|
||||
// This Arc points directly to the queue the session belongs to.
|
||||
pool: Arc<Mutex<VecDeque<BannerSession>>>,
|
||||
}
|
||||
|
||||
impl Deref for PooledSession {
|
||||
type Target = BannerSession;
|
||||
fn deref(&self) -> &Self::Target {
|
||||
// The option is only ever None after drop is called, so this is safe.
|
||||
self.session.as_ref().unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for PooledSession {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
self.session.as_mut().unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
/// The magic happens here: when the guard goes out of scope, this is called.
|
||||
impl Drop for PooledSession {
|
||||
fn drop(&mut self) {
|
||||
if let Some(session) = self.session.take() {
|
||||
// Don't return expired sessions to the pool.
|
||||
if session.is_expired() {
|
||||
debug!("Session {} expired, dropping.", session.unique_session_id);
|
||||
return;
|
||||
}
|
||||
|
||||
// This is a synchronous lock, so it's allowed in drop().
|
||||
// It blocks the current thread briefly to return the session.
|
||||
let mut queue = self.pool.lock().unwrap();
|
||||
queue.push_back(session);
|
||||
debug!(
|
||||
"Session returned to pool. Queue size is now {}.",
|
||||
queue.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SessionPool {
|
||||
sessions: DashMap<Term, Arc<Mutex<VecDeque<BannerSession>>>>,
|
||||
http: ClientWithMiddleware,
|
||||
base_url: String,
|
||||
}
|
||||
|
||||
impl SessionPool {
|
||||
pub fn new(http: ClientWithMiddleware, base_url: String) -> Self {
|
||||
Self {
|
||||
sessions: DashMap::new(),
|
||||
http,
|
||||
base_url,
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquires a session from the pool.
|
||||
/// If no sessions are available, a new one is created on demand.
|
||||
pub async fn acquire(&self, term: Term) -> Result<PooledSession> {
|
||||
// Get the queue for the given term, or insert a new empty one.
|
||||
let pool_entry = self
|
||||
.sessions
|
||||
.entry(term.clone())
|
||||
.or_insert_with(|| Arc::new(Mutex::new(VecDeque::new())))
|
||||
.clone();
|
||||
|
||||
loop {
|
||||
// Lock the specific queue for this term
|
||||
let session_option = {
|
||||
let mut queue = pool_entry.lock().unwrap();
|
||||
queue.pop_front() // Try to get a session
|
||||
};
|
||||
|
||||
if let Some(mut session) = session_option {
|
||||
// We got a session, check if it's expired.
|
||||
if !session.is_expired() {
|
||||
debug!("Reusing session {}", session.unique_session_id);
|
||||
|
||||
session.touch();
|
||||
return Ok(PooledSession {
|
||||
session: Some(session),
|
||||
pool: pool_entry,
|
||||
});
|
||||
} else {
|
||||
debug!(
|
||||
"Popped an expired session {}, discarding.",
|
||||
session.unique_session_id
|
||||
);
|
||||
// The session is expired, so we loop again to try and get another one.
|
||||
}
|
||||
} else {
|
||||
// Queue was empty, so we create a new session.
|
||||
let mut new_session = self.create_session(&term).await?;
|
||||
new_session.touch();
|
||||
|
||||
return Ok(PooledSession {
|
||||
session: Some(new_session),
|
||||
pool: pool_entry,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets up initial session cookies by making required Banner API requests
|
||||
pub async fn setup(&self) -> Result<()> {
|
||||
pub async fn create_session(&self, term: &Term) -> Result<BannerSession> {
|
||||
info!("setting up banner session...");
|
||||
|
||||
let request_paths = ["/registration/registration", "/selfServiceMenu/data"];
|
||||
|
||||
for path in &request_paths {
|
||||
let url = format!("{}{}", self.base_url, path);
|
||||
let response = self
|
||||
.client
|
||||
.get(&url)
|
||||
.query(&[("_", Self::nonce())])
|
||||
.header("User-Agent", user_agent())
|
||||
// The 'register' or 'search' registration page
|
||||
let initial_registration = self
|
||||
.http
|
||||
.get(format!("{}/registration", self.base_url))
|
||||
.send()
|
||||
.await?;
|
||||
// TODO: Validate success
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"Failed to setup session, request to {} returned {}",
|
||||
path,
|
||||
response.status()
|
||||
));
|
||||
let cookies = initial_registration
|
||||
.headers()
|
||||
.get_all("Set-Cookie")
|
||||
.iter()
|
||||
.filter_map(|header_value| {
|
||||
if let Ok(cookie) = Cookie::parse(header_value.to_str().unwrap()) {
|
||||
Some((cookie.name().to_string(), cookie.value().to_string()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect::<HashMap<String, String>>();
|
||||
|
||||
let jsessionid = cookies.get("JSESSIONID").unwrap();
|
||||
let ssb_cookie = cookies.get("SSB_COOKIE").unwrap();
|
||||
|
||||
let data_page_response = self
|
||||
.http
|
||||
.get(format!("{}/selfServiceMenu/data", self.base_url))
|
||||
.send()
|
||||
.await?;
|
||||
// TODO: Validate success
|
||||
|
||||
let term_selection_page_response = self
|
||||
.http
|
||||
.get(format!("{}/term/termSelection", self.base_url))
|
||||
.query(&[("mode", "search")])
|
||||
.send()
|
||||
.await?;
|
||||
// TOOD: Validate success
|
||||
|
||||
let term_search_response = self.get_terms("", 1, 10).await?;
|
||||
// TODO: Validate that the term search response contains the term we want
|
||||
|
||||
let specific_term_search_response = self.get_terms(&term.to_string(), 1, 10).await?;
|
||||
// TODO: Validate that the term response contains the term we want
|
||||
|
||||
let unique_session_id = generate_session_id();
|
||||
self.select_term(&term.to_string(), &unique_session_id)
|
||||
.await?;
|
||||
|
||||
BannerSession::new(&unique_session_id, jsessionid, ssb_cookie).await
|
||||
}
|
||||
|
||||
// Note: Cookie validation would require additional setup in a real implementation
|
||||
debug!("session setup complete");
|
||||
Ok(())
|
||||
/// Retrieves a list of terms from the Banner API.
|
||||
pub async fn get_terms(
|
||||
&self,
|
||||
search: &str,
|
||||
page: i32,
|
||||
max_results: i32,
|
||||
) -> Result<Vec<BannerTerm>> {
|
||||
if page <= 0 {
|
||||
return Err(anyhow::anyhow!("Page must be greater than 0"));
|
||||
}
|
||||
|
||||
let url = format!("{}/classSearch/getTerms", self.base_url);
|
||||
let params = [
|
||||
("searchTerm", search),
|
||||
("offset", &page.to_string()),
|
||||
("max", &max_results.to_string()),
|
||||
("_", &nonce()),
|
||||
];
|
||||
|
||||
let response = self
|
||||
.http
|
||||
.get(&url)
|
||||
.query(¶ms)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("Failed to get terms"))?;
|
||||
|
||||
let terms: Vec<BannerTerm> = response
|
||||
.json()
|
||||
.await
|
||||
.context("Failed to parse terms response")?;
|
||||
|
||||
Ok(terms)
|
||||
}
|
||||
|
||||
/// Selects a term for the current session
|
||||
pub async fn select_term(&self, term: &str) -> Result<()> {
|
||||
let session_id = self.ensure_session()?;
|
||||
|
||||
pub async fn select_term(&self, term: &str, unique_session_id: &str) -> Result<()> {
|
||||
let form_data = [
|
||||
("term", term),
|
||||
("studyPath", ""),
|
||||
("studyPathText", ""),
|
||||
("startDatepicker", ""),
|
||||
("endDatepicker", ""),
|
||||
("uniqueSessionId", &session_id),
|
||||
("uniqueSessionId", unique_session_id),
|
||||
];
|
||||
|
||||
let url = format!("{}/term/search", self.base_url);
|
||||
let response = self
|
||||
.client
|
||||
.http
|
||||
.post(&url)
|
||||
.query(&[("mode", "search")])
|
||||
.form(&form_data)
|
||||
.header("User-Agent", user_agent())
|
||||
.header("Content-Type", "application/x-www-form-urlencoded")
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
@@ -141,20 +316,18 @@ impl SessionManager {
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RedirectResponse {
|
||||
#[serde(rename = "fwdUrl")]
|
||||
#[serde(rename = "fwdURL")]
|
||||
fwd_url: String,
|
||||
}
|
||||
|
||||
let redirect: RedirectResponse = response.json().await?;
|
||||
|
||||
let base_url_path = self.base_url.parse::<Url>().unwrap().path().to_string();
|
||||
let non_overlap_redirect = redirect.fwd_url.strip_prefix(&base_url_path).unwrap();
|
||||
|
||||
// Follow the redirect
|
||||
let redirect_url = format!("{}{}", self.base_url, redirect.fwd_url);
|
||||
let redirect_response = self
|
||||
.client
|
||||
.get(&redirect_url)
|
||||
.header("User-Agent", user_agent())
|
||||
.send()
|
||||
.await?;
|
||||
let redirect_url = format!("{}{}", self.base_url, non_overlap_redirect);
|
||||
let redirect_response = self.http.get(&redirect_url).send().await?;
|
||||
|
||||
if !redirect_response.status().is_success() {
|
||||
return Err(anyhow::anyhow!(
|
||||
@@ -166,33 +339,4 @@ impl SessionManager {
|
||||
debug!("successfully selected term: {}", term);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resets the data form (required before new searches)
|
||||
pub async fn reset_data_form(&self) -> Result<()> {
|
||||
let url = format!("{}/classSearch/resetDataForm", self.base_url);
|
||||
let response = self
|
||||
.client
|
||||
.post(&url)
|
||||
.header("User-Agent", user_agent())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"Failed to reset data form: {}",
|
||||
response.status()
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Generates a timestamp-based nonce
|
||||
pub fn nonce() -> String {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis()
|
||||
.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ pub async fn terms(
|
||||
.data()
|
||||
.app_state
|
||||
.banner_api
|
||||
.sessions
|
||||
.get_terms(&search_term, page_number, max_results)
|
||||
.await?;
|
||||
|
||||
@@ -46,7 +47,11 @@ fn format_term(term: &BannerTerm, current_term_code: &str) -> String {
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let is_archived = if term.is_archived() { " (archived)" } else { "" };
|
||||
let is_archived = if term.is_archived() {
|
||||
" (archived)"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
format!(
|
||||
"- `{}`: {}{}{}",
|
||||
term.code, term.description, is_current, is_archived
|
||||
|
||||
@@ -4,7 +4,6 @@ use crate::error::Error;
|
||||
pub mod commands;
|
||||
pub mod utils;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Data {
|
||||
pub app_state: AppState,
|
||||
} // User data, which is stored and accessible in all command invocations
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
//! Diesel models for the database schema.
|
||||
//! `sqlx` models for the database schema.
|
||||
|
||||
use crate::data::schema::{course_audits, course_metrics, courses, scrape_jobs};
|
||||
use chrono::{DateTime, Utc};
|
||||
use diesel::{Insertable, Queryable, QueryableByName, Selectable};
|
||||
use serde_json::Value;
|
||||
|
||||
#[derive(Queryable, Selectable)]
|
||||
#[diesel(table_name = courses)]
|
||||
#[derive(sqlx::FromRow, Debug, Clone)]
|
||||
pub struct Course {
|
||||
pub id: i32,
|
||||
pub crn: String,
|
||||
@@ -21,24 +18,7 @@ pub struct Course {
|
||||
pub last_scraped_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Insertable)]
|
||||
#[diesel(table_name = courses)]
|
||||
pub struct NewCourse<'a> {
|
||||
pub crn: &'a str,
|
||||
pub subject: &'a str,
|
||||
pub course_number: &'a str,
|
||||
pub title: &'a str,
|
||||
pub term_code: &'a str,
|
||||
pub enrollment: i32,
|
||||
pub max_enrollment: i32,
|
||||
pub wait_count: i32,
|
||||
pub wait_capacity: i32,
|
||||
pub last_scraped_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Queryable, Selectable)]
|
||||
#[diesel(table_name = course_metrics)]
|
||||
#[diesel(belongs_to(Course))]
|
||||
#[derive(sqlx::FromRow, Debug, Clone)]
|
||||
pub struct CourseMetric {
|
||||
pub id: i32,
|
||||
pub course_id: i32,
|
||||
@@ -48,19 +28,7 @@ pub struct CourseMetric {
|
||||
pub seats_available: i32,
|
||||
}
|
||||
|
||||
#[derive(Insertable)]
|
||||
#[diesel(table_name = course_metrics)]
|
||||
pub struct NewCourseMetric {
|
||||
pub course_id: i32,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
pub enrollment: i32,
|
||||
pub wait_count: i32,
|
||||
pub seats_available: i32,
|
||||
}
|
||||
|
||||
#[derive(Queryable, Selectable)]
|
||||
#[diesel(table_name = course_audits)]
|
||||
#[diesel(belongs_to(Course))]
|
||||
#[derive(sqlx::FromRow, Debug, Clone)]
|
||||
pub struct CourseAudit {
|
||||
pub id: i32,
|
||||
pub course_id: i32,
|
||||
@@ -70,18 +38,9 @@ pub struct CourseAudit {
|
||||
pub new_value: String,
|
||||
}
|
||||
|
||||
#[derive(Insertable)]
|
||||
#[diesel(table_name = course_audits)]
|
||||
pub struct NewCourseAudit<'a> {
|
||||
pub course_id: i32,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
pub field_changed: &'a str,
|
||||
pub old_value: &'a str,
|
||||
pub new_value: &'a str,
|
||||
}
|
||||
|
||||
/// The priority level of a scrape job.
|
||||
#[derive(diesel_derive_enum::DbEnum, Copy, Debug, Clone)]
|
||||
#[derive(sqlx::Type, Copy, Debug, Clone)]
|
||||
#[sqlx(type_name = "scrape_priority", rename_all = "PascalCase")]
|
||||
pub enum ScrapePriority {
|
||||
Low,
|
||||
Medium,
|
||||
@@ -90,7 +49,8 @@ pub enum ScrapePriority {
|
||||
}
|
||||
|
||||
/// The type of target for a scrape job, determining how the payload is interpreted.
|
||||
#[derive(diesel_derive_enum::DbEnum, Copy, Debug, Clone)]
|
||||
#[derive(sqlx::Type, Copy, Debug, Clone)]
|
||||
#[sqlx(type_name = "target_type", rename_all = "PascalCase")]
|
||||
pub enum TargetType {
|
||||
Subject,
|
||||
CourseRange,
|
||||
@@ -99,8 +59,7 @@ pub enum TargetType {
|
||||
}
|
||||
|
||||
/// Represents a queryable job from the database.
|
||||
#[derive(Debug, Clone, Queryable, QueryableByName)]
|
||||
#[diesel(table_name = scrape_jobs)]
|
||||
#[derive(sqlx::FromRow, Debug, Clone)]
|
||||
pub struct ScrapeJob {
|
||||
pub id: i32,
|
||||
pub target_type: TargetType,
|
||||
@@ -110,14 +69,3 @@ pub struct ScrapeJob {
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub locked_at: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// Represents a new job to be inserted into the database.
|
||||
#[derive(Debug, Clone, Insertable)]
|
||||
#[diesel(table_name = scrape_jobs)]
|
||||
pub struct NewScrapeJob {
|
||||
pub target_type: TargetType,
|
||||
#[diesel(sql_type = diesel::sql_types::Jsonb)]
|
||||
pub target_payload: Value,
|
||||
pub priority: ScrapePriority,
|
||||
pub execute_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
pub mod sql_types {
|
||||
#[derive(diesel::sql_types::SqlType)]
|
||||
#[diesel(postgres_type(name = "scrape_priority"))]
|
||||
pub struct ScrapePriority;
|
||||
|
||||
#[derive(diesel::sql_types::SqlType)]
|
||||
#[diesel(postgres_type(name = "target_type"))]
|
||||
pub struct TargetType;
|
||||
}
|
||||
|
||||
use super::models::{ScrapePriorityMapping, TargetTypeMapping};
|
||||
|
||||
diesel::table! {
|
||||
use diesel::sql_types::*;
|
||||
use super::{ScrapePriorityMapping, TargetTypeMapping};
|
||||
|
||||
scrape_jobs (id) {
|
||||
id -> Int4,
|
||||
target_type -> TargetTypeMapping,
|
||||
target_payload -> Jsonb,
|
||||
priority -> ScrapePriorityMapping,
|
||||
execute_at -> Timestamptz,
|
||||
created_at -> Timestamptz,
|
||||
locked_at -> Nullable<Timestamptz>,
|
||||
}
|
||||
}
|
||||
|
||||
diesel::table! {
|
||||
courses (id) {
|
||||
id -> Int4,
|
||||
crn -> Varchar,
|
||||
subject -> Varchar,
|
||||
course_number -> Varchar,
|
||||
title -> Varchar,
|
||||
term_code -> Varchar,
|
||||
enrollment -> Int4,
|
||||
max_enrollment -> Int4,
|
||||
wait_count -> Int4,
|
||||
wait_capacity -> Int4,
|
||||
last_scraped_at -> Timestamptz,
|
||||
}
|
||||
}
|
||||
|
||||
diesel::table! {
|
||||
course_metrics (id) {
|
||||
id -> Int4,
|
||||
course_id -> Int4,
|
||||
timestamp -> Timestamptz,
|
||||
enrollment -> Int4,
|
||||
wait_count -> Int4,
|
||||
seats_available -> Int4,
|
||||
}
|
||||
}
|
||||
|
||||
diesel::table! {
|
||||
course_audits (id) {
|
||||
id -> Int4,
|
||||
course_id -> Int4,
|
||||
timestamp -> Timestamptz,
|
||||
field_changed -> Varchar,
|
||||
old_value -> Text,
|
||||
new_value -> Text,
|
||||
}
|
||||
}
|
||||
|
||||
diesel::joinable!(course_metrics -> courses (course_id));
|
||||
diesel::joinable!(course_audits -> courses (course_id));
|
||||
|
||||
diesel::allow_tables_to_appear_in_same_query!(courses, course_metrics, course_audits, scrape_jobs,);
|
||||
@@ -1,6 +1,7 @@
|
||||
pub mod app_state;
|
||||
pub mod banner;
|
||||
pub mod bot;
|
||||
pub mod config;
|
||||
pub mod data;
|
||||
pub mod error;
|
||||
pub mod scraper;
|
||||
|
||||
Reference in New Issue
Block a user