mirror of
https://github.com/Xevion/banner.git
synced 2025-12-16 16:11:14 -06:00
Compare commits
31 Commits
rewrite
...
b3322636a9
| Author | SHA1 | Date | |
|---|---|---|---|
| b3322636a9 | |||
| 878cc5f773 | |||
| 94fb6b4190 | |||
| e3b638a7d8 | |||
| 404a52e64c | |||
| a917315967 | |||
| 9d51fde893 | |||
| 79fc931077 | |||
| f3861a60c4 | |||
| 26b1a88860 | |||
| 27ac9a7302 | |||
| 1d345ed247 | |||
| 6f831f5fa6 | |||
| ac2638dd9a | |||
| cfb847f2e5 | |||
| e7d47f1f96 | |||
| 9a48587479 | |||
| 624247ee14 | |||
| 430e2a255b | |||
| bbc78131ec | |||
| 77ab71d4d5 | |||
| 9d720bb0a7 | |||
| dcc564dee6 | |||
| 4ca55a1fd4 | |||
| a6e7adcaef | |||
| 752c855dec | |||
| 14b02df8f4 | |||
| 00cb209052 | |||
| dfc05a2789 | |||
| fe798e1867 | |||
| 39688f800f |
23
.dockerignore
Normal file
23
.dockerignore
Normal file
@@ -0,0 +1,23 @@
|
||||
# Build artifacts
|
||||
target/
|
||||
**/target/
|
||||
|
||||
# Documentation
|
||||
README.md
|
||||
docs/
|
||||
*.md
|
||||
|
||||
# Old Go codebase
|
||||
go/
|
||||
|
||||
# Development configuration
|
||||
bacon.toml
|
||||
.env
|
||||
|
||||
# Frontend build artifacts and cache
|
||||
web/node_modules/
|
||||
web/dist/
|
||||
web/.vite/
|
||||
web/.tanstack/
|
||||
web/.vscode/
|
||||
.vscode/
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -1,4 +1,5 @@
|
||||
.env
|
||||
/target
|
||||
/go/
|
||||
.cargo/config.toml
|
||||
.cargo/config.toml
|
||||
src/scraper/README.md
|
||||
3
.vscode/settings.json
vendored
Normal file
3
.vscode/settings.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"rust-analyzer.check.command": "clippy"
|
||||
}
|
||||
297
Cargo.lock
generated
297
Cargo.lock
generated
@@ -41,6 +41,56 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "0.6.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
|
||||
dependencies = [
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.99"
|
||||
@@ -168,13 +218,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "banner"
|
||||
version = "0.1.0"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"axum",
|
||||
"bitflags 2.9.4",
|
||||
"chrono",
|
||||
"clap",
|
||||
"compile-time",
|
||||
"cookie",
|
||||
"dashmap 6.1.0",
|
||||
@@ -184,21 +235,26 @@ dependencies = [
|
||||
"futures",
|
||||
"governor",
|
||||
"http 1.3.1",
|
||||
"mime_guess",
|
||||
"num-format",
|
||||
"once_cell",
|
||||
"poise",
|
||||
"rand 0.9.2",
|
||||
"redis",
|
||||
"rapidhash",
|
||||
"regex",
|
||||
"reqwest 0.12.23",
|
||||
"reqwest-middleware",
|
||||
"rust-embed",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_path_to_error",
|
||||
"serenity",
|
||||
"sqlx",
|
||||
"thiserror 2.0.16",
|
||||
"time",
|
||||
"tl",
|
||||
"tokio",
|
||||
"tower-http",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
@@ -246,6 +302,16 @@ dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bstr"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.19.0"
|
||||
@@ -337,19 +403,51 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "combine"
|
||||
version = "4.6.7"
|
||||
name = "clap"
|
||||
version = "4.5.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
|
||||
checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.5.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.5.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.106",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
|
||||
|
||||
[[package]]
|
||||
name = "command_attr"
|
||||
version = "0.5.3"
|
||||
@@ -584,9 +682,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.4.0"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e"
|
||||
checksum = "d630bccd429a5bb5a64b5e94f693bfc48c9f8566418fda4c494cc94f911f87cc"
|
||||
dependencies = [
|
||||
"powerfmt",
|
||||
"serde",
|
||||
@@ -960,6 +1058,19 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
|
||||
|
||||
[[package]]
|
||||
name = "globset"
|
||||
version = "0.4.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"bstr",
|
||||
"log",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "governor"
|
||||
version = "0.10.1"
|
||||
@@ -1142,6 +1253,12 @@ dependencies = [
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-range-header"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c"
|
||||
|
||||
[[package]]
|
||||
name = "httparse"
|
||||
version = "1.10.1"
|
||||
@@ -1453,6 +1570,12 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.15"
|
||||
@@ -1665,16 +1788,6 @@ dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
|
||||
dependencies = [
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint-dig"
|
||||
version = "0.8.4"
|
||||
@@ -1698,6 +1811,16 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
|
||||
|
||||
[[package]]
|
||||
name = "num-format"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.46"
|
||||
@@ -1743,6 +1866,12 @@ version = "1.21.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.73"
|
||||
@@ -2031,17 +2160,6 @@ version = "5.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "r2d2"
|
||||
version = "0.8.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"
|
||||
dependencies = [
|
||||
"log",
|
||||
"parking_lot",
|
||||
"scheduled-thread-pool",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.5"
|
||||
@@ -2101,6 +2219,15 @@ dependencies = [
|
||||
"getrandom 0.3.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rapidhash"
|
||||
version = "4.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "164772177ee16e3b074e6019c63cd92cb3cecf38e8c40d097675958b86dd8084"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "raw-cpuid"
|
||||
version = "11.6.0"
|
||||
@@ -2110,29 +2237,6 @@ dependencies = [
|
||||
"bitflags 2.9.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "0.32.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cd3650deebc68526b304898b192fa4102a4ef0b9ada24da096559cb60e0eef8"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"cfg-if",
|
||||
"combine",
|
||||
"futures-util",
|
||||
"itoa",
|
||||
"num-bigint",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"r2d2",
|
||||
"ryu",
|
||||
"sha1_smol",
|
||||
"socket2 0.6.0",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.17"
|
||||
@@ -2306,6 +2410,41 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed"
|
||||
version = "8.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "025908b8682a26ba8d12f6f2d66b987584a4a87bc024abc5bbc12553a8cd178a"
|
||||
dependencies = [
|
||||
"rust-embed-impl",
|
||||
"rust-embed-utils",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed-impl"
|
||||
version = "8.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6065f1a4392b71819ec1ea1df1120673418bf386f50de1d6f54204d836d4349c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rust-embed-utils",
|
||||
"syn 2.0.106",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed-utils"
|
||||
version = "8.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6cc0c81648b20b70c491ff8cce00c1c3b223bb8ed2b5d41f0e54c6c4c0a3594"
|
||||
dependencies = [
|
||||
"globset",
|
||||
"sha2",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.26"
|
||||
@@ -2454,15 +2593,6 @@ dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scheduled-thread-pool"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"
|
||||
dependencies = [
|
||||
"parking_lot",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
@@ -2641,12 +2771,6 @@ dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1_smol"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
@@ -3159,12 +3283,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.41"
|
||||
version = "0.3.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40"
|
||||
checksum = "83bde6f1ec10e72d583d91623c939f623002284ef622b87de38cfd546cbf2031"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"itoa",
|
||||
"num-conv",
|
||||
"powerfmt",
|
||||
"serde",
|
||||
@@ -3174,15 +3297,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "time-core"
|
||||
version = "0.1.4"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c"
|
||||
checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b"
|
||||
|
||||
[[package]]
|
||||
name = "time-macros"
|
||||
version = "0.2.22"
|
||||
version = "0.2.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49"
|
||||
checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3"
|
||||
dependencies = [
|
||||
"num-conv",
|
||||
"time-core",
|
||||
@@ -3396,14 +3519,24 @@ checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
|
||||
dependencies = [
|
||||
"bitflags 2.9.4",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"http-range-header",
|
||||
"httpdate",
|
||||
"iri-string",
|
||||
"mime",
|
||||
"mime_guess",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3639,6 +3772,12 @@ version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uwl"
|
||||
version = "0.6.0"
|
||||
|
||||
17
Cargo.toml
17
Cargo.toml
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "banner"
|
||||
version = "0.1.0"
|
||||
version = "0.2.3"
|
||||
edition = "2024"
|
||||
default-run = "banner"
|
||||
|
||||
@@ -20,7 +20,6 @@ futures = "0.3"
|
||||
http = "1.3.1"
|
||||
poise = "0.6.1"
|
||||
rand = "0.9.2"
|
||||
redis = { version = "0.32.5", features = ["tokio-comp", "r2d2"] }
|
||||
regex = "1.10"
|
||||
reqwest = { version = "0.12.23", features = ["json", "cookies"] }
|
||||
reqwest-middleware = { version = "0.4.2", features = ["json"] }
|
||||
@@ -35,7 +34,7 @@ sqlx = { version = "0.8.6", features = [
|
||||
"macros",
|
||||
] }
|
||||
thiserror = "2.0.16"
|
||||
time = "0.3.41"
|
||||
time = "0.3.43"
|
||||
tokio = { version = "1.47.1", features = ["full"] }
|
||||
tl = "0.7.8"
|
||||
tracing = "0.1.41"
|
||||
@@ -43,5 +42,17 @@ tracing-subscriber = { version = "0.3.20", features = ["env-filter", "json"] }
|
||||
url = "2.5"
|
||||
governor = "0.10.1"
|
||||
once_cell = "1.21.3"
|
||||
serde_path_to_error = "0.1.17"
|
||||
num-format = "0.4.4"
|
||||
tower-http = { version = "0.6.0", features = ["fs", "cors", "trace"] }
|
||||
rust-embed = { version = "8.0", features = ["debug-embed", "include-exclude"] }
|
||||
mime_guess = "2.0"
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
rapidhash = "4.1.0"
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
# A 'release mode' profile that compiles quickly, but still 'appears' like a release build, useful for debugging
|
||||
[profile.dev-release]
|
||||
inherits = "dev"
|
||||
debug-assertions = false
|
||||
|
||||
39
Dockerfile
39
Dockerfile
@@ -1,5 +1,27 @@
|
||||
# Build Stage
|
||||
ARG RUST_VERSION=1.86.0
|
||||
# Build arguments
|
||||
ARG RUST_VERSION=1.89.0
|
||||
|
||||
# Frontend Build Stage
|
||||
FROM node:22-bookworm-slim AS frontend-builder
|
||||
|
||||
# Install pnpm
|
||||
RUN npm install -g pnpm
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy frontend package files
|
||||
COPY ./web/package.json ./web/pnpm-lock.yaml ./
|
||||
|
||||
# Install dependencies
|
||||
RUN pnpm install --frozen-lockfile
|
||||
|
||||
# Copy frontend source code
|
||||
COPY ./web ./
|
||||
|
||||
# Build frontend
|
||||
RUN pnpm run build
|
||||
|
||||
# Rust Build Stage
|
||||
FROM rust:${RUST_VERSION}-bookworm AS builder
|
||||
|
||||
# Install build dependencies
|
||||
@@ -18,9 +40,14 @@ COPY ./Cargo.toml ./Cargo.lock* ./
|
||||
# Build empty app with downloaded dependencies to produce a stable image layer for next build
|
||||
RUN cargo build --release
|
||||
|
||||
# Build web app with own code
|
||||
# Copy source code
|
||||
RUN rm src/*.rs
|
||||
COPY ./src ./src
|
||||
|
||||
# Copy built frontend assets
|
||||
COPY --from=frontend-builder /app/dist ./web/dist
|
||||
|
||||
# Build web app with embedded assets
|
||||
RUN rm ./target/release/deps/banner*
|
||||
RUN cargo build --release
|
||||
|
||||
@@ -50,9 +77,8 @@ RUN addgroup --gid $GID $APP_USER \
|
||||
&& adduser --uid $UID --disabled-password --gecos "" --ingroup $APP_USER $APP_USER \
|
||||
&& mkdir -p ${APP}
|
||||
|
||||
# Copy application files
|
||||
# Copy application binary
|
||||
COPY --from=builder --chown=$APP_USER:$APP_USER /usr/src/banner/target/release/banner ${APP}/banner
|
||||
COPY --from=builder --chown=$APP_USER:$APP_USER /usr/src/banner/src/fonts ${APP}/fonts
|
||||
|
||||
# Set proper permissions
|
||||
RUN chmod +x ${APP}/banner
|
||||
@@ -74,4 +100,5 @@ HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||
ENV HOSTS=0.0.0.0,[::]
|
||||
|
||||
# Implicitly uses PORT environment variable
|
||||
CMD ["sh", "-c", "exec ./banner --server ${HOSTS}"]
|
||||
# temporary: running without 'scraper' service
|
||||
CMD ["sh", "-c", "exec ./banner --services web,bot"]
|
||||
28
Justfile
Normal file
28
Justfile
Normal file
@@ -0,0 +1,28 @@
|
||||
default_services := "bot,web,scraper"
|
||||
|
||||
# Auto-reloading frontend server
|
||||
frontend:
|
||||
pnpm run -C web dev
|
||||
|
||||
# Production build of frontend
|
||||
build-frontend:
|
||||
pnpm run -C web build
|
||||
|
||||
# Auto-reloading backend server
|
||||
backend services=default_services:
|
||||
bacon --headless run -- -- --services "{{services}}"
|
||||
|
||||
# Production build
|
||||
build:
|
||||
pnpm run -C web build
|
||||
cargo build --release --bin banner
|
||||
|
||||
# Run auto-reloading development build with release characteristics (frontend is embedded, non-auto-reloading)
|
||||
# This is useful for testing backend release-mode details.
|
||||
dev-build services=default_services: build-frontend
|
||||
bacon --headless run -- --profile dev-release -- --services "{{services}}" --tracing pretty
|
||||
|
||||
# Auto-reloading development build for both frontend and backend
|
||||
# Will not notice if either the frontend/backend crashes, but will generally be resistant to stopping on their own.
|
||||
[parallel]
|
||||
dev services=default_services: frontend (backend services)
|
||||
142
README.md
142
README.md
@@ -1,125 +1,51 @@
|
||||
# banner
|
||||
|
||||
A discord bot for executing queries & searches on the Ellucian Banner instance hosting all of UTSA's class data.
|
||||
A complex multi-service system providing a Discord bot and browser-based interface to UTSA's course data.
|
||||
|
||||
## Feature Wishlist
|
||||
## Services
|
||||
|
||||
- Commands
|
||||
- ICS Download (get a ICS download of your classes with location & timing perfectly - set for every class you're in)
|
||||
- Classes Now (find classes happening)
|
||||
- Autocomplete
|
||||
- Class Title
|
||||
- Course Number
|
||||
- Term/Part of Term
|
||||
- Professor
|
||||
- Attribute
|
||||
- Component Pagination
|
||||
- RateMyProfessor Integration (Linked/Embedded)
|
||||
- Smart term selection (i.e. Summer 2024 will be selected automatically when opened)
|
||||
- Rate Limiting (bursting with global/user limits)
|
||||
- DMs Integration (allow usage of the bot in DMs)
|
||||
- Class Change Notifications (get notified when details about a class change)
|
||||
- Multi-term Querying (currently the backend for searching is kinda weird)
|
||||
- Full Autocomplete for Every Search Option
|
||||
- Metrics, Log Query, Privileged Error Feedback
|
||||
- Search for Classes
|
||||
- Major, Professor, Location, Name, Time of Day
|
||||
- Subscribe to Classes
|
||||
- Availability (seat, pre-seat)
|
||||
- Waitlist Movement
|
||||
- Detail Changes (meta, time, location, seats, professor)
|
||||
- `time` Start, End, Days of Week
|
||||
- `seats` Any change in seat/waitlist data
|
||||
- `meta`
|
||||
- Lookup via Course Reference Number (CRN)
|
||||
- Smart Time of Day Handling
|
||||
- "2 PM" -> Start within 2:00 PM to 2:59 PM
|
||||
- "2-3 PM" -> Start within 2:00 PM to 3:59 PM
|
||||
- "ends by 2 PM" -> Ends within 12:00 AM to 2:00 PM
|
||||
- "after 2 PM" -> Start within 2:01 PM to 11:59 PM
|
||||
- "before 2 PM" -> Ends within 12:00 AM to 1:59 PM
|
||||
- Get By Section Command
|
||||
- CS 4393 001 =>
|
||||
- Will require SQL to be able to search for a class by its section number
|
||||
The application consists of three modular services that can be run independently or together:
|
||||
|
||||
## Analysis Required
|
||||
- Discord Bot ([`bot`][src-bot])
|
||||
|
||||
Some of the features and architecture of Ellucian's Banner system are not clear.
|
||||
The follow features, JSON, and more require validation & analysis:
|
||||
- Primary interface for course monitoring and data queries
|
||||
- Built with [Serenity][serenity] and [Poise][poise] frameworks for robust command handling
|
||||
- Uses slash commands with comprehensive error handling and logging
|
||||
|
||||
- Struct Nullability
|
||||
- Much of the responses provided by Ellucian contain nulls, and most of them are uncertain as to when and why they're null.
|
||||
- Analysis must be conducted to be sure of when to use a string and when it should nillable (pointer).
|
||||
- Multiple Professors / Primary Indicator
|
||||
- Multiple Meeting Times
|
||||
- Meeting Schedule Types
|
||||
- AFF vs AIN vs AHB etc.
|
||||
- Do CRNs repeat between years?
|
||||
- Check whether partOfTerm is always filled in, and it's meaning for various class results.
|
||||
- Check which API calls are affected by change in term/sessionID term select
|
||||
- SessionIDs
|
||||
- How long does a session ID work?
|
||||
- Do I really require a separate one per term?
|
||||
- How many can I activate, are there any restrictions?
|
||||
- How should session IDs be checked as 'invalid'?
|
||||
- What action(s) keep a session ID 'active', if any?
|
||||
- Are there any courses with multiple meeting times?
|
||||
- Google Calendar link generation, as an alternative to ICS file generation
|
||||
- Web Server ([`web`][src-web])
|
||||
|
||||
## Change Identification
|
||||
- [Axum][axum]-based server with Vite/React-based frontend
|
||||
- [Embeds static assets][rust-embed] at compile time with E-Tags & Cache-Control headers
|
||||
|
||||
- Important attributes of a class will be parsed on both the old and new data.
|
||||
- These attributes will be compared and given identifiers that can be subscribed to.
|
||||
- When a user subscribes to one of these identifiers, any changes identified will be sent to the user.
|
||||
- Scraper ([`scraper`][src-scraper])
|
||||
|
||||
## Real-time Suggestions
|
||||
- Intelligent data collection system with priority-based queuing inside PostgreSQL via [`sqlx`][sqlx]
|
||||
- Rate-limited scraping with burst handling to respect UTSA's systems
|
||||
- Handles course data updates, availability changes, and metadata synchronization
|
||||
|
||||
Various commands arguments have the ability to have suggestions appear.
|
||||
## Quick Start
|
||||
|
||||
- They must be fast. As ephemeral suggestions that are only relevant for seconds or less, they need to be delivered in less than a second.
|
||||
- They need to be easy to acquire. With as many commands & arguments to search as I do, it is paramount that the API be easy to understand & use.
|
||||
- It cannot be complicated. I only have so much time to develop this.
|
||||
- It does not need to be persistent. Since the data is scraped and rolled periodically from the Banner system, the data used will be deleted and re-requested occasionally.
|
||||
```bash
|
||||
pnpm install -C web # Install frontend dependencies
|
||||
cargo build # Build the backend
|
||||
|
||||
For these reasons, I believe SQLite to be the ideal place for this data to be stored.
|
||||
It is exceptionally fast, works well in-memory, and is less complicated compared to most other solutions.
|
||||
just dev # Runs auto-reloading dev build
|
||||
just dev bot,web # Runs auto-reloading dev build, running only the bot and web services
|
||||
just dev-build # Development build with release characteristics (frontend is embedded, non-auto-reloading)
|
||||
|
||||
- Only required data about the class will be stored, along with the JSON-encoded string.
|
||||
- For now, this would only be the CRN (and possibly the Term).
|
||||
- Potentially, a binary encoding could be used for performance, but it is unlikely to be better.
|
||||
- Database dumping into R2 would be good to ensure that over-scraping of the Banner system does not occur.
|
||||
- Upon a safe close requested
|
||||
- Must be done quickly (<8 seconds)
|
||||
- Every 30 minutes, if any scraping ocurred.
|
||||
- May cause locking of commands.
|
||||
just build # Production build that embeds assets
|
||||
```
|
||||
|
||||
## Scraping
|
||||
## Documentation
|
||||
|
||||
In order to keep the in-memory database of the bot up-to-date with the Banner system, the API must be scraped.
|
||||
Scraping will be separated by major to allow for priority majors (namely, Computer Science) to be scraped more often compared to others.
|
||||
This will lower the overall load on the Banner system while ensuring that data presented by the app is still relevant.
|
||||
Comprehensive documentation is available in the [`docs/`][documentation] folder.
|
||||
|
||||
For now, all majors will be scraped fully every 4 hours with at least 5 minutes between each one.
|
||||
|
||||
- On startup, priority majors will be scraped first (if required).
|
||||
- Other majors will be scraped in arbitrary order (if required).
|
||||
- Scrape timing will be stored in Redis.
|
||||
- CRNs will be the Primary Key within SQLite
|
||||
- If CRNs are duplicated between terms, then the primary key will be (CRN, Term)
|
||||
|
||||
Considerations
|
||||
|
||||
- Change in metadata should decrease the interval
|
||||
- The number of courses scraped should change the interval (2 hours per 500 courses involved)
|
||||
|
||||
## Rate Limiting, Costs & Bursting
|
||||
|
||||
Ideally, this application would implement dynamic rate limiting to ensure overload on the server does not occur.
|
||||
Better, it would also ensure that priority requests (commands) are dispatched faster than background processes (scraping), while making sure different requests are weighted differently.
|
||||
For example, a recent scrape of 350 classes should be weighted 5x more than a search for 8 classes by a user.
|
||||
Still, even if the cap does not normally allow for this request to be processed immediately, the small user search should proceed with a small bursting cap.
|
||||
|
||||
The requirements to this hypothetical system would be:
|
||||
|
||||
- Conditional Bursting: background processes or other requests deemed "low priority" are not allowed to use bursting.
|
||||
- Arbitrary Costs: rate limiting is considered in the form of the request size/speed more or less, such that small simple requests can be made more frequently, unlike large requests.
|
||||
[documentation]: docs/README.md
|
||||
[src-bot]: src/bot
|
||||
[src-web]: src/web
|
||||
[src-scraper]: src/scraper
|
||||
[serenity]: https://github.com/serenity-rs/serenity
|
||||
[poise]: https://github.com/serenity-rs/poise
|
||||
[axum]: https://github.com/tokio-rs/axum
|
||||
[rust-embed]: https://lib.rs/crates/rust-embed
|
||||
[sqlx]: https://github.com/launchbadge/sqlx
|
||||
|
||||
58
bacon.toml
58
bacon.toml
@@ -9,61 +9,20 @@ default_job = "check"
|
||||
env.CARGO_TERM_COLOR = "always"
|
||||
|
||||
[jobs.check]
|
||||
command = ["cargo", "check"]
|
||||
need_stdout = false
|
||||
|
||||
[jobs.check-all]
|
||||
command = ["cargo", "check", "--all-targets"]
|
||||
need_stdout = false
|
||||
|
||||
# Run clippy on the default target
|
||||
[jobs.clippy]
|
||||
command = ["cargo", "clippy"]
|
||||
need_stdout = false
|
||||
|
||||
# Run clippy on all targets
|
||||
# To disable some lints, you may change the job this way:
|
||||
# [jobs.clippy-all]
|
||||
# command = [
|
||||
# "cargo", "clippy",
|
||||
# "--all-targets",
|
||||
# "--",
|
||||
# "-A", "clippy::bool_to_int_with_if",
|
||||
# "-A", "clippy::collapsible_if",
|
||||
# "-A", "clippy::derive_partial_eq_without_eq",
|
||||
# ]
|
||||
# need_stdout = false
|
||||
[jobs.clippy-all]
|
||||
command = ["cargo", "clippy", "--all-targets"]
|
||||
need_stdout = false
|
||||
|
||||
# This job lets you run
|
||||
# - all tests: bacon test
|
||||
# - a specific test: bacon test -- config::test_default_files
|
||||
# - the tests of a package: bacon test -- -- -p config
|
||||
[jobs.test]
|
||||
command = ["cargo", "test"]
|
||||
need_stdout = true
|
||||
|
||||
[jobs.nextest]
|
||||
command = [
|
||||
"cargo", "nextest", "run",
|
||||
"--hide-progress-bar", "--failure-output", "final"
|
||||
]
|
||||
need_stdout = true
|
||||
analyzer = "nextest"
|
||||
|
||||
[jobs.doc]
|
||||
command = ["cargo", "doc", "--no-deps"]
|
||||
need_stdout = false
|
||||
|
||||
# If the doc compiles, then it opens in your browser and bacon switches
|
||||
# to the previous job
|
||||
[jobs.doc-open]
|
||||
command = ["cargo", "doc", "--no-deps", "--open"]
|
||||
need_stdout = false
|
||||
on_success = "back" # so that we don't open the browser at each change
|
||||
|
||||
[jobs.run]
|
||||
command = [
|
||||
"cargo", "run",
|
||||
@@ -74,19 +33,20 @@ background = false
|
||||
on_change_strategy = "kill_then_restart"
|
||||
# kill = ["pkill", "-TERM", "-P"]'
|
||||
|
||||
# This parameterized job runs the example of your choice, as soon
|
||||
# as the code compiles.
|
||||
# Call it as
|
||||
# bacon ex -- my-example
|
||||
[jobs.ex]
|
||||
command = ["cargo", "run", "--example"]
|
||||
[jobs.dev]
|
||||
command = [
|
||||
"just", "dev"
|
||||
]
|
||||
need_stdout = true
|
||||
allow_warnings = true
|
||||
background = false
|
||||
on_change_strategy = "kill_then_restart"
|
||||
|
||||
# You may define here keybindings that would be specific to
|
||||
# a project, for example a shortcut to launch a specific job.
|
||||
# Shortcuts to internal functions (scrolling, toggling, etc.)
|
||||
# should go in your personal global prefs.toml file instead.
|
||||
[keybindings]
|
||||
# alt-m = "job:my-job"
|
||||
c = "job:clippy-all" # comment this to have 'c' run clippy on only the default target
|
||||
c = "job:clippy" # comment this to have 'c' run clippy on only the default target
|
||||
shift-c = "job:check"
|
||||
d = "job:dev"
|
||||
@@ -1,9 +0,0 @@
|
||||
# For documentation on how to configure this file,
|
||||
# see https://diesel.rs/guides/configuring-diesel-cli
|
||||
|
||||
[print_schema]
|
||||
file = "src/data/schema.rs"
|
||||
custom_type_derives = ["diesel::query_builder::QueryId", "Clone"]
|
||||
|
||||
[migrations_directory]
|
||||
dir = "migrations"
|
||||
94
docs/ARCHITECTURE.md
Normal file
94
docs/ARCHITECTURE.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# Architecture
|
||||
|
||||
## System Overview
|
||||
|
||||
The Banner project is built as a multi-service application with the following components:
|
||||
|
||||
- **Discord Bot Service**: Handles Discord interactions and commands
|
||||
- **Web Service**: Serves the React frontend and provides API endpoints
|
||||
- **Scraper Service**: Background data collection and synchronization
|
||||
- **Database Layer**: PostgreSQL for persistent storage
|
||||
|
||||
## Technical Analysis
|
||||
|
||||
### Banner System Integration
|
||||
|
||||
Some of the features and architecture of Ellucian's Banner system are not clear.
|
||||
The following features, JSON, and more require validation & analysis:
|
||||
|
||||
- Struct Nullability
|
||||
- Much of the responses provided by Ellucian contain nulls, and most of them are uncertain as to when and why they're null.
|
||||
- Analysis must be conducted to be sure of when to use a string and when it should nillable (pointer).
|
||||
- Multiple Professors / Primary Indicator
|
||||
- Multiple Meeting Times
|
||||
- Meeting Schedule Types
|
||||
- AFF vs AIN vs AHB etc.
|
||||
- Do CRNs repeat between years?
|
||||
- Check whether partOfTerm is always filled in, and it's meaning for various class results.
|
||||
- Check which API calls are affected by change in term/sessionID term select
|
||||
- SessionIDs
|
||||
- How long does a session ID work?
|
||||
- Do I really require a separate one per term?
|
||||
- How many can I activate, are there any restrictions?
|
||||
- How should session IDs be checked as 'invalid'?
|
||||
- What action(s) keep a session ID 'active', if any?
|
||||
- Are there any courses with multiple meeting times?
|
||||
- Google Calendar link generation, as an alternative to ICS file generation
|
||||
|
||||
## Change Identification
|
||||
|
||||
- Important attributes of a class will be parsed on both the old and new data.
|
||||
- These attributes will be compared and given identifiers that can be subscribed to.
|
||||
- When a user subscribes to one of these identifiers, any changes identified will be sent to the user.
|
||||
|
||||
## Real-time Suggestions
|
||||
|
||||
Various commands arguments have the ability to have suggestions appear.
|
||||
|
||||
- They must be fast. As ephemeral suggestions that are only relevant for seconds or less, they need to be delivered in less than a second.
|
||||
- They need to be easy to acquire. With as many commands & arguments to search as I do, it is paramount that the API be easy to understand & use.
|
||||
- It cannot be complicated. I only have so much time to develop this.
|
||||
- It does not need to be persistent. Since the data is scraped and rolled periodically from the Banner system, the data used will be deleted and re-requested occasionally.
|
||||
|
||||
For these reasons, I believe PostgreSQL to be the ideal place for this data to be stored.
|
||||
It is exceptionally fast, works well in-memory, and is less complicated compared to most other solutions.
|
||||
|
||||
- Only required data about the class will be stored, along with the JSON-encoded string.
|
||||
- For now, this would only be the CRN (and possibly the Term).
|
||||
- Potentially, a binary encoding could be used for performance, but it is unlikely to be better.
|
||||
- Database dumping into R2 would be good to ensure that over-scraping of the Banner system does not occur.
|
||||
- Upon a safe close requested
|
||||
- Must be done quickly (<8 seconds)
|
||||
- Every 30 minutes, if any scraping ocurred.
|
||||
- May cause locking of commands.
|
||||
|
||||
## Scraping System
|
||||
|
||||
In order to keep the in-memory database of the bot up-to-date with the Banner system, the API must be scraped.
|
||||
Scraping will be separated by major to allow for priority majors (namely, Computer Science) to be scraped more often compared to others.
|
||||
This will lower the overall load on the Banner system while ensuring that data presented by the app is still relevant.
|
||||
|
||||
For now, all majors will be scraped fully every 4 hours with at least 5 minutes between each one.
|
||||
|
||||
- On startup, priority majors will be scraped first (if required).
|
||||
- Other majors will be scraped in arbitrary order (if required).
|
||||
- Scrape timing will be stored in database.
|
||||
- CRNs will be the Primary Key within database
|
||||
- If CRNs are duplicated between terms, then the primary key will be (CRN, Term)
|
||||
|
||||
Considerations
|
||||
|
||||
- Change in metadata should decrease the interval
|
||||
- The number of courses scraped should change the interval (2 hours per 500 courses involved)
|
||||
|
||||
## Rate Limiting, Costs & Bursting
|
||||
|
||||
Ideally, this application would implement dynamic rate limiting to ensure overload on the server does not occur.
|
||||
Better, it would also ensure that priority requests (commands) are dispatched faster than background processes (scraping), while making sure different requests are weighted differently.
|
||||
For example, a recent scrape of 350 classes should be weighted 5x more than a search for 8 classes by a user.
|
||||
Still, even if the cap does not normally allow for this request to be processed immediately, the small user search should proceed with a small bursting cap.
|
||||
|
||||
The requirements to this hypothetical system would be:
|
||||
|
||||
- Conditional Bursting: background processes or other requests deemed "low priority" are not allowed to use bursting.
|
||||
- Arbitrary Costs: rate limiting is considered in the form of the request size/speed more or less, such that small simple requests can be made more frequently, unlike large requests.
|
||||
@@ -1,11 +1,17 @@
|
||||
# Sessions
|
||||
# Banner
|
||||
|
||||
All notes on the internal workings of the Banner system by Ellucian.
|
||||
|
||||
## Sessions
|
||||
|
||||
All notes on the internal workings of Sessions in the Banner system.
|
||||
|
||||
- Sessions are generated on demand with a random string of characters.
|
||||
- The format `{5 random characters}{milliseconds since epoch}`
|
||||
- Example: ``
|
||||
- Sessions are invalidated after 30 minutes, but may change.
|
||||
- This delay can be found in the original HTML returned, find `meta[name="maxInactiveInterval"]` and read the `content` attribute.
|
||||
- This is read at runtime by the javascript on initialization.
|
||||
- This is read at runtime (in the browser, by javascript) on initialization.
|
||||
- Multiple timers exist, one is for the Inactivity Timer.
|
||||
- A dialog will appear asking the user to continue their session.
|
||||
- If they click the button, the session will be extended via the keepAliveURL (see `meta[name="keepAliveURL"]`).
|
||||
58
docs/FEATURES.md
Normal file
58
docs/FEATURES.md
Normal file
@@ -0,0 +1,58 @@
|
||||
# Features
|
||||
|
||||
## Current Features
|
||||
|
||||
### Discord Bot Commands
|
||||
|
||||
- **search** - Search for courses with various filters (title, course code, keywords)
|
||||
- **terms** - List available terms or search for a specific term
|
||||
- **time** - Get meeting times for a specific course (CRN)
|
||||
- **ics** - Generate ICS calendar file for a course with holiday exclusions
|
||||
- **gcal** - Generate Google Calendar link for a course
|
||||
|
||||
### Data Pipeline
|
||||
|
||||
- Intelligent scraping system with priority queues
|
||||
- Rate limiting and burst handling
|
||||
- Background data synchronization
|
||||
|
||||
## Feature Wishlist
|
||||
|
||||
### Commands
|
||||
|
||||
- ICS Download (get a ICS download of your classes with location & timing perfectly - set for every class you're in)
|
||||
- Classes Now (find classes happening)
|
||||
- Autocomplete
|
||||
- Class Title
|
||||
- Course Number
|
||||
- Term/Part of Term
|
||||
- Professor
|
||||
- Attribute
|
||||
- Component Pagination
|
||||
- RateMyProfessor Integration (Linked/Embedded)
|
||||
- Smart term selection (i.e. Summer 2024 will be selected automatically when opened)
|
||||
- Rate Limiting (bursting with global/user limits)
|
||||
- DMs Integration (allow usage of the bot in DMs)
|
||||
- Class Change Notifications (get notified when details about a class change)
|
||||
- Multi-term Querying (currently the backend for searching is kinda weird)
|
||||
- Full Autocomplete for Every Search Option
|
||||
- Metrics, Log Query, Privileged Error Feedback
|
||||
- Search for Classes
|
||||
- Major, Professor, Location, Name, Time of Day
|
||||
- Subscribe to Classes
|
||||
- Availability (seat, pre-seat)
|
||||
- Waitlist Movement
|
||||
- Detail Changes (meta, time, location, seats, professor)
|
||||
- `time` Start, End, Days of Week
|
||||
- `seats` Any change in seat/waitlist data
|
||||
- `meta`
|
||||
- Lookup via Course Reference Number (CRN)
|
||||
- Smart Time of Day Handling
|
||||
- "2 PM" -> Start within 2:00 PM to 2:59 PM
|
||||
- "2-3 PM" -> Start within 2:00 PM to 3:59 PM
|
||||
- "ends by 2 PM" -> Ends within 12:00 AM to 2:00 PM
|
||||
- "after 2 PM" -> Start within 2:01 PM to 11:59 PM
|
||||
- "before 2 PM" -> Ends within 12:00 AM to 1:59 PM
|
||||
- Get By Section Command
|
||||
- CS 4393 001 =>
|
||||
- Will require SQL to be able to search for a class by its section number
|
||||
42
docs/README.md
Normal file
42
docs/README.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# Documentation
|
||||
|
||||
This folder contains detailed documentation for the Banner project. This file acts as the index.
|
||||
|
||||
## Files
|
||||
|
||||
- [`FEATURES.md`](FEATURES.md) - Current features, implemented functionality, and future roadmap
|
||||
- [`BANNER.md`](BANNER.md) - General API documentation on the Banner system
|
||||
- [`ARCHITECTURE.md`](ARCHITECTURE.md) - Technical implementation details, system design, and analysis
|
||||
|
||||
## Samples
|
||||
|
||||
The `samples/` folder contains real Banner API response examples:
|
||||
|
||||
- `search/` - Course search API responses with various filters
|
||||
- [`searchResults.json`](samples/search/searchResults.json)
|
||||
- [`searchResults_500.json`](samples/search/searchResults_500.json)
|
||||
- [`searchResults_CS500.json`](samples/search/searchResults_CS500.json)
|
||||
- [`searchResults_malware.json`](samples/search/searchResults_malware.json)
|
||||
- `meta/` - Metadata API responses (terms, subjects, instructors, etc.)
|
||||
- [`get_attribute.json`](samples/meta/get_attribute.json)
|
||||
- [`get_campus.json`](samples/meta/get_campus.json)
|
||||
- [`get_instructionalMethod.json`](samples/meta/get_instructionalMethod.json)
|
||||
- [`get_instructor.json`](samples/meta/get_instructor.json)
|
||||
- [`get_partOfTerm.json`](samples/meta/get_partOfTerm.json)
|
||||
- [`get_subject.json`](samples/meta/get_subject.json)
|
||||
- [`getTerms.json`](samples/meta/getTerms.json)
|
||||
- `course/` - Course detail API responses (HTML and JSON)
|
||||
- [`getFacultyMeetingTimes.json`](samples/course/getFacultyMeetingTimes.json)
|
||||
- [`getClassDetails.html`](samples/course/getClassDetails.html)
|
||||
- [`getCorequisites.html`](samples/course/getCorequisites.html)
|
||||
- [`getCourseDescription.html`](samples/course/getCourseDescription.html)
|
||||
- [`getEnrollmentInfo.html`](samples/course/getEnrollmentInfo.html)
|
||||
- [`getFees.html`](samples/course/getFees.html)
|
||||
- [`getLinkedSections.html`](samples/course/getLinkedSections.html)
|
||||
- [`getRestrictions.html`](samples/course/getRestrictions.html)
|
||||
- [`getSectionAttributes.html`](samples/course/getSectionAttributes.html)
|
||||
- [`getSectionBookstoreDetails.html`](samples/course/getSectionBookstoreDetails.html)
|
||||
- [`getSectionPrerequisites.html`](samples/course/getSectionPrerequisites.html)
|
||||
- [`getXlistSections.html`](samples/course/getXlistSections.html)
|
||||
|
||||
These samples are used for development, testing, and understanding the Banner API structure.
|
||||
@@ -7,8 +7,16 @@ use std::{
|
||||
};
|
||||
|
||||
use crate::banner::{
|
||||
BannerSession, SessionPool, errors::BannerApiError, json::parse_json_with_context,
|
||||
middleware::TransparentMiddleware, models::*, nonce, query::SearchQuery, util::user_agent,
|
||||
BannerSession, SessionPool, create_shared_rate_limiter_with_config,
|
||||
errors::BannerApiError,
|
||||
json::parse_json_with_context,
|
||||
middleware::TransparentMiddleware,
|
||||
models::*,
|
||||
nonce,
|
||||
query::SearchQuery,
|
||||
rate_limit_middleware::RateLimitMiddleware,
|
||||
rate_limiter::{RateLimitConfig, SharedRateLimiter, create_shared_rate_limiter},
|
||||
util::user_agent,
|
||||
};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use cookie::Cookie;
|
||||
@@ -30,6 +38,13 @@ pub struct BannerApi {
|
||||
impl BannerApi {
|
||||
/// Creates a new Banner API client.
|
||||
pub fn new(base_url: String) -> Result<Self> {
|
||||
Self::new_with_config(base_url, RateLimitConfig::default())
|
||||
}
|
||||
|
||||
/// Creates a new Banner API client with custom rate limiting configuration.
|
||||
pub fn new_with_config(base_url: String, rate_limit_config: RateLimitConfig) -> Result<Self> {
|
||||
let rate_limiter = create_shared_rate_limiter_with_config(rate_limit_config);
|
||||
|
||||
let http = ClientBuilder::new(
|
||||
Client::builder()
|
||||
.cookie_store(false)
|
||||
@@ -42,6 +57,7 @@ impl BannerApi {
|
||||
.context("Failed to create HTTP client")?,
|
||||
)
|
||||
.with(TransparentMiddleware)
|
||||
.with(RateLimitMiddleware::new(rate_limiter.clone()))
|
||||
.build();
|
||||
|
||||
Ok(Self {
|
||||
@@ -50,7 +66,6 @@ impl BannerApi {
|
||||
base_url,
|
||||
})
|
||||
}
|
||||
|
||||
/// Validates offset parameter for search methods.
|
||||
fn validate_offset(offset: i32) -> Result<()> {
|
||||
if offset <= 0 {
|
||||
@@ -160,10 +175,9 @@ impl BannerApi {
|
||||
|
||||
debug!(
|
||||
term = term,
|
||||
query = ?query,
|
||||
sort = sort,
|
||||
sort_descending = sort_descending,
|
||||
"Searching for courses with params: {:?}", params
|
||||
subject = query.get_subject().map(|s| s.as_str()).unwrap_or("all"),
|
||||
max_results = query.get_max_results(),
|
||||
"Searching for courses"
|
||||
);
|
||||
|
||||
let response = self
|
||||
@@ -184,7 +198,7 @@ impl BannerApi {
|
||||
|
||||
let search_result: SearchResult = parse_json_with_context(&body).map_err(|e| {
|
||||
BannerApiError::RequestFailed(anyhow!(
|
||||
"Failed to parse search response (status={status}, url={url}): {e}\nBody: {body}"
|
||||
"Failed to parse search response (status={status}, url={url}): {e}"
|
||||
))
|
||||
})?;
|
||||
|
||||
@@ -303,6 +317,12 @@ impl BannerApi {
|
||||
sort: &str,
|
||||
sort_descending: bool,
|
||||
) -> Result<SearchResult, BannerApiError> {
|
||||
debug!(
|
||||
term = term,
|
||||
subject = query.get_subject().map(|s| s.as_str()).unwrap_or("all"),
|
||||
max_results = query.get_max_results(),
|
||||
"Starting course search"
|
||||
);
|
||||
self.perform_search(term, query, sort, sort_descending)
|
||||
.await
|
||||
}
|
||||
@@ -313,6 +333,8 @@ impl BannerApi {
|
||||
term: &str,
|
||||
crn: &str,
|
||||
) -> Result<Option<Course>, BannerApiError> {
|
||||
debug!(term = term, crn = crn, "Looking up course by CRN");
|
||||
|
||||
let query = SearchQuery::new()
|
||||
.course_reference_number(crn)
|
||||
.max_results(1);
|
||||
|
||||
@@ -1,18 +1,34 @@
|
||||
//! JSON parsing utilities for the Banner API client.
|
||||
|
||||
use anyhow::Result;
|
||||
use serde_json;
|
||||
|
||||
/// Attempt to parse JSON and, on failure, include a contextual snippet of the
|
||||
/// line where the error occurred. This prevents dumping huge JSON bodies to logs.
|
||||
pub fn parse_json_with_context<T: serde::de::DeserializeOwned>(body: &str) -> Result<T> {
|
||||
match serde_json::from_str::<T>(body) {
|
||||
let jd = &mut serde_json::Deserializer::from_str(body);
|
||||
match serde_path_to_error::deserialize(jd) {
|
||||
Ok(value) => Ok(value),
|
||||
Err(err) => {
|
||||
let (line, column) = (err.line(), err.column());
|
||||
let snippet = build_error_snippet(body, line, column, 80);
|
||||
Err(anyhow::anyhow!(
|
||||
"{err} at line {line}, column {column}\nSnippet:\n{snippet}",
|
||||
))
|
||||
let inner_err = err.inner();
|
||||
let (line, column) = (inner_err.line(), inner_err.column());
|
||||
let snippet = build_error_snippet(body, line, column, 20);
|
||||
let path = err.path().to_string();
|
||||
|
||||
let msg = inner_err.to_string();
|
||||
let loc = format!(" at line {line} column {column}");
|
||||
let msg_without_loc = msg.strip_suffix(&loc).unwrap_or(&msg).to_string();
|
||||
|
||||
let mut final_err = String::new();
|
||||
if !path.is_empty() && path != "." {
|
||||
final_err.push_str(&format!("for path '{}' ", path));
|
||||
}
|
||||
final_err.push_str(&format!(
|
||||
"({msg_without_loc}) at line {line} column {column}"
|
||||
));
|
||||
final_err.push_str(&format!("\n{snippet}"));
|
||||
|
||||
Err(anyhow::anyhow!(final_err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ impl Middleware for TransparentMiddleware {
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(?error, "Request failed (middleware)");
|
||||
warn!(error = ?error, "Request failed (middleware)");
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
//! This module provides functionality to:
|
||||
//! - Search for courses and retrieve course information
|
||||
//! - Manage Banner API sessions and authentication
|
||||
//! - Scrape course data and cache it in Redis
|
||||
//! - Generate ICS files and calendar links
|
||||
|
||||
pub mod api;
|
||||
@@ -14,6 +13,8 @@ pub mod json;
|
||||
pub mod middleware;
|
||||
pub mod models;
|
||||
pub mod query;
|
||||
pub mod rate_limit_middleware;
|
||||
pub mod rate_limiter;
|
||||
pub mod session;
|
||||
pub mod util;
|
||||
|
||||
@@ -21,4 +22,5 @@ pub use api::*;
|
||||
pub use errors::*;
|
||||
pub use models::*;
|
||||
pub use query::*;
|
||||
pub use rate_limiter::*;
|
||||
pub use session::*;
|
||||
|
||||
@@ -33,7 +33,7 @@ pub struct FacultyItem {
|
||||
#[serde(deserialize_with = "deserialize_string_to_u32")]
|
||||
pub course_reference_number: u32, // CRN, e.g 27294
|
||||
pub display_name: String, // "LastName, FirstName"
|
||||
pub email_address: String, // e.g. FirstName.LastName@utsaedu
|
||||
pub email_address: Option<String>, // e.g. FirstName.LastName@utsaedu
|
||||
pub primary_indicator: bool,
|
||||
pub term: String, // e.g "202420"
|
||||
}
|
||||
@@ -63,7 +63,7 @@ pub struct MeetingTime {
|
||||
pub campus: Option<String>, // campus code, e.g 11
|
||||
pub campus_description: Option<String>, // name of campus, e.g Main Campus
|
||||
pub course_reference_number: String, // CRN, e.g 27294
|
||||
pub credit_hour_session: f64, // e.g. 30
|
||||
pub credit_hour_session: Option<f64>, // e.g. 30
|
||||
pub hours_week: f64, // e.g. 30
|
||||
pub meeting_schedule_type: String, // e.g AFF
|
||||
pub meeting_type: String, // e.g HB, H2, H1, OS, OA, OH, ID, FF
|
||||
@@ -148,6 +148,8 @@ pub enum DayOfWeek {
|
||||
|
||||
impl DayOfWeek {
|
||||
/// Convert to short string representation
|
||||
///
|
||||
/// Do not change these, these are used for ICS generation. Casing does not matter though.
|
||||
pub fn to_short_string(self) -> &'static str {
|
||||
match self {
|
||||
DayOfWeek::Monday => "Mo",
|
||||
|
||||
@@ -160,6 +160,16 @@ impl SearchQuery {
|
||||
self
|
||||
}
|
||||
|
||||
/// Gets the subject field
|
||||
pub fn get_subject(&self) -> Option<&String> {
|
||||
self.subject.as_ref()
|
||||
}
|
||||
|
||||
/// Gets the max_results field
|
||||
pub fn get_max_results(&self) -> i32 {
|
||||
self.max_results
|
||||
}
|
||||
|
||||
/// Converts the query into URL parameters for the Banner API
|
||||
pub fn to_params(&self) -> HashMap<String, String> {
|
||||
let mut params = HashMap::new();
|
||||
|
||||
101
src/banner/rate_limit_middleware.rs
Normal file
101
src/banner/rate_limit_middleware.rs
Normal file
@@ -0,0 +1,101 @@
|
||||
//! HTTP middleware that enforces rate limiting for Banner API requests.
|
||||
|
||||
use crate::banner::rate_limiter::{RequestType, SharedRateLimiter};
|
||||
use http::Extensions;
|
||||
use reqwest::{Request, Response};
|
||||
use reqwest_middleware::{Middleware, Next};
|
||||
use tracing::{debug, trace, warn};
|
||||
use url::Url;
|
||||
|
||||
/// Middleware that enforces rate limiting based on request URL patterns
|
||||
pub struct RateLimitMiddleware {
|
||||
rate_limiter: SharedRateLimiter,
|
||||
}
|
||||
|
||||
impl RateLimitMiddleware {
|
||||
/// Creates a new rate limiting middleware
|
||||
pub fn new(rate_limiter: SharedRateLimiter) -> Self {
|
||||
Self { rate_limiter }
|
||||
}
|
||||
|
||||
/// Determines the request type based on the URL path
|
||||
fn get_request_type(url: &Url) -> RequestType {
|
||||
let path = url.path();
|
||||
|
||||
if path.contains("/registration")
|
||||
|| path.contains("/selfServiceMenu")
|
||||
|| path.contains("/term/termSelection")
|
||||
{
|
||||
RequestType::Session
|
||||
} else if path.contains("/searchResults") || path.contains("/classSearch") {
|
||||
RequestType::Search
|
||||
} else if path.contains("/getTerms")
|
||||
|| path.contains("/getSubjects")
|
||||
|| path.contains("/getCampuses")
|
||||
{
|
||||
RequestType::Metadata
|
||||
} else if path.contains("/resetDataForm") {
|
||||
RequestType::Reset
|
||||
} else {
|
||||
// Default to search for unknown endpoints
|
||||
RequestType::Search
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Middleware for RateLimitMiddleware {
|
||||
async fn handle(
|
||||
&self,
|
||||
req: Request,
|
||||
extensions: &mut Extensions,
|
||||
next: Next<'_>,
|
||||
) -> std::result::Result<Response, reqwest_middleware::Error> {
|
||||
let request_type = Self::get_request_type(req.url());
|
||||
|
||||
trace!(
|
||||
url = %req.url(),
|
||||
request_type = ?request_type,
|
||||
"Rate limiting request"
|
||||
);
|
||||
|
||||
// Wait for permission to make the request
|
||||
self.rate_limiter.wait_for_permission(request_type).await;
|
||||
|
||||
trace!(
|
||||
url = %req.url(),
|
||||
request_type = ?request_type,
|
||||
"Rate limit permission granted, making request"
|
||||
);
|
||||
|
||||
// Make the actual request
|
||||
let response_result = next.run(req, extensions).await;
|
||||
|
||||
match response_result {
|
||||
Ok(response) => {
|
||||
if response.status().is_success() {
|
||||
trace!(
|
||||
url = %response.url(),
|
||||
status = response.status().as_u16(),
|
||||
"Request completed successfully"
|
||||
);
|
||||
} else {
|
||||
warn!(
|
||||
url = %response.url(),
|
||||
status = response.status().as_u16(),
|
||||
"Request completed with error status"
|
||||
);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(
|
||||
url = ?error.url(),
|
||||
error = ?error,
|
||||
"Request failed"
|
||||
);
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
169
src/banner/rate_limiter.rs
Normal file
169
src/banner/rate_limiter.rs
Normal file
@@ -0,0 +1,169 @@
|
||||
//! Rate limiting for Banner API requests to prevent overwhelming the server.
|
||||
|
||||
use governor::{
|
||||
Quota, RateLimiter,
|
||||
clock::DefaultClock,
|
||||
state::{InMemoryState, NotKeyed},
|
||||
};
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
/// Different types of Banner API requests with different rate limits
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum RequestType {
|
||||
/// Session creation and management (very conservative)
|
||||
Session,
|
||||
/// Course search requests (moderate)
|
||||
Search,
|
||||
/// Term and metadata requests (moderate)
|
||||
Metadata,
|
||||
/// Data form resets (low priority)
|
||||
Reset,
|
||||
}
|
||||
|
||||
/// Rate limiter configuration for different request types
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RateLimitConfig {
|
||||
/// Requests per minute for session operations
|
||||
pub session_rpm: u32,
|
||||
/// Requests per minute for search operations
|
||||
pub search_rpm: u32,
|
||||
/// Requests per minute for metadata operations
|
||||
pub metadata_rpm: u32,
|
||||
/// Requests per minute for reset operations
|
||||
pub reset_rpm: u32,
|
||||
/// Burst allowance (extra requests allowed in short bursts)
|
||||
pub burst_allowance: u32,
|
||||
}
|
||||
|
||||
impl Default for RateLimitConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
// Very conservative for session creation
|
||||
session_rpm: 6, // 1 every 10 seconds
|
||||
// Moderate for search operations
|
||||
search_rpm: 30, // 1 every 2 seconds
|
||||
// Moderate for metadata
|
||||
metadata_rpm: 20, // 1 every 3 seconds
|
||||
// Low for resets
|
||||
reset_rpm: 10, // 1 every 6 seconds
|
||||
// Allow small bursts
|
||||
burst_allowance: 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A rate limiter that manages different request types with different limits
|
||||
pub struct BannerRateLimiter {
|
||||
session_limiter: RateLimiter<NotKeyed, InMemoryState, DefaultClock>,
|
||||
search_limiter: RateLimiter<NotKeyed, InMemoryState, DefaultClock>,
|
||||
metadata_limiter: RateLimiter<NotKeyed, InMemoryState, DefaultClock>,
|
||||
reset_limiter: RateLimiter<NotKeyed, InMemoryState, DefaultClock>,
|
||||
config: RateLimitConfig,
|
||||
}
|
||||
|
||||
impl BannerRateLimiter {
|
||||
/// Creates a new rate limiter with the given configuration
|
||||
pub fn new(config: RateLimitConfig) -> Self {
|
||||
let session_quota = Quota::with_period(Duration::from_secs(60) / config.session_rpm)
|
||||
.unwrap()
|
||||
.allow_burst(NonZeroU32::new(config.burst_allowance).unwrap());
|
||||
|
||||
let search_quota = Quota::with_period(Duration::from_secs(60) / config.search_rpm)
|
||||
.unwrap()
|
||||
.allow_burst(NonZeroU32::new(config.burst_allowance).unwrap());
|
||||
|
||||
let metadata_quota = Quota::with_period(Duration::from_secs(60) / config.metadata_rpm)
|
||||
.unwrap()
|
||||
.allow_burst(NonZeroU32::new(config.burst_allowance).unwrap());
|
||||
|
||||
let reset_quota = Quota::with_period(Duration::from_secs(60) / config.reset_rpm)
|
||||
.unwrap()
|
||||
.allow_burst(NonZeroU32::new(config.burst_allowance).unwrap());
|
||||
|
||||
Self {
|
||||
session_limiter: RateLimiter::direct(session_quota),
|
||||
search_limiter: RateLimiter::direct(search_quota),
|
||||
metadata_limiter: RateLimiter::direct(metadata_quota),
|
||||
reset_limiter: RateLimiter::direct(reset_quota),
|
||||
config,
|
||||
}
|
||||
}
|
||||
|
||||
/// Waits for permission to make a request of the given type
|
||||
pub async fn wait_for_permission(&self, request_type: RequestType) {
|
||||
let limiter = match request_type {
|
||||
RequestType::Session => &self.session_limiter,
|
||||
RequestType::Search => &self.search_limiter,
|
||||
RequestType::Metadata => &self.metadata_limiter,
|
||||
RequestType::Reset => &self.reset_limiter,
|
||||
};
|
||||
|
||||
trace!(request_type = ?request_type, "Waiting for rate limit permission");
|
||||
|
||||
// Wait until we can make the request
|
||||
limiter.until_ready().await;
|
||||
|
||||
trace!(request_type = ?request_type, "Rate limit permission granted");
|
||||
}
|
||||
|
||||
/// Checks if a request of the given type would be allowed immediately
|
||||
pub fn check_permission(&self, request_type: RequestType) -> bool {
|
||||
let limiter = match request_type {
|
||||
RequestType::Session => &self.session_limiter,
|
||||
RequestType::Search => &self.search_limiter,
|
||||
RequestType::Metadata => &self.metadata_limiter,
|
||||
RequestType::Reset => &self.reset_limiter,
|
||||
};
|
||||
|
||||
limiter.check().is_ok()
|
||||
}
|
||||
|
||||
/// Gets the current configuration
|
||||
pub fn config(&self) -> &RateLimitConfig {
|
||||
&self.config
|
||||
}
|
||||
|
||||
/// Updates the rate limit configuration
|
||||
pub fn update_config(&mut self, config: RateLimitConfig) {
|
||||
self.config = config;
|
||||
// Note: In a production system, you'd want to recreate the limiters
|
||||
// with the new configuration, but for simplicity we'll just update
|
||||
// the config field here.
|
||||
warn!("Rate limit configuration updated - restart required for full effect");
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for BannerRateLimiter {
|
||||
fn default() -> Self {
|
||||
Self::new(RateLimitConfig::default())
|
||||
}
|
||||
}
|
||||
|
||||
/// A shared rate limiter instance
|
||||
pub type SharedRateLimiter = Arc<BannerRateLimiter>;
|
||||
|
||||
/// Creates a new shared rate limiter with default configuration
|
||||
pub fn create_shared_rate_limiter() -> SharedRateLimiter {
|
||||
Arc::new(BannerRateLimiter::default())
|
||||
}
|
||||
|
||||
/// Creates a new shared rate limiter with custom configuration
|
||||
pub fn create_shared_rate_limiter_with_config(config: RateLimitConfig) -> SharedRateLimiter {
|
||||
Arc::new(BannerRateLimiter::new(config))
|
||||
}
|
||||
|
||||
/// Conversion from config module's RateLimitingConfig to this module's RateLimitConfig
|
||||
impl From<crate::config::RateLimitingConfig> for RateLimitConfig {
|
||||
fn from(config: crate::config::RateLimitingConfig) -> Self {
|
||||
Self {
|
||||
session_rpm: config.session_rpm,
|
||||
search_rpm: config.search_rpm,
|
||||
metadata_rpm: config.metadata_rpm,
|
||||
reset_rpm: config.reset_rpm,
|
||||
burst_allowance: config.burst_allowance,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ use std::ops::{Deref, DerefMut};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::{Mutex, Notify};
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, trace};
|
||||
use url::Url;
|
||||
|
||||
const SESSION_EXPIRY: Duration = Duration::from_secs(25 * 60); // 25 minutes
|
||||
@@ -82,7 +82,7 @@ impl BannerSession {
|
||||
|
||||
/// Updates the last activity timestamp
|
||||
pub fn touch(&mut self) {
|
||||
debug!(id = self.unique_session_id, "Session was used");
|
||||
trace!(id = self.unique_session_id, "Session was used");
|
||||
self.last_activity = Some(Instant::now());
|
||||
}
|
||||
|
||||
@@ -162,7 +162,7 @@ impl TermPool {
|
||||
async fn release(&self, session: BannerSession) {
|
||||
let id = session.unique_session_id.clone();
|
||||
if session.is_expired() {
|
||||
debug!(id = id, "Session is now expired, dropping.");
|
||||
trace!(id = id, "Session is now expired, dropping.");
|
||||
// Wake up a waiter, as it might need to create a new session
|
||||
// if this was the last one.
|
||||
self.notifier.notify_one();
|
||||
@@ -174,10 +174,7 @@ impl TermPool {
|
||||
let queue_size = queue.len();
|
||||
drop(queue); // Release lock before notifying
|
||||
|
||||
debug!(
|
||||
id = id,
|
||||
"Session returned to pool. Queue size is now {queue_size}."
|
||||
);
|
||||
trace!(id = id, queue_size, "Session returned to pool");
|
||||
self.notifier.notify_one();
|
||||
}
|
||||
}
|
||||
@@ -213,13 +210,13 @@ impl SessionPool {
|
||||
let mut queue = term_pool.sessions.lock().await;
|
||||
if let Some(session) = queue.pop_front() {
|
||||
if !session.is_expired() {
|
||||
debug!(id = session.unique_session_id, "Reusing session from pool");
|
||||
trace!(id = session.unique_session_id, "Reusing session from pool");
|
||||
return Ok(PooledSession {
|
||||
session: Some(session),
|
||||
pool: Arc::clone(&term_pool),
|
||||
});
|
||||
} else {
|
||||
debug!(
|
||||
trace!(
|
||||
id = session.unique_session_id,
|
||||
"Popped an expired session, discarding."
|
||||
);
|
||||
@@ -232,7 +229,7 @@ impl SessionPool {
|
||||
if *is_creating_guard {
|
||||
// Another task is already creating a session. Release the lock and wait.
|
||||
drop(is_creating_guard);
|
||||
debug!("Another task is creating a session, waiting for notification...");
|
||||
trace!("Another task is creating a session, waiting for notification...");
|
||||
term_pool.notifier.notified().await;
|
||||
// Loop back to the top to try the fast path again.
|
||||
continue;
|
||||
@@ -243,12 +240,12 @@ impl SessionPool {
|
||||
drop(is_creating_guard);
|
||||
|
||||
// Race: wait for a session to be returned OR for the rate limiter to allow a new one.
|
||||
debug!("Pool empty, racing notifier vs rate limiter...");
|
||||
trace!("Pool empty, racing notifier vs rate limiter...");
|
||||
tokio::select! {
|
||||
_ = term_pool.notifier.notified() => {
|
||||
// A session was returned while we were waiting!
|
||||
// We are no longer the creator. Reset the flag and loop to race for the new session.
|
||||
debug!("Notified that a session was returned. Looping to retry.");
|
||||
trace!("Notified that a session was returned. Looping to retry.");
|
||||
let mut guard = term_pool.is_creating.lock().await;
|
||||
*guard = false;
|
||||
drop(guard);
|
||||
@@ -256,7 +253,7 @@ impl SessionPool {
|
||||
}
|
||||
_ = SESSION_CREATION_RATE_LIMITER.until_ready() => {
|
||||
// The rate limit has elapsed. It's our job to create the session.
|
||||
debug!("Rate limiter ready. Proceeding to create a new session.");
|
||||
trace!("Rate limiter ready. Proceeding to create a new session.");
|
||||
let new_session_result = self.create_session(&term).await;
|
||||
|
||||
// After creation, we are no longer the creator. Reset the flag
|
||||
@@ -286,7 +283,7 @@ impl SessionPool {
|
||||
|
||||
/// Sets up initial session cookies by making required Banner API requests
|
||||
pub async fn create_session(&self, term: &Term) -> Result<BannerSession> {
|
||||
info!("setting up banner session for term {term}");
|
||||
info!(term = %term, "setting up banner session");
|
||||
|
||||
// The 'register' or 'search' registration page
|
||||
let initial_registration = self
|
||||
@@ -317,7 +314,7 @@ impl SessionPool {
|
||||
let ssb_cookie = cookies.get("SSB_COOKIE").unwrap();
|
||||
let cookie_header = format!("JSESSIONID={}; SSB_COOKIE={}", jsessionid, ssb_cookie);
|
||||
|
||||
debug!(
|
||||
trace!(
|
||||
jsessionid = jsessionid,
|
||||
ssb_cookie = ssb_cookie,
|
||||
"New session cookies acquired"
|
||||
@@ -457,7 +454,7 @@ impl SessionPool {
|
||||
));
|
||||
}
|
||||
|
||||
debug!(term = term, "successfully selected term");
|
||||
trace!(term = term, "successfully selected term");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,8 +23,7 @@ async fn main() -> Result<()> {
|
||||
|
||||
// Load configuration
|
||||
let config: Config = Figment::new()
|
||||
.merge(Env::raw().only(&["DATABASE_URL"]))
|
||||
.merge(Env::prefixed("APP_"))
|
||||
.merge(Env::raw())
|
||||
.extract()
|
||||
.expect("Failed to load config");
|
||||
|
||||
@@ -34,7 +33,9 @@ async fn main() -> Result<()> {
|
||||
);
|
||||
|
||||
// Create Banner API client
|
||||
let banner_api = BannerApi::new(config.banner_base_url).expect("Failed to create BannerApi");
|
||||
let banner_api =
|
||||
BannerApi::new_with_config(config.banner_base_url, config.rate_limiting.into())
|
||||
.expect("Failed to create BannerApi");
|
||||
|
||||
// Get current term
|
||||
let term = Term::get_current().inner().to_string();
|
||||
@@ -67,11 +68,11 @@ async fn main() -> Result<()> {
|
||||
),
|
||||
];
|
||||
|
||||
info!("Executing {} concurrent searches", queries.len());
|
||||
info!(query_count = queries.len(), "Executing concurrent searches");
|
||||
|
||||
// Execute all searches concurrently
|
||||
let search_futures = queries.into_iter().map(|(label, query)| {
|
||||
info!("Starting search: {}", label);
|
||||
info!(label = %label, "Starting search");
|
||||
let banner_api = &banner_api;
|
||||
let term = &term;
|
||||
async move {
|
||||
|
||||
@@ -77,7 +77,7 @@ pub async fn gcal(
|
||||
)
|
||||
.await?;
|
||||
|
||||
info!("gcal command completed for CRN: {}", crn);
|
||||
info!(crn = %crn, "gcal command completed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,111 @@
|
||||
//! ICS command implementation for generating calendar files.
|
||||
|
||||
use crate::banner::{Course, MeetingScheduleInfo};
|
||||
use crate::bot::{Context, Error, utils};
|
||||
use chrono::{Datelike, NaiveDate, Utc};
|
||||
use serenity::all::CreateAttachment;
|
||||
use tracing::info;
|
||||
|
||||
/// Represents a holiday or special day that should be excluded from class schedules
|
||||
#[derive(Debug, Clone)]
|
||||
enum Holiday {
|
||||
/// A single-day holiday
|
||||
Single { month: u32, day: u32 },
|
||||
/// A multi-day holiday range
|
||||
Range {
|
||||
month: u32,
|
||||
start_day: u32,
|
||||
end_day: u32,
|
||||
},
|
||||
}
|
||||
|
||||
impl Holiday {
|
||||
/// Check if a specific date falls within this holiday
|
||||
fn contains_date(&self, date: NaiveDate) -> bool {
|
||||
match self {
|
||||
Holiday::Single { month, day, .. } => date.month() == *month && date.day() == *day,
|
||||
Holiday::Range {
|
||||
month,
|
||||
start_day,
|
||||
end_day,
|
||||
..
|
||||
} => date.month() == *month && date.day() >= *start_day && date.day() <= *end_day,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get all dates in this holiday for a given year
|
||||
fn get_dates_for_year(&self, year: i32) -> Vec<NaiveDate> {
|
||||
match self {
|
||||
Holiday::Single { month, day, .. } => {
|
||||
if let Some(date) = NaiveDate::from_ymd_opt(year, *month, *day) {
|
||||
vec![date]
|
||||
} else {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
Holiday::Range {
|
||||
month,
|
||||
start_day,
|
||||
end_day,
|
||||
..
|
||||
} => {
|
||||
let mut dates = Vec::new();
|
||||
for day in *start_day..=*end_day {
|
||||
if let Some(date) = NaiveDate::from_ymd_opt(year, *month, day) {
|
||||
dates.push(date);
|
||||
}
|
||||
}
|
||||
dates
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// University holidays that should be excluded from class schedules
|
||||
const UNIVERSITY_HOLIDAYS: &[(&str, Holiday)] = &[
|
||||
("Labor Day", Holiday::Single { month: 9, day: 1 }),
|
||||
(
|
||||
"Fall Break",
|
||||
Holiday::Range {
|
||||
month: 10,
|
||||
start_day: 13,
|
||||
end_day: 14,
|
||||
},
|
||||
),
|
||||
(
|
||||
"Unspecified Holiday",
|
||||
Holiday::Single { month: 11, day: 26 },
|
||||
),
|
||||
(
|
||||
"Thanksgiving",
|
||||
Holiday::Range {
|
||||
month: 11,
|
||||
start_day: 28,
|
||||
end_day: 29,
|
||||
},
|
||||
),
|
||||
("Student Study Day", Holiday::Single { month: 12, day: 5 }),
|
||||
(
|
||||
"Winter Holiday",
|
||||
Holiday::Range {
|
||||
month: 12,
|
||||
start_day: 23,
|
||||
end_day: 31,
|
||||
},
|
||||
),
|
||||
("New Year's Day", Holiday::Single { month: 1, day: 1 }),
|
||||
("MLK Day", Holiday::Single { month: 1, day: 20 }),
|
||||
(
|
||||
"Spring Break",
|
||||
Holiday::Range {
|
||||
month: 3,
|
||||
start_day: 10,
|
||||
end_day: 15,
|
||||
},
|
||||
),
|
||||
("Student Study Day", Holiday::Single { month: 5, day: 9 }),
|
||||
];
|
||||
|
||||
/// Generate an ICS file for a course
|
||||
#[poise::command(slash_command, prefix_command)]
|
||||
pub async fn ics(
|
||||
@@ -12,14 +115,322 @@ pub async fn ics(
|
||||
ctx.defer().await?;
|
||||
|
||||
let course = utils::get_course_by_crn(&ctx, crn).await?;
|
||||
let term = course.term.clone();
|
||||
|
||||
// TODO: Implement actual ICS file generation
|
||||
ctx.say(format!(
|
||||
"ICS generation for '{}' is not yet implemented.",
|
||||
course.display_title()
|
||||
))
|
||||
// Get meeting times
|
||||
let meeting_times = ctx
|
||||
.data()
|
||||
.app_state
|
||||
.banner_api
|
||||
.get_course_meeting_time(&term, &crn.to_string())
|
||||
.await?;
|
||||
|
||||
if meeting_times.is_empty() {
|
||||
ctx.say("No meeting times found for this course.").await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Sort meeting times by start time
|
||||
let mut sorted_meeting_times = meeting_times.to_vec();
|
||||
sorted_meeting_times.sort_unstable_by(|a, b| match (&a.time_range, &b.time_range) {
|
||||
(Some(a_time), Some(b_time)) => a_time.start.cmp(&b_time.start),
|
||||
(Some(_), None) => std::cmp::Ordering::Less,
|
||||
(None, Some(_)) => std::cmp::Ordering::Greater,
|
||||
(None, None) => a.days.bits().cmp(&b.days.bits()),
|
||||
});
|
||||
|
||||
// Generate ICS content
|
||||
let (ics_content, excluded_holidays) =
|
||||
generate_ics_content(&course, &term, &sorted_meeting_times)?;
|
||||
|
||||
// Create file attachment
|
||||
let filename = format!(
|
||||
"{subject}_{number}_{section}.ics",
|
||||
subject = course.subject.replace(" ", "_"),
|
||||
number = course.course_number,
|
||||
section = course.sequence_number,
|
||||
);
|
||||
|
||||
let file = CreateAttachment::bytes(ics_content.into_bytes(), filename.clone());
|
||||
|
||||
// Build response content
|
||||
let mut response_content = format!(
|
||||
"📅 Generated ICS calendar for **{}**\n\n**Meeting Times:**\n{}",
|
||||
course.display_title(),
|
||||
sorted_meeting_times
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, m)| {
|
||||
let time_info = match &m.time_range {
|
||||
Some(range) => format!(
|
||||
"{} {}",
|
||||
m.days_string().unwrap_or("TBA".to_string()),
|
||||
range.format_12hr()
|
||||
),
|
||||
None => m.days_string().unwrap_or("TBA".to_string()),
|
||||
};
|
||||
format!("{}. {}", i + 1, time_info)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
);
|
||||
|
||||
// Add holiday exclusion information
|
||||
if !excluded_holidays.is_empty() {
|
||||
let count = excluded_holidays.len();
|
||||
let count_text = if count == 1 {
|
||||
"1 date was".to_string()
|
||||
} else {
|
||||
format!("{} dates were", count)
|
||||
};
|
||||
response_content.push_str(&format!("\n\n{} excluded from the ICS file:\n", count_text));
|
||||
response_content.push_str(
|
||||
&excluded_holidays
|
||||
.iter()
|
||||
.map(|s| format!("- {}", s))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n"),
|
||||
);
|
||||
}
|
||||
|
||||
ctx.send(
|
||||
poise::CreateReply::default()
|
||||
.content(response_content)
|
||||
.attachment(file),
|
||||
)
|
||||
.await?;
|
||||
|
||||
info!("ics command completed for CRN: {}", crn);
|
||||
info!(crn = %crn, "ics command completed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Generate ICS content for a course and its meeting times
|
||||
fn generate_ics_content(
|
||||
course: &Course,
|
||||
term: &str,
|
||||
meeting_times: &[MeetingScheduleInfo],
|
||||
) -> Result<(String, Vec<String>), anyhow::Error> {
|
||||
let mut ics_content = String::new();
|
||||
let mut excluded_holidays = Vec::new();
|
||||
|
||||
// ICS header
|
||||
ics_content.push_str("BEGIN:VCALENDAR\r\n");
|
||||
ics_content.push_str("VERSION:2.0\r\n");
|
||||
ics_content.push_str("PRODID:-//Banner Bot//Course Calendar//EN\r\n");
|
||||
ics_content.push_str("CALSCALE:GREGORIAN\r\n");
|
||||
ics_content.push_str("METHOD:PUBLISH\r\n");
|
||||
|
||||
// Calendar name
|
||||
ics_content.push_str(&format!(
|
||||
"X-WR-CALNAME:{} - {}\r\n",
|
||||
course.display_title(),
|
||||
term
|
||||
));
|
||||
|
||||
// Generate events for each meeting time
|
||||
for (index, meeting_time) in meeting_times.iter().enumerate() {
|
||||
let (event_content, holidays) = generate_event_content(course, meeting_time, index)?;
|
||||
ics_content.push_str(&event_content);
|
||||
excluded_holidays.extend(holidays);
|
||||
}
|
||||
|
||||
// ICS footer
|
||||
ics_content.push_str("END:VCALENDAR\r\n");
|
||||
|
||||
Ok((ics_content, excluded_holidays))
|
||||
}
|
||||
|
||||
/// Generate ICS event content for a single meeting time
|
||||
fn generate_event_content(
|
||||
course: &Course,
|
||||
meeting_time: &MeetingScheduleInfo,
|
||||
index: usize,
|
||||
) -> Result<(String, Vec<String>), anyhow::Error> {
|
||||
let course_title = course.display_title();
|
||||
let instructor_name = course.primary_instructor_name();
|
||||
let location = meeting_time.place_string();
|
||||
|
||||
// Create event title with meeting index if multiple meetings
|
||||
let event_title = if index > 0 {
|
||||
format!("{} (Meeting {})", course_title, index + 1)
|
||||
} else {
|
||||
course_title
|
||||
};
|
||||
|
||||
// Create event description
|
||||
let description = format!(
|
||||
"CRN: {}\\nInstructor: {}\\nDays: {}\\nMeeting Type: {}",
|
||||
course.course_reference_number,
|
||||
instructor_name,
|
||||
meeting_time.days_string().unwrap_or("TBA".to_string()),
|
||||
meeting_time.meeting_type.description()
|
||||
);
|
||||
|
||||
// Get start and end times
|
||||
let (start_dt, end_dt) = meeting_time.datetime_range();
|
||||
|
||||
// Format datetimes for ICS (UTC format)
|
||||
let start_utc = start_dt.with_timezone(&Utc);
|
||||
let end_utc = end_dt.with_timezone(&Utc);
|
||||
|
||||
let start_str = start_utc.format("%Y%m%dT%H%M%SZ").to_string();
|
||||
let end_str = end_utc.format("%Y%m%dT%H%M%SZ").to_string();
|
||||
|
||||
// Generate unique ID for the event
|
||||
let uid = format!(
|
||||
"{}-{}-{}@banner-bot.local",
|
||||
course.course_reference_number,
|
||||
index,
|
||||
start_utc.timestamp()
|
||||
);
|
||||
|
||||
let mut event_content = String::new();
|
||||
|
||||
// Event header
|
||||
event_content.push_str("BEGIN:VEVENT\r\n");
|
||||
event_content.push_str(&format!("UID:{}\r\n", uid));
|
||||
event_content.push_str(&format!("DTSTART:{}\r\n", start_str));
|
||||
event_content.push_str(&format!("DTEND:{}\r\n", end_str));
|
||||
event_content.push_str(&format!("SUMMARY:{}\r\n", escape_ics_text(&event_title)));
|
||||
event_content.push_str(&format!(
|
||||
"DESCRIPTION:{}\r\n",
|
||||
escape_ics_text(&description)
|
||||
));
|
||||
event_content.push_str(&format!("LOCATION:{}\r\n", escape_ics_text(&location)));
|
||||
|
||||
// Add recurrence rule if there are specific days and times
|
||||
if !meeting_time.days.is_empty() && meeting_time.time_range.is_some() {
|
||||
let days_of_week = meeting_time.days_of_week();
|
||||
let by_day: Vec<String> = days_of_week
|
||||
.iter()
|
||||
.map(|day| day.to_short_string().to_uppercase())
|
||||
.collect();
|
||||
|
||||
if !by_day.is_empty() {
|
||||
let until_date = meeting_time
|
||||
.date_range
|
||||
.end
|
||||
.format("%Y%m%dT000000Z")
|
||||
.to_string();
|
||||
|
||||
event_content.push_str(&format!(
|
||||
"RRULE:FREQ=WEEKLY;BYDAY={};UNTIL={}\r\n",
|
||||
by_day.join(","),
|
||||
until_date
|
||||
));
|
||||
|
||||
// Add holiday exceptions (EXDATE) if the class would meet on holiday dates
|
||||
let holiday_exceptions = get_holiday_exceptions(meeting_time);
|
||||
if let Some(exdate_property) = generate_exdate_property(&holiday_exceptions, start_utc)
|
||||
{
|
||||
event_content.push_str(&format!("{}\r\n", exdate_property));
|
||||
}
|
||||
|
||||
// Collect holiday names for reporting
|
||||
let mut holiday_names = Vec::new();
|
||||
for (holiday_name, holiday) in UNIVERSITY_HOLIDAYS {
|
||||
for &exception_date in &holiday_exceptions {
|
||||
if holiday.contains_date(exception_date) {
|
||||
holiday_names.push(format!(
|
||||
"{} ({})",
|
||||
holiday_name,
|
||||
exception_date.format("%a, %b %d")
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
holiday_names.sort();
|
||||
holiday_names.dedup();
|
||||
|
||||
return Ok((event_content, holiday_names));
|
||||
}
|
||||
}
|
||||
|
||||
// Event footer
|
||||
event_content.push_str("END:VEVENT\r\n");
|
||||
|
||||
Ok((event_content, Vec::new()))
|
||||
}
|
||||
|
||||
/// Convert chrono::Weekday to the custom DayOfWeek enum
|
||||
fn chrono_weekday_to_day_of_week(weekday: chrono::Weekday) -> crate::banner::meetings::DayOfWeek {
|
||||
use crate::banner::meetings::DayOfWeek;
|
||||
match weekday {
|
||||
chrono::Weekday::Mon => DayOfWeek::Monday,
|
||||
chrono::Weekday::Tue => DayOfWeek::Tuesday,
|
||||
chrono::Weekday::Wed => DayOfWeek::Wednesday,
|
||||
chrono::Weekday::Thu => DayOfWeek::Thursday,
|
||||
chrono::Weekday::Fri => DayOfWeek::Friday,
|
||||
chrono::Weekday::Sat => DayOfWeek::Saturday,
|
||||
chrono::Weekday::Sun => DayOfWeek::Sunday,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a class meets on a specific date based on its meeting days
|
||||
fn class_meets_on_date(meeting_time: &MeetingScheduleInfo, date: NaiveDate) -> bool {
|
||||
let weekday = chrono_weekday_to_day_of_week(date.weekday());
|
||||
let meeting_days = meeting_time.days_of_week();
|
||||
|
||||
meeting_days.contains(&weekday)
|
||||
}
|
||||
|
||||
/// Get holiday dates that fall within the course date range and would conflict with class meetings
|
||||
fn get_holiday_exceptions(meeting_time: &MeetingScheduleInfo) -> Vec<NaiveDate> {
|
||||
let mut exceptions = Vec::new();
|
||||
|
||||
// Get the year range from the course date range
|
||||
let start_year = meeting_time.date_range.start.year();
|
||||
let end_year = meeting_time.date_range.end.year();
|
||||
|
||||
for (_, holiday) in UNIVERSITY_HOLIDAYS {
|
||||
// Check for the holiday in each year of the course
|
||||
for year in start_year..=end_year {
|
||||
let holiday_dates = holiday.get_dates_for_year(year);
|
||||
|
||||
for holiday_date in holiday_dates {
|
||||
// Check if the holiday falls within the course date range
|
||||
if holiday_date >= meeting_time.date_range.start
|
||||
&& holiday_date <= meeting_time.date_range.end
|
||||
{
|
||||
// Check if the class would actually meet on this day
|
||||
if class_meets_on_date(meeting_time, holiday_date) {
|
||||
exceptions.push(holiday_date);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
exceptions
|
||||
}
|
||||
|
||||
/// Generate EXDATE property for holiday exceptions
|
||||
fn generate_exdate_property(
|
||||
exceptions: &[NaiveDate],
|
||||
start_time: chrono::DateTime<Utc>,
|
||||
) -> Option<String> {
|
||||
if exceptions.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut exdate_values = Vec::new();
|
||||
|
||||
for &exception_date in exceptions {
|
||||
// Create a datetime for the exception using the same time as the start time
|
||||
let exception_datetime = exception_date.and_time(start_time.time()).and_utc();
|
||||
|
||||
let exdate_str = exception_datetime.format("%Y%m%dT%H%M%SZ").to_string();
|
||||
exdate_values.push(exdate_str);
|
||||
}
|
||||
|
||||
Some(format!("EXDATE:{}", exdate_values.join(",")))
|
||||
}
|
||||
|
||||
/// Escape text for ICS format
|
||||
fn escape_ics_text(text: &str) -> String {
|
||||
text.replace("\\", "\\\\")
|
||||
.replace(";", "\\;")
|
||||
.replace(",", "\\,")
|
||||
.replace("\n", "\\n")
|
||||
.replace("\r", "")
|
||||
}
|
||||
|
||||
@@ -20,6 +20,6 @@ pub async fn time(
|
||||
))
|
||||
.await?;
|
||||
|
||||
info!("time command completed for CRN: {}", crn);
|
||||
info!(crn = %crn, "time command completed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -13,12 +13,12 @@ pub async fn get_course_by_crn(ctx: &Context<'_>, crn: i32) -> Result<Course> {
|
||||
let current_term_status = Term::get_current();
|
||||
let term = current_term_status.inner();
|
||||
|
||||
// Fetch live course data from Redis cache via AppState
|
||||
// Fetch live course data from database via AppState
|
||||
app_state
|
||||
.get_course_or_fetch(&term.to_string(), &crn.to_string())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(%e, crn, "failed to fetch course data");
|
||||
error!(error = %e, crn = %crn, "failed to fetch course data");
|
||||
e
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ use fundu::{DurationParser, TimeUnit};
|
||||
use serde::{Deserialize, Deserializer};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Application configuration loaded from environment variables
|
||||
/// Main application configuration containing all sub-configurations
|
||||
#[derive(Deserialize)]
|
||||
pub struct Config {
|
||||
/// Log level for the application
|
||||
@@ -20,19 +20,11 @@ pub struct Config {
|
||||
/// Defaults to "info" if not specified
|
||||
#[serde(default = "default_log_level")]
|
||||
pub log_level: String,
|
||||
/// Discord bot token for authentication
|
||||
pub bot_token: String,
|
||||
/// Port for the web server
|
||||
/// Port for the web server (default: 8080)
|
||||
#[serde(default = "default_port")]
|
||||
pub port: u16,
|
||||
/// Database connection URL
|
||||
pub database_url: String,
|
||||
/// Redis connection URL
|
||||
pub redis_url: String,
|
||||
/// Base URL for banner generation service
|
||||
pub banner_base_url: String,
|
||||
/// Target Discord guild ID where the bot operates
|
||||
pub bot_target_guild: u64,
|
||||
/// Graceful shutdown timeout duration
|
||||
///
|
||||
/// Accepts both numeric values (seconds) and duration strings
|
||||
@@ -42,6 +34,19 @@ pub struct Config {
|
||||
deserialize_with = "deserialize_duration"
|
||||
)]
|
||||
pub shutdown_timeout: Duration,
|
||||
/// Discord bot token for authentication
|
||||
pub bot_token: String,
|
||||
/// Target Discord guild ID where the bot operates
|
||||
pub bot_target_guild: u64,
|
||||
|
||||
/// Base URL for banner generation service
|
||||
///
|
||||
/// Defaults to "https://ssbprod.utsa.edu/StudentRegistrationSsb/ssb" if not specified
|
||||
#[serde(default = "default_banner_base_url")]
|
||||
pub banner_base_url: String,
|
||||
/// Rate limiting configuration for Banner API requests
|
||||
#[serde(default = "default_rate_limiting")]
|
||||
pub rate_limiting: RateLimitingConfig,
|
||||
}
|
||||
|
||||
/// Default log level of "info"
|
||||
@@ -49,9 +54,9 @@ fn default_log_level() -> String {
|
||||
"info".to_string()
|
||||
}
|
||||
|
||||
/// Default port of 3000
|
||||
/// Default port of 8080
|
||||
fn default_port() -> u16 {
|
||||
3000
|
||||
8080
|
||||
}
|
||||
|
||||
/// Default shutdown timeout of 8 seconds
|
||||
@@ -59,6 +64,67 @@ fn default_shutdown_timeout() -> Duration {
|
||||
Duration::from_secs(8)
|
||||
}
|
||||
|
||||
/// Default banner base URL
|
||||
fn default_banner_base_url() -> String {
|
||||
"https://ssbprod.utsa.edu/StudentRegistrationSsb/ssb".to_string()
|
||||
}
|
||||
|
||||
/// Rate limiting configuration for Banner API requests
|
||||
#[derive(Deserialize, Clone, Debug)]
|
||||
pub struct RateLimitingConfig {
|
||||
/// Requests per minute for session operations (very conservative)
|
||||
#[serde(default = "default_session_rpm")]
|
||||
pub session_rpm: u32,
|
||||
/// Requests per minute for search operations (moderate)
|
||||
#[serde(default = "default_search_rpm")]
|
||||
pub search_rpm: u32,
|
||||
/// Requests per minute for metadata operations (moderate)
|
||||
#[serde(default = "default_metadata_rpm")]
|
||||
pub metadata_rpm: u32,
|
||||
/// Requests per minute for reset operations (low priority)
|
||||
#[serde(default = "default_reset_rpm")]
|
||||
pub reset_rpm: u32,
|
||||
/// Burst allowance (extra requests allowed in short bursts)
|
||||
#[serde(default = "default_burst_allowance")]
|
||||
pub burst_allowance: u32,
|
||||
}
|
||||
|
||||
/// Default rate limiting configuration
|
||||
fn default_rate_limiting() -> RateLimitingConfig {
|
||||
RateLimitingConfig {
|
||||
session_rpm: default_session_rpm(),
|
||||
search_rpm: default_search_rpm(),
|
||||
metadata_rpm: default_metadata_rpm(),
|
||||
reset_rpm: default_reset_rpm(),
|
||||
burst_allowance: default_burst_allowance(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Default session requests per minute (6 = 1 every 10 seconds)
|
||||
fn default_session_rpm() -> u32 {
|
||||
6
|
||||
}
|
||||
|
||||
/// Default search requests per minute (30 = 1 every 2 seconds)
|
||||
fn default_search_rpm() -> u32 {
|
||||
30
|
||||
}
|
||||
|
||||
/// Default metadata requests per minute (20 = 1 every 3 seconds)
|
||||
fn default_metadata_rpm() -> u32 {
|
||||
20
|
||||
}
|
||||
|
||||
/// Default reset requests per minute (10 = 1 every 6 seconds)
|
||||
fn default_reset_rpm() -> u32 {
|
||||
10
|
||||
}
|
||||
|
||||
/// Default burst allowance (3 extra requests)
|
||||
fn default_burst_allowance() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
/// Duration parser configured to handle various time units with seconds as default
|
||||
///
|
||||
/// Supports:
|
||||
|
||||
243
src/formatter.rs
Normal file
243
src/formatter.rs
Normal file
@@ -0,0 +1,243 @@
|
||||
//! Custom tracing formatter
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value};
|
||||
use std::fmt;
|
||||
use time::macros::format_description;
|
||||
use time::{OffsetDateTime, format_description::FormatItem};
|
||||
use tracing::field::{Field, Visit};
|
||||
use tracing::{Event, Level, Subscriber};
|
||||
use tracing_subscriber::fmt::format::Writer;
|
||||
use tracing_subscriber::fmt::{FmtContext, FormatEvent, FormatFields, FormattedFields};
|
||||
use tracing_subscriber::registry::LookupSpan;
|
||||
|
||||
/// Cached format description for timestamps
|
||||
/// Uses 3 subsecond digits on Emscripten, 5 otherwise for better performance
|
||||
#[cfg(target_os = "emscripten")]
|
||||
const TIMESTAMP_FORMAT: &[FormatItem<'static>] =
|
||||
format_description!("[hour]:[minute]:[second].[subsecond digits:3]");
|
||||
|
||||
#[cfg(not(target_os = "emscripten"))]
|
||||
const TIMESTAMP_FORMAT: &[FormatItem<'static>] =
|
||||
format_description!("[hour]:[minute]:[second].[subsecond digits:5]");
|
||||
|
||||
/// A custom formatter with enhanced timestamp formatting
|
||||
///
|
||||
/// Re-implementation of the Full formatter with improved timestamp display.
|
||||
pub struct CustomPrettyFormatter;
|
||||
|
||||
/// A custom JSON formatter that flattens fields to root level
|
||||
///
|
||||
/// Outputs logs in the format: { "message": "...", "level": "...", "customAttribute": "..." }
|
||||
pub struct CustomJsonFormatter;
|
||||
|
||||
impl<S, N> FormatEvent<S, N> for CustomPrettyFormatter
|
||||
where
|
||||
S: Subscriber + for<'a> LookupSpan<'a>,
|
||||
N: for<'a> FormatFields<'a> + 'static,
|
||||
{
|
||||
fn format_event(
|
||||
&self,
|
||||
ctx: &FmtContext<'_, S, N>,
|
||||
mut writer: Writer<'_>,
|
||||
event: &Event<'_>,
|
||||
) -> fmt::Result {
|
||||
let meta = event.metadata();
|
||||
|
||||
// 1) Timestamp (dimmed when ANSI)
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let formatted_time = now.format(&TIMESTAMP_FORMAT).map_err(|e| {
|
||||
eprintln!("Failed to format timestamp: {}", e);
|
||||
fmt::Error
|
||||
})?;
|
||||
write_dimmed(&mut writer, formatted_time)?;
|
||||
writer.write_char(' ')?;
|
||||
|
||||
// 2) Colored 5-char level like Full
|
||||
write_colored_level(&mut writer, meta.level())?;
|
||||
writer.write_char(' ')?;
|
||||
|
||||
// 3) Span scope chain (bold names, fields in braces, dimmed ':')
|
||||
if let Some(scope) = ctx.event_scope() {
|
||||
let mut saw_any = false;
|
||||
for span in scope.from_root() {
|
||||
write_bold(&mut writer, span.metadata().name())?;
|
||||
saw_any = true;
|
||||
let ext = span.extensions();
|
||||
if let Some(fields) = &ext.get::<FormattedFields<N>>() {
|
||||
if !fields.is_empty() {
|
||||
write_bold(&mut writer, "{")?;
|
||||
write!(writer, "{}", fields)?;
|
||||
write_bold(&mut writer, "}")?;
|
||||
}
|
||||
}
|
||||
if writer.has_ansi_escapes() {
|
||||
write!(writer, "\x1b[2m:\x1b[0m")?;
|
||||
} else {
|
||||
writer.write_char(':')?;
|
||||
}
|
||||
}
|
||||
if saw_any {
|
||||
writer.write_char(' ')?;
|
||||
}
|
||||
}
|
||||
|
||||
// 4) Target (dimmed), then a space
|
||||
if writer.has_ansi_escapes() {
|
||||
write!(writer, "\x1b[2m{}\x1b[0m\x1b[2m:\x1b[0m ", meta.target())?;
|
||||
} else {
|
||||
write!(writer, "{}: ", meta.target())?;
|
||||
}
|
||||
|
||||
// 5) Event fields
|
||||
ctx.format_fields(writer.by_ref(), event)?;
|
||||
|
||||
// 6) Newline
|
||||
writeln!(writer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, N> FormatEvent<S, N> for CustomJsonFormatter
|
||||
where
|
||||
S: Subscriber + for<'a> LookupSpan<'a>,
|
||||
N: for<'a> FormatFields<'a> + 'static,
|
||||
{
|
||||
fn format_event(
|
||||
&self,
|
||||
_ctx: &FmtContext<'_, S, N>,
|
||||
mut writer: Writer<'_>,
|
||||
event: &Event<'_>,
|
||||
) -> fmt::Result {
|
||||
let meta = event.metadata();
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct EventFields {
|
||||
message: String,
|
||||
level: String,
|
||||
target: String,
|
||||
#[serde(flatten)]
|
||||
fields: Map<String, Value>,
|
||||
}
|
||||
|
||||
let (message, fields) = {
|
||||
let mut message: Option<String> = None;
|
||||
let mut fields: Map<String, Value> = Map::new();
|
||||
|
||||
struct FieldVisitor<'a> {
|
||||
message: &'a mut Option<String>,
|
||||
fields: &'a mut Map<String, Value>,
|
||||
}
|
||||
|
||||
impl<'a> Visit for FieldVisitor<'a> {
|
||||
fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
|
||||
let key = field.name();
|
||||
if key == "message" {
|
||||
*self.message = Some(format!("{:?}", value));
|
||||
} else {
|
||||
// Use typed methods for better performance
|
||||
self.fields
|
||||
.insert(key.to_string(), Value::String(format!("{:?}", value)));
|
||||
}
|
||||
}
|
||||
|
||||
fn record_str(&mut self, field: &Field, value: &str) {
|
||||
let key = field.name();
|
||||
if key == "message" {
|
||||
*self.message = Some(value.to_string());
|
||||
} else {
|
||||
self.fields
|
||||
.insert(key.to_string(), Value::String(value.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
fn record_i64(&mut self, field: &Field, value: i64) {
|
||||
let key = field.name();
|
||||
if key != "message" {
|
||||
self.fields.insert(
|
||||
key.to_string(),
|
||||
Value::Number(serde_json::Number::from(value)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn record_u64(&mut self, field: &Field, value: u64) {
|
||||
let key = field.name();
|
||||
if key != "message" {
|
||||
self.fields.insert(
|
||||
key.to_string(),
|
||||
Value::Number(serde_json::Number::from(value)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn record_bool(&mut self, field: &Field, value: bool) {
|
||||
let key = field.name();
|
||||
if key != "message" {
|
||||
self.fields.insert(key.to_string(), Value::Bool(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut visitor = FieldVisitor {
|
||||
message: &mut message,
|
||||
fields: &mut fields,
|
||||
};
|
||||
event.record(&mut visitor);
|
||||
|
||||
(message, fields)
|
||||
};
|
||||
|
||||
let json = EventFields {
|
||||
message: message.unwrap_or_default(),
|
||||
level: meta.level().to_string(),
|
||||
target: meta.target().to_string(),
|
||||
fields,
|
||||
};
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
"{}",
|
||||
serde_json::to_string(&json).unwrap_or_else(|_| "{}".to_string())
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Write the verbosity level with the same coloring/alignment as the Full formatter.
|
||||
fn write_colored_level(writer: &mut Writer<'_>, level: &Level) -> fmt::Result {
|
||||
if writer.has_ansi_escapes() {
|
||||
// Basic ANSI color sequences; reset with \x1b[0m
|
||||
let (color, text) = match *level {
|
||||
Level::TRACE => ("\x1b[35m", "TRACE"), // purple
|
||||
Level::DEBUG => ("\x1b[34m", "DEBUG"), // blue
|
||||
Level::INFO => ("\x1b[32m", " INFO"), // green, note leading space
|
||||
Level::WARN => ("\x1b[33m", " WARN"), // yellow, note leading space
|
||||
Level::ERROR => ("\x1b[31m", "ERROR"), // red
|
||||
};
|
||||
write!(writer, "{}{}\x1b[0m", color, text)
|
||||
} else {
|
||||
// Right-pad to width 5 like Full's non-ANSI mode
|
||||
match *level {
|
||||
Level::TRACE => write!(writer, "{:>5}", "TRACE"),
|
||||
Level::DEBUG => write!(writer, "{:>5}", "DEBUG"),
|
||||
Level::INFO => write!(writer, "{:>5}", " INFO"),
|
||||
Level::WARN => write!(writer, "{:>5}", " WARN"),
|
||||
Level::ERROR => write!(writer, "{:>5}", "ERROR"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_dimmed(writer: &mut Writer<'_>, s: impl fmt::Display) -> fmt::Result {
|
||||
if writer.has_ansi_escapes() {
|
||||
write!(writer, "\x1b[2m{}\x1b[0m", s)
|
||||
} else {
|
||||
write!(writer, "{}", s)
|
||||
}
|
||||
}
|
||||
|
||||
fn write_bold(writer: &mut Writer<'_>, s: impl fmt::Display) -> fmt::Result {
|
||||
if writer.has_ansi_escapes() {
|
||||
write!(writer, "\x1b[1m{}\x1b[0m", s)
|
||||
} else {
|
||||
write!(writer, "{}", s)
|
||||
}
|
||||
}
|
||||
300
src/main.rs
300
src/main.rs
@@ -1,4 +1,7 @@
|
||||
use serenity::all::{ClientBuilder, GatewayIntents};
|
||||
use clap::Parser;
|
||||
use figment::value::UncasedStr;
|
||||
use num_format::{Locale, ToFormattedString};
|
||||
use serenity::all::{ActivityData, ClientBuilder, Context, GatewayIntents};
|
||||
use tokio::signal;
|
||||
use tracing::{error, info, warn};
|
||||
use tracing_subscriber::{EnvFilter, FmtSubscriber};
|
||||
@@ -20,31 +23,183 @@ mod bot;
|
||||
mod config;
|
||||
mod data;
|
||||
mod error;
|
||||
mod formatter;
|
||||
mod scraper;
|
||||
mod services;
|
||||
mod state;
|
||||
mod web;
|
||||
|
||||
#[cfg(debug_assertions)]
|
||||
const DEFAULT_TRACING_FORMAT: TracingFormat = TracingFormat::Pretty;
|
||||
#[cfg(not(debug_assertions))]
|
||||
const DEFAULT_TRACING_FORMAT: TracingFormat = TracingFormat::Json;
|
||||
|
||||
/// Banner Discord Bot - Course availability monitoring
|
||||
///
|
||||
/// This application runs multiple services that can be controlled via CLI arguments:
|
||||
/// - bot: Discord bot for course monitoring commands
|
||||
/// - web: HTTP server for web interface and API
|
||||
/// - scraper: Background service for scraping course data
|
||||
///
|
||||
/// Use --services to specify which services to run, or --disable-services to exclude specific services.
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(author, version, about, long_about = None)]
|
||||
struct Args {
|
||||
/// Log formatter to use
|
||||
#[arg(long, value_enum, default_value_t = DEFAULT_TRACING_FORMAT)]
|
||||
tracing: TracingFormat,
|
||||
|
||||
/// Services to run (comma-separated). Default: all services
|
||||
///
|
||||
/// Examples:
|
||||
/// --services bot,web # Run only bot and web services
|
||||
/// --services scraper # Run only the scraper service
|
||||
#[arg(long, value_delimiter = ',', conflicts_with = "disable_services")]
|
||||
services: Option<Vec<ServiceName>>,
|
||||
|
||||
/// Services to disable (comma-separated)
|
||||
///
|
||||
/// Examples:
|
||||
/// --disable-services bot # Run web and scraper only
|
||||
/// --disable-services bot,web # Run only the scraper service
|
||||
#[arg(long, value_delimiter = ',', conflicts_with = "services")]
|
||||
disable_services: Option<Vec<ServiceName>>,
|
||||
}
|
||||
|
||||
#[derive(clap::ValueEnum, Clone, Debug)]
|
||||
enum TracingFormat {
|
||||
/// Use pretty formatter (default in debug mode)
|
||||
Pretty,
|
||||
/// Use JSON formatter (default in release mode)
|
||||
Json,
|
||||
}
|
||||
|
||||
#[derive(clap::ValueEnum, Clone, Debug, PartialEq)]
|
||||
enum ServiceName {
|
||||
/// Discord bot for course monitoring commands
|
||||
Bot,
|
||||
/// HTTP server for web interface and API
|
||||
Web,
|
||||
/// Background service for scraping course data
|
||||
Scraper,
|
||||
}
|
||||
|
||||
impl ServiceName {
|
||||
/// Get all available services
|
||||
fn all() -> Vec<ServiceName> {
|
||||
vec![ServiceName::Bot, ServiceName::Web, ServiceName::Scraper]
|
||||
}
|
||||
|
||||
/// Convert to string for service registration
|
||||
fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
ServiceName::Bot => "bot",
|
||||
ServiceName::Web => "web",
|
||||
ServiceName::Scraper => "scraper",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Determine which services should be enabled based on CLI arguments
|
||||
fn determine_enabled_services(args: &Args) -> Result<Vec<ServiceName>, anyhow::Error> {
|
||||
match (&args.services, &args.disable_services) {
|
||||
(Some(services), None) => {
|
||||
// User specified which services to run
|
||||
Ok(services.clone())
|
||||
}
|
||||
(None, Some(disabled)) => {
|
||||
// User specified which services to disable
|
||||
let enabled: Vec<ServiceName> = ServiceName::all()
|
||||
.into_iter()
|
||||
.filter(|s| !disabled.contains(s))
|
||||
.collect();
|
||||
Ok(enabled)
|
||||
}
|
||||
(None, None) => {
|
||||
// Default: run all services
|
||||
Ok(ServiceName::all())
|
||||
}
|
||||
(Some(_), Some(_)) => {
|
||||
// This should be prevented by clap's conflicts_with, but just in case
|
||||
Err(anyhow::anyhow!(
|
||||
"Cannot specify both --services and --disable-services"
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn update_bot_status(ctx: &Context, app_state: &AppState) -> Result<(), anyhow::Error> {
|
||||
let course_count = app_state.get_course_count().await?;
|
||||
|
||||
ctx.set_activity(Some(ActivityData::playing(format!(
|
||||
"Querying {:} classes",
|
||||
course_count.to_formatted_string(&Locale::en)
|
||||
))));
|
||||
|
||||
tracing::info!(course_count = course_count, "Updated bot status");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
// Configure logging
|
||||
let filter =
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("warn,banner=debug"));
|
||||
let subscriber = {
|
||||
#[cfg(debug_assertions)]
|
||||
{
|
||||
// Parse CLI arguments
|
||||
let args = Args::parse();
|
||||
|
||||
// Determine which services should be enabled
|
||||
let enabled_services: Vec<ServiceName> =
|
||||
determine_enabled_services(&args).expect("Failed to determine enabled services");
|
||||
|
||||
info!(
|
||||
enabled_services = ?enabled_services,
|
||||
"services configuration loaded"
|
||||
);
|
||||
|
||||
// Load configuration first to get log level
|
||||
let config: Config = Figment::new()
|
||||
.merge(Env::raw().map(|k| {
|
||||
if k == UncasedStr::new("RAILWAY_DEPLOYMENT_DRAINING_SECONDS") {
|
||||
"SHUTDOWN_TIMEOUT".into()
|
||||
} else {
|
||||
k.into()
|
||||
}
|
||||
}))
|
||||
.extract()
|
||||
.expect("Failed to load config");
|
||||
|
||||
// Configure logging based on config
|
||||
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| {
|
||||
let base_level = &config.log_level;
|
||||
EnvFilter::new(format!(
|
||||
"warn,banner={},banner::rate_limiter=warn,banner::session=warn,banner::rate_limit_middleware=warn",
|
||||
base_level
|
||||
))
|
||||
});
|
||||
|
||||
// Select formatter based on CLI args
|
||||
let use_pretty = match args.tracing {
|
||||
TracingFormat::Pretty => true,
|
||||
TracingFormat::Json => false,
|
||||
};
|
||||
|
||||
let subscriber: Box<dyn tracing::Subscriber + Send + Sync> = if use_pretty {
|
||||
Box::new(
|
||||
FmtSubscriber::builder()
|
||||
}
|
||||
#[cfg(not(debug_assertions))]
|
||||
{
|
||||
FmtSubscriber::builder().json()
|
||||
}
|
||||
}
|
||||
.with_env_filter(filter)
|
||||
.with_target(true)
|
||||
.finish();
|
||||
.with_target(true)
|
||||
.event_format(formatter::CustomPrettyFormatter)
|
||||
.with_env_filter(filter)
|
||||
.finish(),
|
||||
)
|
||||
} else {
|
||||
Box::new(
|
||||
FmtSubscriber::builder()
|
||||
.with_target(true)
|
||||
.event_format(formatter::CustomJsonFormatter)
|
||||
.with_env_filter(filter)
|
||||
.finish(),
|
||||
)
|
||||
};
|
||||
tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed");
|
||||
|
||||
// Log application startup context
|
||||
@@ -58,12 +213,6 @@ async fn main() {
|
||||
"starting banner"
|
||||
);
|
||||
|
||||
let config: Config = Figment::new()
|
||||
.merge(Env::raw().only(&["DATABASE_URL"]))
|
||||
.merge(Env::prefixed("APP_"))
|
||||
.extract()
|
||||
.expect("Failed to load config");
|
||||
|
||||
// Create database connection pool
|
||||
let db_pool = PgPoolOptions::new()
|
||||
.max_connections(10)
|
||||
@@ -79,12 +228,14 @@ async fn main() {
|
||||
);
|
||||
|
||||
// Create BannerApi and AppState
|
||||
let banner_api =
|
||||
BannerApi::new(config.banner_base_url.clone()).expect("Failed to create BannerApi");
|
||||
let banner_api = BannerApi::new_with_config(
|
||||
config.banner_base_url.clone(),
|
||||
config.rate_limiting.clone().into(),
|
||||
)
|
||||
.expect("Failed to create BannerApi");
|
||||
|
||||
let banner_api_arc = Arc::new(banner_api);
|
||||
let app_state = AppState::new(banner_api_arc.clone(), &config.redis_url)
|
||||
.expect("Failed to create AppState");
|
||||
let app_state = AppState::new(banner_api_arc.clone(), db_pool.clone());
|
||||
|
||||
// Create BannerState for web service
|
||||
let banner_state = BannerState {
|
||||
@@ -118,7 +269,7 @@ async fn main() {
|
||||
span.record("msg.author", ctx.author().tag().as_str());
|
||||
span.record("msg.id", ctx.id());
|
||||
span.record("msg.channel_id", ctx.channel_id().get());
|
||||
span.record("msg.channel", &channel_name.as_str());
|
||||
span.record("msg.channel", channel_name.as_str());
|
||||
|
||||
tracing::info!(
|
||||
command_name = ctx.command().qualified_name.as_str(),
|
||||
@@ -138,7 +289,7 @@ async fn main() {
|
||||
on_error: |error| {
|
||||
Box::pin(async move {
|
||||
if let Err(e) = poise::builtins::on_error(error).await {
|
||||
tracing::error!("Fatal error while sending error message: {}", e);
|
||||
tracing::error!(error = %e, "Fatal error while sending error message");
|
||||
}
|
||||
// error!(error = ?error, "command error");
|
||||
})
|
||||
@@ -155,6 +306,27 @@ async fn main() {
|
||||
)
|
||||
.await?;
|
||||
poise::builtins::register_globally(ctx, &framework.options().commands).await?;
|
||||
|
||||
// Start status update task
|
||||
let status_app_state = app_state.clone();
|
||||
let status_ctx = ctx.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(30));
|
||||
|
||||
// Update status immediately on startup
|
||||
if let Err(e) = update_bot_status(&status_ctx, &status_app_state).await {
|
||||
tracing::error!(error = %e, "Failed to update status on startup");
|
||||
}
|
||||
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
if let Err(e) = update_bot_status(&status_ctx, &status_app_state).await {
|
||||
tracing::error!(error = %e, "Failed to update bot status");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Data { app_state })
|
||||
})
|
||||
})
|
||||
@@ -172,19 +344,33 @@ async fn main() {
|
||||
// Create service manager
|
||||
let mut service_manager = ServiceManager::new();
|
||||
|
||||
// Register services with the manager
|
||||
let bot_service = Box::new(BotService::new(client));
|
||||
let web_service = Box::new(WebService::new(port, banner_state));
|
||||
let scraper_service = Box::new(ScraperService::new(db_pool.clone(), banner_api_arc.clone()));
|
||||
// Register enabled services with the manager
|
||||
if enabled_services.contains(&ServiceName::Bot) {
|
||||
let bot_service = Box::new(BotService::new(client));
|
||||
service_manager.register_service(ServiceName::Bot.as_str(), bot_service);
|
||||
}
|
||||
|
||||
service_manager.register_service("bot", bot_service);
|
||||
service_manager.register_service("web", web_service);
|
||||
service_manager.register_service("scraper", scraper_service);
|
||||
if enabled_services.contains(&ServiceName::Web) {
|
||||
let web_service = Box::new(WebService::new(port, banner_state));
|
||||
service_manager.register_service(ServiceName::Web.as_str(), web_service);
|
||||
}
|
||||
|
||||
if enabled_services.contains(&ServiceName::Scraper) {
|
||||
let scraper_service =
|
||||
Box::new(ScraperService::new(db_pool.clone(), banner_api_arc.clone()));
|
||||
service_manager.register_service(ServiceName::Scraper.as_str(), scraper_service);
|
||||
}
|
||||
|
||||
// Check if any services are enabled
|
||||
if !service_manager.has_services() {
|
||||
error!("No services enabled. Cannot start application.");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
// Spawn all registered services
|
||||
service_manager.spawn_all();
|
||||
|
||||
// Set up CTRL+C signal handling
|
||||
// Set up signal handling for both SIGINT (Ctrl+C) and SIGTERM
|
||||
let ctrl_c = async {
|
||||
signal::ctrl_c()
|
||||
.await
|
||||
@@ -192,7 +378,23 @@ async fn main() {
|
||||
info!("received ctrl+c, gracefully shutting down...");
|
||||
};
|
||||
|
||||
// Main application loop - wait for services or CTRL+C
|
||||
#[cfg(unix)]
|
||||
let sigterm = async {
|
||||
use tokio::signal::unix::{SignalKind, signal};
|
||||
let mut sigterm_stream =
|
||||
signal(SignalKind::terminate()).expect("Failed to install SIGTERM signal handler");
|
||||
sigterm_stream.recv().await;
|
||||
info!("received SIGTERM, gracefully shutting down...");
|
||||
};
|
||||
|
||||
#[cfg(not(unix))]
|
||||
let sigterm = async {
|
||||
// On non-Unix systems, create a future that never completes
|
||||
// This ensures the select! macro works correctly
|
||||
std::future::pending::<()>().await;
|
||||
};
|
||||
|
||||
// Main application loop - wait for services or signals
|
||||
let mut exit_code = 0;
|
||||
|
||||
tokio::select! {
|
||||
@@ -234,7 +436,7 @@ async fn main() {
|
||||
}
|
||||
}
|
||||
_ = ctrl_c => {
|
||||
// User requested shutdown
|
||||
// User requested shutdown via Ctrl+C
|
||||
info!("user requested shutdown via ctrl+c");
|
||||
match service_manager.shutdown(shutdown_timeout).await {
|
||||
Ok(elapsed) => {
|
||||
@@ -255,6 +457,28 @@ async fn main() {
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = sigterm => {
|
||||
// System requested shutdown via SIGTERM
|
||||
info!("system requested shutdown via SIGTERM");
|
||||
match service_manager.shutdown(shutdown_timeout).await {
|
||||
Ok(elapsed) => {
|
||||
info!(
|
||||
remaining = format!("{:.2?}", shutdown_timeout - elapsed),
|
||||
"graceful shutdown complete"
|
||||
);
|
||||
info!("graceful shutdown complete");
|
||||
}
|
||||
Err(pending_services) => {
|
||||
warn!(
|
||||
pending_count = pending_services.len(),
|
||||
pending_services = ?pending_services,
|
||||
"graceful shutdown elapsed - {} service(s) did not complete",
|
||||
pending_services.len()
|
||||
);
|
||||
exit_code = 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!(exit_code, "application shutdown complete");
|
||||
|
||||
112
src/scraper/jobs/mod.rs
Normal file
112
src/scraper/jobs/mod.rs
Normal file
@@ -0,0 +1,112 @@
|
||||
pub mod subject;
|
||||
|
||||
use crate::banner::BannerApi;
|
||||
use crate::data::models::TargetType;
|
||||
use crate::error::Result;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::PgPool;
|
||||
use std::fmt;
|
||||
|
||||
/// Errors that can occur during job parsing
|
||||
#[derive(Debug)]
|
||||
pub enum JobParseError {
|
||||
InvalidJson(serde_json::Error),
|
||||
UnsupportedTargetType(TargetType),
|
||||
MissingRequiredField(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for JobParseError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
JobParseError::InvalidJson(e) => write!(f, "Invalid JSON in job payload: {}", e),
|
||||
JobParseError::UnsupportedTargetType(t) => {
|
||||
write!(f, "Unsupported target type: {:?}", t)
|
||||
}
|
||||
JobParseError::MissingRequiredField(field) => {
|
||||
write!(f, "Missing required field: {}", field)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for JobParseError {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
match self {
|
||||
JobParseError::InvalidJson(e) => Some(e),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Errors that can occur during job processing
|
||||
#[derive(Debug)]
|
||||
pub enum JobError {
|
||||
Recoverable(anyhow::Error), // API failures, network issues
|
||||
Unrecoverable(anyhow::Error), // Parse errors, corrupted data
|
||||
}
|
||||
|
||||
impl fmt::Display for JobError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
JobError::Recoverable(e) => write!(f, "Recoverable error: {}", e),
|
||||
JobError::Unrecoverable(e) => write!(f, "Unrecoverable error: {}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for JobError {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
match self {
|
||||
JobError::Recoverable(e) => e.source(),
|
||||
JobError::Unrecoverable(e) => e.source(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Common trait interface for all job types
|
||||
#[async_trait::async_trait]
|
||||
pub trait Job: Send + Sync {
|
||||
/// The target type this job handles
|
||||
fn target_type(&self) -> TargetType;
|
||||
|
||||
/// Process the job with the given API client and database pool
|
||||
async fn process(&self, banner_api: &BannerApi, db_pool: &PgPool) -> Result<()>;
|
||||
|
||||
/// Get a human-readable description of the job
|
||||
fn description(&self) -> String;
|
||||
}
|
||||
|
||||
/// Main job enum that dispatches to specific job implementations
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum JobType {
|
||||
Subject(subject::SubjectJob),
|
||||
}
|
||||
|
||||
impl JobType {
|
||||
/// Create a job from the target type and payload
|
||||
pub fn from_target_type_and_payload(
|
||||
target_type: TargetType,
|
||||
payload: serde_json::Value,
|
||||
) -> Result<Self, JobParseError> {
|
||||
match target_type {
|
||||
TargetType::Subject => {
|
||||
let subject_job: subject::SubjectJob =
|
||||
serde_json::from_value(payload).map_err(JobParseError::InvalidJson)?;
|
||||
Ok(JobType::Subject(subject_job))
|
||||
}
|
||||
_ => Err(JobParseError::UnsupportedTargetType(target_type)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert to a Job trait object
|
||||
pub fn as_job(self) -> Box<dyn Job> {
|
||||
match self {
|
||||
JobType::Subject(job) => Box::new(job),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper function to create a subject job
|
||||
pub fn create_subject_job(subject: String) -> JobType {
|
||||
JobType::Subject(subject::SubjectJob::new(subject))
|
||||
}
|
||||
93
src/scraper/jobs/subject.rs
Normal file
93
src/scraper/jobs/subject.rs
Normal file
@@ -0,0 +1,93 @@
|
||||
use super::Job;
|
||||
use crate::banner::{BannerApi, Course, SearchQuery, Term};
|
||||
use crate::data::models::TargetType;
|
||||
use crate::error::Result;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::PgPool;
|
||||
use tracing::{debug, info, trace};
|
||||
|
||||
/// Job implementation for scraping subject data
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SubjectJob {
|
||||
pub subject: String,
|
||||
}
|
||||
|
||||
impl SubjectJob {
|
||||
pub fn new(subject: String) -> Self {
|
||||
Self { subject }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Job for SubjectJob {
|
||||
fn target_type(&self) -> TargetType {
|
||||
TargetType::Subject
|
||||
}
|
||||
|
||||
async fn process(&self, banner_api: &BannerApi, db_pool: &PgPool) -> Result<()> {
|
||||
let subject_code = &self.subject;
|
||||
debug!(subject = subject_code, "Processing subject job");
|
||||
|
||||
// Get the current term
|
||||
let term = Term::get_current().inner().to_string();
|
||||
let query = SearchQuery::new().subject(subject_code).max_results(500);
|
||||
|
||||
let search_result = banner_api
|
||||
.search(&term, &query, "subjectDescription", false)
|
||||
.await?;
|
||||
|
||||
if let Some(courses_from_api) = search_result.data {
|
||||
info!(
|
||||
subject = subject_code,
|
||||
count = courses_from_api.len(),
|
||||
"Found courses"
|
||||
);
|
||||
for course in courses_from_api {
|
||||
self.upsert_course(&course, db_pool).await?;
|
||||
}
|
||||
}
|
||||
|
||||
debug!(subject = subject_code, "Subject job completed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn description(&self) -> String {
|
||||
format!("Scrape subject: {}", self.subject)
|
||||
}
|
||||
}
|
||||
|
||||
impl SubjectJob {
|
||||
async fn upsert_course(&self, course: &Course, db_pool: &PgPool) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO courses (crn, subject, course_number, title, term_code, enrollment, max_enrollment, wait_count, wait_capacity, last_scraped_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
ON CONFLICT (crn, term_code) DO UPDATE SET
|
||||
subject = EXCLUDED.subject,
|
||||
course_number = EXCLUDED.course_number,
|
||||
title = EXCLUDED.title,
|
||||
enrollment = EXCLUDED.enrollment,
|
||||
max_enrollment = EXCLUDED.max_enrollment,
|
||||
wait_count = EXCLUDED.wait_count,
|
||||
wait_capacity = EXCLUDED.wait_capacity,
|
||||
last_scraped_at = EXCLUDED.last_scraped_at
|
||||
"#,
|
||||
)
|
||||
.bind(&course.course_reference_number)
|
||||
.bind(&course.subject)
|
||||
.bind(&course.course_number)
|
||||
.bind(&course.course_title)
|
||||
.bind(&course.term)
|
||||
.bind(course.enrollment)
|
||||
.bind(course.maximum_enrollment)
|
||||
.bind(course.wait_count)
|
||||
.bind(course.wait_capacity)
|
||||
.bind(chrono::Utc::now())
|
||||
.execute(db_pool)
|
||||
.await
|
||||
.map(|result| {
|
||||
trace!(subject = course.subject, crn = course.course_reference_number, result = ?result, "Course upserted");
|
||||
})
|
||||
.map_err(|e| anyhow::anyhow!("Failed to upsert course: {e}"))
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod jobs;
|
||||
pub mod scheduler;
|
||||
pub mod worker;
|
||||
|
||||
@@ -35,14 +36,14 @@ impl ScraperService {
|
||||
|
||||
/// Starts the scheduler and a pool of workers.
|
||||
pub fn start(&mut self) {
|
||||
info!("ScraperService starting...");
|
||||
info!("ScraperService starting");
|
||||
|
||||
let scheduler = Scheduler::new(self.db_pool.clone(), self.banner_api.clone());
|
||||
let scheduler_handle = tokio::spawn(async move {
|
||||
scheduler.run().await;
|
||||
});
|
||||
self.scheduler_handle = Some(scheduler_handle);
|
||||
info!("Scheduler task spawned.");
|
||||
info!("Scheduler task spawned");
|
||||
|
||||
let worker_count = 4; // This could be configurable
|
||||
for i in 0..worker_count {
|
||||
@@ -52,19 +53,22 @@ impl ScraperService {
|
||||
});
|
||||
self.worker_handles.push(worker_handle);
|
||||
}
|
||||
info!("Spawned {} worker tasks.", self.worker_handles.len());
|
||||
info!(
|
||||
worker_count = self.worker_handles.len(),
|
||||
"Spawned worker tasks"
|
||||
);
|
||||
}
|
||||
|
||||
/// Signals all child tasks to gracefully shut down.
|
||||
pub async fn shutdown(&mut self) {
|
||||
info!("Shutting down scraper service...");
|
||||
info!("Shutting down scraper service");
|
||||
if let Some(handle) = self.scheduler_handle.take() {
|
||||
handle.abort();
|
||||
}
|
||||
for handle in self.worker_handles.drain(..) {
|
||||
handle.abort();
|
||||
}
|
||||
info!("Scraper service shutdown.");
|
||||
info!("Scraper service shutdown");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
use crate::banner::{BannerApi, Term};
|
||||
use crate::data::models::{ScrapePriority, TargetType};
|
||||
use crate::error::Result;
|
||||
use crate::scraper::jobs::subject::SubjectJob;
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::time;
|
||||
use tracing::{error, info};
|
||||
use tracing::{debug, error, info, trace};
|
||||
|
||||
/// Periodically analyzes data and enqueues prioritized scrape jobs.
|
||||
pub struct Scheduler {
|
||||
@@ -24,12 +25,12 @@ impl Scheduler {
|
||||
|
||||
/// Runs the scheduler's main loop.
|
||||
pub async fn run(&self) {
|
||||
info!("Scheduler service started.");
|
||||
info!("Scheduler service started");
|
||||
let mut interval = time::interval(Duration::from_secs(60)); // Runs every minute
|
||||
|
||||
loop {
|
||||
interval.tick().await;
|
||||
info!("Scheduler waking up to analyze and schedule jobs...");
|
||||
// Scheduler analyzing data...
|
||||
if let Err(e) = self.schedule_jobs().await {
|
||||
error!(error = ?e, "Failed to schedule jobs");
|
||||
}
|
||||
@@ -40,46 +41,80 @@ impl Scheduler {
|
||||
async fn schedule_jobs(&self) -> Result<()> {
|
||||
// For now, we will implement a simple baseline scheduling strategy:
|
||||
// 1. Get a list of all subjects from the Banner API.
|
||||
// 2. For each subject, check if an active (not locked, not completed) job already exists.
|
||||
// 3. If no job exists, create a new, low-priority job to be executed in the near future.
|
||||
// 2. Query existing jobs for all subjects in a single query.
|
||||
// 3. Create new jobs only for subjects that don't have existing jobs.
|
||||
let term = Term::get_current().inner().to_string();
|
||||
|
||||
info!(
|
||||
term = term,
|
||||
"[Scheduler] Enqueuing baseline subject scrape jobs..."
|
||||
);
|
||||
debug!(term = term, "Enqueuing subject jobs");
|
||||
|
||||
let subjects = self.banner_api.get_subjects("", &term, 1, 500).await?;
|
||||
debug!(
|
||||
subject_count = subjects.len(),
|
||||
"Retrieved subjects from API"
|
||||
);
|
||||
|
||||
for subject in subjects {
|
||||
let payload = json!({ "subject": subject.code });
|
||||
// Create payloads for all subjects
|
||||
let subject_payloads: Vec<_> = subjects
|
||||
.iter()
|
||||
.map(|subject| json!({ "subject": subject.code }))
|
||||
.collect();
|
||||
|
||||
let existing_job: Option<(i32,)> = sqlx::query_as(
|
||||
"SELECT id FROM scrape_jobs WHERE target_type = $1 AND target_payload = $2 AND locked_at IS NULL"
|
||||
)
|
||||
.bind(TargetType::Subject)
|
||||
.bind(&payload)
|
||||
.fetch_optional(&self.db_pool)
|
||||
.await?;
|
||||
// Query existing jobs for all subjects in a single query
|
||||
let existing_jobs: Vec<(serde_json::Value,)> = sqlx::query_as(
|
||||
"SELECT target_payload FROM scrape_jobs
|
||||
WHERE target_type = $1 AND target_payload = ANY($2) AND locked_at IS NULL",
|
||||
)
|
||||
.bind(TargetType::Subject)
|
||||
.bind(&subject_payloads)
|
||||
.fetch_all(&self.db_pool)
|
||||
.await?;
|
||||
|
||||
if existing_job.is_some() {
|
||||
continue;
|
||||
// Convert to a HashSet for efficient lookup
|
||||
let existing_payloads: std::collections::HashSet<String> = existing_jobs
|
||||
.into_iter()
|
||||
.map(|(payload,)| payload.to_string())
|
||||
.collect();
|
||||
|
||||
// Filter out subjects that already have jobs and prepare new jobs
|
||||
let new_jobs: Vec<_> = subjects
|
||||
.into_iter()
|
||||
.filter_map(|subject| {
|
||||
let job = SubjectJob::new(subject.code.clone());
|
||||
let payload = serde_json::to_value(&job).unwrap();
|
||||
let payload_str = payload.to_string();
|
||||
|
||||
if existing_payloads.contains(&payload_str) {
|
||||
trace!(subject = subject.code, "Job already exists, skipping");
|
||||
None
|
||||
} else {
|
||||
Some((payload, subject.code))
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Insert all new jobs in a single batch
|
||||
if !new_jobs.is_empty() {
|
||||
let now = chrono::Utc::now();
|
||||
let mut tx = self.db_pool.begin().await?;
|
||||
|
||||
for (payload, subject_code) in new_jobs {
|
||||
sqlx::query(
|
||||
"INSERT INTO scrape_jobs (target_type, target_payload, priority, execute_at) VALUES ($1, $2, $3, $4)"
|
||||
)
|
||||
.bind(TargetType::Subject)
|
||||
.bind(&payload)
|
||||
.bind(ScrapePriority::Low)
|
||||
.bind(now)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
debug!(subject = subject_code, "New job enqueued for subject");
|
||||
}
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO scrape_jobs (target_type, target_payload, priority, execute_at) VALUES ($1, $2, $3, $4)"
|
||||
)
|
||||
.bind(TargetType::Subject)
|
||||
.bind(&payload)
|
||||
.bind(ScrapePriority::Low)
|
||||
.bind(chrono::Utc::now())
|
||||
.execute(&self.db_pool)
|
||||
.await?;
|
||||
|
||||
info!(subject = subject.code, "[Scheduler] Enqueued new job");
|
||||
tx.commit().await?;
|
||||
}
|
||||
|
||||
info!("[Scheduler] Job scheduling complete.");
|
||||
debug!("Job scheduling complete");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
use crate::banner::{BannerApi, BannerApiError, Course, SearchQuery, Term};
|
||||
use crate::banner::{BannerApi, BannerApiError};
|
||||
use crate::data::models::ScrapeJob;
|
||||
use crate::error::Result;
|
||||
use serde_json::Value;
|
||||
use crate::scraper::jobs::{JobError, JobType};
|
||||
use sqlx::PgPool;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::time;
|
||||
use tracing::{error, info, warn};
|
||||
use tracing::{debug, error, info, trace, warn};
|
||||
|
||||
/// A single worker instance.
|
||||
///
|
||||
@@ -34,44 +34,65 @@ impl Worker {
|
||||
match self.fetch_and_lock_job().await {
|
||||
Ok(Some(job)) => {
|
||||
let job_id = job.id;
|
||||
info!(worker_id = self.id, job_id = job.id, "Processing job");
|
||||
if let Err(e) = self.process_job(job).await {
|
||||
// Check if the error is due to an invalid session
|
||||
if let Some(BannerApiError::InvalidSession(_)) =
|
||||
e.downcast_ref::<BannerApiError>()
|
||||
{
|
||||
warn!(
|
||||
worker_id = self.id,
|
||||
job_id, "Invalid session detected. Forcing session refresh."
|
||||
);
|
||||
} else {
|
||||
error!(worker_id = self.id, job_id, error = ?e, "Failed to process job");
|
||||
debug!(worker_id = self.id, job_id = job.id, "Processing job");
|
||||
match self.process_job(job).await {
|
||||
Ok(()) => {
|
||||
debug!(worker_id = self.id, job_id, "Job completed");
|
||||
// If successful, delete the job.
|
||||
if let Err(delete_err) = self.delete_job(job_id).await {
|
||||
error!(
|
||||
worker_id = self.id,
|
||||
job_id,
|
||||
?delete_err,
|
||||
"Failed to delete job"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(JobError::Recoverable(e)) => {
|
||||
// Check if the error is due to an invalid session
|
||||
if let Some(BannerApiError::InvalidSession(_)) =
|
||||
e.downcast_ref::<BannerApiError>()
|
||||
{
|
||||
warn!(
|
||||
worker_id = self.id,
|
||||
job_id, "Invalid session detected. Forcing session refresh."
|
||||
);
|
||||
} else {
|
||||
error!(worker_id = self.id, job_id, error = ?e, "Failed to process job");
|
||||
}
|
||||
|
||||
// Unlock the job so it can be retried
|
||||
if let Err(unlock_err) = self.unlock_job(job_id).await {
|
||||
error!(
|
||||
worker_id = self.id,
|
||||
job_id,
|
||||
?unlock_err,
|
||||
"Failed to unlock job"
|
||||
);
|
||||
// Unlock the job so it can be retried
|
||||
if let Err(unlock_err) = self.unlock_job(job_id).await {
|
||||
error!(
|
||||
worker_id = self.id,
|
||||
job_id,
|
||||
?unlock_err,
|
||||
"Failed to unlock job"
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
info!(worker_id = self.id, job_id, "Job processed successfully");
|
||||
// If successful, delete the job.
|
||||
if let Err(delete_err) = self.delete_job(job_id).await {
|
||||
Err(JobError::Unrecoverable(e)) => {
|
||||
error!(
|
||||
worker_id = self.id,
|
||||
job_id,
|
||||
?delete_err,
|
||||
"Failed to delete job"
|
||||
error = ?e,
|
||||
"Job corrupted, deleting"
|
||||
);
|
||||
// Parse errors are unrecoverable - delete the job
|
||||
if let Err(delete_err) = self.delete_job(job_id).await {
|
||||
error!(
|
||||
worker_id = self.id,
|
||||
job_id,
|
||||
?delete_err,
|
||||
"Failed to delete corrupted job"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
// No job found, wait for a bit before polling again.
|
||||
trace!(worker_id = self.id, "No jobs available, waiting");
|
||||
time::sleep(Duration::from_secs(5)).await;
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -108,79 +129,26 @@ impl Worker {
|
||||
Ok(job)
|
||||
}
|
||||
|
||||
async fn process_job(&self, job: ScrapeJob) -> Result<()> {
|
||||
match job.target_type {
|
||||
crate::data::models::TargetType::Subject => {
|
||||
self.process_subject_job(&job.target_payload).await
|
||||
}
|
||||
_ => {
|
||||
warn!(worker_id = self.id, job_id = job.id, "unhandled job type");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
async fn process_job(&self, job: ScrapeJob) -> Result<(), JobError> {
|
||||
// Convert the database job to our job type
|
||||
let job_type = JobType::from_target_type_and_payload(job.target_type, job.target_payload)
|
||||
.map_err(|e| JobError::Unrecoverable(anyhow::anyhow!(e)))?; // Parse errors are unrecoverable
|
||||
|
||||
async fn process_subject_job(&self, payload: &Value) -> Result<()> {
|
||||
let subject_code = payload["subject"]
|
||||
.as_str()
|
||||
.ok_or_else(|| anyhow::anyhow!("Invalid subject payload"))?;
|
||||
info!(
|
||||
// Get the job implementation
|
||||
let job_impl = job_type.as_job();
|
||||
|
||||
debug!(
|
||||
worker_id = self.id,
|
||||
subject = subject_code,
|
||||
"Processing subject job"
|
||||
job_id = job.id,
|
||||
description = job_impl.description(),
|
||||
"Processing job"
|
||||
);
|
||||
|
||||
let term = Term::get_current().inner().to_string();
|
||||
let query = SearchQuery::new().subject(subject_code).max_results(500);
|
||||
|
||||
let search_result = self
|
||||
.banner_api
|
||||
.search(&term, &query, "subjectDescription", false)
|
||||
.await?;
|
||||
|
||||
if let Some(courses_from_api) = search_result.data {
|
||||
info!(
|
||||
worker_id = self.id,
|
||||
subject = subject_code,
|
||||
count = courses_from_api.len(),
|
||||
"Found courses to upsert"
|
||||
);
|
||||
for course in courses_from_api {
|
||||
self.upsert_course(&course).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn upsert_course(&self, course: &Course) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO courses (crn, subject, course_number, title, term_code, enrollment, max_enrollment, wait_count, wait_capacity, last_scraped_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
ON CONFLICT (crn, term_code) DO UPDATE SET
|
||||
subject = EXCLUDED.subject,
|
||||
course_number = EXCLUDED.course_number,
|
||||
title = EXCLUDED.title,
|
||||
enrollment = EXCLUDED.enrollment,
|
||||
max_enrollment = EXCLUDED.max_enrollment,
|
||||
wait_count = EXCLUDED.wait_count,
|
||||
wait_capacity = EXCLUDED.wait_capacity,
|
||||
last_scraped_at = EXCLUDED.last_scraped_at
|
||||
"#,
|
||||
)
|
||||
.bind(&course.course_reference_number)
|
||||
.bind(&course.subject)
|
||||
.bind(&course.course_number)
|
||||
.bind(&course.course_title)
|
||||
.bind(&course.term)
|
||||
.bind(course.enrollment)
|
||||
.bind(course.maximum_enrollment)
|
||||
.bind(course.wait_count)
|
||||
.bind(course.wait_capacity)
|
||||
.bind(chrono::Utc::now())
|
||||
.execute(&self.db_pool)
|
||||
.await?;
|
||||
// Process the job - API errors are recoverable
|
||||
job_impl
|
||||
.process(&self.banner_api, &self.db_pool)
|
||||
.await
|
||||
.map_err(JobError::Recoverable)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -190,7 +158,6 @@ impl Worker {
|
||||
.bind(job_id)
|
||||
.execute(&self.db_pool)
|
||||
.await?;
|
||||
info!(worker_id = self.id, job_id, "Job deleted");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -199,7 +166,7 @@ impl Worker {
|
||||
.bind(job_id)
|
||||
.execute(&self.db_pool)
|
||||
.await?;
|
||||
info!(worker_id = self.id, job_id, "Job unlocked after failure");
|
||||
info!(worker_id = self.id, job_id, "Job unlocked for retry");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use super::Service;
|
||||
use serenity::Client;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error};
|
||||
use tracing::{error, warn};
|
||||
|
||||
/// Discord bot service implementation
|
||||
pub struct BotService {
|
||||
@@ -28,7 +28,7 @@ impl Service for BotService {
|
||||
async fn run(&mut self) -> Result<(), anyhow::Error> {
|
||||
match self.client.start().await {
|
||||
Ok(()) => {
|
||||
debug!(service = "bot", "stopped early.");
|
||||
warn!(service = "bot", "stopped early");
|
||||
Err(anyhow::anyhow!("bot stopped early"))
|
||||
}
|
||||
Err(e) => {
|
||||
|
||||
@@ -34,6 +34,11 @@ impl ServiceManager {
|
||||
self.registered_services.insert(name.to_string(), service);
|
||||
}
|
||||
|
||||
/// Check if there are any registered services
|
||||
pub fn has_services(&self) -> bool {
|
||||
!self.registered_services.is_empty()
|
||||
}
|
||||
|
||||
/// Spawn all registered services
|
||||
pub fn spawn_all(&mut self) {
|
||||
let service_count = self.registered_services.len();
|
||||
@@ -42,7 +47,7 @@ impl ServiceManager {
|
||||
for (name, service) in self.registered_services.drain() {
|
||||
let shutdown_rx = self.shutdown_tx.subscribe();
|
||||
let handle = tokio::spawn(run_service(service, shutdown_rx));
|
||||
trace!(service = name, id = ?handle.id(), "service spawned",);
|
||||
debug!(service = name, id = ?handle.id(), "service spawned");
|
||||
self.running_services.insert(name, handle);
|
||||
}
|
||||
|
||||
@@ -127,7 +132,7 @@ impl ServiceManager {
|
||||
for (name, handle) in self.running_services.drain() {
|
||||
match tokio::time::timeout(timeout, handle).await {
|
||||
Ok(Ok(_)) => {
|
||||
debug!(service = name, "service shutdown completed");
|
||||
trace!(service = name, "service shutdown completed");
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
warn!(service = name, error = ?e, "service shutdown failed");
|
||||
|
||||
@@ -3,7 +3,7 @@ use crate::web::{BannerState, create_router};
|
||||
use std::net::SocketAddr;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing::{info, warn, trace};
|
||||
|
||||
/// Web server service implementation
|
||||
pub struct WebService {
|
||||
@@ -40,10 +40,10 @@ impl Service for WebService {
|
||||
);
|
||||
|
||||
let listener = TcpListener::bind(addr).await?;
|
||||
debug!(
|
||||
info!(
|
||||
service = "web",
|
||||
"web server listening on {}",
|
||||
format!("http://{}", addr)
|
||||
address = %addr,
|
||||
"web server listening"
|
||||
);
|
||||
|
||||
// Create internal shutdown channel for axum graceful shutdown
|
||||
@@ -54,7 +54,7 @@ impl Service for WebService {
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
let _ = shutdown_rx.recv().await;
|
||||
debug!(
|
||||
trace!(
|
||||
service = "web",
|
||||
"received shutdown signal, starting graceful shutdown"
|
||||
);
|
||||
|
||||
46
src/state.rs
46
src/state.rs
@@ -3,46 +3,36 @@
|
||||
use crate::banner::BannerApi;
|
||||
use crate::banner::Course;
|
||||
use anyhow::Result;
|
||||
use redis::AsyncCommands;
|
||||
use redis::Client;
|
||||
use sqlx::PgPool;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
pub banner_api: Arc<BannerApi>,
|
||||
pub redis: Arc<Client>,
|
||||
pub db_pool: PgPool,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
pub fn new(
|
||||
banner_api: Arc<BannerApi>,
|
||||
redis_url: &str,
|
||||
) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let redis_client = Client::open(redis_url)?;
|
||||
|
||||
Ok(Self {
|
||||
pub fn new(banner_api: Arc<BannerApi>, db_pool: PgPool) -> Self {
|
||||
Self {
|
||||
banner_api,
|
||||
redis: Arc::new(redis_client),
|
||||
})
|
||||
db_pool,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a course by CRN with Redis cache fallback to Banner API
|
||||
/// Get a course by CRN directly from Banner API
|
||||
pub async fn get_course_or_fetch(&self, term: &str, crn: &str) -> Result<Course> {
|
||||
let mut conn = self.redis.get_multiplexed_async_connection().await?;
|
||||
self.banner_api
|
||||
.get_course_by_crn(term, crn)
|
||||
.await?
|
||||
.ok_or_else(|| anyhow::anyhow!("Course not found for CRN {crn}"))
|
||||
}
|
||||
|
||||
let key = format!("class:{crn}");
|
||||
if let Some(serialized) = conn.get::<_, Option<String>>(&key).await? {
|
||||
let course: Course = serde_json::from_str(&serialized)?;
|
||||
return Ok(course);
|
||||
}
|
||||
|
||||
// Fallback: fetch from Banner API
|
||||
if let Some(course) = self.banner_api.get_course_by_crn(term, crn).await? {
|
||||
let serialized = serde_json::to_string(&course)?;
|
||||
let _: () = conn.set(&key, serialized).await?;
|
||||
return Ok(course);
|
||||
}
|
||||
|
||||
Err(anyhow::anyhow!("Course not found for CRN {crn}"))
|
||||
/// Get the total number of courses in the database
|
||||
pub async fn get_course_count(&self) -> Result<i64> {
|
||||
let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM courses")
|
||||
.fetch_one(&self.db_pool)
|
||||
.await?;
|
||||
Ok(count.0)
|
||||
}
|
||||
}
|
||||
|
||||
96
src/web/assets.rs
Normal file
96
src/web/assets.rs
Normal file
@@ -0,0 +1,96 @@
|
||||
//! Embedded assets for the web frontend
|
||||
//!
|
||||
//! This module handles serving static assets that are embedded into the binary
|
||||
//! at compile time using rust-embed.
|
||||
|
||||
use dashmap::DashMap;
|
||||
use once_cell::sync::Lazy;
|
||||
use rapidhash::v3::rapidhash_v3;
|
||||
use rust_embed::RustEmbed;
|
||||
use std::fmt;
|
||||
|
||||
/// Embedded web assets from the dist directory
|
||||
#[derive(RustEmbed)]
|
||||
#[folder = "web/dist/"]
|
||||
#[include = "*"]
|
||||
#[exclude = "*.map"]
|
||||
pub struct WebAssets;
|
||||
|
||||
/// RapidHash hash type for asset content (u64 native output size)
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct AssetHash(u64);
|
||||
|
||||
impl AssetHash {
|
||||
/// Create a new AssetHash from u64 value
|
||||
pub fn new(hash: u64) -> Self {
|
||||
Self(hash)
|
||||
}
|
||||
|
||||
/// Get the hash as a hex string
|
||||
pub fn to_hex(&self) -> String {
|
||||
format!("{:016x}", self.0)
|
||||
}
|
||||
|
||||
/// Get the hash as a quoted hex string
|
||||
pub fn quoted(&self) -> String {
|
||||
format!("\"{}\"", self.to_hex())
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for AssetHash {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.to_hex())
|
||||
}
|
||||
}
|
||||
|
||||
/// Metadata for an asset including MIME type and RapidHash hash
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AssetMetadata {
|
||||
pub mime_type: Option<String>,
|
||||
pub hash: AssetHash,
|
||||
}
|
||||
|
||||
impl AssetMetadata {
|
||||
/// Check if the etag matches the asset hash
|
||||
pub fn etag_matches(&self, etag: &str) -> bool {
|
||||
// Remove quotes if present (ETags are typically quoted)
|
||||
let etag = etag.trim_matches('"');
|
||||
|
||||
// ETags generated from u64 hex should be 16 characters
|
||||
etag.len() == 16
|
||||
// Parse the hexadecimal, compare if it matches
|
||||
&& etag.parse::<u64>()
|
||||
.map(|parsed| parsed == self.hash.0)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
/// Global cache for asset metadata to avoid repeated calculations
|
||||
static ASSET_CACHE: Lazy<DashMap<String, AssetMetadata>> = Lazy::new(DashMap::new);
|
||||
|
||||
/// Get cached asset metadata for a file path, caching on-demand
|
||||
/// Returns AssetMetadata containing MIME type and RapidHash hash
|
||||
pub fn get_asset_metadata_cached(path: &str, content: &[u8]) -> AssetMetadata {
|
||||
// Check cache first
|
||||
if let Some(cached) = ASSET_CACHE.get(path) {
|
||||
return cached.value().clone();
|
||||
}
|
||||
|
||||
// Calculate MIME type
|
||||
let mime_type = mime_guess::from_path(path)
|
||||
.first()
|
||||
.map(|mime| mime.to_string());
|
||||
|
||||
// Calculate RapidHash hash (using u64 native output size)
|
||||
let hash_value = rapidhash_v3(content);
|
||||
let hash = AssetHash::new(hash_value);
|
||||
|
||||
let metadata = AssetMetadata { mime_type, hash };
|
||||
|
||||
// Only cache if we haven't exceeded the limit
|
||||
if ASSET_CACHE.len() < 1000 {
|
||||
ASSET_CACHE.insert(path.to_string(), metadata.clone());
|
||||
}
|
||||
|
||||
metadata
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! Web API module for the banner application.
|
||||
|
||||
pub mod assets;
|
||||
pub mod routes;
|
||||
|
||||
pub use routes::*;
|
||||
|
||||
@@ -1,12 +1,61 @@
|
||||
//! Web API endpoints for Banner bot monitoring and metrics.
|
||||
|
||||
use axum::{Router, extract::State, response::Json, routing::get};
|
||||
use axum::{
|
||||
Router,
|
||||
extract::{Request, State},
|
||||
http::{HeaderMap, HeaderValue, StatusCode, Uri},
|
||||
response::{Html, IntoResponse, Json, Response},
|
||||
routing::get,
|
||||
};
|
||||
use http::header;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
use tower_http::{
|
||||
cors::{Any, CorsLayer},
|
||||
trace::TraceLayer,
|
||||
};
|
||||
use tracing::info;
|
||||
|
||||
use crate::web::assets::{WebAssets, get_asset_metadata_cached};
|
||||
|
||||
use crate::banner::BannerApi;
|
||||
|
||||
/// Set appropriate caching headers based on asset type
|
||||
fn set_caching_headers(response: &mut Response, path: &str, etag: &str) {
|
||||
let headers = response.headers_mut();
|
||||
|
||||
// Set ETag
|
||||
if let Ok(etag_value) = HeaderValue::from_str(etag) {
|
||||
headers.insert(header::ETAG, etag_value);
|
||||
}
|
||||
|
||||
// Set Cache-Control based on asset type
|
||||
let cache_control = if path.starts_with("assets/") {
|
||||
// Static assets with hashed filenames - long-term cache
|
||||
"public, max-age=31536000, immutable"
|
||||
} else if path == "index.html" {
|
||||
// HTML files - short-term cache
|
||||
"public, max-age=300"
|
||||
} else {
|
||||
match path.split_once('.').map(|(_, extension)| extension) {
|
||||
Some(ext) => match ext {
|
||||
// CSS/JS files - medium-term cache
|
||||
"css" | "js" => "public, max-age=86400",
|
||||
// Images - long-term cache
|
||||
"png" | "jpg" | "jpeg" | "gif" | "svg" | "ico" => "public, max-age=2592000",
|
||||
// Default for other files
|
||||
_ => "public, max-age=3600",
|
||||
},
|
||||
// Default for files without an extension
|
||||
None => "public, max-age=3600",
|
||||
}
|
||||
};
|
||||
|
||||
if let Ok(cache_control_value) = HeaderValue::from_str(cache_control) {
|
||||
headers.insert(header::CACHE_CONTROL, cache_control_value);
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared application state for web server
|
||||
#[derive(Clone)]
|
||||
pub struct BannerState {
|
||||
@@ -15,24 +64,124 @@ pub struct BannerState {
|
||||
|
||||
/// Creates the web server router
|
||||
pub fn create_router(state: BannerState) -> Router {
|
||||
Router::new()
|
||||
.route("/", get(root))
|
||||
let api_router = Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/status", get(status))
|
||||
.route("/metrics", get(metrics))
|
||||
.with_state(state)
|
||||
.with_state(state);
|
||||
|
||||
if cfg!(debug_assertions) {
|
||||
// Development mode: API routes only, frontend served by Vite dev server
|
||||
Router::new()
|
||||
.route("/", get(root))
|
||||
.nest("/api", api_router)
|
||||
.layer(
|
||||
CorsLayer::new()
|
||||
.allow_origin(Any)
|
||||
.allow_methods(Any)
|
||||
.allow_headers(Any),
|
||||
)
|
||||
.layer(TraceLayer::new_for_http())
|
||||
} else {
|
||||
// Production mode: serve embedded assets and handle SPA routing
|
||||
Router::new()
|
||||
.route("/", get(root))
|
||||
.nest("/api", api_router)
|
||||
.fallback(fallback)
|
||||
.layer(TraceLayer::new_for_http())
|
||||
}
|
||||
}
|
||||
|
||||
async fn root() -> Json<Value> {
|
||||
Json(json!({
|
||||
"message": "Banner Discord Bot API",
|
||||
"version": "0.1.0",
|
||||
"endpoints": {
|
||||
"health": "/health",
|
||||
"status": "/status",
|
||||
"metrics": "/metrics"
|
||||
async fn root() -> Response {
|
||||
if cfg!(debug_assertions) {
|
||||
// Development mode: return API info
|
||||
Json(json!({
|
||||
"message": "Banner Discord Bot API",
|
||||
"version": "0.2.1",
|
||||
"mode": "development",
|
||||
"frontend": "http://localhost:3000",
|
||||
"endpoints": {
|
||||
"health": "/api/health",
|
||||
"status": "/api/status",
|
||||
"metrics": "/api/metrics"
|
||||
}
|
||||
}))
|
||||
.into_response()
|
||||
} else {
|
||||
// Production mode: serve the SPA index.html
|
||||
handle_spa_fallback_with_headers(Uri::from_static("/"), HeaderMap::new()).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Handler that extracts request information for caching
|
||||
async fn fallback(request: Request) -> Response {
|
||||
let uri = request.uri().clone();
|
||||
let headers = request.headers().clone();
|
||||
handle_spa_fallback_with_headers(uri, headers).await
|
||||
}
|
||||
|
||||
/// Handles SPA routing by serving index.html for non-API, non-asset requests
|
||||
/// This version includes HTTP caching headers and ETag support
|
||||
async fn handle_spa_fallback_with_headers(uri: Uri, request_headers: HeaderMap) -> Response {
|
||||
let path = uri.path().trim_start_matches('/');
|
||||
|
||||
if let Some(content) = WebAssets::get(path) {
|
||||
// Get asset metadata (MIME type and hash) with caching
|
||||
let metadata = get_asset_metadata_cached(path, &content.data);
|
||||
|
||||
// Check if client has a matching ETag (conditional request)
|
||||
if let Some(etag) = request_headers.get(header::IF_NONE_MATCH)
|
||||
&& metadata.etag_matches(etag.to_str().unwrap())
|
||||
{
|
||||
return StatusCode::NOT_MODIFIED.into_response();
|
||||
}
|
||||
}))
|
||||
|
||||
// Use cached MIME type, only set Content-Type if we have a valid MIME type
|
||||
let mut response = (
|
||||
[(
|
||||
header::CONTENT_TYPE,
|
||||
// For unknown types, set to application/octet-stream
|
||||
metadata
|
||||
.mime_type
|
||||
.unwrap_or("application/octet-stream".to_string()),
|
||||
)],
|
||||
content.data,
|
||||
)
|
||||
.into_response();
|
||||
|
||||
// Set caching headers
|
||||
set_caching_headers(&mut response, path, &metadata.hash.quoted());
|
||||
|
||||
return response;
|
||||
} else {
|
||||
// Any assets that are not found should be treated as a 404, not falling back to the SPA index.html
|
||||
if path.starts_with("assets/") {
|
||||
return (StatusCode::NOT_FOUND, "Asset not found").into_response();
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to the SPA index.html
|
||||
match WebAssets::get("index.html") {
|
||||
Some(content) => {
|
||||
let metadata = get_asset_metadata_cached("index.html", &content.data);
|
||||
|
||||
// Check if client has a matching ETag for index.html
|
||||
if let Some(etag) = request_headers.get(header::IF_NONE_MATCH)
|
||||
&& metadata.etag_matches(etag.to_str().unwrap())
|
||||
{
|
||||
return StatusCode::NOT_MODIFIED.into_response();
|
||||
}
|
||||
|
||||
let mut response = Html(content.data).into_response();
|
||||
set_caching_headers(&mut response, "index.html", &metadata.hash.quoted());
|
||||
response
|
||||
}
|
||||
None => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"Failed to load index.html",
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Health check endpoint
|
||||
@@ -69,18 +218,8 @@ async fn status(State(_state): State<BannerState>) -> Json<Value> {
|
||||
async fn metrics(State(_state): State<BannerState>) -> Json<Value> {
|
||||
// For now, return basic metrics structure
|
||||
Json(json!({
|
||||
"redis": {
|
||||
"status": "connected",
|
||||
"connected_clients": "TODO: implement client counting",
|
||||
"used_memory": "TODO: implement memory tracking"
|
||||
},
|
||||
"cache": {
|
||||
"courses": {
|
||||
"count": "TODO: implement course counting"
|
||||
},
|
||||
"subjects": {
|
||||
"count": "TODO: implement subject counting"
|
||||
}
|
||||
"banner_api": {
|
||||
"status": "connected"
|
||||
},
|
||||
"timestamp": chrono::Utc::now().to_rfc3339()
|
||||
}))
|
||||
|
||||
9
web/.gitignore
vendored
Normal file
9
web/.gitignore
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
node_modules
|
||||
.DS_Store
|
||||
dist
|
||||
dist-ssr
|
||||
*.local
|
||||
count.txt
|
||||
.env
|
||||
.nitro
|
||||
.tanstack
|
||||
11
web/.vscode/settings.json
vendored
Normal file
11
web/.vscode/settings.json
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"files.watcherExclude": {
|
||||
"**/routeTree.gen.ts": true
|
||||
},
|
||||
"search.exclude": {
|
||||
"**/routeTree.gen.ts": true
|
||||
},
|
||||
"files.readonlyInclude": {
|
||||
"**/routeTree.gen.ts": true
|
||||
}
|
||||
}
|
||||
20
web/index.html
Normal file
20
web/index.html
Normal file
@@ -0,0 +1,20 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<link rel="icon" href="/favicon.ico" />
|
||||
<meta name="theme-color" content="#000000" />
|
||||
<meta
|
||||
name="description"
|
||||
content="Web site created using create-tsrouter-app"
|
||||
/>
|
||||
<link rel="apple-touch-icon" href="/logo192.png" />
|
||||
<link rel="manifest" href="/manifest.json" />
|
||||
<title>Create TanStack App - web-template</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
36
web/package.json
Normal file
36
web/package.json
Normal file
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"name": "web-template",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite --port 3000",
|
||||
"start": "vite --port 3000",
|
||||
"build": "vite build && tsc",
|
||||
"serve": "vite preview",
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@radix-ui/themes": "^3.2.1",
|
||||
"@tanstack/react-devtools": "^0.2.2",
|
||||
"@tanstack/react-router": "^1.130.2",
|
||||
"@tanstack/react-router-devtools": "^1.131.5",
|
||||
"@tanstack/router-plugin": "^1.121.2",
|
||||
"lucide-react": "^0.544.0",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"recharts": "^3.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@testing-library/dom": "^10.4.0",
|
||||
"@testing-library/react": "^16.2.0",
|
||||
"@types/node": "^24.3.3",
|
||||
"@types/react": "^19.0.8",
|
||||
"@types/react-dom": "^19.0.3",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
"jsdom": "^26.0.0",
|
||||
"typescript": "^5.7.2",
|
||||
"vite": "^6.3.5",
|
||||
"vitest": "^3.0.5",
|
||||
"web-vitals": "^4.2.4"
|
||||
}
|
||||
}
|
||||
4594
web/pnpm-lock.yaml
generated
Normal file
4594
web/pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load Diff
BIN
web/public/favicon.ico
Normal file
BIN
web/public/favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 3.8 KiB |
BIN
web/public/logo192.png
Normal file
BIN
web/public/logo192.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 5.2 KiB |
BIN
web/public/logo512.png
Normal file
BIN
web/public/logo512.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 9.4 KiB |
25
web/public/manifest.json
Normal file
25
web/public/manifest.json
Normal file
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"short_name": "TanStack App",
|
||||
"name": "Create TanStack App Sample",
|
||||
"icons": [
|
||||
{
|
||||
"src": "favicon.ico",
|
||||
"sizes": "64x64 32x32 24x24 16x16",
|
||||
"type": "image/x-icon"
|
||||
},
|
||||
{
|
||||
"src": "logo192.png",
|
||||
"type": "image/png",
|
||||
"sizes": "192x192"
|
||||
},
|
||||
{
|
||||
"src": "logo512.png",
|
||||
"type": "image/png",
|
||||
"sizes": "512x512"
|
||||
}
|
||||
],
|
||||
"start_url": ".",
|
||||
"display": "standalone",
|
||||
"theme_color": "#000000",
|
||||
"background_color": "#ffffff"
|
||||
}
|
||||
3
web/public/robots.txt
Normal file
3
web/public/robots.txt
Normal file
@@ -0,0 +1,3 @@
|
||||
# https://www.robotstxt.org/robotstxt.html
|
||||
User-agent: *
|
||||
Disallow:
|
||||
38
web/src/App.css
Normal file
38
web/src/App.css
Normal file
@@ -0,0 +1,38 @@
|
||||
.App {
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.App-logo {
|
||||
height: 40vmin;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: no-preference) {
|
||||
.App-logo {
|
||||
animation: App-logo-spin infinite 20s linear;
|
||||
}
|
||||
}
|
||||
|
||||
.App-header {
|
||||
background-color: #282c34;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: calc(10px + 2vmin);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.App-link {
|
||||
color: #61dafb;
|
||||
}
|
||||
|
||||
@keyframes App-logo-spin {
|
||||
from {
|
||||
transform: rotate(0deg);
|
||||
}
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
63
web/src/lib/api.test.ts
Normal file
63
web/src/lib/api.test.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { BannerApiClient } from "./api";
|
||||
|
||||
// Mock fetch
|
||||
global.fetch = vi.fn();
|
||||
|
||||
describe("BannerApiClient", () => {
|
||||
let apiClient: BannerApiClient;
|
||||
|
||||
beforeEach(() => {
|
||||
apiClient = new BannerApiClient();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("should fetch health data", async () => {
|
||||
const mockHealth = {
|
||||
status: "healthy",
|
||||
timestamp: "2024-01-01T00:00:00Z",
|
||||
};
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: () => Promise.resolve(mockHealth),
|
||||
} as Response);
|
||||
|
||||
const result = await apiClient.getHealth();
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith("/api/health");
|
||||
expect(result).toEqual(mockHealth);
|
||||
});
|
||||
|
||||
it("should fetch status data", async () => {
|
||||
const mockStatus = {
|
||||
status: "operational",
|
||||
bot: { status: "running", uptime: "1h" },
|
||||
cache: { status: "connected", courses: "100", subjects: "50" },
|
||||
banner_api: { status: "connected" },
|
||||
timestamp: "2024-01-01T00:00:00Z",
|
||||
};
|
||||
|
||||
vi.mocked(fetch).mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: () => Promise.resolve(mockStatus),
|
||||
} as Response);
|
||||
|
||||
const result = await apiClient.getStatus();
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith("/api/status");
|
||||
expect(result).toEqual(mockStatus);
|
||||
});
|
||||
|
||||
it("should handle API errors", async () => {
|
||||
vi.mocked(fetch).mockResolvedValueOnce({
|
||||
ok: false,
|
||||
status: 500,
|
||||
statusText: "Internal Server Error",
|
||||
} as Response);
|
||||
|
||||
await expect(apiClient.getHealth()).rejects.toThrow(
|
||||
"API request failed: 500 Internal Server Error"
|
||||
);
|
||||
});
|
||||
});
|
||||
66
web/src/lib/api.ts
Normal file
66
web/src/lib/api.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
// API client for Banner backend
|
||||
const API_BASE_URL = "/api";
|
||||
|
||||
export interface HealthResponse {
|
||||
status: string;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface StatusResponse {
|
||||
status: string;
|
||||
bot: {
|
||||
status: string;
|
||||
uptime: string;
|
||||
};
|
||||
cache: {
|
||||
status: string;
|
||||
courses: string;
|
||||
subjects: string;
|
||||
};
|
||||
banner_api: {
|
||||
status: string;
|
||||
};
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface MetricsResponse {
|
||||
banner_api: {
|
||||
status: string;
|
||||
};
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export class BannerApiClient {
|
||||
private baseUrl: string;
|
||||
|
||||
constructor(baseUrl: string = API_BASE_URL) {
|
||||
this.baseUrl = baseUrl;
|
||||
}
|
||||
|
||||
private async request<T>(endpoint: string): Promise<T> {
|
||||
const response = await fetch(`${this.baseUrl}${endpoint}`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`API request failed: ${response.status} ${response.statusText}`
|
||||
);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async getHealth(): Promise<HealthResponse> {
|
||||
return this.request<HealthResponse>("/health");
|
||||
}
|
||||
|
||||
async getStatus(): Promise<StatusResponse> {
|
||||
return this.request<StatusResponse>("/status");
|
||||
}
|
||||
|
||||
async getMetrics(): Promise<MetricsResponse> {
|
||||
return this.request<MetricsResponse>("/metrics");
|
||||
}
|
||||
}
|
||||
|
||||
// Export a default instance
|
||||
export const apiClient = new BannerApiClient();
|
||||
44
web/src/logo.svg
Normal file
44
web/src/logo.svg
Normal file
@@ -0,0 +1,44 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg id="Layer_1"
|
||||
xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 841.9 595.3">
|
||||
<!-- Generator: Adobe Illustrator 29.3.0, SVG Export Plug-In . SVG Version: 2.1.0 Build 146) -->
|
||||
<defs>
|
||||
<style>
|
||||
.st0 {
|
||||
fill: #9ae7fc;
|
||||
}
|
||||
|
||||
.st1 {
|
||||
fill: #61dafb;
|
||||
}
|
||||
</style>
|
||||
</defs>
|
||||
<g>
|
||||
<path class="st1" d="M666.3,296.5c0-32.5-40.7-63.3-103.1-82.4,14.4-63.6,8-114.2-20.2-130.4-6.5-3.8-14.1-5.6-22.4-5.6v22.3c4.6,0,8.3.9,11.4,2.6,13.6,7.8,19.5,37.5,14.9,75.7-1.1,9.4-2.9,19.3-5.1,29.4-19.6-4.8-41-8.5-63.5-10.9-13.5-18.5-27.5-35.3-41.6-50,32.6-30.3,63.2-46.9,84-46.9v-22.3c-27.5,0-63.5,19.6-99.9,53.6-36.4-33.8-72.4-53.2-99.9-53.2v22.3c20.7,0,51.4,16.5,84,46.6-14,14.7-28,31.4-41.3,49.9-22.6,2.4-44,6.1-63.6,11-2.3-10-4-19.7-5.2-29-4.7-38.2,1.1-67.9,14.6-75.8,3-1.8,6.9-2.6,11.5-2.6v-22.3c-8.4,0-16,1.8-22.6,5.6-28.1,16.2-34.4,66.7-19.9,130.1-62.2,19.2-102.7,49.9-102.7,82.3s40.7,63.3,103.1,82.4c-14.4,63.6-8,114.2,20.2,130.4,6.5,3.8,14.1,5.6,22.5,5.6,27.5,0,63.5-19.6,99.9-53.6,36.4,33.8,72.4,53.2,99.9,53.2,8.4,0,16-1.8,22.6-5.6,28.1-16.2,34.4-66.7,19.9-130.1,62-19.1,102.5-49.9,102.5-82.3zm-130.2-66.7c-3.7,12.9-8.3,26.2-13.5,39.5-4.1-8-8.4-16-13.1-24-4.6-8-9.5-15.8-14.4-23.4,14.2,2.1,27.9,4.7,41,7.9zm-45.8,106.5c-7.8,13.5-15.8,26.3-24.1,38.2-14.9,1.3-30,2-45.2,2s-30.2-.7-45-1.9c-8.3-11.9-16.4-24.6-24.2-38-7.6-13.1-14.5-26.4-20.8-39.8,6.2-13.4,13.2-26.8,20.7-39.9,7.8-13.5,15.8-26.3,24.1-38.2,14.9-1.3,30-2,45.2-2s30.2.7,45,1.9c8.3,11.9,16.4,24.6,24.2,38,7.6,13.1,14.5,26.4,20.8,39.8-6.3,13.4-13.2,26.8-20.7,39.9zm32.3-13c5.4,13.4,10,26.8,13.8,39.8-13.1,3.2-26.9,5.9-41.2,8,4.9-7.7,9.8-15.6,14.4-23.7,4.6-8,8.9-16.1,13-24.1zm-101.4,106.7c-9.3-9.6-18.6-20.3-27.8-32,9,.4,18.2.7,27.5.7s18.7-.2,27.8-.7c-9,11.7-18.3,22.4-27.5,32zm-74.4-58.9c-14.2-2.1-27.9-4.7-41-7.9,3.7-12.9,8.3-26.2,13.5-39.5,4.1,8,8.4,16,13.1,24s9.5,15.8,14.4,23.4zm73.9-208.1c9.3,9.6,18.6,20.3,27.8,32-9-.4-18.2-.7-27.5-.7s-18.7.2-27.8.7c9-11.7,18.3-22.4,27.5-32zm-74,58.9c-4.9,7.7-9.8,15.6-14.4,23.7-4.6,8-8.9,16-13,24-5.4-13.4-10-26.8-13.8-39.8,13.1-3.1,26.9-5.8,41.2-7.9zm-90.5,125.2c-35.4-15.1-58.3-34.9-58.3-50.6s22.9-35.6,58.3-50.6c8.6-3.7,18-7,27.7-10.1,5.7,19.6,13.2,40,22.5,60.9-9.2,20.8-16.6,41.1-22.2,60.6-9.9-3.1-19.3-6.5-28-10.2zm53.8,142.9c-13.6-7.8-19.5-37.5-14.9-75.7,1.1-9.4,2.9-19.3,5.1-29.4,19.6,4.8,41,8.5,63.5,10.9,13.5,18.5,27.5,35.3,41.6,50-32.6,30.3-63.2,46.9-84,46.9-4.5-.1-8.3-1-11.3-2.7zm237.2-76.2c4.7,38.2-1.1,67.9-14.6,75.8-3,1.8-6.9,2.6-11.5,2.6-20.7,0-51.4-16.5-84-46.6,14-14.7,28-31.4,41.3-49.9,22.6-2.4,44-6.1,63.6-11,2.3,10.1,4.1,19.8,5.2,29.1zm38.5-66.7c-8.6,3.7-18,7-27.7,10.1-5.7-19.6-13.2-40-22.5-60.9,9.2-20.8,16.6-41.1,22.2-60.6,9.9,3.1,19.3,6.5,28.1,10.2,35.4,15.1,58.3,34.9,58.3,50.6,0,15.7-23,35.6-58.4,50.6zm-264.9-268.7z"/>
|
||||
<circle class="st1" cx="420.9" cy="296.5" r="45.7"/>
|
||||
<path class="st1" d="M520.5,78.1"/>
|
||||
</g>
|
||||
<circle class="st0" cx="420.8" cy="296.6" r="43"/>
|
||||
<path class="st1" d="M466.1,296.6c0,25-20.2,45.2-45.2,45.2s-45.2-20.2-45.2-45.2,20.2-45.2,45.2-45.2,45.2,20.2,45.2,45.2ZM386,295.6v-6.3c0-1.1,1.2-5.1,1.8-6.2,1-1.9,2.9-3.5,4.6-4.7l-3.4-3.4c4-3.6,9.4-3.7,13.7-.7,1.9-4.7,6.6-7.1,11.6-6.7l-.8,4.2c5.9.2,13.1,4.1,13.1,10.8s0,.5-.7.7c-1.7.3-3.4-.4-5-.6s-1.2-.4-1.2.3,2.5,4.1,3,5.5,1,3.5.8,5.3c-5.6-.8-10.5-3.2-14.8-6.7.3,2.6,4.1,21.7,5.3,21.9s.8-.6,1-1.1,1.3-6.3,1.3-6.7c0-1-1.7-1.8-2.2-2.8-1.2-2.7,1.3-4.7,3.7-3.3s5.2,6.2,7.5,7.3,13,1.4,14.8,3.3-2.9,4.6-1.5,7.6c6.7-2.6,13.5-3.3,20.6-2.5,3.1-9.7,3.1-20.3-.9-29.8-7.3,0-14.7-3.6-17.2-10.8-2.5-7.2-.7-8.6-1.3-9.3-.8-1-6.3.6-7.4-1.5s.3-1.1-.2-1.4-1.9-.6-2.6-.8c-26-6.4-51.3,15.7-49.7,42.1,0,1.6,1.6,10.3,2.4,11.1s4.8,0,6.3,0,3.7.3,5,.5c2.9.4,7.2,2.4,9.4,2.5s2.4-.8,2.7-2.4c.4-2.6.5-7.4.5-10.1s-1-7.8-1.3-11.6c-.9-.2-.7,0-.9.5-.7,1.3-1.1,3.2-1.9,4.8s-5.2,8.7-5.7,9-.7-.5-.8-.8c-1.6-3.5-2-7.9-1.9-11.8-.9-1-5.4,4.9-6.7,5.3l-.8-.4v-.3h-.2ZM455.6,276.4c1.1-1.2-6-8.9-7.2-10-3-2.7-5.4-4.5-3.5,1.4s5.7,7.8,10.6,8.5h.1ZM410.9,270.1c-.4-.5-6.1,2.9-5.5,4.6,1.9-1.3,5.9-1.7,5.5-4.6ZM400.4,276.4c-.3-2.4-6.3-2.7-7.2-1s1.6,1.4,1.9,1.4c1.8.3,3.5-.6,5.2-.4h.1ZM411.3,276.8c3.8,1.3,6.6,3.6,10.9,3.7s0-3-1.2-3.9c-2.2-1.7-5.1-2.4-7.8-2.4s-1.6-.3-1.4.4c2.8.6,7.3.7,8.4,3.8-2.3-.3-3.9-1.6-6.2-2s-2.5-.5-2.6.3h0ZM420.6,290.3c-.8-5.1-5.7-10.8-10.9-11.6s-1.3-.4-.8.5,4.7,3.2,5.7,4,4.5,4.2,2.1,3.8-8.4-7.8-9.4-6.7c.2.9,1.1,1.9,1.7,2.7,3,3.8,6.9,6.8,11.8,7.4h-.2ZM395.3,279.8c-5,1.1-6.9,6.3-6.7,11,.7.8,5-3.8,5.4-4.5s2.7-4.6,1.1-4-2.9,4.4-4.2,4.6.2-2.1.4-2.5c1.1-1.6,2.9-3.1,4-4.6h0ZM400.4,281.5c-.4-.5-2,1.3-2.3,1.7-2.9,3.9-2.6,10.2-1.5,14.8.8.2.8-.3,1.2-.7,3-3.8,5.5-10.5,4.5-15.4-2.1,3.1-3.1,7.3-3.6,11h-1.3c0-4,1.9-7.7,3-11.4h0ZM426.9,305.9c0-1.7-1.7-1.4-2.5-1.9s-1.3-1.9-3-1.4c1.3,2.1,3,3.2,5.5,3.4h0ZM417.2,308.5c7.6.7,5.5-1.9,1.4-5.5-1.3-.3-1.5,4.5-1.4,5.5ZM437,309.7c-3.5-.3-7.8-2-11.2-2.1s-1.3,0-1.9.7c4,1.3,8.4,1.7,12.1,4l1-2.5h0ZM420.5,312.8c-7.3,0-15.1,3.7-20.4,8.8s-4.8,5.3-4.8,6.2c0,1.8,8.6,6.2,10.5,6.8,12.1,4.8,27.5,3.5,38.2-4.2s3.1-2.7,0-6.2c-5.7-6.6-14.7-11.4-23.4-11.3h-.1ZM398.7,316.9c-1.4-1.4-5-1.9-7-2.1s-5.3-.3-6.9.6l13.9,1.4h0ZM456.9,314.8h-7.4c-.9,0-4.9,1.1-6,1.6s-.8.6,0,.5c2.4,0,5.1-1,7.6-1.3s3.5.2,5.1,0,1.3-.3.6-.8h0Z"/>
|
||||
<path class="st0" d="M386,295.6l.8.4c1.3-.3,5.8-6.2,6.7-5.3,0,3.9.3,8.3,1.9,11.8s0,1.2.8.8,5.1-7.8,5.7-9,1.3-3.5,1.9-4.8,0-.7.9-.5c.3,3.8,1.2,7.8,1.3,11.6s0,7.5-.5,10.1-1.1,2.4-2.7,2.4-6.5-2.1-9.4-2.5-3.7-.5-5-.5-5.4,1.1-6.3,0-2.2-9.5-2.4-11.1c-1.5-26.4,23.7-48.5,49.7-42.1s2.2.4,2.6.8,0,1,.2,1.4c1.1,2,6.5.5,7.4,1.5s.4,6.9,1.3,9.3c2.5,7.2,10,10.9,17.2,10.8,4,9.4,4,20.1.9,29.8-7.2-.7-13.9,0-20.6,2.5-1.3-3.1,4.1-5.1,1.5-7.6s-11.8-1.9-14.8-3.3-5.4-6.1-7.5-7.3-4.9.6-3.7,3.3,2.1,1.8,2.2,2.8-1,6.2-1.3,6.7-.3,1.3-1,1.1c-1.1-.3-5-19.3-5.3-21.9,4.3,3.5,9.2,5.9,14.8,6.7.2-1.9-.3-3.5-.8-5.3s-3-5.1-3-5.5c0-.8.9-.3,1.2-.3,1.6,0,3.3.8,5,.6s.7.3.7-.7c0-6.6-7.2-10.6-13.1-10.8l.8-4.2c-5.1-.3-9.6,2-11.6,6.7-4.3-3-9.8-3-13.7.7l3.4,3.4c-1.8,1.3-3.5,2.8-4.6,4.7s-1.8,5.1-1.8,6.2v6.6h.2ZM431.6,265c7.8,2.1,8.7-3.5.2-1.3l-.2,1.3ZM432.4,270.9c.3.6,6.4-.4,5.8-2.3s-4.6.6-5.7.6l-.2,1.7h.1ZM434.5,276c.8,1.2,5.7-1.8,5.5-2.7-.4-1.9-6.6,1.2-5.5,2.7ZM442.9,276.4c-.9-.9-5,2.8-4.6,4,.6,2.4,5.7-3,4.6-4ZM445.1,279.9c-.3.2-3.1,4.6-1.5,5s3.5-3.4,3.5-4-1.3-1.3-2-.9h0ZM448.9,287.4c2.1.8,3.8-5.1,2.3-5.5-1.9-.6-2.6,5.1-2.3,5.5ZM457.3,288.6c.5-1.7,1.1-4.7-1-5.5-1,.3-.6,3.9-.6,4.8l.3.5,1.3.2h0Z"/>
|
||||
<path class="st0" d="M455.6,276.4c-5-.8-9.1-3.6-10.6-8.5s.5-4,3.5-1.4,8.3,8.7,7.2,10h-.1Z"/>
|
||||
<path class="st0" d="M420.6,290.3c-4.9-.6-8.9-3.6-11.8-7.4s-1.5-1.8-1.7-2.7c1-1,8.5,6.6,9.4,6.7,2.4.4-1.8-3.5-2.1-3.8-1-.8-5.4-3.5-5.7-4-.4-.8.5-.5.8-.5,5.2.8,10.1,6.6,10.9,11.6h.2Z"/>
|
||||
<path class="st0" d="M400.4,281.5c-1.1,3.7-3,7.3-3,11.4h1.3c.5-3.7,1.5-7.8,3.6-11,1,4.8-1.5,11.6-4.5,15.4s-.4.8-1.2.7c-1.1-4.5-1.3-10.8,1.5-14.8s1.9-2.2,2.3-1.7h0Z"/>
|
||||
<path class="st0" d="M411.3,276.8c0-.8,2.1-.4,2.6-.3,2.4.4,4,1.7,6.2,2-1.2-3.1-5.7-3.2-8.4-3.8,0-.8.9-.4,1.4-.4,2.8,0,5.6.7,7.8,2.4,2.2,1.7,4,4,1.2,3.9-4.3,0-7.1-2.4-10.9-3.7h0Z"/>
|
||||
<path class="st0" d="M395.3,279.8c-1.1,1.6-3,3-4,4.6s-1.9,2.8-.4,2.5,2.8-4,4.2-4.6-.9,3.6-1.1,4c-.4.7-4.7,5.2-5.4,4.5-.2-4.6,1.8-9.9,6.7-11h0Z"/>
|
||||
<path class="st0" d="M437,309.7l-1,2.5c-3.6-2.3-8-2.8-12.1-4,.5-.7,1.1-.7,1.9-.7,3.4,0,7.8,1.8,11.2,2.1h0Z"/>
|
||||
<path class="st0" d="M417.2,308.5c0-1,0-5.8,1.4-5.5,4,3.5,6.1,6.2-1.4,5.5Z"/>
|
||||
<path class="st0" d="M400.4,276.4c-1.8-.3-3.5.7-5.2.4s-2.3-.8-1.9-1.4c.8-1.6,6.9-1.4,7.2,1h-.1Z"/>
|
||||
<path class="st0" d="M410.9,270.1c.4,3-3.6,3.3-5.5,4.6-.6-1.8,5-5.1,5.5-4.6Z"/>
|
||||
<path class="st0" d="M426.9,305.9c-2.5-.2-4.1-1.3-5.5-3.4,1.7-.4,2,.8,3,1.4s2.6.3,2.5,1.9h0Z"/>
|
||||
<path class="st1" d="M432.4,270.9l.2-1.7c1.1,0,5.1-2.2,5.7-.6s-5.5,2.9-5.8,2.3h-.1Z"/>
|
||||
<path class="st1" d="M431.6,265l.2-1.3c8.4-2.1,7.7,3.4-.2,1.3Z"/>
|
||||
<path class="st1" d="M434.5,276c-1.1-1.5,5.1-4.6,5.5-2.7s-4.6,4-5.5,2.7Z"/>
|
||||
<path class="st1" d="M442.9,276.4c1.1,1.1-4,6.4-4.6,4s3.7-4.9,4.6-4Z"/>
|
||||
<path class="st1" d="M445.1,279.9c.7-.4,2.1,0,2,.9s-2.4,4.4-3.5,4,1.3-4.8,1.5-5h0Z"/>
|
||||
<path class="st1" d="M448.9,287.4c-.3-.3.4-6.1,2.3-5.5,1.4.4-.2,6.2-2.3,5.5Z"/>
|
||||
<path class="st1" d="M457.3,288.6l-1.3-.2-.3-.5c0-.9-.4-4.6.6-4.8,2.1.8,1.5,3.8,1,5.5h0Z"/>
|
||||
<path class="st0" d="M420.5,312.8c8.9,0,17.9,4.7,23.4,11.3,5.6,6.6,3.8,3.5,0,6.2-10.7,7.7-26.1,9-38.2,4.2-1.9-.8-10.5-5.1-10.5-6.8s4-5.3,4.8-6.2c5.3-5,13.1-8.6,20.4-8.8h.1Z"/>
|
||||
<path class="st0" d="M398.7,316.9l-13.9-1.4c1.7-1,5-.8,6.9-.6s5.6.7,7,2.1h0Z"/>
|
||||
<path class="st0" d="M456.9,314.8c.7.5,0,.8-.6.8-1.6.2-3.5-.2-5.1,0-2.4.3-5.2,1.2-7.6,1.3s-1.1,0,0-.5,5.1-1.6,6-1.6h7.4,0Z"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 8.4 KiB |
42
web/src/main.tsx
Normal file
42
web/src/main.tsx
Normal file
@@ -0,0 +1,42 @@
|
||||
import { StrictMode } from 'react'
|
||||
import ReactDOM from 'react-dom/client'
|
||||
import { RouterProvider, createRouter } from '@tanstack/react-router'
|
||||
|
||||
// Import the generated route tree
|
||||
import { routeTree } from './routeTree.gen'
|
||||
|
||||
import './styles.css'
|
||||
import reportWebVitals from './reportWebVitals.ts'
|
||||
|
||||
// Create a new router instance
|
||||
const router = createRouter({
|
||||
routeTree,
|
||||
context: {},
|
||||
defaultPreload: 'intent',
|
||||
scrollRestoration: true,
|
||||
defaultStructuralSharing: true,
|
||||
defaultPreloadStaleTime: 0,
|
||||
})
|
||||
|
||||
// Register the router instance for type safety
|
||||
declare module '@tanstack/react-router' {
|
||||
interface Register {
|
||||
router: typeof router
|
||||
}
|
||||
}
|
||||
|
||||
// Render the app
|
||||
const rootElement = document.getElementById('app')
|
||||
if (rootElement && !rootElement.innerHTML) {
|
||||
const root = ReactDOM.createRoot(rootElement)
|
||||
root.render(
|
||||
<StrictMode>
|
||||
<RouterProvider router={router} />
|
||||
</StrictMode>,
|
||||
)
|
||||
}
|
||||
|
||||
// If you want to start measuring performance in your app, pass a function
|
||||
// to log results (for example: reportWebVitals(console.log))
|
||||
// or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals
|
||||
reportWebVitals()
|
||||
13
web/src/reportWebVitals.ts
Normal file
13
web/src/reportWebVitals.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
const reportWebVitals = (onPerfEntry?: () => void) => {
|
||||
if (onPerfEntry && onPerfEntry instanceof Function) {
|
||||
import('web-vitals').then(({ onCLS, onINP, onFCP, onLCP, onTTFB }) => {
|
||||
onCLS(onPerfEntry)
|
||||
onINP(onPerfEntry)
|
||||
onFCP(onPerfEntry)
|
||||
onLCP(onPerfEntry)
|
||||
onTTFB(onPerfEntry)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export default reportWebVitals
|
||||
59
web/src/routeTree.gen.ts
Normal file
59
web/src/routeTree.gen.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
/* eslint-disable */
|
||||
|
||||
// @ts-nocheck
|
||||
|
||||
// noinspection JSUnusedGlobalSymbols
|
||||
|
||||
// This file was automatically generated by TanStack Router.
|
||||
// You should NOT make any changes in this file as it will be overwritten.
|
||||
// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified.
|
||||
|
||||
import { Route as rootRouteImport } from './routes/__root'
|
||||
import { Route as IndexRouteImport } from './routes/index'
|
||||
|
||||
const IndexRoute = IndexRouteImport.update({
|
||||
id: '/',
|
||||
path: '/',
|
||||
getParentRoute: () => rootRouteImport,
|
||||
} as any)
|
||||
|
||||
export interface FileRoutesByFullPath {
|
||||
'/': typeof IndexRoute
|
||||
}
|
||||
export interface FileRoutesByTo {
|
||||
'/': typeof IndexRoute
|
||||
}
|
||||
export interface FileRoutesById {
|
||||
__root__: typeof rootRouteImport
|
||||
'/': typeof IndexRoute
|
||||
}
|
||||
export interface FileRouteTypes {
|
||||
fileRoutesByFullPath: FileRoutesByFullPath
|
||||
fullPaths: '/'
|
||||
fileRoutesByTo: FileRoutesByTo
|
||||
to: '/'
|
||||
id: '__root__' | '/'
|
||||
fileRoutesById: FileRoutesById
|
||||
}
|
||||
export interface RootRouteChildren {
|
||||
IndexRoute: typeof IndexRoute
|
||||
}
|
||||
|
||||
declare module '@tanstack/react-router' {
|
||||
interface FileRoutesByPath {
|
||||
'/': {
|
||||
id: '/'
|
||||
path: '/'
|
||||
fullPath: '/'
|
||||
preLoaderRoute: typeof IndexRouteImport
|
||||
parentRoute: typeof rootRouteImport
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const rootRouteChildren: RootRouteChildren = {
|
||||
IndexRoute: IndexRoute,
|
||||
}
|
||||
export const routeTree = rootRouteImport
|
||||
._addFileChildren(rootRouteChildren)
|
||||
._addFileTypes<FileRouteTypes>()
|
||||
22
web/src/routes/__root.tsx
Normal file
22
web/src/routes/__root.tsx
Normal file
@@ -0,0 +1,22 @@
|
||||
import { Outlet, createRootRoute } from '@tanstack/react-router'
|
||||
import { TanStackRouterDevtoolsPanel } from '@tanstack/react-router-devtools'
|
||||
import { TanstackDevtools } from '@tanstack/react-devtools'
|
||||
|
||||
export const Route = createRootRoute({
|
||||
component: () => (
|
||||
<>
|
||||
<Outlet />
|
||||
<TanstackDevtools
|
||||
config={{
|
||||
position: 'bottom-left',
|
||||
}}
|
||||
plugins={[
|
||||
{
|
||||
name: 'Tanstack Router',
|
||||
render: <TanStackRouterDevtoolsPanel />,
|
||||
},
|
||||
]}
|
||||
/>
|
||||
</>
|
||||
),
|
||||
})
|
||||
87
web/src/routes/index.tsx
Normal file
87
web/src/routes/index.tsx
Normal file
@@ -0,0 +1,87 @@
|
||||
import { createFileRoute } from "@tanstack/react-router";
|
||||
import { useState, useEffect } from "react";
|
||||
import {
|
||||
apiClient,
|
||||
type HealthResponse,
|
||||
type StatusResponse,
|
||||
} from "../lib/api";
|
||||
import logo from "../logo.svg";
|
||||
import "../App.css";
|
||||
|
||||
export const Route = createFileRoute("/")({
|
||||
component: App,
|
||||
});
|
||||
|
||||
function App() {
|
||||
const [health, setHealth] = useState<HealthResponse | null>(null);
|
||||
const [status, setStatus] = useState<StatusResponse | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const fetchData = async () => {
|
||||
try {
|
||||
setLoading(true);
|
||||
const [healthData, statusData] = await Promise.all([
|
||||
apiClient.getHealth(),
|
||||
apiClient.getStatus(),
|
||||
]);
|
||||
setHealth(healthData);
|
||||
setStatus(statusData);
|
||||
setError(null);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Failed to fetch data");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
fetchData();
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="App">
|
||||
<header className="App-header">
|
||||
<img src={logo} className="App-logo" alt="logo" />
|
||||
<h1>Banner Discord Bot Dashboard</h1>
|
||||
|
||||
{loading && <p>Loading...</p>}
|
||||
|
||||
{error && (
|
||||
<div style={{ color: "red", margin: "20px 0" }}>
|
||||
<p>Error: {error}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{health && (
|
||||
<div style={{ margin: "20px 0", textAlign: "left" }}>
|
||||
<h3>Health Status</h3>
|
||||
<p>Status: {health.status}</p>
|
||||
<p>Timestamp: {new Date(health.timestamp).toLocaleString()}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{status && (
|
||||
<div style={{ margin: "20px 0", textAlign: "left" }}>
|
||||
<h3>System Status</h3>
|
||||
<p>Overall: {status.status}</p>
|
||||
<p>Bot: {status.bot.status}</p>
|
||||
<p>Cache: {status.cache.status}</p>
|
||||
<p>Banner API: {status.banner_api.status}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div style={{ marginTop: "40px" }}>
|
||||
<a
|
||||
className="App-link"
|
||||
href="https://tanstack.com"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
Learn TanStack Router
|
||||
</a>
|
||||
</div>
|
||||
</header>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
14
web/src/styles.css
Normal file
14
web/src/styles.css
Normal file
@@ -0,0 +1,14 @@
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Roboto", "Oxygen",
|
||||
"Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue",
|
||||
sans-serif;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
}
|
||||
|
||||
code {
|
||||
font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New",
|
||||
monospace;
|
||||
}
|
||||
28
web/tsconfig.json
Normal file
28
web/tsconfig.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"include": ["**/*.ts", "**/*.tsx"],
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"jsx": "react-jsx",
|
||||
"module": "ESNext",
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"types": ["vite/client"],
|
||||
|
||||
/* Bundler mode */
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"noEmit": true,
|
||||
|
||||
/* Linting */
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedSideEffectImports": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"],
|
||||
}
|
||||
}
|
||||
}
|
||||
32
web/vite.config.ts
Normal file
32
web/vite.config.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
import { defineConfig } from "vite";
|
||||
import viteReact from "@vitejs/plugin-react";
|
||||
import tanstackRouter from "@tanstack/router-plugin/vite";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
// https://vitejs.dev/config/
|
||||
export default defineConfig({
|
||||
plugins: [tanstackRouter({ autoCodeSplitting: true }), viteReact()],
|
||||
test: {
|
||||
globals: true,
|
||||
environment: "jsdom",
|
||||
},
|
||||
resolve: {
|
||||
alias: {
|
||||
"@": resolve(__dirname, "./src"),
|
||||
},
|
||||
},
|
||||
server: {
|
||||
port: 3000,
|
||||
proxy: {
|
||||
"/api": {
|
||||
target: "http://localhost:8080",
|
||||
changeOrigin: true,
|
||||
secure: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
build: {
|
||||
outDir: "dist",
|
||||
sourcemap: true,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user