From 16fba6aabc264e9c59387174284007df2a996633 Mon Sep 17 00:00:00 2001 From: Xevion Date: Mon, 29 Dec 2025 16:43:37 -0600 Subject: [PATCH] build(docker): consolidate WASM build into multi-stage Dockerfile - Move WASM compilation from GitHub Actions into Docker build stages - Add emsdkVersion to package.json config as single source of truth - Implement cargo-chef for dependency caching in both WASM and server builds - Update .dockerignore to include packed game assets while excluding unpacked - Simplify deploy workflow by removing local WASM build steps --- .dockerignore | 12 ++- .github/workflows/deploy.yaml | 68 +++------------ README.md | 16 +++- package.json | 3 + pacman-server/Dockerfile | 155 +++++++++++++++++++++++++++------- pacman/Cargo.toml | 2 +- pacman/web.build.ts | 22 +++-- 7 files changed, 176 insertions(+), 102 deletions(-) diff --git a/.dockerignore b/.dockerignore index db7776b..1552319 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,8 +3,9 @@ /emsdk *.exe -/pacman/assets -/assets +# Exclude unpacked assets but keep packed game assets +/pacman/assets/unpacked +/pacman/assets/site # Web/Node artifacts node_modules @@ -16,8 +17,13 @@ node_modules # Development files /.git +/.github /*.md /Justfile /bacon.toml -/rust-toolchain.toml /rustfmt.toml +/.pre-commit-config.yaml + +# Test files (not needed in Docker builds) +/pacman/tests +/pacman-server/tests diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 8de85dc..edd4ef4 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -24,67 +24,18 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - name: Setup Emscripten SDK - uses: pyodide/setup-emsdk@v15 - with: - version: latest - actions-cache-folder: "emsdk-cache-b" - - - name: Setup Rust (WASM32 Emscripten) - uses: dtolnay/rust-toolchain@master - with: - target: wasm32-unknown-emscripten - toolchain: 1.86.0 - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - - - name: Setup Bun - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - # ========== WASM Build ========== - - name: Build WASM with Emscripten - shell: bash + - name: Read emsdk version from package.json + id: versions run: | - # Retry mechanism for Emscripten build - only retry on specific hash errors - MAX_RETRIES=3 - RETRY_DELAY=30 - - for attempt in $(seq 1 $MAX_RETRIES); do - echo "Build attempt $attempt of $MAX_RETRIES" - - # Capture output and check for specific error while preserving real-time output - if bun run -i pacman/web.build.ts 2>&1 | tee /tmp/build_output.log; then - echo "Build successful on attempt $attempt" - break - else - echo "Build failed on attempt $attempt" - - # Check if the failure was due to the specific hash error - if grep -q "emcc: error: Unexpected hash:" /tmp/build_output.log; then - echo "::warning::Detected 'emcc: error: Unexpected hash:' error - will retry (attempt $attempt of $MAX_RETRIES)" - - if [ $attempt -eq $MAX_RETRIES ]; then - echo "::error::All retry attempts failed. Exiting with error." - exit 1 - fi - - echo "Waiting $RETRY_DELAY seconds before retry..." - sleep $RETRY_DELAY - - # Exponential backoff: double the delay for next attempt - RETRY_DELAY=$((RETRY_DELAY * 2)) - else - echo "Build failed but not due to hash error - not retrying" - exit 1 - fi - fi - done + EMSDK_VERSION=$(jq -r '.config.emsdkVersion' package.json) + if [ -z "$EMSDK_VERSION" ] || [ "$EMSDK_VERSION" = "null" ]; then + echo "::error::emsdkVersion not found in package.json config" + exit 1 + fi + echo "emsdk=$EMSDK_VERSION" >> $GITHUB_OUTPUT # ========== Docker Build and Push ========== - # Note: Frontend is now built inside Docker using multi-stage build + # Note: WASM and frontend are built inside Docker using multi-stage build - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -118,6 +69,7 @@ jobs: cache-to: type=gha,mode=max build-args: | GIT_COMMIT_SHA=${{ github.sha }} + EMSDK_VERSION=${{ steps.versions.outputs.emsdk }} # Wait for ghcr.io propagation (paranoid safety) - name: Wait for registry propagation diff --git a/README.md b/README.md index 773173c..95da055 100644 --- a/README.md +++ b/README.md @@ -91,11 +91,23 @@ Since this project is still in progress, I'm only going to cover non-obvious bui - We use rustc 1.86.0 for the build, due to bulk-memory-opt related issues on wasm32-unknown-emscripten. - Technically, we could probably use stable or even nightly on desktop targets, but using different versions for different targets is a pain, mainly because of clippy warnings changing between versions. - For the WASM build, you need to have the Emscripten SDK cloned; you can do so with `git clone https://github.com/emscripten-core/emsdk.git` - - The first time you clone, you'll need to install the SDK with `./emsdk install latest` and then activate it with `./emsdk activate latest`. On Windows, use `./emsdk/emsdk.ps1` instead. - - Occasionally, the build will fail due to dependencies failing to download. I even have a retry mechanism in the build workflow due to this. + - The project's emsdk version is specified in `package.json` (`config.emsdkVersion`, currently 4.0.22) + - To install manually: + ```bash + # Linux/macOS + EMSDK_VERSION=$(jq -r '.config.emsdkVersion' package.json) + ./emsdk install $EMSDK_VERSION && ./emsdk activate $EMSDK_VERSION + + # Windows PowerShell + $EMSDK_VERSION = (Get-Content package.json | ConvertFrom-Json).config.emsdkVersion + ./emsdk/emsdk.ps1 install $EMSDK_VERSION + ./emsdk/emsdk.ps1 activate $EMSDK_VERSION + ``` + - Or simply run `bun run pacman/web.build.ts` which automatically installs the correct version - You can then activate the Emscripten SDK with `source ./emsdk/emsdk_env.sh` or `./emsdk/emsdk_env.ps1` or `./emsdk/emsdk_env.bat` depending on your OS/terminal. - While using the `web.build.ts` is not technically required, it simplifies the build process and is very helpful. - It is intended to be run with `bun`, which you can acquire at [bun.sh](https://bun.sh/) + - Occasionally, the build will fail due to dependencies failing to download. - Tip: You can launch a fileserver with `python` or `caddy` to serve the files in the `dist` folder. - `python3 -m http.server 8080 -d dist` - `caddy file-server --root dist` (install with `[sudo apt|brew|choco] install caddy` or [a dozen other ways](https://caddyserver.com/docs/install)) diff --git a/package.json b/package.json index 3edfdb6..a1b0517 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,9 @@ "name": "pacman", "type": "module", "packageManager": "bun@^1.3.5", + "config": { + "emsdkVersion": "4.0.22" + }, "engines": { "bun": ">=1.3.5" }, diff --git a/pacman-server/Dockerfile b/pacman-server/Dockerfile index be7a96d..839dd98 100644 --- a/pacman-server/Dockerfile +++ b/pacman-server/Dockerfile @@ -1,67 +1,158 @@ -ARG RUST_VERSION=1.89.0 +ARG RUST_VERSION=1.86.0 +ARG EMSDK_VERSION=4.0.22 ARG GIT_COMMIT_SHA +# ========== Stage 1: WASM Planner ========== +FROM emscripten/emsdk:${EMSDK_VERSION} AS wasm-planner +ARG RUST_VERSION +WORKDIR /app + +# Install Rust and cargo-chef for dependency caching +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + --default-toolchain ${RUST_VERSION} \ + --profile minimal +ENV PATH="/root/.cargo/bin:${PATH}" +RUN cargo install cargo-chef --locked + +# Copy workspace for recipe generation +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY .cargo/ ./.cargo/ +COPY pacman-common/ ./pacman-common/ +COPY pacman/ ./pacman/ +COPY pacman-server/ ./pacman-server/ + +# Generate dependency recipe +RUN cargo chef prepare --bin pacman --recipe-path recipe.json + +# ========== Stage 2: WASM Dependency Builder ========== +FROM emscripten/emsdk:${EMSDK_VERSION} AS wasm-deps +ARG RUST_VERSION +WORKDIR /app + +# Install Rust with WASM target and cargo-chef +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + --default-toolchain ${RUST_VERSION} \ + --profile minimal +ENV PATH="/root/.cargo/bin:${PATH}" +RUN rustup target add wasm32-unknown-emscripten && \ + cargo install cargo-chef --locked + +# Cook dependencies only (this layer is cached until Cargo.toml/Cargo.lock change) +# Note: Assets are required during linking due to --preload-file in rustflags +COPY --from=wasm-planner /app/recipe.json recipe.json +COPY .cargo/ ./.cargo/ +COPY pacman/assets/game/ ./pacman/assets/game/ +RUN cargo chef cook --release --target wasm32-unknown-emscripten --recipe-path recipe.json + +# ========== Stage 3: WASM Builder ========== +FROM emscripten/emsdk:${EMSDK_VERSION} AS wasm-builder +ARG RUST_VERSION +WORKDIR /app + +# Install Rust with WASM target (minimal, no cargo-chef needed) +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + --default-toolchain ${RUST_VERSION} \ + --profile minimal +ENV PATH="/root/.cargo/bin:${PATH}" +RUN rustup target add wasm32-unknown-emscripten + +# Copy cached dependencies from wasm-deps +COPY --from=wasm-deps /app/target target +COPY --from=wasm-deps /root/.cargo /root/.cargo + +# Copy workspace source (build from workspace root so preload-file paths work) +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY .cargo/ ./.cargo/ +COPY pacman-common/ ./pacman-common/ +COPY pacman/ ./pacman/ +COPY pacman-server/ ./pacman-server/ + +# Build WASM binary (dependencies already cached) +RUN cargo build --release --target wasm32-unknown-emscripten --bin pacman + +# ========== Stage 4: Frontend Builder ========== +FROM oven/bun:1 AS frontend-builder +WORKDIR /app + +# Copy package files for dependency installation +COPY web/package.json web/bun.lock* ./ +RUN bun install --frozen-lockfile + +# Copy WASM artifacts from wasm-builder stage +# Note: .wasm and .js are in release/, but .data (preloaded assets) is in release/deps/ +RUN mkdir -p ./public +COPY --from=wasm-builder /app/target/wasm32-unknown-emscripten/release/pacman.wasm ./public/pacman.wasm +COPY --from=wasm-builder /app/target/wasm32-unknown-emscripten/release/pacman.js ./public/pacman.js +COPY --from=wasm-builder /app/target/wasm32-unknown-emscripten/release/deps/pacman.data ./public/pacman.data + +# Verify WASM artifacts exist and have reasonable sizes +RUN test -f ./public/pacman.wasm && \ + test -f ./public/pacman.js && \ + test -f ./public/pacman.data && \ + [ $(stat -c%s ./public/pacman.wasm) -gt $((1024 * 1024)) ] && \ + [ $(stat -c%s ./public/pacman.js) -gt $((100 * 1024)) ] && \ + [ $(stat -c%s ./public/pacman.data) -gt $((10 * 1024)) ] && \ + echo "WASM artifacts verified (wasm >1MiB, js >100KiB, data >10KiB)" || \ + (echo "WASM artifacts missing or too small!" && exit 1) + +# Copy frontend source +COPY web/ ./ + +# Build frontend (Vite bundles WASM files from public/) +RUN bun run build + +# ========== Stage 5: Backend Chef ========== FROM lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION} AS chef WORKDIR /app -# -- Planner stage -- +# ========== Stage 6: Backend Planner ========== FROM chef AS planner COPY . . RUN cargo chef prepare --bin pacman-server --recipe-path recipe.json -# -- Frontend builder stage -- -FROM oven/bun:1 AS frontend-builder -WORKDIR /app - -# Copy frontend package files first for layer caching -COPY web/package.json web/bun.lock* ./ -RUN bun install --frozen-lockfile - -# Copy all frontend source including public directory (contains WASM files) -COPY web/ ./ - -# Build the frontend (Vite will copy public/ contents to dist/client/) -RUN bun run build - -# -- Backend builder stage -- +# ========== Stage 7: Backend Builder ========== FROM chef AS builder COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --release --bin pacman-server --recipe-path recipe.json -# Copy the source code AFTER, so that dependencies are already cached +# Copy source code COPY . . -# Install build dependencies, then build the server -RUN apt-get update && apt-get install -y pkg-config libssl-dev && rm -rf /var/lib/apt/lists/* +# Install build dependencies and compile server +RUN apt-get update && \ + apt-get install -y pkg-config libssl-dev && \ + rm -rf /var/lib/apt/lists/* RUN cargo build --package pacman-server --release --bin pacman-server -# -- Runtime stage -- +# ========== Stage 8: Runtime ========== FROM debian:bookworm-slim AS runtime WORKDIR /app + +# Install runtime dependencies (curl needed for healthcheck) +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates tzdata curl && \ + rm -rf /var/lib/apt/lists/* + +# Copy compiled server binary COPY --from=builder /app/target/release/pacman-server /usr/local/bin/pacman-server -# Copy frontend static files from frontend-builder stage +# Copy frontend static files (includes WASM) COPY --from=frontend-builder /app/dist/client /app/static -# Install runtime dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - tzdata \ - && rm -rf /var/lib/apt/lists/* - +# Environment configuration ARG TZ=Etc/UTC ENV TZ=${TZ} -# Optional build-time environment variable for embedding the Git commit SHA ARG GIT_COMMIT_SHA ENV GIT_COMMIT_SHA=${GIT_COMMIT_SHA} -# Specify PORT at build-time or run-time, default to 3000 ARG PORT=3000 ENV PORT=${PORT} EXPOSE ${PORT} -# Set static files directory for the server to serve ENV STATIC_FILES_DIR=/app/static -CMD ["sh", "-c", "exec /usr/local/bin/pacman-server"] +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD curl --fail http://localhost:${PORT}/api/health || exit 1 + +CMD ["pacman-server"] diff --git a/pacman/Cargo.toml b/pacman/Cargo.toml index 1f89c1d..bec0fdb 100644 --- a/pacman/Cargo.toml +++ b/pacman/Cargo.toml @@ -30,7 +30,7 @@ anyhow = "1.0" smallvec = "1.15.1" bitflags = "2.9.4" micromap = "0.1.0" -circular-buffer = "=1.1.0" +circular-buffer = "1.1.0" parking_lot = "0.12.3" strum = "0.27.2" strum_macros = "0.27.2" diff --git a/pacman/web.build.ts b/pacman/web.build.ts index 3f18e26..12f2583 100644 --- a/pacman/web.build.ts +++ b/pacman/web.build.ts @@ -21,6 +21,16 @@ await configure({ const logger = getLogger("web"); +// Read emsdk version from package.json (single source of truth) +const packageJson = JSON.parse( + await fs.readFile(resolve(__dirname, "../package.json"), "utf-8") +); +const EMSDK_VERSION = packageJson.config?.emsdkVersion; +if (!EMSDK_VERSION) { + logger.error("emsdkVersion not found in package.json config"); + process.exit(1); +} + type Os = | { type: "linux"; wsl: boolean } | { type: "windows" } @@ -163,19 +173,19 @@ async function activateEmsdk( }; } - // Install latest version - logger.debug("Installing latest Emscripten version..."); + // Install emsdk version from package.json + logger.debug(`Installing emsdk version ${EMSDK_VERSION}...`); const emsdkBinary = join(emsdkDir, os.type === "windows" ? "emsdk.bat" : "emsdk"); - const installResult = await $`${emsdkBinary} install latest`.quiet(); + const installResult = await $`${emsdkBinary} install ${EMSDK_VERSION}`.quiet(); if (installResult.exitCode !== 0) { return { err: `Failed to install emsdk: ${installResult.stderr.toString()}`, }; } - // Activate latest version - logger.debug("Activating latest Emscripten version..."); - const activateResult = await $`${emsdkBinary} activate latest`.quiet(); + // Activate emsdk version from package.json + logger.debug(`Activating emsdk version ${EMSDK_VERSION}...`); + const activateResult = await $`${emsdkBinary} activate ${EMSDK_VERSION}`.quiet(); if (activateResult.exitCode !== 0) { return { err: `Failed to activate emsdk: ${activateResult.stderr.toString()}`,