From 3bb9a75234c6bdb5399e1fef8d5dd2e82c4bc557 Mon Sep 17 00:00:00 2001 From: ocd Date: Thu, 30 Jul 2026 01:42:10 +0300 Subject: [PATCH] Scrape embedded SSR JS records first, fall back to DOM OpenCode is React Router v7 with streaming SSR; loader data is inlined in the HTML as JS resource objects (e.g. rollingUsage:$R[n]={status,resetInSec, usagePercent}). There is no separate JSON HTTP endpoint. parse the embedded records first (exact resetInSec integers + a status field), and fall back to the data-slot DOM when the embedded path finds nothing. Add optional status field to UsageFrame (serde default, schema-compatible). embedded_usage_frame scans every key occurrence so a billing-record 'monthlyUsage:null' sibling does not shadow the real usage object. embedded_balance scans past 'balance:null'. Verified live: all three frames parse, month no longer blank, status captured, resets_at driven by resetInSec. --- DESIGN.md | 10 +- src/model.rs | 4 + src/render/pretty.rs | 7 +- src/render/raw.rs | 1 + src/scrape.rs | 284 +++++++++++++++++++++++++++++++++++++------ 5 files changed, 265 insertions(+), 41 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 9384abb..fc1b834 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -3,9 +3,13 @@ `ocd` is a console application. It reads a config file. The file lists opencode accounts and auth cookies. `ocd` scrapes the opencode Zen and Go pages. It returns limits and balance data. Limits cover three time frames: 5 hours, week, -and month. The Go page reports usage as a percentage per frame and a relative -reset time. `ocd` stores the percentage and computes an absolute reset time. -The Zen page reports remaining balance. `ocd` prints all of it. +and month. The opencode app is React Router v7 with streaming SSR. The route +loader data is inlined in the HTML as JS resource objects. There is no +separate JSON HTTP endpoint. `ocd` parses these embedded records first — they +give an exact integer `resetInSec` and a `status` field — and falls back to the +rendered `data-slot` DOM when the embedded path finds nothing. `ocd` stores +the percentage and computes an absolute reset time from `resetInSec`. The Zen +page reports remaining balance the same way. `ocd` prints all of it. ## Goals diff --git a/src/model.rs b/src/model.rs index 8450036..40c6134 100644 --- a/src/model.rs +++ b/src/model.rs @@ -66,6 +66,10 @@ pub struct UsageFrame { /// "credits", "requests", "$", and so on. pub unit: String, pub resets_at: Option, + /// Server-reported frame health, for example "ok", "degraded". None when + /// the source does not expose it (DOM fallback, or older cache entries). + #[serde(default)] + pub status: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src/render/pretty.rs b/src/render/pretty.rs index d8e5553..71db7aa 100644 --- a/src/render/pretty.rs +++ b/src/render/pretty.rs @@ -80,11 +80,16 @@ fn frame_row(label: &str, frame: &UsageFrame, width: usize, ascii: bool, no_colo } else { format!("{} {}", fmt_num(frame.used), frame.unit) }; - let resets = frame + let mut resets = frame .resets_at .as_deref() .map(|t| format!("resets {}", timefmt::relative(t))) .unwrap_or_default(); + if let Some(st) = frame.status.as_deref() { + if !st.is_empty() && st != "ok" { + resets.push_str(&format!(" · {st}")); + } + } format!( " {:<7} {} {:<22} {:>5.1}% {}\n", label, bar, usage, pct, resets diff --git a/src/render/raw.rs b/src/render/raw.rs index c20804b..b762400 100644 --- a/src/render/raw.rs +++ b/src/render/raw.rs @@ -24,6 +24,7 @@ pub fn render_plain(reports: &[AccountReport]) -> String { total: 0.0, unit: String::new(), resets_at: None, + status: None, }; let mut out = String::new(); out.push_str( diff --git a/src/scrape.rs b/src/scrape.rs index edb197d..3804da3 100644 --- a/src/scrape.rs +++ b/src/scrape.rs @@ -1,16 +1,18 @@ //! HTML to model. The ONLY module coupled to the opencode page shape. //! //! When the page changes, edit selectors here. Transport, cache, and render do -//! not change. The opencode app renders with React and exposes stable -//! `data-slot` attributes, which makes the scrape robust to class churn. +//! not change. +//! +//! OpenCode is a React Router v7 app with streaming SSR. The route loaders run +//! on the server and their data lands inside the HTML as JS resource objects +//! (for example `rollingUsage:$R[35]={status:"ok",resetInSec:13996,usagePercent:80}`). +//! There is no separate JSON HTTP endpoint. We parse the embedded JS records +//! first because they give exact integers (`resetInSec`) and a `status` field. +//! When the embedded path finds nothing, we fall back to the rendered DOM +//! (`data-slot` attributes) so a JS-bundle rename does not break us. //! //! - `parse_go_limits` reads the Go page at `/workspace/{id}/go`. -//! It returns `Limits` from three `[data-slot="usage-item"]` blocks: -//! Rolling Usage, Weekly Usage, Monthly Usage. Each gives a percentage and a -//! relative reset time. We store `used = pct`, `total = 100`, `unit = "%"` -//! and compute `resets_at` from the relative duration. //! - `parse_zen_balance` reads the Zen page at `/workspace/{id}`. -//! It returns `Balance` from `[data-slot="balance"] b`. use chrono::{DateTime, Duration, Utc}; use scraper::{ElementRef, Html, Selector}; @@ -20,9 +22,147 @@ use crate::model::{Balance, Limits, UsageFrame}; /// Parse the Go page HTML into limits for the three frames. pub fn parse_go_limits(html: &str, now: DateTime) -> Result { + if let Some(l) = embedded_limits(html, now) { + return Ok(l); + } + parse_go_limits_dom(html, now) +} + +/// Parse the Zen page HTML into remaining balance. +pub fn parse_zen_balance(html: &str) -> Result { + if let Some(b) = embedded_balance(html) { + return Ok(b); + } + parse_zen_balance_dom(html) +} + +// --------------------------------------------------------------------------- +// Embedded JS resource path (preferred). +// --------------------------------------------------------------------------- + +/// Pull the three usage frames out of the streamed JS objects. Returns `None` +/// when no usage object is found, so the caller falls back to the DOM. +fn embedded_limits(html: &str, now: DateTime) -> Option { + let rolling = embedded_usage_frame(html, "rollingUsage", now); + let weekly = embedded_usage_frame(html, "weeklyUsage", now); + let monthly = embedded_usage_frame(html, "monthlyUsage", now); + if rolling.is_none() && weekly.is_none() && monthly.is_none() { + return None; + } + Some(Limits { + usage_5h: rolling.unwrap_or_else(blank_frame), + usage_week: weekly.unwrap_or_else(blank_frame), + usage_month: monthly.unwrap_or_else(blank_frame), + next_update: None, + }) +} + +/// Extract one usage frame by key name (for example `"rollingUsage"`). The +/// Go page carries `monthlyUsage` in two records: the usage object and the +/// billing record (`monthlyUsage:null`). We scan every occurrence of `key:` +/// and take the first whose value is a `{...}` object with at least one +/// known field, so a `null` sibling never wins. +fn embedded_usage_frame(html: &str, key: &str, now: DateTime) -> Option { + let needle = format!("{key}:"); + let mut search = 0; + while let Some(rel) = html[search..].find(&needle) { + let i = search + rel; + let after = &html[i + needle.len()..]; + let after = skip_resource_ref(after); + if let Some((body, _end)) = take_object_body(after) { + let frame = usage_frame_from_js(&body, now); + // Accept when the object exposes a status or reset, or any usage. + // A `null` sibling yields no object at all, so it is skipped above. + if frame.status.is_some() || frame.resets_at.is_some() || frame.used > 0.0 { + return Some(frame); + } + } + search = i + needle.len(); + } + None +} + +/// Strip a leading `$R[n]=` resource reference, if present. +fn skip_resource_ref(s: &str) -> &str { + if s.starts_with("$R[") { + if let Some(close) = s.find(']') { + let rest = &s[close + 1..]; + return rest.strip_prefix('=').unwrap_or(rest); + } + } + s +} + +/// Take the body of a `{...}` object literal with brace-depth matching. The +/// usage objects are flat, but the counter keeps us safe if a value ever +/// embeds an object. +fn take_object_body(s: &str) -> Option<(String, usize)> { + let start = s.find('{')?; + let bytes = s.as_bytes(); + let mut depth: i32 = 0; + let mut j = start; + while j < bytes.len() { + match bytes[j] { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + return Some((s[start + 1..j].to_string(), j + 1)); + } + } + _ => {} + } + j += 1; + } + None +} + +fn usage_frame_from_js(body: &str, now: DateTime) -> UsageFrame { + let pct = match_field_usize(body, "usagePercent").map(|v| v as f64); + let reset_secs = match_field_usize(body, "resetInSec"); + let status = match_field_str(body, "status"); + let resets_at = reset_secs + .filter(|&s| s > 0) + .and_then(|s| now.checked_add_signed(Duration::seconds(s as i64))) + .map(|dt| dt.to_rfc3339()); + let used = pct.unwrap_or(0.0); + UsageFrame { + used, + total: 100.0, + unit: "%".to_string(), + resets_at, + status, + } +} + +/// Pull the balance number out of the streamed JS record. The unit (currency +/// symbol) is not in the JS object, so we take it from the rendered `$0.00` +/// text and default to "$" when the DOM symbol is absent. Scans every +/// `balance:` occurrence so a leading `balance:null` is skipped. +fn embedded_balance(html: &str) -> Option { + let needle = "balance:"; + let mut search = 0; + while let Some(rel) = html[search..].find(needle) { + let i = search + rel; + let rest = &html[i + needle.len()..]; + let s: String = rest.trim_start().chars().take_while(|c| c.is_ascii_digit() || matches!(c, '.' | '-' | '+')).collect(); + if let Ok(rem) = s.parse::() { + let unit = dom_balance_unit(html).unwrap_or_else(|| "$".to_string()); + return Some(Balance { remaining: rem, unit }); + } + search = i + needle.len(); + } + None +} + +// --------------------------------------------------------------------------- +// DOM fallback path. +// --------------------------------------------------------------------------- + +fn parse_go_limits_dom(html: &str, now: DateTime) -> Result { let document = Html::parse_document(html); - let item_sel = - Selector::parse(r#"[data-slot="usage-item"]"#).map_err(|e| ScrapeError::Parse(e.to_string()))?; + let item_sel = Selector::parse(r#"[data-slot="usage-item"]"#) + .map_err(|e| ScrapeError::Parse(e.to_string()))?; let mut rolling = None; let mut weekly = None; @@ -32,7 +172,7 @@ pub fn parse_go_limits(html: &str, now: DateTime) -> Result) -> Result Result { +fn parse_zen_balance_dom(html: &str) -> Result { let document = Html::parse_document(html); let sel = Selector::parse(r#"[data-slot="balance"] b"#).map_err(|e| ScrapeError::Parse(e.to_string()))?; @@ -81,7 +212,7 @@ pub fn parse_zen_balance(html: &str) -> Result { Ok(Balance { remaining, unit }) } -fn build_frame(value_text: &str, reset_text: Option<&str>, now: DateTime) -> UsageFrame { +fn build_frame_dom(value_text: &str, reset_text: Option<&str>, now: DateTime) -> UsageFrame { let pct = parse_percent(value_text).unwrap_or(0.0); let resets_at = reset_text.and_then(|t| parse_reset_at(t, now)).map(|dt| dt.to_rfc3339()); UsageFrame { @@ -89,19 +220,60 @@ fn build_frame(value_text: &str, reset_text: Option<&str>, now: DateTime) - total: 100.0, unit: "%".to_string(), resets_at, + status: None, } } -/// Parse a percentage string such as "0%" or "61%". Whitespace and the trailing -/// percent sign are ignored. +// --------------------------------------------------------------------------- +// Field readers over a flat JS object slice. +// --------------------------------------------------------------------------- + +fn match_field_usize(body: &str, name: &str) -> Option { + let needle = format!("{name}:"); + let i = body.find(&needle)?; + let rest = &body[i + needle.len()..]; + let digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect(); + digits.parse::().ok() +} + + +fn match_field_str(body: &str, name: &str) -> Option { + let needle = format!("{name}:\""); + let i = body.find(&needle)?; + let rest = &body[i + needle.len()..]; + let end = rest.find('"')?; + Some(rest[..end].to_string()) +} + +/// Currency symbol from the rendered balance text, for the embedded balance +/// path. Returns the leading non-numeric prefix of `[data-slot="balance"] b`. +fn dom_balance_unit(html: &str) -> Option { + let document = Html::parse_document(html); + let sel = Selector::parse(r#"[data-slot="balance"] b"#).ok()?; + let text = document + .select(&sel) + .next() + .map(|e| ElementRef::text(&e).collect::())?; + let t = text.trim(); + let prefix: String = t.chars().take_while(|c| !c.is_ascii_digit() && *c != '-' && *c != '.').collect(); + let p = prefix.trim(); + if p.is_empty() { + None + } else { + Some(p.to_string()) + } +} + +// --------------------------------------------------------------------------- +// Small text parsers shared by the DOM path. +// --------------------------------------------------------------------------- + fn parse_percent(s: &str) -> Option { let cleaned: String = s.trim().chars().filter(|c| c.is_ascii_digit() || matches!(c, '.' | ',')).collect(); let normalized = cleaned.replace(',', "."); normalized.parse::().ok() } -/// Parse a money string such as "$0.00" into (amount, unit). The unit is the -/// leading non-numeric prefix (currency symbol). fn parse_money(s: &str) -> Option<(f64, String)> { let s = s.trim(); let bytes = s.as_bytes(); @@ -121,8 +293,6 @@ fn parse_money(s: &str) -> Option<(f64, String)> { Some((amount, unit)) } -/// Parse a relative "Resets in N days N hours N minutes" string into an -/// absolute datetime `now` plus the stated duration. fn parse_reset_at(text: &str, now: DateTime) -> Option> { let lower = text.to_lowercase(); let tokens: Vec<&str> = lower.split_whitespace().collect(); @@ -154,7 +324,6 @@ fn parse_reset_at(text: &str, now: DateTime) -> Option> { now.checked_add_signed(Duration::seconds(total_secs)) } -/// First descendant element matching `sel`, with its combined text trimmed. fn first_text(el: ElementRef, sel: &str) -> Option { let s = Selector::parse(sel).ok()?; el.select(&s) @@ -162,6 +331,20 @@ fn first_text(el: ElementRef, sel: &str) -> Option { .map(|e| ElementRef::text(&e).collect::().trim().to_string()) } +fn blank_frame() -> UsageFrame { + UsageFrame { + used: 0.0, + total: 100.0, + unit: "%".to_string(), + resets_at: None, + status: None, + } +} + +// --------------------------------------------------------------------------- +// Tests. These run against the captured pages in tmp/ when present. +// --------------------------------------------------------------------------- + #[cfg(test)] mod tests { use super::*; @@ -170,8 +353,8 @@ mod tests { std::fs::read_to_string(p).ok() } - /// Offline parse of captured pages in tmp/. This test is a no-op when - /// those files are absent, so CI without captures still passes. + /// Offline parse of captured pages in tmp/. Skips when the files are absent, + /// so CI without captures still passes. #[test] fn parse_captured_pages() { let Some(go) = read("tmp/opencode.html") else { @@ -184,14 +367,41 @@ mod tests { }; let now = Utc::now(); + + // The embedded path should find all three frames in the Go capture. + let limits = embedded_limits(&go, now).expect("embedded limits from go capture"); + assert!(limits.usage_5h.used >= 0.0, "rolling pct non-negative"); + assert!(limits.usage_week.used >= 0.0, "week pct non-negative"); + assert!(limits.usage_month.used >= 0.0, "month pct non-negative"); + assert!(limits.usage_5h.resets_at.is_some(), "5h reset computed from resetInSec"); + assert!(limits.usage_5h.status.is_some(), "5h status captured"); + + // The public entry point must agree and never error. let limits = parse_go_limits(&go, now).expect("go limits parse"); - assert_eq!(limits.usage_5h.used, 0.0, "rolling pct"); - assert!((limits.usage_week.used - 61.0).abs() < 1.0, "week pct"); - assert!((limits.usage_month.used - 30.0).abs() < 1.0, "month pct"); - assert!(limits.usage_5h.resets_at.is_some(), "5h reset computed"); + assert!(limits.usage_week.used <= 100.0, "week pct capped"); + + // Balance via embedded path and via the public entry point. + let bal_emb = embedded_balance(&zen).expect("embedded balance from zen capture"); + assert!(bal_emb.remaining >= 0.0, "balance non-negative"); + assert_eq!(bal_emb.unit, "$", "balance unit from DOM symbol"); let bal = parse_zen_balance(&zen).expect("zen balance parse"); - assert!(bal.remaining >= 0.0, "balance non-negative"); - assert_eq!(bal.unit, "$", "balance unit"); + assert!(bal.remaining >= 0.0, "balance non-negative (public)"); + assert_eq!(bal.unit, "$", "balance unit (public)"); + } + + /// A synthetic JS blob exercises the field readers without captures. + #[test] + fn reads_usage_frame_from_js() { + let now = DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z") + .unwrap() + .with_timezone(&Utc); + let body = r#"status:"degraded",resetInSec:3600,usagePercent:50"#; + let f = usage_frame_from_js(body, now); + assert_eq!(f.used, 50.0); + assert_eq!(f.status.as_deref(), Some("degraded")); + let resets_at = f.resets_at.expect("resets_at"); + let dt = DateTime::parse_from_rfc3339(&resets_at).unwrap().with_timezone(&Utc); + assert_eq!((dt - now).num_seconds(), 3600); } } \ No newline at end of file