diff --git a/DESIGN.md b/DESIGN.md index ef30b6f..9384abb 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -268,7 +268,8 @@ empty in the model. - Crate name: `ocd`. - Live watch mode for `htop`-like refresh. A later task. -- Send `If-Modified-Since` and `If-None-Match`. The cache already stores the - slots. The HTTP layer needs to attach the headers once we confirm OpenCode - emits ETag or Last-Modified. The TTL cache keeps politeness intact without it. - from TTL alone. \ No newline at end of file +- Conditional GET: `get_text` sends `If-None-Match` and `If-Modified-Since` + from cached validators and handles a 304 by reusing the cached parse. + OpenCode (via Cloudflare) currently emits no `ETag` or `Last-Modified` on + either page, so the slots stay empty and no headers go out. The machinery + is idle but correct; it works the day OpenCode adds validators. \ No newline at end of file diff --git a/src/cache.rs b/src/cache.rs index a23e769..5aa5b9f 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -46,6 +46,15 @@ pub fn read(account: &str, workspace: &str, ttl_secs: u32) -> Option Option { + let path = entry_path(account, workspace); + let body = std::fs::read_to_string(&path).ok()?; + serde_json::from_str(&body).ok() +} + /// Write a cache entry. Creates the cache dir on demand. pub fn write(account: &str, workspace: &str, record: &CachedRecord) -> std::io::Result<()> { let path = entry_path(account, workspace); diff --git a/src/client.rs b/src/client.rs index 5b8abad..08d850d 100644 --- a/src/client.rs +++ b/src/client.rs @@ -54,13 +54,23 @@ impl HttpClient { Ok(Self { client, retries }) } - /// GET a page as text with the account cookie. Retries transient errors - /// with exponential backoff and jitter. Returns the response body. - pub async fn get_text(&self, url: &str, cookie: &str) -> Result { + /// GET a page with optional conditional-GET validators. Retries transient + /// errors with exponential backoff and jitter. + /// + /// Returns the body plus any `ETag`/`Last-Modified` the server emitted. A + /// 304 Not Modified returns `body = None`; the caller reuses the cached + /// parse. `body.is_none()` is the 304 signal. + pub async fn get_text( + &self, + url: &str, + cookie: &str, + etag: Option<&str>, + last_modified: Option<&str>, + ) -> Result { let mut attempt: u8 = 0; loop { - match self.get_once(url, cookie).await { - Ok(body) => return Ok(body), + match self.get_once(url, cookie, etag, last_modified).await { + Ok(r) => return Ok(r), Err(e) if e.is_transient() && attempt < self.retries => { let backoff = backoff_for(attempt); tracing::warn!(attempt, backoff_ms = backoff.as_millis(), error = %e, "retry"); @@ -73,29 +83,44 @@ impl HttpClient { } } - async fn get_once(&self, url: &str, cookie: &str) -> Result { - let resp = self + async fn get_once( + &self, + url: &str, + cookie: &str, + etag: Option<&str>, + last_modified: Option<&str>, + ) -> Result { + let mut req = self .client .get(url) .header("cookie", cookie_header(cookie)) .header("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") - .header("accept-language", "en-US,en;q=0.9") - .send() - .await - .map_err(|e| { - if e.is_timeout() { - FetchError::Timeout - } else { - FetchError::Send(e.to_string()) - } - })?; + .header("accept-language", "en-US,en;q=0.9"); + if let Some(e) = etag { + req = req.header("if-none-match", e); + } + if let Some(lm) = last_modified { + req = req.header("if-modified-since", lm); + } + + let resp = req.send().await.map_err(|e| { + if e.is_timeout() { + FetchError::Timeout + } else { + FetchError::Send(e.to_string()) + } + })?; let status = resp.status().as_u16(); if status == 429 { return Err(FetchError::RateLimited); } if status == 304 { - return Err(FetchError::Send("unexpected 304".to_string())); + return Ok(FetchResult { + body: None, + etag: header_str(&resp, "etag"), + last_modified: header_str(&resp, "last-modified"), + }); } if !(200..300).contains(&status) { return Err(FetchError::Status { @@ -104,12 +129,25 @@ impl HttpClient { }); } - resp.text() - .await - .map_err(|e| FetchError::Body(e.to_string())) + let etag = header_str(&resp, "etag"); + let last_modified = header_str(&resp, "last-modified"); + let body = resp.text().await.map_err(|e| FetchError::Body(e.to_string()))?; + Ok(FetchResult { body: Some(body), etag, last_modified }) } } +/// Result of one GET. `body` is `None` when the server returned 304. +#[derive(Debug, Clone)] +pub struct FetchResult { + pub body: Option, + pub etag: Option, + pub last_modified: Option, +} + +fn header_str(resp: &reqwest::Response, name: &str) -> Option { + resp.headers().get(name).and_then(|v| v.to_str().ok()).map(|s| s.to_string()) +} + /// Backoff for `attempt` (0-based): 500 ms, 1 s, 2 s, capped at 8 s, plus /// jitter up to 25 per cent of each step. fn backoff_for(attempt: u8) -> Duration { diff --git a/src/fetch.rs b/src/fetch.rs index 95d01d6..c145397 100644 --- a/src/fetch.rs +++ b/src/fetch.rs @@ -74,9 +74,11 @@ pub async fn run(cfg: &crate::config::Config, opts: &FetchOptions) -> anyhow::Re Ok(reports) } -/// Fetch one target. Cache short-circuits the network. +/// Fetch one target. A fresh cache entry short-circuits the network. An expired +/// cache entry still supplies conditional-GET validators so the server can +/// answer 304 instead of resending the full page. async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> AccountReport { - // Cache hit path. + // Fresh cache hit: no network at all. if !opts.force_refresh && !opts.no_cache { if let Some(rec) = cache::read(&tgt.account, &tgt.workspace, opts.ttl_secs) { let mut r = rec.report; @@ -85,45 +87,65 @@ async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> A } } - // Network path. Fire Zen and Go at once. + // Recover validators and last-known parses from a stale entry, if any. + let stale = if opts.no_cache { None } else { cache::read_raw(&tgt.account, &tgt.workspace) }; + let (zen_etag, zen_lm) = stale.as_ref().map(|s| (s.zen_etag.clone(), s.zen_last_modified.clone())).unwrap_or((None, None)); + let (go_etag, go_lm) = stale.as_ref().map(|s| (s.go_etag.clone(), s.go_last_modified.clone())).unwrap_or((None, None)); + let stale_balance = stale.as_ref().and_then(|s| s.report.balance.clone()); + let stale_limits = stale.as_ref().and_then(|s| s.report.limits.clone()); + let zen_url = client::zen_url(&tgt.workspace); let go_url = client::go_url(&tgt.workspace); let (zen_res, go_res) = tokio::join!( - http.get_text(&zen_url, &tgt.cookie), - http.get_text(&go_url, &tgt.cookie), + http.get_text(&zen_url, &tgt.cookie, zen_etag.as_deref(), zen_lm.as_deref()), + http.get_text(&go_url, &tgt.cookie, go_etag.as_deref(), go_lm.as_deref()), ); let now = Utc::now(); let fetched_at = Some(now.to_rfc3339()); - let balance = match &zen_res { - Ok(html) => match scrape::parse_zen_balance(html) { - Ok(b) => Some(b), - Err(e) => { - tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "zen parse failed"); - None - } - }, + + // Balance: parse a fresh body, reuse the stale parse on 304, else None. + let (balance, zen_etag_new, zen_lm_new) = match &zen_res { + Ok(fr) => { + let bal = match fr.body.as_deref() { + Some(html) => match scrape::parse_zen_balance(html) { + Ok(b) => Some(b), + Err(e) => { + tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "zen parse failed"); + None + } + }, + None => stale_balance.clone(), + }; + (bal, fr.etag.clone(), fr.last_modified.clone()) + } Err(e) => { tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "zen fetch failed"); - None - } - }; - let limits = match &go_res { - Ok(html) => match scrape::parse_go_limits(html, now) { - Ok(l) => Some(l), - Err(e) => { - tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "go parse failed"); - None - } - }, - Err(e) => { - tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "go fetch failed"); - None + (None, None, None) + } + }; + + // Limits: parse a fresh body, reuse the stale parse on 304, else None. + let (limits, go_etag_new, go_lm_new) = match &go_res { + Ok(fr) => { + let lim = match fr.body.as_deref() { + Some(html) => match scrape::parse_go_limits(html, now) { + Ok(l) => Some(l), + Err(e) => { + tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "go parse failed"); + None + } + }, + None => stale_limits.clone(), + }; + (lim, fr.etag.clone(), fr.last_modified.clone()) + } + Err(e) => { + tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "go fetch failed"); + (None, None, None) } }; - // Decide the status. RateLimited and Timeout take priority over generic - // Error so callers can react. let status = report_status(&zen_res, &go_res, limits.as_ref(), balance.as_ref()); let report = AccountReport { @@ -135,14 +157,15 @@ async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> A balance: balance.clone(), }; - // Write cache only on a clean Ok, so errors never poison the cache. + // Write cache on a clean Ok so errors never poison the cache. Carry the + // latest validators so the next run can do conditional GET. if status == ReportStatus::Ok { let record = CachedRecord { report: report.clone(), - go_etag: None, - go_last_modified: None, - zen_etag: None, - zen_last_modified: None, + go_etag: go_etag_new, + go_last_modified: go_lm_new, + zen_etag: zen_etag_new, + zen_last_modified: zen_lm_new, }; if let Err(e) = cache::write(&tgt.account, &tgt.workspace, &record) { tracing::warn!(error = %e, "cache write failed"); @@ -154,8 +177,8 @@ async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> A /// Fold the two fetch results into one status. fn report_status( - zen: &Result, - go: &Result, + zen: &Result, + go: &Result, limits: Option<&Limits>, balance: Option<&Balance>, ) -> ReportStatus {