Implement conditional GET (inert until OpenCode emits validators)

HttpClient.get_text now accepts etag/last_modified and sends
If-None-Match/If-Modified-Since; a 304 returns no body and the caller reuses
the cached parse. fetch_target recovers validators from the stale cache entry
(via new cache::read_raw, TTL-agnostic) and stores whatever the server emits.
Verified live: OpenCode/Cloudflare returns no ETag or Last-Modified on either
page, so the slots stay None, no headers go out, and 304 never fires. The
machinery is correct and ready for the day they add validators.
This commit is contained in:
ocd 2026-07-30 01:27:30 +03:00
parent 4463a9b51d
commit 4bf3eb1e06
4 changed files with 132 additions and 61 deletions

View File

@ -268,7 +268,8 @@ empty in the model.
- Crate name: `ocd`.
- Live watch mode for `htop`-like refresh. A later task.
- Send `If-Modified-Since` and `If-None-Match`. The cache already stores the
slots. The HTTP layer needs to attach the headers once we confirm OpenCode
emits ETag or Last-Modified. The TTL cache keeps politeness intact without it.
from TTL alone.
- Conditional GET: `get_text` sends `If-None-Match` and `If-Modified-Since`
from cached validators and handles a 304 by reusing the cached parse.
OpenCode (via Cloudflare) currently emits no `ETag` or `Last-Modified` on
either page, so the slots stay empty and no headers go out. The machinery
is idle but correct; it works the day OpenCode adds validators.

View File

@ -46,6 +46,15 @@ pub fn read(account: &str, workspace: &str, ttl_secs: u32) -> Option<CachedRecor
serde_json::from_str(&body).ok()
}
/// Read a cache entry ignoring TTL. Used to recover conditional-GET validators
/// (etag, last_modified) from an expired entry so the next request can still
/// ask the server whether anything changed.
pub fn read_raw(account: &str, workspace: &str) -> Option<CachedRecord> {
let path = entry_path(account, workspace);
let body = std::fs::read_to_string(&path).ok()?;
serde_json::from_str(&body).ok()
}
/// Write a cache entry. Creates the cache dir on demand.
pub fn write(account: &str, workspace: &str, record: &CachedRecord) -> std::io::Result<()> {
let path = entry_path(account, workspace);

View File

@ -54,13 +54,23 @@ impl HttpClient {
Ok(Self { client, retries })
}
/// GET a page as text with the account cookie. Retries transient errors
/// with exponential backoff and jitter. Returns the response body.
pub async fn get_text(&self, url: &str, cookie: &str) -> Result<String, FetchError> {
/// GET a page with optional conditional-GET validators. Retries transient
/// errors with exponential backoff and jitter.
///
/// Returns the body plus any `ETag`/`Last-Modified` the server emitted. A
/// 304 Not Modified returns `body = None`; the caller reuses the cached
/// parse. `body.is_none()` is the 304 signal.
pub async fn get_text(
&self,
url: &str,
cookie: &str,
etag: Option<&str>,
last_modified: Option<&str>,
) -> Result<FetchResult, FetchError> {
let mut attempt: u8 = 0;
loop {
match self.get_once(url, cookie).await {
Ok(body) => return Ok(body),
match self.get_once(url, cookie, etag, last_modified).await {
Ok(r) => return Ok(r),
Err(e) if e.is_transient() && attempt < self.retries => {
let backoff = backoff_for(attempt);
tracing::warn!(attempt, backoff_ms = backoff.as_millis(), error = %e, "retry");
@ -73,29 +83,44 @@ impl HttpClient {
}
}
async fn get_once(&self, url: &str, cookie: &str) -> Result<String, FetchError> {
let resp = self
async fn get_once(
&self,
url: &str,
cookie: &str,
etag: Option<&str>,
last_modified: Option<&str>,
) -> Result<FetchResult, FetchError> {
let mut req = self
.client
.get(url)
.header("cookie", cookie_header(cookie))
.header("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
.header("accept-language", "en-US,en;q=0.9")
.send()
.await
.map_err(|e| {
if e.is_timeout() {
FetchError::Timeout
} else {
FetchError::Send(e.to_string())
}
})?;
.header("accept-language", "en-US,en;q=0.9");
if let Some(e) = etag {
req = req.header("if-none-match", e);
}
if let Some(lm) = last_modified {
req = req.header("if-modified-since", lm);
}
let resp = req.send().await.map_err(|e| {
if e.is_timeout() {
FetchError::Timeout
} else {
FetchError::Send(e.to_string())
}
})?;
let status = resp.status().as_u16();
if status == 429 {
return Err(FetchError::RateLimited);
}
if status == 304 {
return Err(FetchError::Send("unexpected 304".to_string()));
return Ok(FetchResult {
body: None,
etag: header_str(&resp, "etag"),
last_modified: header_str(&resp, "last-modified"),
});
}
if !(200..300).contains(&status) {
return Err(FetchError::Status {
@ -104,12 +129,25 @@ impl HttpClient {
});
}
resp.text()
.await
.map_err(|e| FetchError::Body(e.to_string()))
let etag = header_str(&resp, "etag");
let last_modified = header_str(&resp, "last-modified");
let body = resp.text().await.map_err(|e| FetchError::Body(e.to_string()))?;
Ok(FetchResult { body: Some(body), etag, last_modified })
}
}
/// Result of one GET. `body` is `None` when the server returned 304.
#[derive(Debug, Clone)]
pub struct FetchResult {
pub body: Option<String>,
pub etag: Option<String>,
pub last_modified: Option<String>,
}
fn header_str(resp: &reqwest::Response, name: &str) -> Option<String> {
resp.headers().get(name).and_then(|v| v.to_str().ok()).map(|s| s.to_string())
}
/// Backoff for `attempt` (0-based): 500 ms, 1 s, 2 s, capped at 8 s, plus
/// jitter up to 25 per cent of each step.
fn backoff_for(attempt: u8) -> Duration {

View File

@ -74,9 +74,11 @@ pub async fn run(cfg: &crate::config::Config, opts: &FetchOptions) -> anyhow::Re
Ok(reports)
}
/// Fetch one target. Cache short-circuits the network.
/// Fetch one target. A fresh cache entry short-circuits the network. An expired
/// cache entry still supplies conditional-GET validators so the server can
/// answer 304 instead of resending the full page.
async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> AccountReport {
// Cache hit path.
// Fresh cache hit: no network at all.
if !opts.force_refresh && !opts.no_cache {
if let Some(rec) = cache::read(&tgt.account, &tgt.workspace, opts.ttl_secs) {
let mut r = rec.report;
@ -85,45 +87,65 @@ async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> A
}
}
// Network path. Fire Zen and Go at once.
// Recover validators and last-known parses from a stale entry, if any.
let stale = if opts.no_cache { None } else { cache::read_raw(&tgt.account, &tgt.workspace) };
let (zen_etag, zen_lm) = stale.as_ref().map(|s| (s.zen_etag.clone(), s.zen_last_modified.clone())).unwrap_or((None, None));
let (go_etag, go_lm) = stale.as_ref().map(|s| (s.go_etag.clone(), s.go_last_modified.clone())).unwrap_or((None, None));
let stale_balance = stale.as_ref().and_then(|s| s.report.balance.clone());
let stale_limits = stale.as_ref().and_then(|s| s.report.limits.clone());
let zen_url = client::zen_url(&tgt.workspace);
let go_url = client::go_url(&tgt.workspace);
let (zen_res, go_res) = tokio::join!(
http.get_text(&zen_url, &tgt.cookie),
http.get_text(&go_url, &tgt.cookie),
http.get_text(&zen_url, &tgt.cookie, zen_etag.as_deref(), zen_lm.as_deref()),
http.get_text(&go_url, &tgt.cookie, go_etag.as_deref(), go_lm.as_deref()),
);
let now = Utc::now();
let fetched_at = Some(now.to_rfc3339());
let balance = match &zen_res {
Ok(html) => match scrape::parse_zen_balance(html) {
Ok(b) => Some(b),
Err(e) => {
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "zen parse failed");
None
}
},
// Balance: parse a fresh body, reuse the stale parse on 304, else None.
let (balance, zen_etag_new, zen_lm_new) = match &zen_res {
Ok(fr) => {
let bal = match fr.body.as_deref() {
Some(html) => match scrape::parse_zen_balance(html) {
Ok(b) => Some(b),
Err(e) => {
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "zen parse failed");
None
}
},
None => stale_balance.clone(),
};
(bal, fr.etag.clone(), fr.last_modified.clone())
}
Err(e) => {
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "zen fetch failed");
None
}
};
let limits = match &go_res {
Ok(html) => match scrape::parse_go_limits(html, now) {
Ok(l) => Some(l),
Err(e) => {
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "go parse failed");
None
}
},
Err(e) => {
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "go fetch failed");
None
(None, None, None)
}
};
// Limits: parse a fresh body, reuse the stale parse on 304, else None.
let (limits, go_etag_new, go_lm_new) = match &go_res {
Ok(fr) => {
let lim = match fr.body.as_deref() {
Some(html) => match scrape::parse_go_limits(html, now) {
Ok(l) => Some(l),
Err(e) => {
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "go parse failed");
None
}
},
None => stale_limits.clone(),
};
(lim, fr.etag.clone(), fr.last_modified.clone())
}
Err(e) => {
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "go fetch failed");
(None, None, None)
}
};
// Decide the status. RateLimited and Timeout take priority over generic
// Error so callers can react.
let status = report_status(&zen_res, &go_res, limits.as_ref(), balance.as_ref());
let report = AccountReport {
@ -135,14 +157,15 @@ async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> A
balance: balance.clone(),
};
// Write cache only on a clean Ok, so errors never poison the cache.
// Write cache on a clean Ok so errors never poison the cache. Carry the
// latest validators so the next run can do conditional GET.
if status == ReportStatus::Ok {
let record = CachedRecord {
report: report.clone(),
go_etag: None,
go_last_modified: None,
zen_etag: None,
zen_last_modified: None,
go_etag: go_etag_new,
go_last_modified: go_lm_new,
zen_etag: zen_etag_new,
zen_last_modified: zen_lm_new,
};
if let Err(e) = cache::write(&tgt.account, &tgt.workspace, &record) {
tracing::warn!(error = %e, "cache write failed");
@ -154,8 +177,8 @@ async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> A
/// Fold the two fetch results into one status.
fn report_status(
zen: &Result<String, crate::error::FetchError>,
go: &Result<String, crate::error::FetchError>,
zen: &Result<crate::client::FetchResult, crate::error::FetchError>,
go: &Result<crate::client::FetchResult, crate::error::FetchError>,
limits: Option<&Limits>,
balance: Option<&Balance>,
) -> ReportStatus {