Implement conditional GET (inert until OpenCode emits validators)
HttpClient.get_text now accepts etag/last_modified and sends If-None-Match/If-Modified-Since; a 304 returns no body and the caller reuses the cached parse. fetch_target recovers validators from the stale cache entry (via new cache::read_raw, TTL-agnostic) and stores whatever the server emits. Verified live: OpenCode/Cloudflare returns no ETag or Last-Modified on either page, so the slots stay None, no headers go out, and 304 never fires. The machinery is correct and ready for the day they add validators.
This commit is contained in:
parent
4463a9b51d
commit
4bf3eb1e06
@ -268,7 +268,8 @@ empty in the model.
|
|||||||
|
|
||||||
- Crate name: `ocd`.
|
- Crate name: `ocd`.
|
||||||
- Live watch mode for `htop`-like refresh. A later task.
|
- Live watch mode for `htop`-like refresh. A later task.
|
||||||
- Send `If-Modified-Since` and `If-None-Match`. The cache already stores the
|
- Conditional GET: `get_text` sends `If-None-Match` and `If-Modified-Since`
|
||||||
slots. The HTTP layer needs to attach the headers once we confirm OpenCode
|
from cached validators and handles a 304 by reusing the cached parse.
|
||||||
emits ETag or Last-Modified. The TTL cache keeps politeness intact without it.
|
OpenCode (via Cloudflare) currently emits no `ETag` or `Last-Modified` on
|
||||||
from TTL alone.
|
either page, so the slots stay empty and no headers go out. The machinery
|
||||||
|
is idle but correct; it works the day OpenCode adds validators.
|
||||||
@ -46,6 +46,15 @@ pub fn read(account: &str, workspace: &str, ttl_secs: u32) -> Option<CachedRecor
|
|||||||
serde_json::from_str(&body).ok()
|
serde_json::from_str(&body).ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read a cache entry ignoring TTL. Used to recover conditional-GET validators
|
||||||
|
/// (etag, last_modified) from an expired entry so the next request can still
|
||||||
|
/// ask the server whether anything changed.
|
||||||
|
pub fn read_raw(account: &str, workspace: &str) -> Option<CachedRecord> {
|
||||||
|
let path = entry_path(account, workspace);
|
||||||
|
let body = std::fs::read_to_string(&path).ok()?;
|
||||||
|
serde_json::from_str(&body).ok()
|
||||||
|
}
|
||||||
|
|
||||||
/// Write a cache entry. Creates the cache dir on demand.
|
/// Write a cache entry. Creates the cache dir on demand.
|
||||||
pub fn write(account: &str, workspace: &str, record: &CachedRecord) -> std::io::Result<()> {
|
pub fn write(account: &str, workspace: &str, record: &CachedRecord) -> std::io::Result<()> {
|
||||||
let path = entry_path(account, workspace);
|
let path = entry_path(account, workspace);
|
||||||
|
|||||||
@ -54,13 +54,23 @@ impl HttpClient {
|
|||||||
Ok(Self { client, retries })
|
Ok(Self { client, retries })
|
||||||
}
|
}
|
||||||
|
|
||||||
/// GET a page as text with the account cookie. Retries transient errors
|
/// GET a page with optional conditional-GET validators. Retries transient
|
||||||
/// with exponential backoff and jitter. Returns the response body.
|
/// errors with exponential backoff and jitter.
|
||||||
pub async fn get_text(&self, url: &str, cookie: &str) -> Result<String, FetchError> {
|
///
|
||||||
|
/// Returns the body plus any `ETag`/`Last-Modified` the server emitted. A
|
||||||
|
/// 304 Not Modified returns `body = None`; the caller reuses the cached
|
||||||
|
/// parse. `body.is_none()` is the 304 signal.
|
||||||
|
pub async fn get_text(
|
||||||
|
&self,
|
||||||
|
url: &str,
|
||||||
|
cookie: &str,
|
||||||
|
etag: Option<&str>,
|
||||||
|
last_modified: Option<&str>,
|
||||||
|
) -> Result<FetchResult, FetchError> {
|
||||||
let mut attempt: u8 = 0;
|
let mut attempt: u8 = 0;
|
||||||
loop {
|
loop {
|
||||||
match self.get_once(url, cookie).await {
|
match self.get_once(url, cookie, etag, last_modified).await {
|
||||||
Ok(body) => return Ok(body),
|
Ok(r) => return Ok(r),
|
||||||
Err(e) if e.is_transient() && attempt < self.retries => {
|
Err(e) if e.is_transient() && attempt < self.retries => {
|
||||||
let backoff = backoff_for(attempt);
|
let backoff = backoff_for(attempt);
|
||||||
tracing::warn!(attempt, backoff_ms = backoff.as_millis(), error = %e, "retry");
|
tracing::warn!(attempt, backoff_ms = backoff.as_millis(), error = %e, "retry");
|
||||||
@ -73,29 +83,44 @@ impl HttpClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_once(&self, url: &str, cookie: &str) -> Result<String, FetchError> {
|
async fn get_once(
|
||||||
let resp = self
|
&self,
|
||||||
|
url: &str,
|
||||||
|
cookie: &str,
|
||||||
|
etag: Option<&str>,
|
||||||
|
last_modified: Option<&str>,
|
||||||
|
) -> Result<FetchResult, FetchError> {
|
||||||
|
let mut req = self
|
||||||
.client
|
.client
|
||||||
.get(url)
|
.get(url)
|
||||||
.header("cookie", cookie_header(cookie))
|
.header("cookie", cookie_header(cookie))
|
||||||
.header("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
.header("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||||
.header("accept-language", "en-US,en;q=0.9")
|
.header("accept-language", "en-US,en;q=0.9");
|
||||||
.send()
|
if let Some(e) = etag {
|
||||||
.await
|
req = req.header("if-none-match", e);
|
||||||
.map_err(|e| {
|
}
|
||||||
if e.is_timeout() {
|
if let Some(lm) = last_modified {
|
||||||
FetchError::Timeout
|
req = req.header("if-modified-since", lm);
|
||||||
} else {
|
}
|
||||||
FetchError::Send(e.to_string())
|
|
||||||
}
|
let resp = req.send().await.map_err(|e| {
|
||||||
})?;
|
if e.is_timeout() {
|
||||||
|
FetchError::Timeout
|
||||||
|
} else {
|
||||||
|
FetchError::Send(e.to_string())
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
|
||||||
let status = resp.status().as_u16();
|
let status = resp.status().as_u16();
|
||||||
if status == 429 {
|
if status == 429 {
|
||||||
return Err(FetchError::RateLimited);
|
return Err(FetchError::RateLimited);
|
||||||
}
|
}
|
||||||
if status == 304 {
|
if status == 304 {
|
||||||
return Err(FetchError::Send("unexpected 304".to_string()));
|
return Ok(FetchResult {
|
||||||
|
body: None,
|
||||||
|
etag: header_str(&resp, "etag"),
|
||||||
|
last_modified: header_str(&resp, "last-modified"),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
if !(200..300).contains(&status) {
|
if !(200..300).contains(&status) {
|
||||||
return Err(FetchError::Status {
|
return Err(FetchError::Status {
|
||||||
@ -104,12 +129,25 @@ impl HttpClient {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
resp.text()
|
let etag = header_str(&resp, "etag");
|
||||||
.await
|
let last_modified = header_str(&resp, "last-modified");
|
||||||
.map_err(|e| FetchError::Body(e.to_string()))
|
let body = resp.text().await.map_err(|e| FetchError::Body(e.to_string()))?;
|
||||||
|
Ok(FetchResult { body: Some(body), etag, last_modified })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Result of one GET. `body` is `None` when the server returned 304.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct FetchResult {
|
||||||
|
pub body: Option<String>,
|
||||||
|
pub etag: Option<String>,
|
||||||
|
pub last_modified: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn header_str(resp: &reqwest::Response, name: &str) -> Option<String> {
|
||||||
|
resp.headers().get(name).and_then(|v| v.to_str().ok()).map(|s| s.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
/// Backoff for `attempt` (0-based): 500 ms, 1 s, 2 s, capped at 8 s, plus
|
/// Backoff for `attempt` (0-based): 500 ms, 1 s, 2 s, capped at 8 s, plus
|
||||||
/// jitter up to 25 per cent of each step.
|
/// jitter up to 25 per cent of each step.
|
||||||
fn backoff_for(attempt: u8) -> Duration {
|
fn backoff_for(attempt: u8) -> Duration {
|
||||||
|
|||||||
95
src/fetch.rs
95
src/fetch.rs
@ -74,9 +74,11 @@ pub async fn run(cfg: &crate::config::Config, opts: &FetchOptions) -> anyhow::Re
|
|||||||
Ok(reports)
|
Ok(reports)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch one target. Cache short-circuits the network.
|
/// Fetch one target. A fresh cache entry short-circuits the network. An expired
|
||||||
|
/// cache entry still supplies conditional-GET validators so the server can
|
||||||
|
/// answer 304 instead of resending the full page.
|
||||||
async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> AccountReport {
|
async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> AccountReport {
|
||||||
// Cache hit path.
|
// Fresh cache hit: no network at all.
|
||||||
if !opts.force_refresh && !opts.no_cache {
|
if !opts.force_refresh && !opts.no_cache {
|
||||||
if let Some(rec) = cache::read(&tgt.account, &tgt.workspace, opts.ttl_secs) {
|
if let Some(rec) = cache::read(&tgt.account, &tgt.workspace, opts.ttl_secs) {
|
||||||
let mut r = rec.report;
|
let mut r = rec.report;
|
||||||
@ -85,45 +87,65 @@ async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> A
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Network path. Fire Zen and Go at once.
|
// Recover validators and last-known parses from a stale entry, if any.
|
||||||
|
let stale = if opts.no_cache { None } else { cache::read_raw(&tgt.account, &tgt.workspace) };
|
||||||
|
let (zen_etag, zen_lm) = stale.as_ref().map(|s| (s.zen_etag.clone(), s.zen_last_modified.clone())).unwrap_or((None, None));
|
||||||
|
let (go_etag, go_lm) = stale.as_ref().map(|s| (s.go_etag.clone(), s.go_last_modified.clone())).unwrap_or((None, None));
|
||||||
|
let stale_balance = stale.as_ref().and_then(|s| s.report.balance.clone());
|
||||||
|
let stale_limits = stale.as_ref().and_then(|s| s.report.limits.clone());
|
||||||
|
|
||||||
let zen_url = client::zen_url(&tgt.workspace);
|
let zen_url = client::zen_url(&tgt.workspace);
|
||||||
let go_url = client::go_url(&tgt.workspace);
|
let go_url = client::go_url(&tgt.workspace);
|
||||||
let (zen_res, go_res) = tokio::join!(
|
let (zen_res, go_res) = tokio::join!(
|
||||||
http.get_text(&zen_url, &tgt.cookie),
|
http.get_text(&zen_url, &tgt.cookie, zen_etag.as_deref(), zen_lm.as_deref()),
|
||||||
http.get_text(&go_url, &tgt.cookie),
|
http.get_text(&go_url, &tgt.cookie, go_etag.as_deref(), go_lm.as_deref()),
|
||||||
);
|
);
|
||||||
|
|
||||||
let now = Utc::now();
|
let now = Utc::now();
|
||||||
let fetched_at = Some(now.to_rfc3339());
|
let fetched_at = Some(now.to_rfc3339());
|
||||||
let balance = match &zen_res {
|
|
||||||
Ok(html) => match scrape::parse_zen_balance(html) {
|
// Balance: parse a fresh body, reuse the stale parse on 304, else None.
|
||||||
Ok(b) => Some(b),
|
let (balance, zen_etag_new, zen_lm_new) = match &zen_res {
|
||||||
Err(e) => {
|
Ok(fr) => {
|
||||||
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "zen parse failed");
|
let bal = match fr.body.as_deref() {
|
||||||
None
|
Some(html) => match scrape::parse_zen_balance(html) {
|
||||||
}
|
Ok(b) => Some(b),
|
||||||
},
|
Err(e) => {
|
||||||
|
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "zen parse failed");
|
||||||
|
None
|
||||||
|
}
|
||||||
|
},
|
||||||
|
None => stale_balance.clone(),
|
||||||
|
};
|
||||||
|
(bal, fr.etag.clone(), fr.last_modified.clone())
|
||||||
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "zen fetch failed");
|
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "zen fetch failed");
|
||||||
None
|
(None, None, None)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let limits = match &go_res {
|
|
||||||
Ok(html) => match scrape::parse_go_limits(html, now) {
|
// Limits: parse a fresh body, reuse the stale parse on 304, else None.
|
||||||
Ok(l) => Some(l),
|
let (limits, go_etag_new, go_lm_new) = match &go_res {
|
||||||
Err(e) => {
|
Ok(fr) => {
|
||||||
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "go parse failed");
|
let lim = match fr.body.as_deref() {
|
||||||
None
|
Some(html) => match scrape::parse_go_limits(html, now) {
|
||||||
}
|
Ok(l) => Some(l),
|
||||||
},
|
Err(e) => {
|
||||||
Err(e) => {
|
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "go parse failed");
|
||||||
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "go fetch failed");
|
None
|
||||||
None
|
}
|
||||||
|
},
|
||||||
|
None => stale_limits.clone(),
|
||||||
|
};
|
||||||
|
(lim, fr.etag.clone(), fr.last_modified.clone())
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(account = %tgt.account, workspace = %tgt.workspace, error = %e, "go fetch failed");
|
||||||
|
(None, None, None)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Decide the status. RateLimited and Timeout take priority over generic
|
|
||||||
// Error so callers can react.
|
|
||||||
let status = report_status(&zen_res, &go_res, limits.as_ref(), balance.as_ref());
|
let status = report_status(&zen_res, &go_res, limits.as_ref(), balance.as_ref());
|
||||||
|
|
||||||
let report = AccountReport {
|
let report = AccountReport {
|
||||||
@ -135,14 +157,15 @@ async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> A
|
|||||||
balance: balance.clone(),
|
balance: balance.clone(),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Write cache only on a clean Ok, so errors never poison the cache.
|
// Write cache on a clean Ok so errors never poison the cache. Carry the
|
||||||
|
// latest validators so the next run can do conditional GET.
|
||||||
if status == ReportStatus::Ok {
|
if status == ReportStatus::Ok {
|
||||||
let record = CachedRecord {
|
let record = CachedRecord {
|
||||||
report: report.clone(),
|
report: report.clone(),
|
||||||
go_etag: None,
|
go_etag: go_etag_new,
|
||||||
go_last_modified: None,
|
go_last_modified: go_lm_new,
|
||||||
zen_etag: None,
|
zen_etag: zen_etag_new,
|
||||||
zen_last_modified: None,
|
zen_last_modified: zen_lm_new,
|
||||||
};
|
};
|
||||||
if let Err(e) = cache::write(&tgt.account, &tgt.workspace, &record) {
|
if let Err(e) = cache::write(&tgt.account, &tgt.workspace, &record) {
|
||||||
tracing::warn!(error = %e, "cache write failed");
|
tracing::warn!(error = %e, "cache write failed");
|
||||||
@ -154,8 +177,8 @@ async fn fetch_target(tgt: &Target, opts: &FetchOptions, http: &HttpClient) -> A
|
|||||||
|
|
||||||
/// Fold the two fetch results into one status.
|
/// Fold the two fetch results into one status.
|
||||||
fn report_status(
|
fn report_status(
|
||||||
zen: &Result<String, crate::error::FetchError>,
|
zen: &Result<crate::client::FetchResult, crate::error::FetchError>,
|
||||||
go: &Result<String, crate::error::FetchError>,
|
go: &Result<crate::client::FetchResult, crate::error::FetchError>,
|
||||||
limits: Option<&Limits>,
|
limits: Option<&Limits>,
|
||||||
balance: Option<&Balance>,
|
balance: Option<&Balance>,
|
||||||
) -> ReportStatus {
|
) -> ReportStatus {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user