diff --git a/src/main.rs b/src/main.rs index 55e9f61..a643011 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ #![allow(non_upper_case_globals)] -use std::{ env, error::Error, fs::File, io::Read, sync::LazyLock, time::Duration }; -use anyhow::Result; +use std::{ env, fs::File, io::Read, thread::sleep, time::{ Duration, Instant } }; +use anyhow::{ bail, Error as _Error, Result }; use const_format::formatcp; use serde::{ Serialize, Deserialize }; use ureq::{ http::Response, Agent, Body }; @@ -21,14 +21,16 @@ const user_agent: &'static str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53 #[derive(Debug, Clone, Deserialize, Serialize)] enum RequestSort { #[serde(rename = "nameAsc")] - NameAsc + NameAsc, + #[serde(rename = "regAsc")] + RegAsc } #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] struct Request { - pub page: u16, + pub page: u32, pub page_size: u8, pub sort: RequestSort, pub project_id: u32, @@ -36,28 +38,28 @@ struct Request { impl Default for Request { fn default () -> Self { - Self { page: 1, page_size: MAX_USERS_PER_PAGE, sort: RequestSort::NameAsc, project_id } + Self { page: 1, page_size: MAX_USERS_PER_PAGE, sort: RequestSort::RegAsc, project_id } } } #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(untagged)] -enum Resp { - Ok (OkResp), - Err (ErrResp), +enum TildaResponse { + Ok (OkTildaResponse), + Err (ErrorTildaResponse), } #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] -struct OkResp { +struct OkTildaResponse { pub status: String, pub data: ResponseData } #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] -struct ErrResp { +struct ErrorTildaResponse { pub status: String, pub code: String } @@ -100,7 +102,7 @@ pub struct Group { } impl Request { - fn for_page (page: u16) -> Self { + fn for_page (page: u32) -> Self { let mut _self = Self::default(); _self.page = page; _self @@ -144,22 +146,123 @@ fn read_cookie (filename: &str) -> Option { } - -fn retrieve_users (cookie: &str) { +#[derive(Debug)] +struct Retriever { + pub tries: usize, + pub tries_left: usize, + pub interval: Duration, + pub cookie: String, + pub trace: Vec<_Error>, + pub out: Vec, } -fn main () { - let cookie = read_cookie(cookie_filename).expect(&format!("Directory with executable should contain `{cookie_filename}` file containing a valid cookie for accessing Tilda control panel")); - let req = Request::default().send(&cookie); - - if let Ok(mut req) = req { - - let mut body = req.body_mut(); - - - println!("{:?}", body.read_json::()); +impl Retriever { + pub fn run (&mut self) -> Result { + match self.retrieve() { + Ok(members) => { + self.out = members; + return Ok(self.out.len()); + }, + Err(error) => { + self.trace.push(error); + self.tries_left = self.tries_left.saturating_sub(1); + if self.tries == 0 { + bail!("Wasn't able to retrieve data in {} tries. Trace log:\n{:?}", self.tries, self.trace); + } else { + sleep(self.interval); + return self.run(); + } + } + } } - // println!("{} {} {} {} {} {}", host, project_id, &*origin, &*referer, user_agent, cookie); + pub fn new (cookie: T) -> Self where T: Into { + Self { tries_left: 5, tries: 5, cookie: cookie.into(), trace: vec![], interval: Duration::from_secs(5), out: vec![] } + } + + pub fn with_tries (mut self, tries: usize) -> Self { + self.tries_left = tries; + self.tries = tries; + self + } + + pub fn with_interval (mut self, interval: Duration) -> Self { + self.interval = interval; + self + } + + fn retrieve (&mut self) -> Result> { + let mut members = vec![]; + + if let Ok(first_page) = self.retrieve_page(1) { + let expected = first_page.total; + members.reserve(expected as usize); + members.extend(first_page.members); + for page in 2..=first_page.pages { + sleep(self.interval); + match self.retrieve_page(page) { + Ok(page) => members.extend(page.members), + Err(err) => { + eprintln!("Can't get page {} out of {}. Bailing…", {page}, {first_page.pages}); + bail!(err) + } + } + } + + if members.len() as u32 != first_page.total { + bail!("User count changed during the retrieving process. Bailing…") + } + + Ok(members) + } else { + bail!("Can't retrieve users from Tilda; check cookies, permissions, and if there's a sudden influx of a million new users per second, and try again"); // w + } + } + + fn retrieve_page (&self, page: u32) -> Result { + let request = Request::for_page(page).send(&self.cookie); + + if let Ok(mut response) = request { + if let Ok(response) = response.body_mut().read_json::() { + if let TildaResponse::Ok(data) = response { + return Ok(data.data); + } else { + bail!("Error response from Tilda: {:?}", response) + } + } else { + bail!("Can't parse response from Tilda: it's likely that the data format was changed. Make the necessary corrections and run the script again.") + } + + } else { + bail!("Error during requesting Tilda: {:?}", request) + } + } + +} + + +fn main () { + dotenvy::dotenv().ok(); + let cookie = read_cookie(cookie_filename).expect(&format!("Directory with executable should contain `{cookie_filename}` file containing a valid cookie for accessing Tilda control panel")); + + + let out = env::var("OUT_LOCATION").unwrap_or_else(|_| String::from("./data.json")); + + let benchmark = Instant::now(); + let mut retriever = Retriever::new(cookie).with_tries(5).with_interval(Duration::from_secs(7)); + let run = retriever.run(); + + if let Ok(fetched) = run { + println!("Successfully fetched {} users in {} ms, writing out to {}…", fetched, benchmark.elapsed().as_millis(), out); + // @TODO write + } else { + eprintln!("Wasn't able to retrieve data. Logs are below:"); + eprintln!("---"); + for line in retriever.trace { + eprintln!("{}", line); + } + eprintln!("---"); + eprintln!("Exiting…") + } }