implement more advanced fetching, error handling, retrying

still not happy with the code, but there are at least some structs here
and there now

src/main.rs

@@ -1,7 +1,7 @@
 #![allow(non_upper_case_globals)]
 
-use std::{ env, error::Error, fs::File, io::Read, sync::LazyLock, time::Duration };
-use anyhow::Result;
+use std::{ env, fs::File, io::Read, thread::sleep, time::{ Duration, Instant } };
+use anyhow::{ bail, Error as _Error, Result };
 use const_format::formatcp;
 use serde::{ Serialize, Deserialize };
 use ureq::{ http::Response, Agent, Body };
@@ -21,14 +21,16 @@ const user_agent: &'static str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53
 #[derive(Debug, Clone, Deserialize, Serialize)]
 enum RequestSort {
     #[serde(rename = "nameAsc")]
-    NameAsc
+    NameAsc,
+    #[serde(rename = "regAsc")]
+    RegAsc
 }
 
 
 #[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct Request {
-    pub page: u16,
+    pub page: u32,
     pub page_size: u8,
     pub sort: RequestSort,
     pub project_id: u32,
@@ -36,28 +38,28 @@ struct Request {
 
 impl Default for Request {
     fn default () -> Self {
-        Self { page: 1, page_size: MAX_USERS_PER_PAGE, sort: RequestSort::NameAsc, project_id }
+        Self { page: 1, page_size: MAX_USERS_PER_PAGE, sort: RequestSort::RegAsc, project_id }
     }
 }
 
 
 #[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(untagged)]
-enum Resp {
-    Ok (OkResp),
-    Err (ErrResp),
+enum TildaResponse {
+    Ok (OkTildaResponse),
+    Err (ErrorTildaResponse),
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(rename_all = "camelCase")]
-struct OkResp {
+struct OkTildaResponse {
     pub status: String,
     pub data: ResponseData
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(rename_all = "camelCase")]
-struct ErrResp {
+struct ErrorTildaResponse {
     pub status: String,
     pub code: String
 }
@@ -100,7 +102,7 @@ pub struct Group {
 }
 
 impl Request {
-    fn for_page (page: u16) -> Self {
+    fn for_page (page: u32) -> Self {
         let mut _self = Self::default();
         _self.page = page;
         _self
@@ -144,22 +146,123 @@ fn read_cookie (filename: &str) -> Option<String> {
 }
 
 
-
-fn retrieve_users (cookie: &str) {
+#[derive(Debug)]
+struct Retriever {
+    pub tries: usize,
+    pub tries_left: usize,
+    pub interval: Duration,
+    pub cookie: String,
+    pub trace: Vec<_Error>,
+    pub out: Vec<Member>,
 }
 
-fn main () {
-    let cookie = read_cookie(cookie_filename).expect(&format!("Directory with executable should contain `{cookie_filename}` file containing a valid cookie for accessing Tilda control panel"));
-    let req = Request::default().send(&cookie);
-
-    if let Ok(mut req) = req {
-
-        let mut body = req.body_mut();
-
-
-        println!("{:?}", body.read_json::<Resp>());
+impl Retriever {
+    pub fn run (&mut self) -> Result<usize> {
+        match self.retrieve() {
+            Ok(members) => {
+                self.out = members;
+                return Ok(self.out.len());
+            },
+            Err(error) => {
+                self.trace.push(error);
+                self.tries_left = self.tries_left.saturating_sub(1);
+                if self.tries == 0 {
+                    bail!("Wasn't able to retrieve data in {} tries. Trace log:\n{:?}", self.tries, self.trace);
+                } else {
+                    sleep(self.interval);
+                    return self.run();
+                }
+            }
+        }
     }
 
-    // println!("{} {} {} {} {} {}", host, project_id, &*origin, &*referer, user_agent, cookie);
+    pub fn new <T> (cookie: T) -> Self where T: Into<String> {
+        Self { tries_left: 5, tries: 5, cookie: cookie.into(), trace: vec![], interval: Duration::from_secs(5), out: vec![] }
+    }
+
+    pub fn with_tries (mut self, tries: usize) -> Self {
+        self.tries_left = tries;
+        self.tries = tries;
+        self
+    }
+
+    pub fn with_interval (mut self, interval: Duration) -> Self {
+        self.interval = interval;
+        self
+    }
+
+    fn retrieve (&mut self) -> Result<Vec<Member>> {
+        let mut members = vec![];
+
+        if let Ok(first_page) = self.retrieve_page(1) {
+            let expected = first_page.total;
+            members.reserve(expected as usize);
+            members.extend(first_page.members);
+            for page in 2..=first_page.pages {
+                sleep(self.interval);
+                match self.retrieve_page(page) {
+                    Ok(page) => members.extend(page.members),
+                    Err(err) => {
+                        eprintln!("Can't get page {} out of {}. Bailing…", {page}, {first_page.pages});
+                        bail!(err)
+                    }
+                }
+            }
+
+            if members.len() as u32 != first_page.total {
+                bail!("User count changed during the retrieving process. Bailing…")
+            }
+
+            Ok(members)
+        } else {
+            bail!("Can't retrieve users from Tilda; check cookies, permissions, and if there's a sudden influx of a million new users per second, and try again"); // w
+        }
+    }
+
+    fn retrieve_page (&self, page: u32) -> Result<ResponseData> {
+        let request = Request::for_page(page).send(&self.cookie);
+
+        if let Ok(mut response) = request {
+            if let Ok(response) = response.body_mut().read_json::<TildaResponse>() {
+                if let TildaResponse::Ok(data) = response {
+                    return Ok(data.data);
+                } else {
+                    bail!("Error response from Tilda: {:?}", response)
+                }
+            } else {
+                bail!("Can't parse response from Tilda: it's likely that the data format was changed. Make the necessary corrections and run the script again.")
+            }
+
+        } else {
+            bail!("Error during requesting Tilda: {:?}", request)
+        }
+    }
+
+}
+
+
+fn main () {
+    dotenvy::dotenv().ok();
+    let cookie = read_cookie(cookie_filename).expect(&format!("Directory with executable should contain `{cookie_filename}` file containing a valid cookie for accessing Tilda control panel"));
+
+
+    let out = env::var("OUT_LOCATION").unwrap_or_else(|_| String::from("./data.json"));
+
+    let benchmark = Instant::now();
+    let mut retriever = Retriever::new(cookie).with_tries(5).with_interval(Duration::from_secs(7));
+    let run = retriever.run();
+
+    if let Ok(fetched) = run {
+        println!("Successfully fetched {} users in {} ms, writing out to {}…", fetched, benchmark.elapsed().as_millis(), out);
+        // @TODO write
+    } else {
+        eprintln!("Wasn't able to retrieve data. Logs are below:");
+        eprintln!("---");
+        for line in retriever.trace {
+            eprintln!("{}", line);
+        }
+        eprintln!("---");
+        eprintln!("Exiting…")
+    }
 
 }