Prevent race conditions

- This is a massive cope, I don't know how to code

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>

src/database/assets.rs

@@ -3,15 +3,15 @@ use clickhouse::Client;
 
 pub async fn select(clickhouse_client: &Client) -> Vec<Asset> {
     clickhouse_client
-        .query("SELECT ?fields FROM assets")
+        .query("SELECT ?fields FROM assets FINAL")
         .fetch_all::<Asset>()
         .await
         .unwrap()
 }
 
-pub async fn select_where_class(clickhouse_client: &Client, class: Class) -> Vec<Asset> {
+pub async fn select_where_class(clickhouse_client: &Client, class: &Class) -> Vec<Asset> {
     clickhouse_client
-        .query("SELECT ?fields FROM assets WHERE class = ?")
+        .query("SELECT ?fields FROM assets FINAL WHERE class = ?")
         .bind(class)
         .fetch_all::<Asset>()
         .await
@@ -20,23 +20,25 @@ pub async fn select_where_class(clickhouse_client: &Client, class: Class) -> Vec
 
 pub async fn select_where_symbol(clickhouse_client: &Client, symbol: &str) -> Option<Asset> {
     clickhouse_client
-        .query("SELECT ?fields FROM assets WHERE symbol = ?")
+        .query("SELECT ?fields FROM assets FINAL WHERE symbol = ?")
         .bind(symbol)
         .fetch_optional::<Asset>()
         .await
         .unwrap()
 }
 
-pub async fn upsert(clickhouse_client: &Client, asset: &Asset) {
+pub async fn upsert_batch(clickhouse_client: &Client, assets: &Vec<Asset>) {
     let mut insert = clickhouse_client.insert("assets").unwrap();
-    insert.write(asset).await.unwrap();
+    for asset in assets {
+        insert.write(asset).await.unwrap();
+    }
     insert.end().await.unwrap();
 }
 
-pub async fn delete_where_symbol(clickhouse_client: &Client, symbol: &str) {
+pub async fn delete_where_symbols(clickhouse_client: &Client, symbols: &Vec<String>) {
     clickhouse_client
-        .query("DELETE FROM assets WHERE symbol = ?")
-        .bind(symbol)
+        .query("DELETE FROM assets WHERE symbol IN ?")
+        .bind(symbols)
         .execute()
         .await
         .unwrap();

src/database/backfills.rs

@@ -0,0 +1,38 @@
+use crate::types::Backfill;
+use clickhouse::Client;
+
+pub async fn select_latest_where_symbol(
+    clickhouse_client: &Client,
+    symbol: &str,
+) -> Option<Backfill> {
+    clickhouse_client
+        .query("SELECT ?fields FROM backfills FINAL WHERE symbol = ? ORDER BY time DESC LIMIT 1")
+        .bind(symbol)
+        .fetch_optional::<Backfill>()
+        .await
+        .unwrap()
+}
+
+pub async fn upsert(clickhouse_client: &Client, backfill: &Backfill) {
+    let mut insert = clickhouse_client.insert("backfills").unwrap();
+    insert.write(backfill).await.unwrap();
+    insert.end().await.unwrap();
+}
+
+pub async fn delete_where_symbols(clickhouse_client: &Client, symbols: &Vec<String>) {
+    clickhouse_client
+        .query("DELETE FROM backfills WHERE symbol IN ?")
+        .bind(symbols)
+        .execute()
+        .await
+        .unwrap();
+}
+
+pub async fn delete_where_not_symbols(clickhouse_client: &Client, symbols: &Vec<String>) {
+    clickhouse_client
+        .query("DELETE FROM backfills WHERE symbol NOT IN ?")
+        .bind(symbols)
+        .execute()
+        .await
+        .unwrap();
+}

src/database/bars.rs

@@ -1,4 +1,4 @@
-use crate::types::{Bar, BarValidity};
+use crate::types::Bar;
 use clickhouse::Client;
 
 pub async fn upsert(clickhouse_client: &Client, bar: &Bar) {
@@ -15,46 +15,19 @@ pub async fn upsert_batch(clickhouse_client: &Client, bars: &[Bar]) {
     insert.end().await.unwrap();
 }
 
-pub async fn delete_where_symbol(clickhouse_client: &Client, symbol: &str) {
+pub async fn delete_where_symbols(clickhouse_client: &Client, symbols: &Vec<String>) {
     clickhouse_client
-        .query("DELETE FROM bars WHERE symbol = ?")
-        .bind(symbol)
+        .query("DELETE FROM bars WHERE symbol IN ?")
+        .bind(symbols)
         .execute()
         .await
         .unwrap();
 }
 
-pub async fn select_validity_where_symbol(
-    clickhouse_client: &Client,
-    symbol: &str,
-) -> Option<BarValidity> {
+pub async fn delete_where_not_symbols(clickhouse_client: &Client, symbols: &Vec<String>) {
     clickhouse_client
-        .query("SELECT ?fields FROM bars_validity FINAL WHERE symbol = ?")
-        .bind(symbol)
-        .fetch_optional::<BarValidity>()
-        .await
-        .unwrap()
-}
-
-pub async fn upsert_validity(clickhouse_client: &Client, bar_validity: &BarValidity) {
-    let mut insert = clickhouse_client.insert("bars_validity").unwrap();
-    insert.write(bar_validity).await.unwrap();
-    insert.end().await.unwrap();
-}
-
-pub async fn insert_validity_if_not_exists(clickhouse_client: &Client, bar_validity: &BarValidity) {
-    if select_validity_where_symbol(clickhouse_client, &bar_validity.symbol)
-        .await
-        .is_none()
-    {
-        upsert_validity(clickhouse_client, bar_validity).await;
-    }
-}
-
-pub async fn delete_validity_where_symbol(clickhouse_client: &Client, symbol: &str) {
-    clickhouse_client
-        .query("DELETE FROM bars_validity WHERE symbol = ?")
-        .bind(symbol)
+        .query("DELETE FROM bars WHERE symbol NOT IN ?")
+        .bind(symbols)
         .execute()
         .await
         .unwrap();

src/database/mod.rs

@@ -1,2 +1,3 @@
 pub mod assets;
+pub mod backfills;
 pub mod bars;