Add news data support

- Refactor everything in the process, oops

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>

src/database/assets.rs

@@ -1,4 +1,4 @@
-use crate::types::{Asset, Class};
+use crate::types::Asset;
 use clickhouse::Client;
 use serde::Serialize;
 
@@ -10,21 +10,13 @@ pub async fn select(clickhouse_client: &Client) -> Vec<Asset> {
         .unwrap()
 }
 
-pub async fn select_where_class(clickhouse_client: &Client, class: &Class) -> Vec<Asset> {
-    clickhouse_client
-        .query("SELECT ?fields FROM assets FINAL WHERE class = ?")
-        .bind(class)
-        .fetch_all::<Asset>()
-        .await
-        .unwrap()
-}
-
 pub async fn select_where_symbol<T>(clickhouse_client: &Client, symbol: &T) -> Option<Asset>
 where
     T: AsRef<str> + Serialize + Send + Sync,
 {
     clickhouse_client
-        .query("SELECT ?fields FROM assets FINAL WHERE symbol = ?")
+        .query("SELECT ?fields FROM assets FINAL WHERE symbol = ? OR abbreviation = ?")
+        .bind(symbol)
         .bind(symbol)
         .fetch_optional::<Asset>()
         .await

src/database/backfills.rs

@@ -1,48 +1,93 @@
-use crate::types::Backfill;
+use crate::{database::assets, threads::data::ThreadType, types::Backfill};
 use clickhouse::Client;
 use serde::Serialize;
+use tokio::join;
 
 pub async fn select_latest_where_symbol<T>(
     clickhouse_client: &Client,
+    thread_type: &ThreadType,
     symbol: &T,
 ) -> Option<Backfill>
 where
     T: AsRef<str> + Serialize + Send + Sync,
 {
     clickhouse_client
-        .query("SELECT ?fields FROM backfills FINAL WHERE symbol = ? ORDER BY time DESC LIMIT 1")
+        .query(&format!(
+            "SELECT ?fields FROM {} FINAL WHERE symbol = ? ORDER BY time DESC LIMIT 1",
+            match thread_type {
+                ThreadType::Bars(_) => "backfills_bars",
+                ThreadType::News => "backfills_news",
+            }
+        ))
         .bind(symbol)
         .fetch_optional::<Backfill>()
         .await
         .unwrap()
 }
 
-pub async fn upsert(clickhouse_client: &Client, backfill: &Backfill) {
-    let mut insert = clickhouse_client.insert("backfills").unwrap();
+pub async fn upsert(clickhouse_client: &Client, thread_type: &ThreadType, backfill: &Backfill) {
+    let mut insert = clickhouse_client
+        .insert(match thread_type {
+            ThreadType::Bars(_) => "backfills_bars",
+            ThreadType::News => "backfills_news",
+        })
+        .unwrap();
     insert.write(backfill).await.unwrap();
     insert.end().await.unwrap();
 }
 
-pub async fn delete_where_symbols<T>(clickhouse_client: &Client, symbols: &[T])
-where
+pub async fn delete_where_symbols<T>(
+    clickhouse_client: &Client,
+    thread_type: &ThreadType,
+    symbols: &[T],
+) where
     T: AsRef<str> + Serialize + Send + Sync,
 {
     clickhouse_client
-        .query("DELETE FROM backfills WHERE symbol IN ?")
+        .query(&format!(
+            "DELETE FROM {} WHERE symbol IN ?",
+            match thread_type {
+                ThreadType::Bars(_) => "backfills_bars",
+                ThreadType::News => "backfills_news",
+            }
+        ))
         .bind(symbols)
         .execute()
         .await
         .unwrap();
 }
 
-pub async fn delete_where_not_symbols<T>(clickhouse_client: &Client, symbols: &[T])
-where
-    T: AsRef<str> + Serialize + Send + Sync,
-{
-    clickhouse_client
-        .query("DELETE FROM backfills WHERE symbol NOT IN ?")
-        .bind(symbols)
-        .execute()
-        .await
-        .unwrap();
+pub async fn cleanup(clickhouse_client: &Client) {
+    let assets = assets::select(clickhouse_client).await;
+
+    let bars_symbols = assets
+        .clone()
+        .into_iter()
+        .map(|asset| asset.symbol)
+        .collect::<Vec<_>>();
+
+    let news_symbols = assets
+        .into_iter()
+        .map(|asset| asset.abbreviation)
+        .collect::<Vec<_>>();
+
+    let delete_bars_future = async {
+        clickhouse_client
+            .query("DELETE FROM backfills_bars WHERE symbol NOT IN ?")
+            .bind(bars_symbols)
+            .execute()
+            .await
+            .unwrap();
+    };
+
+    let delete_news_future = async {
+        clickhouse_client
+            .query("DELETE FROM backfills_news WHERE symbol NOT IN ?")
+            .bind(news_symbols)
+            .execute()
+            .await
+            .unwrap();
+    };
+
+    join!(delete_bars_future, delete_news_future);
 }

src/database/bars.rs

@@ -1,3 +1,4 @@
+use super::assets;
 use crate::types::Bar;
 use clickhouse::Client;
 use serde::Serialize;
@@ -32,10 +33,14 @@ where
         .unwrap();
 }
 
-pub async fn delete_where_not_symbols<T>(clickhouse_client: &Client, symbols: &[T])
-where
-    T: AsRef<str> + Serialize + Send + Sync,
-{
+pub async fn cleanup(clickhouse_client: &Client) {
+    let assets = assets::select(clickhouse_client).await;
+
+    let symbols = assets
+        .into_iter()
+        .map(|asset| asset.symbol)
+        .collect::<Vec<_>>();
+
     clickhouse_client
         .query("DELETE FROM bars WHERE symbol NOT IN ?")
         .bind(symbols)

src/database/mod.rs

@@ -1,3 +1,4 @@
 pub mod assets;
 pub mod backfills;
 pub mod bars;
+pub mod news;

src/database/news.rs

@@ -0,0 +1,50 @@
+use super::assets;
+use crate::types::News;
+use clickhouse::Client;
+use serde::Serialize;
+
+pub async fn upsert(clickhouse_client: &Client, news: &News) {
+    let mut insert = clickhouse_client.insert("news").unwrap();
+    insert.write(news).await.unwrap();
+    insert.end().await.unwrap();
+}
+
+pub async fn upsert_batch<T>(clickhouse_client: &Client, news: T)
+where
+    T: IntoIterator<Item = News> + Send + Sync,
+    T::IntoIter: Send,
+{
+    let mut insert = clickhouse_client.insert("news").unwrap();
+    for news in news {
+        insert.write(&news).await.unwrap();
+    }
+    insert.end().await.unwrap();
+}
+
+pub async fn delete_where_symbols<T>(clickhouse_client: &Client, symbols: &[T])
+where
+    T: AsRef<str> + Serialize + Send + Sync,
+{
+    clickhouse_client
+        .query("DELETE FROM news WHERE hasAny(symbols, ?)")
+        .bind(symbols)
+        .execute()
+        .await
+        .unwrap();
+}
+
+pub async fn cleanup(clickhouse_client: &Client) {
+    let assets = assets::select(clickhouse_client).await;
+
+    let symbols = assets
+        .into_iter()
+        .map(|asset| asset.abbreviation)
+        .collect::<Vec<_>>();
+
+    clickhouse_client
+        .query("DELETE FROM news WHERE NOT hasAny(symbols, ?)")
+        .bind(symbols)
+        .execute()
+        .await
+        .unwrap();
+}