Add order/position management

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>

src/database/assets.rs

@@ -1,50 +1,14 @@
-use crate::types::Asset;
+use crate::{
+    delete_where_symbols, optimize, select, select_where_symbol, types::Asset, upsert_batch,
+};
 use clickhouse::{error::Error, Client};
 use serde::Serialize;
 
-pub async fn select(clickhouse_client: &Client) -> Result<Vec<Asset>, Error> {
-    clickhouse_client
-        .query("SELECT ?fields FROM assets FINAL")
-        .fetch_all::<Asset>()
-        .await
-}
-
-pub async fn select_where_symbol<T>(
-    clickhouse_client: &Client,
-    symbol: &T,
-) -> Result<Option<Asset>, Error>
-where
-    T: AsRef<str> + Serialize + Send + Sync,
-{
-    clickhouse_client
-        .query("SELECT ?fields FROM assets FINAL WHERE symbol = ?")
-        .bind(symbol)
-        .fetch_optional::<Asset>()
-        .await
-}
-
-pub async fn upsert_batch<T>(clickhouse_client: &Client, assets: T) -> Result<(), Error>
-where
-    T: IntoIterator<Item = Asset> + Send + Sync,
-    T::IntoIter: Send,
-{
-    let mut insert = clickhouse_client.insert("assets")?;
-    for asset in assets {
-        insert.write(&asset).await?;
-    }
-    insert.end().await
-}
-
-pub async fn delete_where_symbols<T>(clickhouse_client: &Client, symbols: &[T]) -> Result<(), Error>
-where
-    T: AsRef<str> + Serialize + Send + Sync,
-{
-    clickhouse_client
-        .query("DELETE FROM assets WHERE symbol IN ?")
-        .bind(symbols)
-        .execute()
-        .await
-}
+select!(Asset, "assets");
+select_where_symbol!(Asset, "assets");
+upsert_batch!(Asset, "assets");
+delete_where_symbols!("assets");
+optimize!("assets");
 
 pub async fn update_status_where_symbol<T>(
     clickhouse_client: &Client,
@@ -61,3 +25,19 @@ where
         .execute()
         .await
 }
+
+pub async fn update_qty_where_symbol<T>(
+    clickhouse_client: &Client,
+    symbol: &T,
+    qty: f64,
+) -> Result<(), Error>
+where
+    T: AsRef<str> + Serialize + Send + Sync,
+{
+    clickhouse_client
+        .query("ALTER TABLE assets UPDATE qty = ? WHERE symbol = ?")
+        .bind(qty)
+        .bind(symbol)
+        .execute()
+        .await
+}

src/database/backfills.rs

@@ -1,79 +0,0 @@
-use crate::types::Backfill;
-use clickhouse::{error::Error, Client};
-use serde::Serialize;
-use std::fmt::{Display, Formatter};
-use tokio::try_join;
-
-pub enum Table {
-    Bars,
-    News,
-}
-
-impl Display for Table {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Self::Bars => write!(f, "backfills_bars"),
-            Self::News => write!(f, "backfills_news"),
-        }
-    }
-}
-
-pub async fn select_latest_where_symbol<T>(
-    clickhouse_client: &Client,
-    table: &Table,
-    symbol: &T,
-) -> Result<Option<Backfill>, Error>
-where
-    T: AsRef<str> + Serialize + Send + Sync,
-{
-    clickhouse_client
-        .query(&format!(
-            "SELECT ?fields FROM {table} FINAL WHERE symbol = ? ORDER BY time DESC LIMIT 1",
-        ))
-        .bind(symbol)
-        .fetch_optional::<Backfill>()
-        .await
-}
-
-pub async fn upsert(
-    clickhouse_client: &Client,
-    table: &Table,
-    backfill: &Backfill,
-) -> Result<(), Error> {
-    let mut insert = clickhouse_client.insert(&table.to_string())?;
-    insert.write(backfill).await?;
-    insert.end().await
-}
-
-pub async fn delete_where_symbols<T>(
-    clickhouse_client: &Client,
-    table: &Table,
-    symbols: &[T],
-) -> Result<(), Error>
-where
-    T: AsRef<str> + Serialize + Send + Sync,
-{
-    clickhouse_client
-        .query(&format!("DELETE FROM {table} WHERE symbol IN ?"))
-        .bind(symbols)
-        .execute()
-        .await
-}
-
-pub async fn cleanup(clickhouse_client: &Client) -> Result<(), Error> {
-    let delete_bars_future = async {
-        clickhouse_client
-            .query("DELETE FROM backfills_bars WHERE symbol NOT IN (SELECT symbol FROM assets)")
-            .execute()
-            .await
-    };
-
-    let delete_news_future = async {
-        clickhouse_client
-            .query("DELETE FROM backfills_news WHERE symbol NOT IN (SELECT symbol FROM assets)")
-            .execute()
-            .await
-    };
-
-    try_join!(delete_bars_future, delete_news_future).map(|_| ())
-}

src/database/backfills_bars.rs

@@ -0,0 +1,17 @@
+use crate::{
+    cleanup, delete_where_symbols, optimize, select_where_symbol, types::Backfill, upsert,
+};
+use clickhouse::{error::Error, Client};
+
+select_where_symbol!(Backfill, "backfills_bars");
+upsert!(Backfill, "backfills_bars");
+delete_where_symbols!("backfills_bars");
+cleanup!("backfills_bars");
+optimize!("backfills_bars");
+
+pub async fn unfresh(clickhouse_client: &Client) -> Result<(), Error> {
+    clickhouse_client
+        .query("ALTER TABLE backfills_bars UPDATE fresh = false WHERE true")
+        .execute()
+        .await
+}

src/database/backfills_news.rs

@@ -0,0 +1,17 @@
+use crate::{
+    cleanup, delete_where_symbols, optimize, select_where_symbol, types::Backfill, upsert,
+};
+use clickhouse::{error::Error, Client};
+
+select_where_symbol!(Backfill, "backfills_news");
+upsert!(Backfill, "backfills_news");
+delete_where_symbols!("backfills_news");
+cleanup!("backfills_news");
+optimize!("backfills_news");
+
+pub async fn unfresh(clickhouse_client: &Client) -> Result<(), Error> {
+    clickhouse_client
+        .query("ALTER TABLE backfills_news UPDATE fresh = false WHERE true")
+        .execute()
+        .await
+}

src/database/bars.rs

@@ -1,39 +1,7 @@
-use crate::types::Bar;
-use clickhouse::{error::Error, Client};
-use serde::Serialize;
+use crate::{cleanup, delete_where_symbols, optimize, types::Bar, upsert, upsert_batch};
 
-pub async fn upsert(clickhouse_client: &Client, bar: &Bar) -> Result<(), Error> {
-    let mut insert = clickhouse_client.insert("bars")?;
-    insert.write(bar).await?;
-    insert.end().await
-}
-
-pub async fn upsert_batch<T>(clickhouse_client: &Client, bars: T) -> Result<(), Error>
-where
-    T: IntoIterator<Item = Bar> + Send + Sync,
-    T::IntoIter: Send,
-{
-    let mut insert = clickhouse_client.insert("bars")?;
-    for bar in bars {
-        insert.write(&bar).await?;
-    }
-    insert.end().await
-}
-
-pub async fn delete_where_symbols<T>(clickhouse_client: &Client, symbols: &[T]) -> Result<(), Error>
-where
-    T: AsRef<str> + Serialize + Send + Sync,
-{
-    clickhouse_client
-        .query("DELETE FROM bars WHERE symbol IN ?")
-        .bind(symbols)
-        .execute()
-        .await
-}
-
-pub async fn cleanup(clickhouse_client: &Client) -> Result<(), Error> {
-    clickhouse_client
-        .query("DELETE FROM bars WHERE symbol NOT IN (SELECT symbol FROM assets)")
-        .execute()
-        .await
-}
+upsert!(Bar, "bars");
+upsert_batch!(Bar, "bars");
+delete_where_symbols!("bars");
+cleanup!("bars");
+optimize!("bars");

src/database/mod.rs

@@ -1,6 +1,150 @@
 pub mod assets;
-pub mod backfills;
+pub mod backfills_bars;
+pub mod backfills_news;
 pub mod bars;
 pub mod news;
 pub mod orders;
-pub mod positions;
+
+use clickhouse::{error::Error, Client};
+use log::info;
+use tokio::try_join;
+
+#[macro_export]
+macro_rules! select {
+    ($record:ty, $table_name:expr) => {
+        pub async fn select(
+            client: &clickhouse::Client,
+        ) -> Result<Vec<$record>, clickhouse::error::Error> {
+            client
+                .query(&format!("SELECT ?fields FROM {} FINAL", $table_name))
+                .fetch_all::<$record>()
+                .await
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! select_where_symbol {
+    ($record:ty, $table_name:expr) => {
+        pub async fn select_where_symbol<T>(
+            client: &clickhouse::Client,
+            symbol: &T,
+        ) -> Result<Option<$record>, clickhouse::error::Error>
+        where
+            T: AsRef<str> + serde::Serialize + Send + Sync,
+        {
+            client
+                .query(&format!(
+                    "SELECT ?fields FROM {} FINAL WHERE symbol = ?",
+                    $table_name
+                ))
+                .bind(symbol)
+                .fetch_optional::<$record>()
+                .await
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! upsert {
+    ($record:ty, $table_name:expr) => {
+        pub async fn upsert(
+            client: &clickhouse::Client,
+            record: &$record,
+        ) -> Result<(), clickhouse::error::Error> {
+            let mut insert = client.insert($table_name)?;
+            insert.write(record).await?;
+            insert.end().await
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! upsert_batch {
+    ($record:ty, $table_name:expr) => {
+        pub async fn upsert_batch<'a, T>(
+            client: &clickhouse::Client,
+            records: T,
+        ) -> Result<(), clickhouse::error::Error>
+        where
+            T: IntoIterator<Item = &'a $record> + Send + Sync,
+            T::IntoIter: Send,
+        {
+            let mut insert = client.insert($table_name)?;
+            for record in records {
+                insert.write(record).await?;
+            }
+            insert.end().await
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! delete_where_symbols {
+    ($table_name:expr) => {
+        pub async fn delete_where_symbols<T>(
+            client: &clickhouse::Client,
+            symbols: &[T],
+        ) -> Result<(), clickhouse::error::Error>
+        where
+            T: AsRef<str> + serde::Serialize + Send + Sync,
+        {
+            client
+                .query(&format!("DELETE FROM {} WHERE symbol IN ?", $table_name))
+                .bind(symbols)
+                .execute()
+                .await
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! cleanup {
+    ($table_name:expr) => {
+        pub async fn cleanup(client: &clickhouse::Client) -> Result<(), clickhouse::error::Error> {
+            client
+                .query(&format!(
+                    "DELETE FROM {} WHERE symbol NOT IN (SELECT symbol FROM assets)",
+                    $table_name
+                ))
+                .execute()
+                .await
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! optimize {
+    ($table_name:expr) => {
+        pub async fn optimize(client: &clickhouse::Client) -> Result<(), clickhouse::error::Error> {
+            client
+                .query(&format!("OPTIMIZE TABLE {} FINAL", $table_name))
+                .execute()
+                .await
+        }
+    };
+}
+
+pub async fn cleanup_all(clickhouse_client: &Client) -> Result<(), Error> {
+    info!("Cleaning up database.");
+    try_join!(
+        bars::cleanup(clickhouse_client),
+        news::cleanup(clickhouse_client),
+        backfills_bars::cleanup(clickhouse_client),
+        backfills_news::cleanup(clickhouse_client)
+    )
+    .map(|_| ())
+}
+
+pub async fn optimize_all(clickhouse_client: &Client) -> Result<(), Error> {
+    info!("Optimizing database.");
+    try_join!(
+        assets::optimize(clickhouse_client),
+        bars::optimize(clickhouse_client),
+        news::optimize(clickhouse_client),
+        backfills_bars::optimize(clickhouse_client),
+        backfills_news::optimize(clickhouse_client),
+        orders::optimize(clickhouse_client)
+    )
+    .map(|_| ())
+}

src/database/news.rs

@@ -1,24 +1,10 @@
-use crate::types::News;
+use crate::{optimize, types::News, upsert, upsert_batch};
 use clickhouse::{error::Error, Client};
 use serde::Serialize;
 
-pub async fn upsert(clickhouse_client: &Client, news: &News) -> Result<(), Error> {
-    let mut insert = clickhouse_client.insert("news")?;
-    insert.write(news).await?;
-    insert.end().await
-}
-
-pub async fn upsert_batch<T>(clickhouse_client: &Client, news: T) -> Result<(), Error>
-where
-    T: IntoIterator<Item = News> + Send + Sync,
-    T::IntoIter: Send,
-{
-    let mut insert = clickhouse_client.insert("news")?;
-    for news in news {
-        insert.write(&news).await?;
-    }
-    insert.end().await
-}
+upsert!(News, "news");
+upsert_batch!(News, "news");
+optimize!("news");
 
 pub async fn delete_where_symbols<T>(clickhouse_client: &Client, symbols: &[T]) -> Result<(), Error>
 where

src/database/orders.rs

@@ -0,0 +1,5 @@
+use crate::{optimize, types::Order, upsert, upsert_batch};
+
+upsert!(Order, "orders");
+upsert_batch!(Order, "orders");
+optimize!("orders");