Add multiple asset adding route

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>

src/database/assets.rs

@@ -1,8 +1,11 @@
+use std::sync::Arc;
+
 use crate::{
     delete_where_symbols, optimize, select, select_where_symbol, types::Asset, upsert_batch,
 };
 use clickhouse::{error::Error, Client};
 use serde::Serialize;
+use tokio::sync::Semaphore;
 
 select!(Asset, "assets");
 select_where_symbol!(Asset, "assets");
@@ -11,14 +14,16 @@ delete_where_symbols!("assets");
 optimize!("assets");
 
 pub async fn update_status_where_symbol<T>(
-    clickhouse_client: &Client,
+    client: &Client,
+    concurrency_limiter: &Arc<Semaphore>,
     symbol: &T,
     status: bool,
 ) -> Result<(), Error>
 where
     T: AsRef<str> + Serialize + Send + Sync,
 {
-    clickhouse_client
+    let _ = concurrency_limiter.acquire().await.unwrap();
+    client
         .query("ALTER TABLE assets UPDATE status = ? WHERE symbol = ?")
         .bind(status)
         .bind(symbol)
@@ -27,14 +32,16 @@ where
 }
 
 pub async fn update_qty_where_symbol<T>(
-    clickhouse_client: &Client,
+    client: &Client,
+    concurrency_limiter: &Arc<Semaphore>,
     symbol: &T,
     qty: f64,
 ) -> Result<(), Error>
 where
     T: AsRef<str> + Serialize + Send + Sync,
 {
-    clickhouse_client
+    let _ = concurrency_limiter.acquire().await.unwrap();
+    client
         .query("ALTER TABLE assets UPDATE qty = ? WHERE symbol = ?")
         .bind(qty)
         .bind(symbol)

src/database/backfills_bars.rs

@@ -1,16 +1,20 @@
+use std::sync::Arc;
+
 use crate::{
-    cleanup, delete_where_symbols, optimize, select_where_symbol, types::Backfill, upsert,
+    cleanup, delete_where_symbols, optimize, select_where_symbols, types::Backfill, upsert,
 };
 use clickhouse::{error::Error, Client};
+use tokio::sync::Semaphore;
 
-select_where_symbol!(Backfill, "backfills_bars");
+select_where_symbols!(Backfill, "backfills_bars");
 upsert!(Backfill, "backfills_bars");
 delete_where_symbols!("backfills_bars");
 cleanup!("backfills_bars");
 optimize!("backfills_bars");
 
-pub async fn unfresh(clickhouse_client: &Client) -> Result<(), Error> {
-    clickhouse_client
+pub async fn unfresh(client: &Client, concurrency_limiter: &Arc<Semaphore>) -> Result<(), Error> {
+    let _ = concurrency_limiter.acquire().await.unwrap();
+    client
         .query("ALTER TABLE backfills_bars UPDATE fresh = false WHERE true")
         .execute()
         .await

src/database/backfills_news.rs

@@ -1,16 +1,20 @@
+use std::sync::Arc;
+
 use crate::{
-    cleanup, delete_where_symbols, optimize, select_where_symbol, types::Backfill, upsert,
+    cleanup, delete_where_symbols, optimize, select_where_symbols, types::Backfill, upsert,
 };
 use clickhouse::{error::Error, Client};
+use tokio::sync::Semaphore;
 
-select_where_symbol!(Backfill, "backfills_news");
+select_where_symbols!(Backfill, "backfills_news");
 upsert!(Backfill, "backfills_news");
 delete_where_symbols!("backfills_news");
 cleanup!("backfills_news");
 optimize!("backfills_news");
 
-pub async fn unfresh(clickhouse_client: &Client) -> Result<(), Error> {
-    clickhouse_client
+pub async fn unfresh(client: &Client, concurrency_limiter: &Arc<Semaphore>) -> Result<(), Error> {
+    let _ = concurrency_limiter.acquire().await.unwrap();
+    client
         .query("ALTER TABLE backfills_news UPDATE fresh = false WHERE true")
         .execute()
         .await

src/database/bars.rs

@@ -1,7 +1,21 @@
-use crate::{cleanup, delete_where_symbols, optimize, types::Bar, upsert, upsert_batch};
+use std::sync::Arc;
+
+use crate::{delete_where_symbols, optimize, types::Bar, upsert, upsert_batch};
+use clickhouse::Client;
+use tokio::sync::Semaphore;
 
 upsert!(Bar, "bars");
 upsert_batch!(Bar, "bars");
 delete_where_symbols!("bars");
-cleanup!("bars");
 optimize!("bars");
+
+pub async fn cleanup(
+    client: &Client,
+    concurrency_limiter: &Arc<Semaphore>,
+) -> Result<(), clickhouse::error::Error> {
+    let _ = concurrency_limiter.acquire().await.unwrap();
+    client
+        .query("DELETE FROM bars WHERE symbol NOT IN (SELECT symbol FROM assets) OR symbol NOT IN (SELECT symbol FROM backfills_bars)")
+        .execute()
+        .await
+}

src/database/calendar.rs

@@ -1,11 +1,14 @@
+use std::sync::Arc;
+
 use crate::{optimize, types::Calendar};
-use clickhouse::error::Error;
-use tokio::try_join;
+use clickhouse::{error::Error, Client};
+use tokio::{sync::Semaphore, try_join};
 
 optimize!("calendar");
 
 pub async fn upsert_batch_and_delete<'a, T>(
-    client: &clickhouse::Client,
+    client: &Client,
+    concurrency_limiter: &Arc<Semaphore>,
     records: T,
 ) -> Result<(), Error>
 where
@@ -34,5 +37,6 @@ where
             .await
     };
 
+    let _ = concurrency_limiter.acquire_many(2).await.unwrap();
     try_join!(upsert_future, delete_future).map(|_| ())
 }

src/database/mod.rs

@@ -15,7 +15,9 @@ macro_rules! select {
     ($record:ty, $table_name:expr) => {
         pub async fn select(
             client: &clickhouse::Client,
+            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
         ) -> Result<Vec<$record>, clickhouse::error::Error> {
+            let _ = concurrency_limiter.acquire().await.unwrap();
             client
                 .query(&format!("SELECT ?fields FROM {} FINAL", $table_name))
                 .fetch_all::<$record>()
@@ -29,11 +31,13 @@ macro_rules! select_where_symbol {
     ($record:ty, $table_name:expr) => {
         pub async fn select_where_symbol<T>(
             client: &clickhouse::Client,
+            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
             symbol: &T,
         ) -> Result<Option<$record>, clickhouse::error::Error>
         where
             T: AsRef<str> + serde::Serialize + Send + Sync,
         {
+            let _ = concurrency_limiter.acquire().await.unwrap();
             client
                 .query(&format!(
                     "SELECT ?fields FROM {} FINAL WHERE symbol = ?",
@@ -46,13 +50,39 @@ macro_rules! select_where_symbol {
     };
 }
 
+#[macro_export]
+macro_rules! select_where_symbols {
+    ($record:ty, $table_name:expr) => {
+        pub async fn select_where_symbols<T>(
+            client: &clickhouse::Client,
+            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
+            symbols: &[T],
+        ) -> Result<Vec<$record>, clickhouse::error::Error>
+        where
+            T: AsRef<str> + serde::Serialize + Send + Sync,
+        {
+            let _ = concurrency_limiter.acquire().await.unwrap();
+            client
+                .query(&format!(
+                    "SELECT ?fields FROM {} FINAL WHERE symbol IN ?",
+                    $table_name
+                ))
+                .bind(symbols)
+                .fetch_all::<$record>()
+                .await
+        }
+    };
+}
+
 #[macro_export]
 macro_rules! upsert {
     ($record:ty, $table_name:expr) => {
         pub async fn upsert(
             client: &clickhouse::Client,
+            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
             record: &$record,
         ) -> Result<(), clickhouse::error::Error> {
+            let _ = concurrency_limiter.acquire().await.unwrap();
             let mut insert = client.insert($table_name)?;
             insert.write(record).await?;
             insert.end().await
@@ -65,12 +95,14 @@ macro_rules! upsert_batch {
     ($record:ty, $table_name:expr) => {
         pub async fn upsert_batch<'a, T>(
             client: &clickhouse::Client,
+            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
             records: T,
         ) -> Result<(), clickhouse::error::Error>
         where
             T: IntoIterator<Item = &'a $record> + Send + Sync,
             T::IntoIter: Send,
         {
+            let _ = concurrency_limiter.acquire().await.unwrap();
             let mut insert = client.insert($table_name)?;
             for record in records {
                 insert.write(record).await?;
@@ -85,11 +117,13 @@ macro_rules! delete_where_symbols {
     ($table_name:expr) => {
         pub async fn delete_where_symbols<T>(
             client: &clickhouse::Client,
+            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
             symbols: &[T],
         ) -> Result<(), clickhouse::error::Error>
         where
             T: AsRef<str> + serde::Serialize + Send + Sync,
         {
+            let _ = concurrency_limiter.acquire().await.unwrap();
             client
                 .query(&format!("DELETE FROM {} WHERE symbol IN ?", $table_name))
                 .bind(symbols)
@@ -102,7 +136,11 @@ macro_rules! delete_where_symbols {
 #[macro_export]
 macro_rules! cleanup {
     ($table_name:expr) => {
-        pub async fn cleanup(client: &clickhouse::Client) -> Result<(), clickhouse::error::Error> {
+        pub async fn cleanup(
+            client: &clickhouse::Client,
+            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
+        ) -> Result<(), clickhouse::error::Error> {
+            let _ = concurrency_limiter.acquire().await.unwrap();
             client
                 .query(&format!(
                     "DELETE FROM {} WHERE symbol NOT IN (SELECT symbol FROM assets)",
@@ -117,7 +155,11 @@ macro_rules! cleanup {
 #[macro_export]
 macro_rules! optimize {
     ($table_name:expr) => {
-        pub async fn optimize(client: &clickhouse::Client) -> Result<(), clickhouse::error::Error> {
+        pub async fn optimize(
+            client: &clickhouse::Client,
+            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
+        ) -> Result<(), clickhouse::error::Error> {
+            let _ = concurrency_limiter.acquire().await.unwrap();
             client
                 .query(&format!("OPTIMIZE TABLE {} FINAL", $table_name))
                 .execute()
@@ -126,27 +168,33 @@ macro_rules! optimize {
     };
 }
 
-pub async fn cleanup_all(clickhouse_client: &Client) -> Result<(), Error> {
+pub async fn cleanup_all(
+    clickhouse_client: &Client,
+    concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
+) -> Result<(), Error> {
     info!("Cleaning up database.");
     try_join!(
-        bars::cleanup(clickhouse_client),
-        news::cleanup(clickhouse_client),
-        backfills_bars::cleanup(clickhouse_client),
-        backfills_news::cleanup(clickhouse_client)
+        bars::cleanup(clickhouse_client, concurrency_limiter),
+        news::cleanup(clickhouse_client, concurrency_limiter),
+        backfills_bars::cleanup(clickhouse_client, concurrency_limiter),
+        backfills_news::cleanup(clickhouse_client, concurrency_limiter)
     )
     .map(|_| ())
 }
 
-pub async fn optimize_all(clickhouse_client: &Client) -> Result<(), Error> {
+pub async fn optimize_all(
+    clickhouse_client: &Client,
+    concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
+) -> Result<(), Error> {
     info!("Optimizing database.");
     try_join!(
-        assets::optimize(clickhouse_client),
-        bars::optimize(clickhouse_client),
-        news::optimize(clickhouse_client),
-        backfills_bars::optimize(clickhouse_client),
-        backfills_news::optimize(clickhouse_client),
-        orders::optimize(clickhouse_client),
-        calendar::optimize(clickhouse_client)
+        assets::optimize(clickhouse_client, concurrency_limiter),
+        bars::optimize(clickhouse_client, concurrency_limiter),
+        news::optimize(clickhouse_client, concurrency_limiter),
+        backfills_bars::optimize(clickhouse_client, concurrency_limiter),
+        backfills_news::optimize(clickhouse_client, concurrency_limiter),
+        orders::optimize(clickhouse_client, concurrency_limiter),
+        calendar::optimize(clickhouse_client, concurrency_limiter)
     )
     .map(|_| ())
 }

src/database/news.rs

@@ -1,24 +1,33 @@
+use std::sync::Arc;
+
 use crate::{optimize, types::News, upsert, upsert_batch};
 use clickhouse::{error::Error, Client};
 use serde::Serialize;
+use tokio::sync::Semaphore;
 
 upsert!(News, "news");
 upsert_batch!(News, "news");
 optimize!("news");
 
-pub async fn delete_where_symbols<T>(clickhouse_client: &Client, symbols: &[T]) -> Result<(), Error>
+pub async fn delete_where_symbols<T>(
+    client: &Client,
+    concurrency_limiter: &Arc<Semaphore>,
+    symbols: &[T],
+) -> Result<(), Error>
 where
     T: AsRef<str> + Serialize + Send + Sync,
 {
-    clickhouse_client
+    let _ = concurrency_limiter.acquire().await.unwrap();
+    client
         .query("DELETE FROM news WHERE hasAny(symbols, ?) AND NOT hasAny(symbols, (SELECT groupArray(symbol) FROM assets))")
         .bind(symbols)
         .execute()
         .await
 }
 
-pub async fn cleanup(clickhouse_client: &Client) -> Result<(), Error> {
-    clickhouse_client
+pub async fn cleanup(client: &Client, concurrency_limiter: &Arc<Semaphore>) -> Result<(), Error> {
+    let _ = concurrency_limiter.acquire().await.unwrap();
+    client
         .query(
             "DELETE FROM news WHERE NOT hasAny(symbols, (SELECT groupArray(symbol) FROM assets))",
         )