Add pipelined backfilling

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>

src/database/backfills_bars.rs

@@ -1,13 +1,13 @@
 use std::sync::Arc;
 
 use crate::{
-    cleanup, delete_where_symbols, optimize, select_where_symbols, types::Backfill, upsert,
+    cleanup, delete_where_symbols, optimize, select_where_symbols, types::Backfill, upsert_batch,
 };
 use clickhouse::{error::Error, Client};
 use tokio::sync::Semaphore;
 
 select_where_symbols!(Backfill, "backfills_bars");
-upsert!(Backfill, "backfills_bars");
+upsert_batch!(Backfill, "backfills_bars");
 delete_where_symbols!("backfills_bars");
 cleanup!("backfills_bars");
 optimize!("backfills_bars");

src/database/backfills_news.rs

@@ -1,13 +1,13 @@
 use std::sync::Arc;
 
 use crate::{
-    cleanup, delete_where_symbols, optimize, select_where_symbols, types::Backfill, upsert,
+    cleanup, delete_where_symbols, optimize, select_where_symbols, types::Backfill, upsert_batch,
 };
 use clickhouse::{error::Error, Client};
 use tokio::sync::Semaphore;
 
 select_where_symbols!(Backfill, "backfills_news");
-upsert!(Backfill, "backfills_news");
+upsert_batch!(Backfill, "backfills_news");
 delete_where_symbols!("backfills_news");
 cleanup!("backfills_news");
 optimize!("backfills_news");

src/database/bars.rs

@@ -4,6 +4,8 @@ use crate::{delete_where_symbols, optimize, types::Bar, upsert, upsert_batch};
 use clickhouse::Client;
 use tokio::sync::Semaphore;
 
+pub const BATCH_FLUSH_SIZE: usize = 100_000;
+
 upsert!(Bar, "bars");
 upsert_batch!(Bar, "bars");
 delete_where_symbols!("bars");

src/database/news.rs

@@ -5,6 +5,8 @@ use clickhouse::{error::Error, Client};
 use serde::Serialize;
 use tokio::sync::Semaphore;
 
+pub const BATCH_FLUSH_SIZE: usize = 500;
+
 upsert!(News, "news");
 upsert_batch!(News, "news");
 optimize!("news");