karaolidis/qrust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add backfill early saving

Browse Source 
Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>
This commit is contained in:
Nick Karaolidis
2024-03-11 16:53:12 +00:00
parent 2de86b46f7
commit b60cbc891d
7 changed files with 87 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
src/threads/data/backfill/news.rs
View File

@@ -5,7 +5,6 @@ use crate::{
};
use async_trait::async_trait;
use futures_util::future::join_all;
use itertools::{Either, Itertools};
use log::{error, info};
use qrust::{
types::{
@@ -78,7 +77,7 @@ impl super::Handler for Handler {
info!("Backfilling news for {:?}.", symbols);
let mut news = vec![];
let mut news = Vec::with_capacity(database::news::BATCH_FLUSH_SIZE);
let mut last_times = symbols
.iter()
.map(|symbol| (symbol.clone(), None))
@@ -160,44 +159,39 @@ impl super::Handler for Handler {
)
.await
.unwrap();
news = vec![];
}
if message.next_page_token.is_none() {
break;
}
next_page_token = message.next_page_token;
}
let backfilled = last_times
.into_iter()
.filter_map(|(symbol, time)| {
if let Some(time) = time {
return Some(Backfill { symbol, time });
}
None
})
.collect::<Vec<_>>();
let (backfilled, skipped): (Vec<_>, Vec<_>) =
last_times.into_iter().partition_map(|(symbol, time)| {
if let Some(time) = time {
Either::Left(Backfill { symbol, time })
} else {
Either::Right(symbol)
database::backfills_news::upsert_batch(
&self.config.clickhouse_client,
&self.config.clickhouse_concurrency_limiter,
&backfilled,
)
.await
.unwrap();
if message.next_page_token.is_none() {
break;
}
});
database::backfills_news::upsert_batch(
&self.config.clickhouse_client,
&self.config.clickhouse_concurrency_limiter,
&backfilled,
)
.await
.unwrap();
let backfilled = backfilled
.into_iter()
.map(|backfill| backfill.symbol)
.collect::<Vec<_>>();
if !skipped.is_empty() {
info!("No news to backfill for {:?}.", skipped);
next_page_token = message.next_page_token;
news = Vec::with_capacity(database::news::BATCH_FLUSH_SIZE);
last_times = symbols
.iter()
.map(|symbol| (symbol.clone(), None))
.collect::<HashMap<_, _>>();
}
}
if !backfilled.is_empty() {
info!("Backfilled news for {:?}.", backfilled);
}
info!("Backfilled news for {:?}.", symbols);
}
fn max_limit(&self) -> i64 {