karaolidis/qrust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize backfill early saving allocations

Browse Source 
Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>
This commit is contained in:
Nick Karaolidis
2024-03-11 20:41:59 +00:00
parent 2d8972dce2
commit d02f958865
2 changed files with 12 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/threads/data/backfill/bars.rs
View File

@@ -120,10 +120,7 @@ impl super::Handler for Handler {
info!("Backfilling bars for {:?}.", symbols); info!("Backfilling bars for {:?}.", symbols);
let mut bars = Vec::with_capacity(database::bars::BATCH_FLUSH_SIZE); let mut bars = Vec::with_capacity(database::bars::BATCH_FLUSH_SIZE);
let mut last_times = symbols let mut last_times = HashMap::new();
.iter()
.map(|symbol| (symbol.clone(), None))
.collect::<HashMap<_, _>>();
let mut next_page_token = None; let mut next_page_token = None;
loop { loop {
@@ -150,7 +147,7 @@ impl super::Handler for Handler {
for (symbol, bar_vec) in message.bars { for (symbol, bar_vec) in message.bars {
if let Some(last) = bar_vec.last() { if let Some(last) = bar_vec.last() {
last_times.insert(symbol.clone(), Some(last.time)); last_times.insert(symbol.clone(), last.time);
} }
for bar in bar_vec { for bar in bar_vec {
@@ -168,13 +165,8 @@ impl super::Handler for Handler {
.unwrap(); .unwrap();
let backfilled = last_times let backfilled = last_times
.into_iter() .drain()
.filter_map(|(symbol, time)| { .map(|(symbol, time)| Backfill { symbol, time })
if let Some(time) = time {
return Some(Backfill { symbol, time });
}
None
})
.collect::<Vec<_>>(); .collect::<Vec<_>>();
database::backfills_bars::upsert_batch( database::backfills_bars::upsert_batch(
@@ -190,11 +182,7 @@ impl super::Handler for Handler {
} }
next_page_token = message.next_page_token; next_page_token = message.next_page_token;
bars = Vec::with_capacity(database::bars::BATCH_FLUSH_SIZE); bars.clear();
last_times = symbols
.iter()
.map(|symbol| (symbol.clone(), None))
.collect::<HashMap<_, _>>();
} }
} }

26
src/threads/data/backfill/news.rs
View File

@@ -69,23 +69,20 @@ impl super::Handler for Handler {
sleep(run_delay).await; sleep(run_delay).await;
} }
#[allow(clippy::too_many_lines)]
async fn backfill(&self, jobs: HashMap<String, Job>) { async fn backfill(&self, jobs: HashMap<String, Job>) {
if jobs.is_empty() { if jobs.is_empty() {
return; return;
} }
let symbols = jobs.keys().cloned().collect::<Vec<_>>(); let symbols = jobs.keys().cloned().collect::<Vec<_>>();
let symbols_set = symbols.iter().collect::<std::collections::HashSet<_>>();
let fetch_from = jobs.values().map(|job| job.fetch_from).min().unwrap(); let fetch_from = jobs.values().map(|job| job.fetch_from).min().unwrap();
let fetch_to = jobs.values().map(|job| job.fetch_to).max().unwrap(); let fetch_to = jobs.values().map(|job| job.fetch_to).max().unwrap();
info!("Backfilling news for {:?}.", symbols); info!("Backfilling news for {:?}.", symbols);
let mut news = Vec::with_capacity(database::news::BATCH_FLUSH_SIZE); let mut news = Vec::with_capacity(database::news::BATCH_FLUSH_SIZE);
let mut last_times = symbols let mut last_times = HashMap::new();
.iter()
.map(|symbol| (symbol.clone(), None))
.collect::<HashMap<_, _>>();
let mut next_page_token = None; let mut next_page_token = None;
loop { loop {
@@ -115,8 +112,8 @@ impl super::Handler for Handler {
let news_item = News::from(news_item); let news_item = News::from(news_item);
for symbol in &news_item.symbols { for symbol in &news_item.symbols {
if last_times.contains_key(symbol) { if symbols_set.contains(symbol) {
last_times.insert(symbol.clone(), Some(news_item.time_created)); last_times.insert(symbol.clone(), news_item.time_created);
} }
} }
@@ -165,13 +162,8 @@ impl super::Handler for Handler {
.unwrap(); .unwrap();
let backfilled = last_times let backfilled = last_times
.into_iter() .drain()
.filter_map(|(symbol, time)| { .map(|(symbol, time)| Backfill { symbol, time })
if let Some(time) = time {
return Some(Backfill { symbol, time });
}
None
})
.collect::<Vec<_>>(); .collect::<Vec<_>>();
database::backfills_news::upsert_batch( database::backfills_news::upsert_batch(
@@ -187,11 +179,7 @@ impl super::Handler for Handler {
} }
next_page_token = message.next_page_token; next_page_token = message.next_page_token;
news = Vec::with_capacity(database::news::BATCH_FLUSH_SIZE); news.clear();
last_times = symbols
.iter()
.map(|symbol| (symbol.clone(), None))
.collect::<HashMap<_, _>>();
} }
} }