karaolidis/qrust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up error propagation

Browse Source 
Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>
This commit is contained in:
Nick Karaolidis
2024-02-08 18:13:52 +00:00
parent 52e88f4bc9
commit 76bf2fddcb
24 changed files with 465 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
src/threads/data/mod.rs
View File

@@ -6,9 +6,9 @@ use crate::{
config::{
Config, ALPACA_CRYPTO_WEBSOCKET_URL, ALPACA_NEWS_WEBSOCKET_URL, ALPACA_STOCK_WEBSOCKET_URL,
},
database,
create_send_await, database,
types::{alpaca, Asset, Class},
utils::{authenticate, cleanup},
utils::{authenticate, backoff, cleanup},
};
use futures_util::{future::join_all, StreamExt};
use itertools::{Either, Itertools};
@@ -52,22 +52,22 @@ pub enum ThreadType {
}
pub async fn run(
app_config: Arc<Config>,
config: Arc<Config>,
mut receiver: mpsc::Receiver<Message>,
mut clock_receiver: mpsc::Receiver<clock::Message>,
) {
let (bars_us_equity_websocket_sender, bars_us_equity_backfill_sender) =
init_thread(app_config.clone(), ThreadType::Bars(Class::UsEquity)).await;
init_thread(config.clone(), ThreadType::Bars(Class::UsEquity)).await;
let (bars_crypto_websocket_sender, bars_crypto_backfill_sender) =
init_thread(app_config.clone(), ThreadType::Bars(Class::Crypto)).await;
init_thread(config.clone(), ThreadType::Bars(Class::Crypto)).await;
let (news_websocket_sender, news_backfill_sender) =
init_thread(app_config.clone(), ThreadType::News).await;
init_thread(config.clone(), ThreadType::News).await;
loop {
select! {
Some(message) = receiver.recv() => {
spawn(handle_message(
app_config.clone(),
config.clone(),
bars_us_equity_websocket_sender.clone(),
bars_us_equity_backfill_sender.clone(),
bars_crypto_websocket_sender.clone(),
@@ -79,7 +79,7 @@ pub async fn run(
}
Some(_) = clock_receiver.recv() => {
spawn(handle_clock_message(
app_config.clone(),
config.clone(),
bars_us_equity_backfill_sender.clone(),
bars_crypto_backfill_sender.clone(),
news_backfill_sender.clone(),
@@ -91,34 +91,33 @@ pub async fn run(
}
async fn init_thread(
app_config: Arc<Config>,
config: Arc<Config>,
thread_type: ThreadType,
) -> (
mpsc::Sender<websocket::Message>,
mpsc::Sender<backfill::Message>,
) {
let websocket_url = match thread_type {
ThreadType::Bars(Class::UsEquity) => format!(
"{}/{}",
ALPACA_STOCK_WEBSOCKET_URL, &app_config.alpaca_source
),
ThreadType::Bars(Class::UsEquity) => {
format!("{}/{}", ALPACA_STOCK_WEBSOCKET_URL, &config.alpaca_source)
}
ThreadType::Bars(Class::Crypto) => ALPACA_CRYPTO_WEBSOCKET_URL.into(),
ThreadType::News => ALPACA_NEWS_WEBSOCKET_URL.into(),
};
let (websocket, _) = connect_async(websocket_url).await.unwrap();
let (mut websocket_sink, mut websocket_stream) = websocket.split();
authenticate(&app_config, &mut websocket_sink, &mut websocket_stream).await;
authenticate(&config, &mut websocket_sink, &mut websocket_stream).await;
let (backfill_sender, backfill_receiver) = mpsc::channel(100);
spawn(backfill::run(
Arc::new(backfill::create_handler(thread_type, app_config.clone())),
Arc::new(backfill::create_handler(thread_type, config.clone())),
backfill_receiver,
));
let (websocket_sender, websocket_receiver) = mpsc::channel(100);
spawn(websocket::run(
Arc::new(websocket::create_handler(thread_type, app_config.clone())),
Arc::new(websocket::create_handler(thread_type, config.clone())),
websocket_receiver,
websocket_stream,
websocket_sink,
@@ -127,17 +126,9 @@ async fn init_thread(
(websocket_sender, backfill_sender)
}
macro_rules! create_send_await {
($sender:expr, $action:expr, $($contents:expr),*) => {
let (message, receiver) = $action($($contents),*);
$sender.send(message).await.unwrap();
receiver.await.unwrap();
};
}
#[allow(clippy::too_many_arguments)]
async fn handle_message(
app_config: Arc<Config>,
config: Arc<Config>,
bars_us_equity_websocket_sender: mpsc::Sender<websocket::Message>,
bars_us_equity_backfill_sender: mpsc::Sender<backfill::Message>,
bars_crypto_websocket_sender: mpsc::Sender<websocket::Message>,
@@ -221,22 +212,30 @@ async fn handle_message(
match message.action {
Action::Add => {
let assets =
join_all(symbols.into_iter().map(|symbol| {
let app_config = app_config.clone();
async move {
alpaca::api::incoming::asset::get_by_symbol(&app_config, &symbol).await
}
}))
.await
.into_iter()
.map(|result| Asset::from(result.unwrap()))
.collect::<Vec<_>>();
let assets = join_all(symbols.into_iter().map(|symbol| {
let config = config.clone();
async move {
Asset::from(
alpaca::api::incoming::asset::get_by_symbol(
&config,
&symbol,
Some(backoff::infinite()),
)
.await
.unwrap(),
)
}
}))
.await;
database::assets::upsert_batch(&app_config.clickhouse_client, assets).await;
database::assets::upsert_batch(&config.clickhouse_client, assets)
.await
.unwrap();
}
Action::Remove => {
database::assets::delete_where_symbols(&app_config.clickhouse_client, &symbols).await;
database::assets::delete_where_symbols(&config.clickhouse_client, &symbols)
.await
.unwrap();
}
}
@@ -244,14 +243,16 @@ async fn handle_message(
}
async fn handle_clock_message(
app_config: Arc<Config>,
config: Arc<Config>,
bars_us_equity_backfill_sender: mpsc::Sender<backfill::Message>,
bars_crypto_backfill_sender: mpsc::Sender<backfill::Message>,
news_backfill_sender: mpsc::Sender<backfill::Message>,
) {
cleanup(&app_config.clickhouse_client).await;
cleanup(&config.clickhouse_client).await.unwrap();
let assets = database::assets::select(&app_config.clickhouse_client).await;
let assets = database::assets::select(&config.clickhouse_client)
.await
.unwrap();
let (us_equity_symbols, crypto_symbols): (Vec<_>, Vec<_>) = assets
.clone()