karaolidis/qrust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix bars_validity for market close

Browse Source 
Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>
This commit is contained in:
Nick Karaolidis
2024-01-16 15:46:22 +00:00
parent ad7a6407ff
commit 3ee72a0e1b
13 changed files with 125 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
src/data/market.rs
View File

@@ -1,16 +1,19 @@
use crate::{
config::{
Config, ALPACA_CRYPTO_DATA_URL, ALPACA_CRYPTO_WEBSOCKET_URL, ALPACA_STOCK_DATA_URL,
ALPACA_STOCK_WEBSOCKET_URL, ALPACA_TIMESTAMP_FORMAT,
Config, ALPACA_CLOCK_API_URL, ALPACA_CRYPTO_DATA_URL, ALPACA_CRYPTO_WEBSOCKET_URL,
ALPACA_STOCK_DATA_URL, ALPACA_STOCK_WEBSOCKET_URL,
},
data::authenticate_websocket,
database,
types::{
alpaca::{api::incoming, websocket},
alpaca::{
api::{incoming, outgoing},
websocket, Source,
},
asset::{self, Asset},
Bar, BarValidity, BroadcastMessage, Class,
},
utils::{duration_until, last_minute, next_minute, ONE_MINUTE},
utils::{duration_until, last_minute, FIFTEEN_MINUTES, ONE_MINUTE},
};
use core::panic;
use futures_util::{
@@ -188,7 +191,11 @@ async fn websocket_handle_message(
backfilled.write().await.insert(asset.symbol.clone(), false);
let bar_validity = BarValidity::none(asset.symbol.clone());
database::bars::upsert_validity(&app_config.clickhouse_client, &bar_validity).await;
database::bars::insert_validity_if_not_exists(
&app_config.clickhouse_client,
&bar_validity,
)
.await;
spawn(backfill(
app_config.clone(),
@@ -240,14 +247,39 @@ pub async fn backfill(
.unwrap();
let fetch_from = bar_validity.time_last + ONE_MINUTE;
let fetch_until = last_minute();
let fetch_until = if app_config.alpaca_source == Source::Iex {
app_config.alpaca_rate_limit.until_ready().await;
let clock = app_config
.alpaca_client
.get(ALPACA_CLOCK_API_URL)
.send()
.await
.unwrap()
.json::<incoming::clock::Clock>()
.await
.unwrap();
if clock.is_open {
last_minute()
} else {
clock.next_open
}
} else {
last_minute()
};
if fetch_from > fetch_until {
return;
}
info!("Queing historical data backfill for {}...", asset.symbol);
let task_run_offsetdatetime = next_minute() + app_config.alpaca_historical_offset;
sleep(duration_until(task_run_offsetdatetime)).await;
if app_config.alpaca_source == Source::Iex {
let task_run_delay = duration_until(fetch_until + FIFTEEN_MINUTES + ONE_MINUTE);
info!(
"Queing historical data backfill for {} in {:?}.",
asset.symbol, task_run_delay
);
sleep(task_run_delay).await;
}
info!("Running historical data backfill for {}...", asset.symbol);
@@ -255,59 +287,38 @@ pub async fn backfill(
let mut next_page_token = None;
loop {
let request = app_config
app_config.alpaca_rate_limit.until_ready().await;
let message = app_config
.alpaca_client
.get(match asset.class {
Class::UsEquity => ALPACA_STOCK_DATA_URL,
Class::Crypto => ALPACA_CRYPTO_DATA_URL,
})
.query(&[
("symbols", &asset.symbol),
("timeframe", &String::from("1Min")),
(
"start",
&fetch_from
.format(ALPACA_TIMESTAMP_FORMAT)
.unwrap()
.to_string(),
),
(
"end",
&fetch_until
.format(ALPACA_TIMESTAMP_FORMAT)
.unwrap()
.to_string(),
),
("limit", &String::from("10000")),
("page_token", &next_page_token.clone().unwrap_or_default()),
]);
app_config.alpaca_rate_limit.until_ready().await;
let response = request.send().await.unwrap();
let response = if response.status() == reqwest::StatusCode::OK {
response.json::<incoming::bar::Message>().await.unwrap()
} else {
error!(
"Failed to backfill historical data for {} from {} to {}: {}",
asset.symbol,
.query(&outgoing::bar::Bar::new(
vec![asset.symbol.clone()],
String::from("1Min"),
fetch_from,
fetch_until,
response.text().await.unwrap()
);
break;
};
10000,
next_page_token,
))
.send()
.await
.unwrap()
.json::<incoming::bar::Message>()
.await
.unwrap();
response.bars.into_iter().for_each(|(symbol, bar_vec)| {
message.bars.into_iter().for_each(|(symbol, bar_vec)| {
bar_vec.unwrap_or_default().into_iter().for_each(|bar| {
bars.push(Bar::from((bar, symbol.clone())));
});
});
if response.next_page_token.is_none() {
if message.next_page_token.is_none() {
break;
}
next_page_token = response.next_page_token;
next_page_token = message.next_page_token;
}
database::bars::upsert_batch(&app_config.clickhouse_client, &bars).await;