karaolidis/qrust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix the Sin of Man

Browse Source 
- Migrate to ClickHouse
- Simplify serde renaming
- Simplify backfill logic
- Compartmentalize database columns

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>
This commit is contained in:
Nick Karaolidis
2024-01-15 23:51:53 +00:00
parent 63a9ca950f
commit de3989ec35
45 changed files with 1120 additions and 2718 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
src/routes/assets.rs
View File

@@ -1,29 +1,28 @@
use crate::config::{
Config, ALPACA_ASSET_API_URL, ALPACA_CRYPTO_DATA_URL, ALPACA_STOCK_DATA_URL,
ALPACA_TIMESTAMP_FORMAT,
};
use crate::config::{Config, ALPACA_ASSET_API_URL};
use crate::database;
use crate::types::Class;
use crate::types::{api::incoming, asset, Asset, BroadcastMessage, Status};
use axum::{extract::Path, http::StatusCode, Extension, Json};
use crate::types::{
api::incoming::{self, asset::Status},
asset, Asset, BroadcastMessage,
};
use axum::{extract::Path, Extension, Json};
use http::StatusCode;
use log::info;
use serde::Deserialize;
use std::sync::Arc;
use time::OffsetDateTime;
use tokio::sync::broadcast::Sender;
pub async fn get_all(
Extension(app_config): Extension<Arc<Config>>,
) -> Result<(StatusCode, Json<Vec<Asset>>), StatusCode> {
let assets = database::assets::select(&app_config.postgres_pool).await;
Ok((StatusCode::OK, Json(assets)))
}
pub async fn get(
Extension(app_config): Extension<Arc<Config>>,
) -> Result<(StatusCode, Json<Vec<Asset>>), StatusCode> {
let assets = database::assets::select(&app_config.clickhouse_client).await;
Ok((StatusCode::OK, Json(assets)))
}
pub async fn get_where_symbol(
Extension(app_config): Extension<Arc<Config>>,
Path(symbol): Path<String>,
) -> Result<(StatusCode, Json<Asset>), StatusCode> {
let asset = database::assets::select_where_symbol(&app_config.postgres_pool, &symbol).await;
let asset = database::assets::select_where_symbol(&app_config.clickhouse_client, &symbol).await;
asset.map_or(Err(StatusCode::NOT_FOUND), |asset| {
Ok((StatusCode::OK, Json(asset)))
})
@@ -32,15 +31,14 @@ pub async fn get(
#[derive(Deserialize)]
pub struct AddAssetRequest {
symbol: String,
trading: Option<bool>,
}
pub async fn add(
Extension(app_config): Extension<Arc<Config>>,
Extension(asset_broadcast_sender): Extension<Sender<BroadcastMessage>>,
Extension(broadcast_sender): Extension<Sender<BroadcastMessage>>,
Json(request): Json<AddAssetRequest>,
) -> Result<(StatusCode, Json<Asset>), StatusCode> {
if database::assets::select_where_symbol(&app_config.postgres_pool, &request.symbol)
if database::assets::select_where_symbol(&app_config.clickhouse_client, &request.symbol)
.await
.is_some()
{
@@ -53,66 +51,25 @@ pub async fn add(
.get(&format!("{}/{}", ALPACA_ASSET_API_URL, request.symbol))
.send()
.await
.map_err(|e| match e.status() {
Some(StatusCode::NOT_FOUND) => StatusCode::NOT_FOUND,
_ => panic!(),
})?;
.map_err(|e| {
if e.status() == Some(reqwest::StatusCode::NOT_FOUND) {
StatusCode::NOT_FOUND
} else {
panic!()
}
})
.unwrap();
let asset = asset.json::<incoming::Asset>().await.unwrap();
let asset = asset.json::<incoming::asset::Asset>().await.unwrap();
if asset.status != Status::Active || !asset.tradable || !asset.fractionable {
return Err(StatusCode::FORBIDDEN);
}
let mut earliest_bar_request = app_config
.alpaca_client
.get(match asset.class {
Class::UsEquity => ALPACA_STOCK_DATA_URL,
Class::Crypto => ALPACA_CRYPTO_DATA_URL,
})
.query(&[
("symbols", &asset.symbol),
("timeframe", &String::from("1Min")),
(
"start",
&OffsetDateTime::UNIX_EPOCH
.format(ALPACA_TIMESTAMP_FORMAT)
.unwrap(),
),
("limit", &String::from("1")),
]);
let asset = Asset::from(asset);
database::assets::insert(&app_config.clickhouse_client, &asset).await;
if asset.class == Class::UsEquity {
earliest_bar_request =
earliest_bar_request.query(&[("feed", &app_config.alpaca_source.to_string())]);
}
let earliest_bar = earliest_bar_request
.send()
.await
.unwrap()
.json::<incoming::bar::Message>()
.await
.unwrap();
let earliest_bar = earliest_bar
.bars
.get(&asset.symbol)
.ok_or(StatusCode::NOT_FOUND)?
.as_ref()
.ok_or(StatusCode::NOT_FOUND)?
.first()
.ok_or(StatusCode::NOT_FOUND)?;
let asset = Asset::from((
asset,
request.trading.unwrap_or(false),
earliest_bar.timestamp,
));
database::assets::insert(&app_config.postgres_pool, &asset).await;
asset_broadcast_sender
broadcast_sender
.send(BroadcastMessage::Asset(asset::BroadcastMessage::Added(
asset.clone(),
)))
@@ -122,50 +79,24 @@ pub async fn add(
Ok((StatusCode::CREATED, Json(asset)))
}
#[allow(dead_code)]
#[derive(Deserialize)]
pub struct UpdateAssetRequest {
trading: bool,
}
pub async fn update(
Extension(app_config): Extension<Arc<Config>>,
Extension(asset_broadcast_sender): Extension<Sender<BroadcastMessage>>,
Path(symbol): Path<String>,
Json(request): Json<UpdateAssetRequest>,
) -> Result<(StatusCode, Json<Asset>), StatusCode> {
let asset = database::assets::update_trading_where_symbol(
&app_config.postgres_pool,
&symbol,
&request.trading,
)
.await;
asset.map_or(Err(StatusCode::NOT_FOUND), |asset| {
asset_broadcast_sender
.send(BroadcastMessage::Asset(asset::BroadcastMessage::Updated(
asset.clone(),
)))
.unwrap();
info!("Updated asset {}.", symbol);
Ok((StatusCode::OK, Json(asset)))
})
}
pub async fn delete(
Extension(app_config): Extension<Arc<Config>>,
Extension(asset_broadcast_sender): Extension<Sender<BroadcastMessage>>,
Extension(broadcast_sender): Extension<Sender<BroadcastMessage>>,
Path(symbol): Path<String>,
) -> Result<StatusCode, StatusCode> {
let asset = database::assets::delete_where_symbol(&app_config.postgres_pool, &symbol).await;
let asset = database::assets::select_where_symbol(&app_config.clickhouse_client, &symbol)
.await
.ok_or(StatusCode::NOT_FOUND)
.unwrap();
asset.map_or(Err(StatusCode::NOT_FOUND), |asset| {
asset_broadcast_sender
.send(BroadcastMessage::Asset(asset::BroadcastMessage::Deleted(
asset,
)))
.unwrap();
info!("Deleted asset {}.", symbol);
Ok(StatusCode::NO_CONTENT)
})
broadcast_sender
.send(BroadcastMessage::Asset(asset::BroadcastMessage::Deleted(
asset,
)))
.unwrap();
database::assets::delete_where_symbol(&app_config.clickhouse_client, &symbol).await;
info!("Deleted asset {}.", symbol);
Ok(StatusCode::NO_CONTENT)
}

20
src/routes/mod.rs
View File

@@ -1,30 +1,26 @@
use crate::{config::Config, types::BroadcastMessage};
use axum::{
routing::{delete, get, post},
Extension, Router, Server,
serve, Extension, Router,
};
use log::info;
use std::{net::SocketAddr, sync::Arc};
use tokio::sync::broadcast::Sender;
use tokio::{net::TcpListener, sync::broadcast::Sender};
pub mod assets;
pub async fn run(app_config: Arc<Config>, asset_broadcast_sender: Sender<BroadcastMessage>) {
pub async fn run(app_config: Arc<Config>, broadcast_sender: Sender<BroadcastMessage>) {
let app = Router::new()
.route("/assets", get(assets::get_all))
.route("/assets/:symbol", get(assets::get))
.route("/assets", get(assets::get))
.route("/assets/:symbol", get(assets::get_where_symbol))
.route("/assets", post(assets::add))
.route("/assets/:symbol", post(assets::update))
.route("/assets/:symbol", delete(assets::delete))
.layer(Extension(app_config))
.layer(Extension(asset_broadcast_sender));
.layer(Extension(broadcast_sender));
let addr = SocketAddr::from(([0, 0, 0, 0], 7878));
let listener = TcpListener::bind(addr).await.unwrap();
info!("Listening on {}.", addr);
Server::bind(&addr)
.serve(app.into_make_service())
.await
.unwrap();
serve(listener, app).await.unwrap();
unreachable!()
}