Fix the Sin of Man
- Migrate to ClickHouse - Simplify serde renaming - Simplify backfill logic - Compartmentalize database columns Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>
This commit is contained in:
@@ -1,92 +1,43 @@
|
||||
use crate::types::{Asset, Class, Exchange};
|
||||
use sqlx::{query_as, PgPool};
|
||||
use std::convert::Into;
|
||||
use time::OffsetDateTime;
|
||||
use crate::types::{Asset, Class};
|
||||
use clickhouse::Client;
|
||||
|
||||
pub async fn select(postgres_pool: &PgPool) -> Vec<Asset> {
|
||||
query_as!(
|
||||
Asset,
|
||||
r#"SELECT symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last FROM assets"#
|
||||
)
|
||||
.fetch_all(postgres_pool)
|
||||
.await
|
||||
.unwrap()
|
||||
pub async fn select(clickhouse_client: &Client) -> Vec<Asset> {
|
||||
clickhouse_client
|
||||
.query("SELECT ?fields FROM assets")
|
||||
.fetch_all::<Asset>()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub async fn select_where_class(postgres_pool: &PgPool, class: Class) -> Vec<Asset> {
|
||||
query_as!(
|
||||
Asset,
|
||||
r#"SELECT symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last FROM assets WHERE class = $1::CLASS"#,
|
||||
class as Class
|
||||
)
|
||||
.fetch_all(postgres_pool)
|
||||
.await
|
||||
.unwrap()
|
||||
pub async fn select_where_class(clickhouse_client: &Client, class: Class) -> Vec<Asset> {
|
||||
clickhouse_client
|
||||
.query("SELECT ?fields FROM assets WHERE class = ?")
|
||||
.bind(class)
|
||||
.fetch_all::<Asset>()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub async fn select_where_symbol(postgres_pool: &PgPool, symbol: &str) -> Option<Asset> {
|
||||
query_as!(
|
||||
Asset,
|
||||
r#"SELECT symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last FROM assets WHERE symbol = $1"#,
|
||||
symbol
|
||||
)
|
||||
.fetch_optional(postgres_pool)
|
||||
.await
|
||||
.unwrap()
|
||||
pub async fn select_where_symbol(clickhouse_client: &Client, symbol: &str) -> Option<Asset> {
|
||||
clickhouse_client
|
||||
.query("SELECT ?fields FROM assets WHERE symbol = ?")
|
||||
.bind(symbol)
|
||||
.fetch_optional::<Asset>()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub async fn insert(postgres_pool: &PgPool, asset: &Asset) -> Asset {
|
||||
query_as!(
|
||||
Asset,
|
||||
r#"INSERT INTO assets (symbol, class, exchange, trading, timestamp_added, timestamp_first, timestamp_last) VALUES ($1, $2::CLASS, $3::EXCHANGE, $4, $5, $6, $7)
|
||||
RETURNING symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last"#,
|
||||
asset.symbol, asset.class as Class, asset.exchange as Exchange, asset.trading, asset.timestamp_added, asset.timestamp_first, asset.timestamp_last
|
||||
)
|
||||
.fetch_one(postgres_pool)
|
||||
.await
|
||||
.unwrap()
|
||||
pub async fn insert(clickhouse_client: &Client, asset: &Asset) {
|
||||
let mut insert = clickhouse_client.insert("assets").unwrap();
|
||||
insert.write(asset).await.unwrap();
|
||||
insert.end().await.unwrap();
|
||||
}
|
||||
|
||||
pub async fn update_trading_where_symbol(
|
||||
postgres_pool: &PgPool,
|
||||
symbol: &str,
|
||||
trading: &bool,
|
||||
) -> Option<Asset> {
|
||||
query_as!(
|
||||
Asset,
|
||||
r#"UPDATE assets SET trading = $1 WHERE symbol = $2
|
||||
RETURNING symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last"#,
|
||||
trading, symbol
|
||||
)
|
||||
.fetch_optional(postgres_pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub async fn update_timestamp_last_where_symbol(
|
||||
postgres_pool: &PgPool,
|
||||
symbol: &str,
|
||||
timestamp_last: &OffsetDateTime,
|
||||
) -> Option<Asset> {
|
||||
query_as!(
|
||||
Asset,
|
||||
r#"UPDATE assets SET timestamp_last = $1 WHERE symbol = $2
|
||||
RETURNING symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last"#,
|
||||
timestamp_last, symbol
|
||||
)
|
||||
.fetch_optional(postgres_pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub async fn delete_where_symbol(postgres_pool: &PgPool, symbol: &str) -> Option<Asset> {
|
||||
query_as!(
|
||||
Asset,
|
||||
r#"DELETE FROM assets WHERE symbol = $1
|
||||
RETURNING symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last"#,
|
||||
symbol
|
||||
)
|
||||
.fetch_optional(postgres_pool)
|
||||
.await
|
||||
.unwrap()
|
||||
pub async fn delete_where_symbol(clickhouse_client: &Client, symbol: &str) {
|
||||
clickhouse_client
|
||||
.query("DELETE FROM assets WHERE symbol = ?")
|
||||
.bind(symbol)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
@@ -1,161 +1,52 @@
|
||||
use crate::types::Bar;
|
||||
use sqlx::{query_as, PgPool, Postgres};
|
||||
use time::OffsetDateTime;
|
||||
use crate::types::{Bar, BarValidity};
|
||||
use clickhouse::Client;
|
||||
|
||||
pub async fn select_not_null_where_symbol_where_timestamp_smaller_than_order_by_timestamp_desc_limit_one(
|
||||
postgres_pool: &PgPool,
|
||||
pub async fn upsert(clickhouse_client: &Client, bar: &Bar) {
|
||||
let mut insert = clickhouse_client.insert("bars").unwrap();
|
||||
insert.write(bar).await.unwrap();
|
||||
insert.end().await.unwrap();
|
||||
}
|
||||
|
||||
pub async fn upsert_batch(clickhouse_client: &Client, bars: &[Bar]) {
|
||||
let mut insert = clickhouse_client.insert("bars").unwrap();
|
||||
for bar in bars {
|
||||
insert.write(bar).await.unwrap();
|
||||
}
|
||||
insert.end().await.unwrap();
|
||||
}
|
||||
|
||||
pub async fn delete_where_symbol(clickhouse_client: &Client, symbol: &str) {
|
||||
clickhouse_client
|
||||
.query("DELETE FROM bars WHERE symbol = ?")
|
||||
.bind(symbol)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
pub async fn select_validity_where_symbol(
|
||||
clickhouse_client: &Client,
|
||||
symbol: &str,
|
||||
timestamp: &OffsetDateTime,
|
||||
) -> Bar {
|
||||
query_as!(
|
||||
Bar,
|
||||
r#"SELECT * FROM bars WHERE asset_symbol = $1 AND timestamp < $2 AND open IS NOT NULL AND high IS NOT NULL AND low IS NOT NULL AND close IS NOT NULL ORDER BY timestamp DESC LIMIT 1"#,
|
||||
symbol,
|
||||
timestamp
|
||||
)
|
||||
.fetch_one(postgres_pool)
|
||||
.await
|
||||
.unwrap()
|
||||
) -> Option<BarValidity> {
|
||||
clickhouse_client
|
||||
.query("SELECT ?fields FROM bars_validity FINAL WHERE symbol = ?")
|
||||
.bind(symbol)
|
||||
.fetch_optional::<BarValidity>()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub async fn select_where_symbol_where_timestamp_larger_than(
|
||||
postgres_pool: &PgPool,
|
||||
symbol: &str,
|
||||
timestamp: &OffsetDateTime,
|
||||
) -> Vec<Bar> {
|
||||
query_as!(
|
||||
Bar,
|
||||
r#"SELECT * FROM bars WHERE asset_symbol = $1 AND timestamp > $2 ORDER BY timestamp ASC"#,
|
||||
symbol,
|
||||
timestamp
|
||||
)
|
||||
.fetch_all(postgres_pool)
|
||||
.await
|
||||
.unwrap()
|
||||
pub async fn upsert_validity(clickhouse_client: &Client, bar_validity: &BarValidity) {
|
||||
let mut insert = clickhouse_client.insert("bars_validity").unwrap();
|
||||
insert.write(bar_validity).await.unwrap();
|
||||
insert.end().await.unwrap();
|
||||
}
|
||||
|
||||
pub async fn upsert(postgres_pool: &PgPool, bar: &Bar, backfill: bool) -> Bar {
|
||||
let mut bar = bar.clone();
|
||||
|
||||
if backfill
|
||||
&& (bar.open.is_none() || bar.high.is_none() || bar.low.is_none() || bar.close.is_none())
|
||||
{
|
||||
let filled_bar = select_not_null_where_symbol_where_timestamp_smaller_than_order_by_timestamp_desc_limit_one(
|
||||
postgres_pool,
|
||||
&bar.asset_symbol,
|
||||
&bar.timestamp,
|
||||
).await;
|
||||
bar.merge_empty(&filled_bar);
|
||||
}
|
||||
|
||||
query_as!(
|
||||
Bar,
|
||||
r#"INSERT INTO bars (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (timestamp, asset_symbol) DO UPDATE SET open = $3, high = $4, low = $5, close = $6, volume = $7, num_trades = $8, volume_weighted = $9
|
||||
RETURNING timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted"#,
|
||||
bar.timestamp, bar.asset_symbol, bar.open, bar.high, bar.low, bar.close, bar.volume, bar.num_trades, bar.volume_weighted
|
||||
)
|
||||
.fetch_one(postgres_pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub async fn insert_or_skip(postgres_pool: &PgPool, bar: &Bar, backfill: bool) {
|
||||
let mut bar = bar.clone();
|
||||
|
||||
if backfill
|
||||
&& (bar.open.is_none() || bar.high.is_none() || bar.low.is_none() || bar.close.is_none())
|
||||
{
|
||||
let filled_bar = select_not_null_where_symbol_where_timestamp_smaller_than_order_by_timestamp_desc_limit_one(
|
||||
postgres_pool,
|
||||
&bar.asset_symbol,
|
||||
&bar.timestamp,
|
||||
).await;
|
||||
bar.merge_empty(&filled_bar);
|
||||
}
|
||||
|
||||
query_as!(
|
||||
Bar,
|
||||
r#"INSERT INTO bars (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (timestamp, asset_symbol) DO NOTHING"#,
|
||||
bar.timestamp, bar.asset_symbol, bar.open, bar.high, bar.low, bar.close, bar.volume, bar.num_trades, bar.volume_weighted
|
||||
)
|
||||
.execute(postgres_pool)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
pub async fn upsert_batch(postgres_pool: &PgPool, bars: &[Bar], backfill: bool) -> Vec<Bar> {
|
||||
let mut bars = bars.to_vec();
|
||||
|
||||
if bars.is_empty() {
|
||||
return bars;
|
||||
}
|
||||
|
||||
if backfill
|
||||
&& (bars[0].open.is_none()
|
||||
|| bars[0].high.is_none()
|
||||
|| bars[0].low.is_none()
|
||||
|| bars[0].close.is_none())
|
||||
{
|
||||
let filled_bar = select_not_null_where_symbol_where_timestamp_smaller_than_order_by_timestamp_desc_limit_one(
|
||||
postgres_pool,
|
||||
&bars[0].asset_symbol,
|
||||
&bars[0].timestamp,
|
||||
).await;
|
||||
bars[0].merge_empty(&filled_bar);
|
||||
}
|
||||
|
||||
let mut timestamp = Vec::with_capacity(bars.len());
|
||||
let mut asset_symbol = Vec::with_capacity(bars.len());
|
||||
let mut open = Vec::with_capacity(bars.len());
|
||||
let mut high = Vec::with_capacity(bars.len());
|
||||
let mut low = Vec::with_capacity(bars.len());
|
||||
let mut close = Vec::with_capacity(bars.len());
|
||||
let mut volume = Vec::with_capacity(bars.len());
|
||||
let mut num_trades = Vec::with_capacity(bars.len());
|
||||
let mut volume_weighted = Vec::with_capacity(bars.len());
|
||||
|
||||
let mut last_filled_bar = bars[0].clone();
|
||||
|
||||
for mut bar in bars {
|
||||
if backfill {
|
||||
if bar.open.is_none() || bar.high.is_none() || bar.low.is_none() || bar.close.is_none()
|
||||
{
|
||||
bar.merge_empty(&last_filled_bar);
|
||||
} else {
|
||||
last_filled_bar = bar.clone();
|
||||
}
|
||||
}
|
||||
|
||||
timestamp.push(bar.timestamp);
|
||||
asset_symbol.push(bar.asset_symbol.clone());
|
||||
open.push(bar.open);
|
||||
high.push(bar.high);
|
||||
low.push(bar.low);
|
||||
close.push(bar.close);
|
||||
volume.push(bar.volume);
|
||||
num_trades.push(bar.num_trades);
|
||||
volume_weighted.push(bar.volume_weighted);
|
||||
}
|
||||
|
||||
// No type-safety here because of NULLABLE bulk insert
|
||||
query_as::<Postgres, Bar>(
|
||||
r#"INSERT INTO bars (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted)
|
||||
SELECT * FROM UNNEST($1::timestamptz[], $2::text[], $3::float8[], $4::float8[], $5::float8[], $6::float8[], $7::float8[], $8::int8[], $9::float8[])
|
||||
ON CONFLICT (timestamp, asset_symbol) DO UPDATE SET open = EXCLUDED.open, high = EXCLUDED.high, low = EXCLUDED.low, close = EXCLUDED.close, volume = EXCLUDED.volume, num_trades = EXCLUDED.num_trades, volume_weighted = EXCLUDED.volume_weighted
|
||||
RETURNING timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted"#,
|
||||
)
|
||||
.bind(timestamp)
|
||||
.bind(asset_symbol)
|
||||
.bind(open)
|
||||
.bind(high)
|
||||
.bind(low)
|
||||
.bind(close)
|
||||
.bind(volume)
|
||||
.bind(num_trades)
|
||||
.bind(volume_weighted)
|
||||
.fetch_all(postgres_pool)
|
||||
.await
|
||||
.unwrap()
|
||||
pub async fn delete_validity_where_symbol(clickhouse_client: &Client, symbol: &str) {
|
||||
clickhouse_client
|
||||
.query("DELETE FROM bars_validity WHERE symbol = ?")
|
||||
.bind(symbol)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user