Add market data backfilling

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>
2023-09-09 17:48:49 +03:00
parent 548a8e42d5
commit e26d2b95e7
72 changed files with 1847 additions and 1044 deletions
--- a/src/database/assets.rs
+++ b/src/database/assets.rs
@@ -0,0 +1,92 @@
+use crate::types::{Asset, Class, Exchange};
+use sqlx::{query_as, PgPool};
+use std::convert::Into;
+use time::OffsetDateTime;
+
+pub async fn select(postgres_pool: &PgPool) -> Vec<Asset> {
+    query_as!(
+        Asset,
+        r#"SELECT symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last FROM assets"#
+    )
+    .fetch_all(postgres_pool)
+    .await
+    .unwrap()
+}
+
+pub async fn select_where_class(postgres_pool: &PgPool, class: Class) -> Vec<Asset> {
+    query_as!(
+        Asset,
+        r#"SELECT symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last FROM assets WHERE class = $1::CLASS"#,
+        class as Class
+    )
+    .fetch_all(postgres_pool)
+    .await
+    .unwrap()
+}
+
+pub async fn select_where_symbol(postgres_pool: &PgPool, symbol: &str) -> Option<Asset> {
+    query_as!(
+        Asset,
+        r#"SELECT symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last FROM assets WHERE symbol = $1"#,
+        symbol
+    )
+    .fetch_optional(postgres_pool)
+    .await
+    .unwrap()
+}
+
+pub async fn insert(postgres_pool: &PgPool, asset: &Asset) -> Asset {
+    query_as!(
+        Asset,
+        r#"INSERT INTO assets (symbol, class, exchange, trading, timestamp_added, timestamp_first, timestamp_last) VALUES ($1, $2::CLASS, $3::EXCHANGE, $4, $5, $6, $7)
+        RETURNING symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last"#,
+        asset.symbol, asset.class as Class, asset.exchange as Exchange, asset.trading, asset.timestamp_added, asset.timestamp_first, asset.timestamp_last
+    )
+    .fetch_one(postgres_pool)
+    .await
+    .unwrap()
+}
+
+pub async fn update_trading_where_symbol(
+    postgres_pool: &PgPool,
+    symbol: &str,
+    trading: &bool,
+) -> Option<Asset> {
+    query_as!(
+        Asset,
+        r#"UPDATE assets SET trading = $1 WHERE symbol = $2
+        RETURNING symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last"#,
+        trading, symbol
+    )
+    .fetch_optional(postgres_pool)
+    .await
+    .unwrap()
+}
+
+pub async fn update_timestamp_last_where_symbol(
+    postgres_pool: &PgPool,
+    symbol: &str,
+    timestamp_last: &OffsetDateTime,
+) -> Option<Asset> {
+    query_as!(
+        Asset,
+        r#"UPDATE assets SET timestamp_last = $1 WHERE symbol = $2
+        RETURNING symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last"#,
+        timestamp_last, symbol
+    )
+    .fetch_optional(postgres_pool)
+    .await
+    .unwrap()
+}
+
+pub async fn delete_where_symbol(postgres_pool: &PgPool, symbol: &str) -> Option<Asset> {
+    query_as!(
+        Asset,
+        r#"DELETE FROM assets WHERE symbol = $1
+        RETURNING symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last"#,
+        symbol
+    )
+    .fetch_optional(postgres_pool)
+    .await
+    .unwrap()
+}
--- a/src/database/bars.rs
+++ b/src/database/bars.rs
@@ -0,0 +1,89 @@
+use crate::types::Bar;
+use sqlx::{query_as, PgPool, Postgres};
+use std::convert::Into;
+use time::OffsetDateTime;
+
+pub async fn upsert(postgres_pool: &PgPool, bar: &Bar) -> Bar {
+    query_as!(
+        Bar,
+        r#"INSERT INTO bars (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+        ON CONFLICT (timestamp, asset_symbol) DO UPDATE SET open = $3, high = $4, low = $5, close = $6, volume = $7, num_trades = $8, volume_weighted = $9
+        RETURNING timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted"#,
+        bar.timestamp, bar.asset_symbol, bar.open, bar.high, bar.low, bar.close, bar.volume, bar.num_trades, bar.volume_weighted
+    )
+    .fetch_one(postgres_pool)
+    .await
+    .unwrap()
+}
+
+pub async fn insert_or_skip(postgres_pool: &PgPool, bar: &Bar) {
+    query_as!(
+        Bar,
+        r#"INSERT INTO bars (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+        ON CONFLICT (timestamp, asset_symbol) DO NOTHING"#,
+        bar.timestamp, bar.asset_symbol, bar.open, bar.high, bar.low, bar.close, bar.volume, bar.num_trades, bar.volume_weighted
+    )
+    .execute(postgres_pool)
+    .await
+    .unwrap();
+}
+
+pub async fn upsert_batch(postgres_pool: &PgPool, bars: &[Bar]) -> Vec<Bar> {
+    let mut timestamp = Vec::with_capacity(bars.len());
+    let mut asset_symbol = Vec::with_capacity(bars.len());
+    let mut open = Vec::with_capacity(bars.len());
+    let mut high = Vec::with_capacity(bars.len());
+    let mut low = Vec::with_capacity(bars.len());
+    let mut close = Vec::with_capacity(bars.len());
+    let mut volume = Vec::with_capacity(bars.len());
+    let mut num_trades = Vec::with_capacity(bars.len());
+    let mut volume_weighted = Vec::with_capacity(bars.len());
+
+    for bar in bars {
+        timestamp.push(bar.timestamp);
+        asset_symbol.push(bar.asset_symbol.clone());
+        open.push(bar.open);
+        high.push(bar.high);
+        low.push(bar.low);
+        close.push(bar.close);
+        volume.push(bar.volume);
+        num_trades.push(bar.num_trades);
+        volume_weighted.push(bar.volume_weighted);
+    }
+
+    // No type-safety here because of NULLABLE bulk insert
+    query_as::<Postgres, Bar>(
+        r#"INSERT INTO bars (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted)
+        SELECT * FROM UNNEST($1::timestamptz[], $2::text[], $3::float8[], $4::float8[], $5::float8[], $6::float8[], $7::float8[], $8::int8[], $9::float8[])
+        ON CONFLICT (timestamp, asset_symbol) DO UPDATE SET open = EXCLUDED.open, high = EXCLUDED.high, low = EXCLUDED.low, close = EXCLUDED.close, volume = EXCLUDED.volume, num_trades = EXCLUDED.num_trades, volume_weighted = EXCLUDED.volume_weighted
+        RETURNING timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted"#,
+    )
+    .bind(timestamp)
+    .bind(asset_symbol)
+    .bind(open)
+    .bind(high)
+    .bind(low)
+    .bind(close)
+    .bind(volume)
+    .bind(num_trades)
+    .bind(volume_weighted)
+    .fetch_all(postgres_pool)
+    .await
+    .unwrap()
+}
+
+pub async fn select_where_symbol_where_timestamp_larger_than(
+    postgres_pool: &PgPool,
+    symbol: &str,
+    timestamp: &OffsetDateTime,
+) -> Vec<Bar> {
+    query_as!(
+        Bar,
+        r#"SELECT * FROM bars WHERE asset_symbol = $1 AND timestamp > $2 ORDER BY timestamp ASC"#,
+        symbol,
+        timestamp
+    )
+    .fetch_all(postgres_pool)
+    .await
+    .unwrap()
+}
--- a/src/database/bars_filled.rs
+++ b/src/database/bars_filled.rs
@@ -0,0 +1,133 @@
+use crate::types::Bar;
+use sqlx::{query_as, PgPool, Postgres};
+use std::convert::Into;
+
+pub async fn upsert(postgres_pool: &PgPool, bar: &Bar) -> Bar {
+    let mut bar = bar.clone();
+
+    if bar.open.is_none() || bar.high.is_none() || bar.low.is_none() || bar.close.is_none() {
+        let filled_bar = query_as!(
+            Bar,
+            r#"SELECT * FROM bars_filled WHERE timestamp < $1 AND asset_symbol = $2 ORDER BY timestamp DESC LIMIT 1"#,
+            bar.timestamp,
+            bar.asset_symbol
+        )
+        .fetch_one(postgres_pool)
+        .await
+        .unwrap();
+        bar.merge_empty(&filled_bar);
+    }
+
+    query_as!(
+        Bar,
+        r#"INSERT INTO bars_filled (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+        ON CONFLICT (timestamp, asset_symbol) DO UPDATE SET open = $3, high = $4, low = $5, close = $6, volume = $7, num_trades = $8, volume_weighted = $9
+        RETURNING timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted"#,
+        bar.timestamp, bar.asset_symbol, bar.open, bar.high, bar.low, bar.close, bar.volume, bar.num_trades, bar.volume_weighted
+    )
+    .fetch_one(postgres_pool)
+    .await
+    .unwrap()
+}
+
+pub async fn insert_or_skip(postgres_pool: &PgPool, bar: &Bar) {
+    let mut bar = bar.clone();
+
+    if bar.open.is_none() || bar.high.is_none() || bar.low.is_none() || bar.close.is_none() {
+        let filled_bar = query_as!(
+            Bar,
+            r#"SELECT * FROM bars_filled WHERE timestamp < $1 AND asset_symbol = $2 ORDER BY timestamp DESC LIMIT 1"#,
+            bar.timestamp,
+            bar.asset_symbol
+        )
+        .fetch_one(postgres_pool)
+        .await
+        .unwrap();
+        bar.merge_empty(&filled_bar);
+    }
+
+    query_as!(
+        Bar,
+        r#"INSERT INTO bars_filled (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+        ON CONFLICT (timestamp, asset_symbol) DO NOTHING"#,
+        bar.timestamp, bar.asset_symbol, bar.open, bar.high, bar.low, bar.close, bar.volume, bar.num_trades, bar.volume_weighted
+    )
+    .execute(postgres_pool)
+    .await
+    .unwrap();
+}
+
+pub async fn upsert_batch(postgres_pool: &PgPool, bars: &[Bar]) -> Vec<Bar> {
+    let mut bars = bars.to_vec();
+
+    if bars.is_empty() {
+        return bars;
+    }
+
+    if bars[0].open.is_none()
+        || bars[0].high.is_none()
+        || bars[0].low.is_none()
+        || bars[0].close.is_none()
+    {
+        let filled_bar = &query_as!(
+            Bar,
+            r#"SELECT * FROM bars_filled WHERE timestamp < $1 AND asset_symbol = $2 ORDER BY timestamp DESC LIMIT 1"#,
+            bars[0].timestamp,
+            bars[0].asset_symbol
+        )
+        .fetch_one(postgres_pool)
+        .await
+        .unwrap();
+        bars[0].merge_empty(filled_bar);
+    }
+
+    let mut timestamp = Vec::with_capacity(bars.len());
+    let mut asset_symbol = Vec::with_capacity(bars.len());
+    let mut open = Vec::with_capacity(bars.len());
+    let mut high = Vec::with_capacity(bars.len());
+    let mut low = Vec::with_capacity(bars.len());
+    let mut close = Vec::with_capacity(bars.len());
+    let mut volume = Vec::with_capacity(bars.len());
+    let mut num_trades = Vec::with_capacity(bars.len());
+    let mut volume_weighted = Vec::with_capacity(bars.len());
+
+    let mut last_filled_bar = bars[0].clone();
+
+    for mut bar in bars {
+        if bar.open.is_none() || bar.high.is_none() || bar.low.is_none() || bar.close.is_none() {
+            bar.merge_empty(&last_filled_bar);
+        } else {
+            last_filled_bar = bar.clone();
+        }
+
+        timestamp.push(bar.timestamp);
+        asset_symbol.push(bar.asset_symbol.clone());
+        open.push(bar.open);
+        high.push(bar.high);
+        low.push(bar.low);
+        close.push(bar.close);
+        volume.push(bar.volume);
+        num_trades.push(bar.num_trades);
+        volume_weighted.push(bar.volume_weighted);
+    }
+
+    // No type-safety here because of NULLABLE bulk insert
+    query_as::<Postgres, Bar>(
+        r#"INSERT INTO bars_filled (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted)
+        SELECT * FROM UNNEST($1::timestamptz[], $2::text[], $3::float8[], $4::float8[], $5::float8[], $6::float8[], $7::float8[], $8::int8[], $9::float8[])
+        ON CONFLICT (timestamp, asset_symbol) DO UPDATE SET open = EXCLUDED.open, high = EXCLUDED.high, low = EXCLUDED.low, close = EXCLUDED.close, volume = EXCLUDED.volume, num_trades = EXCLUDED.num_trades, volume_weighted = EXCLUDED.volume_weighted
+        RETURNING timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted"#,
+    )
+    .bind(timestamp)
+    .bind(asset_symbol)
+    .bind(open)
+    .bind(high)
+    .bind(low)
+    .bind(close)
+    .bind(volume)
+    .bind(num_trades)
+    .bind(volume_weighted)
+    .fetch_all(postgres_pool)
+    .await
+    .unwrap()
+}
--- a/src/database/mod.rs
+++ b/src/database/mod.rs
@@ -0,0 +1,3 @@
+pub mod assets;
+pub mod bars;
+pub mod bars_filled;