Add market data backfilling

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>
2023-09-09 17:48:49 +03:00
parent 548a8e42d5
commit e26d2b95e7
72 changed files with 1847 additions and 1044 deletions
--- a/src/config.rs
+++ b/src/config.rs
@@ -0,0 +1,73 @@
+use crate::types::Source;
+use governor::{DefaultDirectRateLimiter, Quota, RateLimiter};
+use http::HeaderMap;
+use reqwest::Client;
+use sqlx::{postgres::PgPoolOptions, PgPool};
+use std::{env, num::NonZeroU32, sync::Arc};
+use time::{format_description::FormatItem, macros::format_description};
+use tokio::time::Duration;
+
+pub const ALPACA_ASSET_API_URL: &str = "https://api.alpaca.markets/v2/assets";
+pub const ALPACA_STOCK_DATA_URL: &str = "https://data.alpaca.markets/v2/stocks/bars";
+pub const ALPACA_CRYPTO_DATA_URL: &str = "https://data.alpaca.markets/v1beta3/crypto/us/bars";
+pub const ALPACA_STOCK_WEBSOCKET_URL: &str = "wss://stream.data.alpaca.markets/v2";
+pub const ALPACA_CRYPTO_WEBSOCKET_URL: &str = "wss://stream.data.alpaca.markets/v1beta3/crypto/us";
+pub const ALPACA_TIMESTAMP_FORMAT: &[FormatItem] =
+    format_description!("[year]-[month]-[day]T[hour]:[minute]:[second]Z");
+
+const NUM_CLIENTS: u32 = 10;
+
+pub struct Config {
+    pub alpaca_api_key: String,
+    pub alpaca_api_secret: String,
+    pub alpaca_client: Client,
+    pub alpaca_rate_limit: DefaultDirectRateLimiter,
+    pub alpaca_historical_offset: Duration,
+    pub alpaca_source: Source,
+    pub postgres_pool: PgPool,
+}
+
+impl Config {
+    pub async fn from_env() -> Self {
+        let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set.");
+        let alpaca_api_key = env::var("ALPACA_API_KEY").expect("ALPACA_API_KEY must be set.");
+        let alpaca_api_secret =
+            env::var("ALPACA_API_SECRET").expect("ALPACA_API_SECRET must be set.");
+        let alpaca_source: Source = env::var("ALPACA_SOURCE")
+            .expect("ALPACA_SOURCE must be set.")
+            .parse()
+            .expect("ALPACA_SOURCE must be a either 'iex' or 'sip'.");
+
+        Self {
+            alpaca_api_key: alpaca_api_key.clone(),
+            alpaca_api_secret: alpaca_api_secret.clone(),
+            alpaca_client: Client::builder()
+                .default_headers({
+                    let mut headers = HeaderMap::new();
+                    headers.insert("APCA-API-KEY-ID", alpaca_api_key.parse().unwrap());
+                    headers.insert("APCA-API-SECRET-KEY", alpaca_api_secret.parse().unwrap());
+                    headers
+                })
+                .build()
+                .unwrap(),
+            alpaca_rate_limit: RateLimiter::direct(Quota::per_minute(match alpaca_source {
+                Source::Iex => NonZeroU32::new(200).unwrap(),
+                Source::Sip => NonZeroU32::new(1000).unwrap(),
+            })),
+            alpaca_historical_offset: Duration::from_secs(match alpaca_source {
+                Source::Iex => 900,
+                Source::Sip => 0,
+            }),
+            alpaca_source,
+            postgres_pool: PgPoolOptions::new()
+                .max_connections(NUM_CLIENTS)
+                .connect(&database_url)
+                .await
+                .unwrap(),
+        }
+    }
+
+    pub async fn arc_from_env() -> Arc<Self> {
+        Arc::new(Self::from_env().await)
+    }
+}