Add Ollama news sentiment analysis

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>

.gitignore

@@ -2,7 +2,6 @@
 # will have compiled files and executables
 debug/
 target/
-log/
 
 # These are backup files generated by rustfmt
 **/*.rs.bk

.gitlab-ci.yml

@@ -24,13 +24,13 @@ build:
   script:
     - cargo +nightly build
 
-# test:
-#   image: registry.karaolidis.com/karaolidis/qrust/rust
-#   stage: test
-#   cache:
-#     <<: *global_cache
-#   script:
-#     - cargo +nightly test
+test:
+  image: registry.karaolidis.com/karaolidis/qrust/rust
+  stage: test
+  cache:
+    <<: *global_cache
+  script:
+    - cargo +nightly test
 
 lint:
   image: registry.karaolidis.com/karaolidis/qrust/rust
@@ -48,7 +48,7 @@ depcheck:
     <<: *global_cache
   script:
     - cargo +nightly outdated
-    - cargo +nightly udeps --all-targets
+    - cargo +nightly udeps
 
 build-release:
   image: registry.karaolidis.com/karaolidis/qrust/rust

Cargo.toml

@@ -3,14 +3,6 @@ name    = "qrust"
 version = "0.1.0"
 edition = "2021"
 
-[lib]
-name = "qrust"
-path = "src/lib/mod.rs"
-
-[[bin]]
-name = "qrust"
-path = "src/main.rs"
-
 [profile.release]
 panic         = 'abort'
 strip         = true
@@ -35,8 +27,6 @@ log4rs = "1.2.0"
 serde = "1.0.188"
 serde_json = "1.0.105"
 serde_repr = "0.1.18"
-serde_with = "3.6.1"
-serde-aux = "4.4.0"
 futures-util = "0.3.28"
 reqwest = { version = "0.11.20", features = [
     "json",
@@ -49,25 +39,15 @@ clickhouse = { version = "0.11.6", features = [
     "time",
     "uuid",
 ] }
-uuid = { version = "1.6.1", features = [
-    "serde",
-    "v4",
-] }
+uuid = "1.6.1"
 time = { version = "0.3.31", features = [
     "serde",
-    "serde-well-known",
-    "serde-human-readable",
     "formatting",
     "macros",
-    "local-offset",
+    "serde-well-known",
 ] }
 backoff = { version = "0.4.0", features = [
     "tokio",
 ] }
 regex = "1.10.3"
-async-trait = "0.1.77"
-itertools = "0.12.1"
-lazy_static = "1.4.0"
-nonempty = { version = "0.10.0", features = [
-    "serialize",
-] }
+html-escape = "0.2.13"

README.md

@@ -1,5 +1,3 @@
-# qrust
+# QRust
 
-![XKCD - Engineer Syllogism](./static/engineer-syllogism.png)
-
-`qrust` (/kɹʌst/, QuantitativeRust) is an algorithmic trading library written in Rust.
+QRust (/kɹʌst/, QuantitativeRust) is an algorithmic trading library written in Rust.

docker-compose.yml

@@ -4,6 +4,11 @@ services:
       file: support/clickhouse/docker-compose.yml
       service: clickhouse
 
+  ollama:
+    extends:
+      file: support/ollama/docker-compose.yml
+      service: ollama
+
   grafana:
     extends:
       file: support/grafana/docker-compose.yml
@@ -19,10 +24,12 @@ services:
       - 7878:7878
     depends_on:
       - clickhouse
+      - ollama
     env_file:
       - .env.docker
 
 volumes:
   clickhouse-lib:
   clickhouse-log:
+  ollama:
   grafana-lib:

log4rs.yaml

@@ -4,14 +4,7 @@ appenders:
     encoder:
       pattern: "{d} {h({l})} {M}::{L} - {m}{n}"
 
-  file:
-    kind: file
-    path: "./log/output.log"
-    encoder:
-      pattern: "{d} {l} {M}::{L} - {m}{n}"
-
 root:
   level: info
   appenders:
     - stdout
-    - file

src/config.rs

@@ -1,82 +1,87 @@
+use crate::types::alpaca::Source;
 use governor::{DefaultDirectRateLimiter, Quota, RateLimiter};
-use lazy_static::lazy_static;
-use qrust::types::alpaca::shared::{Mode, Source};
 use reqwest::{
     header::{HeaderMap, HeaderName, HeaderValue},
     Client,
 };
-use std::{env, num::NonZeroU32, sync::Arc};
-use tokio::sync::Semaphore;
+use std::{env, num::NonZeroU32, sync::Arc, time::Duration};
 
-lazy_static! {
-    pub static ref ALPACA_MODE: Mode = env::var("ALPACA_MODE")
-        .expect("ALPACA_MODE must be set.")
-        .parse()
-        .expect("ALPACA_MODE must be 'live' or 'paper'");
-    pub static ref ALPACA_API_BASE: String = match *ALPACA_MODE {
-        Mode::Live => String::from("api"),
-        Mode::Paper => String::from("paper-api"),
-    };
-    pub static ref ALPACA_SOURCE: Source = env::var("ALPACA_SOURCE")
-        .expect("ALPACA_SOURCE must be set.")
-        .parse()
-        .expect("ALPACA_SOURCE must be 'iex', 'sip', or 'otc'");
-    pub static ref ALPACA_API_KEY: String =
-        env::var("ALPACA_API_KEY").expect("ALPACA_API_KEY must be set.");
-    pub static ref ALPACA_API_SECRET: String =
-        env::var("ALPACA_API_SECRET").expect("ALPACA_API_SECRET must be set.");
-    pub static ref CLICKHOUSE_BATCH_BARS_SIZE: usize = env::var("BATCH_BACKFILL_BARS_SIZE")
-        .expect("BATCH_BACKFILL_BARS_SIZE must be set.")
-        .parse()
-        .expect("BATCH_BACKFILL_BARS_SIZE must be a positive integer.");
-    pub static ref CLICKHOUSE_BATCH_NEWS_SIZE: usize = env::var("BATCH_BACKFILL_NEWS_SIZE")
-        .expect("BATCH_BACKFILL_NEWS_SIZE must be set.")
-        .parse()
-        .expect("BATCH_BACKFILL_NEWS_SIZE must be a positive integer.");
-    pub static ref CLICKHOUSE_MAX_CONNECTIONS: usize = env::var("CLICKHOUSE_MAX_CONNECTIONS")
-        .expect("CLICKHOUSE_MAX_CONNECTIONS must be set.")
-        .parse()
-        .expect("CLICKHOUSE_MAX_CONNECTIONS must be a positive integer.");
-}
+pub const ALPACA_ASSET_API_URL: &str = "https://api.alpaca.markets/v2/assets";
+pub const ALPACA_CLOCK_API_URL: &str = "https://api.alpaca.markets/v2/clock";
+pub const ALPACA_STOCK_DATA_URL: &str = "https://data.alpaca.markets/v2/stocks/bars";
+pub const ALPACA_CRYPTO_DATA_URL: &str = "https://data.alpaca.markets/v1beta3/crypto/us/bars";
+pub const ALPACA_NEWS_DATA_URL: &str = "https://data.alpaca.markets/v1beta1/news";
+
+pub const ALPACA_STOCK_WEBSOCKET_URL: &str = "wss://stream.data.alpaca.markets/v2";
+pub const ALPACA_CRYPTO_WEBSOCKET_URL: &str = "wss://stream.data.alpaca.markets/v1beta3/crypto/us";
+pub const ALPACA_NEWS_WEBSOCKET_URL: &str = "wss://stream.data.alpaca.markets/v1beta1/news";
 
 pub struct Config {
+    pub alpaca_api_key: String,
+    pub alpaca_api_secret: String,
+    pub alpaca_rate_limit: DefaultDirectRateLimiter,
+    pub alpaca_source: Source,
     pub alpaca_client: Client,
-    pub alpaca_rate_limiter: DefaultDirectRateLimiter,
+    pub ollama_url: String,
+    pub ollama_model: String,
+    pub ollama_client: Client,
     pub clickhouse_client: clickhouse::Client,
-    pub clickhouse_concurrency_limiter: Arc<Semaphore>,
 }
 
 impl Config {
     pub fn from_env() -> Self {
+        let alpaca_api_key = env::var("ALPACA_API_KEY").expect("ALPACA_API_KEY must be set.");
+        let alpaca_api_secret =
+            env::var("ALPACA_API_SECRET").expect("ALPACA_API_SECRET must be set.");
+        let alpaca_source: Source = env::var("ALPACA_SOURCE")
+            .expect("ALPACA_SOURCE must be set.")
+            .parse()
+            .expect("ALPACA_SOURCE must be a either 'iex' or 'sip'.");
+
+        let ollama_url = env::var("OLLAMA_URL").expect("OLLAMA_URL must be set.");
+        let ollama_model = env::var("OLLAMA_MODEL").expect("OLLAMA_MODEL must be set.");
+
+        let clickhouse_url = env::var("CLICKHOUSE_URL").expect("CLICKHOUSE_URL must be set.");
+        let clickhouse_user = env::var("CLICKHOUSE_USER").expect("CLICKHOUSE_USER must be set.");
+        let clickhouse_password =
+            env::var("CLICKHOUSE_PASSWORD").expect("CLICKHOUSE_PASSWORD must be set.");
+        let clickhouse_db = env::var("CLICKHOUSE_DB").expect("CLICKHOUSE_DB must be set.");
+
         Self {
             alpaca_client: Client::builder()
                 .default_headers(HeaderMap::from_iter([
                     (
                         HeaderName::from_static("apca-api-key-id"),
-                        HeaderValue::from_str(&ALPACA_API_KEY)
+                        HeaderValue::from_str(&alpaca_api_key)
                             .expect("Alpaca API key must not contain invalid characters."),
                     ),
                     (
                         HeaderName::from_static("apca-api-secret-key"),
-                        HeaderValue::from_str(&ALPACA_API_SECRET)
+                        HeaderValue::from_str(&alpaca_api_secret)
                             .expect("Alpaca API secret must not contain invalid characters."),
                     ),
                 ]))
+                .timeout(Duration::from_secs(60))
                 .build()
                 .unwrap(),
-            alpaca_rate_limiter: RateLimiter::direct(Quota::per_minute(match *ALPACA_SOURCE {
-                Source::Iex => unsafe { NonZeroU32::new_unchecked(200) },
-                Source::Sip => unsafe { NonZeroU32::new_unchecked(10_000) },
-                Source::Otc => unimplemented!("OTC rate limit not implemented."),
+            alpaca_rate_limit: RateLimiter::direct(Quota::per_minute(match alpaca_source {
+                Source::Iex => unsafe { NonZeroU32::new_unchecked(190) },
+                Source::Sip => unsafe { NonZeroU32::new_unchecked(9990) },
             })),
+            alpaca_source,
+            alpaca_api_key,
+            alpaca_api_secret,
+            ollama_url,
+            ollama_model,
+            ollama_client: Client::builder()
+                .timeout(Duration::from_secs(15))
+                .build()
+                .unwrap(),
             clickhouse_client: clickhouse::Client::default()
-                .with_url(env::var("CLICKHOUSE_URL").expect("CLICKHOUSE_URL must be set."))
-                .with_user(env::var("CLICKHOUSE_USER").expect("CLICKHOUSE_USER must be set."))
-                .with_password(
-                    env::var("CLICKHOUSE_PASSWORD").expect("CLICKHOUSE_PASSWORD must be set."),
-                )
-                .with_database(env::var("CLICKHOUSE_DB").expect("CLICKHOUSE_DB must be set.")),
-            clickhouse_concurrency_limiter: Arc::new(Semaphore::new(*CLICKHOUSE_MAX_CONNECTIONS)),
+                .with_url(clickhouse_url)
+                .with_user(clickhouse_user)
+                .with_password(clickhouse_password)
+                .with_database(clickhouse_db),
         }
     }
 

src/database/assets.rs

@@ -0,0 +1,48 @@
+use crate::types::Asset;
+use clickhouse::Client;
+use serde::Serialize;
+
+pub async fn select(clickhouse_client: &Client) -> Vec<Asset> {
+    clickhouse_client
+        .query("SELECT ?fields FROM assets FINAL")
+        .fetch_all::<Asset>()
+        .await
+        .unwrap()
+}
+
+pub async fn select_where_symbol<T>(clickhouse_client: &Client, symbol: &T) -> Option<Asset>
+where
+    T: AsRef<str> + Serialize + Send + Sync,
+{
+    clickhouse_client
+        .query("SELECT ?fields FROM assets FINAL WHERE symbol = ? OR abbreviation = ?")
+        .bind(symbol)
+        .bind(symbol)
+        .fetch_optional::<Asset>()
+        .await
+        .unwrap()
+}
+
+pub async fn upsert_batch<T>(clickhouse_client: &Client, assets: T)
+where
+    T: IntoIterator<Item = Asset> + Send + Sync,
+    T::IntoIter: Send,
+{
+    let mut insert = clickhouse_client.insert("assets").unwrap();
+    for asset in assets {
+        insert.write(&asset).await.unwrap();
+    }
+    insert.end().await.unwrap();
+}
+
+pub async fn delete_where_symbols<T>(clickhouse_client: &Client, symbols: &[T])
+where
+    T: AsRef<str> + Serialize + Send + Sync,
+{
+    clickhouse_client
+        .query("DELETE FROM assets WHERE symbol IN ?")
+        .bind(symbols)
+        .execute()
+        .await
+        .unwrap();
+}

src/database/backfills.rs

@@ -0,0 +1,93 @@
+use crate::{database::assets, threads::data::ThreadType, types::Backfill};
+use clickhouse::Client;
+use serde::Serialize;
+use tokio::join;
+
+pub async fn select_latest_where_symbol<T>(
+    clickhouse_client: &Client,
+    thread_type: &ThreadType,
+    symbol: &T,
+) -> Option<Backfill>
+where
+    T: AsRef<str> + Serialize + Send + Sync,
+{
+    clickhouse_client
+        .query(&format!(
+            "SELECT ?fields FROM {} FINAL WHERE symbol = ? ORDER BY time DESC LIMIT 1",
+            match thread_type {
+                ThreadType::Bars(_) => "backfills_bars",
+                ThreadType::News => "backfills_news",
+            }
+        ))
+        .bind(symbol)
+        .fetch_optional::<Backfill>()
+        .await
+        .unwrap()
+}
+
+pub async fn upsert(clickhouse_client: &Client, thread_type: &ThreadType, backfill: &Backfill) {
+    let mut insert = clickhouse_client
+        .insert(match thread_type {
+            ThreadType::Bars(_) => "backfills_bars",
+            ThreadType::News => "backfills_news",
+        })
+        .unwrap();
+    insert.write(backfill).await.unwrap();
+    insert.end().await.unwrap();
+}
+
+pub async fn delete_where_symbols<T>(
+    clickhouse_client: &Client,
+    thread_type: &ThreadType,
+    symbols: &[T],
+) where
+    T: AsRef<str> + Serialize + Send + Sync,
+{
+    clickhouse_client
+        .query(&format!(
+            "DELETE FROM {} WHERE symbol IN ?",
+            match thread_type {
+                ThreadType::Bars(_) => "backfills_bars",
+                ThreadType::News => "backfills_news",
+            }
+        ))
+        .bind(symbols)
+        .execute()
+        .await
+        .unwrap();
+}
+
+pub async fn cleanup(clickhouse_client: &Client) {
+    let assets = assets::select(clickhouse_client).await;
+
+    let bars_symbols = assets
+        .clone()
+        .into_iter()
+        .map(|asset| asset.symbol)
+        .collect::<Vec<_>>();
+
+    let news_symbols = assets
+        .into_iter()
+        .map(|asset| asset.abbreviation)
+        .collect::<Vec<_>>();
+
+    let delete_bars_future = async {
+        clickhouse_client
+            .query("DELETE FROM backfills_bars WHERE symbol NOT IN ?")
+            .bind(bars_symbols)
+            .execute()
+            .await
+            .unwrap();
+    };
+
+    let delete_news_future = async {
+        clickhouse_client
+            .query("DELETE FROM backfills_news WHERE symbol NOT IN ?")
+            .bind(news_symbols)
+            .execute()
+            .await
+            .unwrap();
+    };
+
+    join!(delete_bars_future, delete_news_future);
+}

src/database/bars.rs

@@ -0,0 +1,50 @@
+use super::assets;
+use crate::types::Bar;
+use clickhouse::Client;
+use serde::Serialize;
+
+pub async fn upsert(clickhouse_client: &Client, bar: &Bar) {
+    let mut insert = clickhouse_client.insert("bars").unwrap();
+    insert.write(bar).await.unwrap();
+    insert.end().await.unwrap();
+}
+
+pub async fn upsert_batch<T>(clickhouse_client: &Client, bars: T)
+where
+    T: IntoIterator<Item = Bar> + Send + Sync,
+    T::IntoIter: Send,
+{
+    let mut insert = clickhouse_client.insert("bars").unwrap();
+    for bar in bars {
+        insert.write(&bar).await.unwrap();
+    }
+    insert.end().await.unwrap();
+}
+
+pub async fn delete_where_symbols<T>(clickhouse_client: &Client, symbols: &[T])
+where
+    T: AsRef<str> + Serialize + Send + Sync,
+{
+    clickhouse_client
+        .query("DELETE FROM bars WHERE symbol IN ?")
+        .bind(symbols)
+        .execute()
+        .await
+        .unwrap();
+}
+
+pub async fn cleanup(clickhouse_client: &Client) {
+    let assets = assets::select(clickhouse_client).await;
+
+    let symbols = assets
+        .into_iter()
+        .map(|asset| asset.symbol)
+        .collect::<Vec<_>>();
+
+    clickhouse_client
+        .query("DELETE FROM bars WHERE symbol NOT IN ?")
+        .bind(symbols)
+        .execute()
+        .await
+        .unwrap();
+}

src/database/mod.rs

@@ -0,0 +1,4 @@
+pub mod assets;
+pub mod backfills;
+pub mod bars;
+pub mod news;

src/database/news.rs

@@ -0,0 +1,50 @@
+use super::assets;
+use crate::types::News;
+use clickhouse::Client;
+use serde::Serialize;
+
+pub async fn upsert(clickhouse_client: &Client, news: &News) {
+    let mut insert = clickhouse_client.insert("news").unwrap();
+    insert.write(news).await.unwrap();
+    insert.end().await.unwrap();
+}
+
+pub async fn upsert_batch<T>(clickhouse_client: &Client, news: T)
+where
+    T: IntoIterator<Item = News> + Send + Sync,
+    T::IntoIter: Send,
+{
+    let mut insert = clickhouse_client.insert("news").unwrap();
+    for news in news {
+        insert.write(&news).await.unwrap();
+    }
+    insert.end().await.unwrap();
+}
+
+pub async fn delete_where_symbols<T>(clickhouse_client: &Client, symbols: &[T])
+where
+    T: AsRef<str> + Serialize + Send + Sync,
+{
+    clickhouse_client
+        .query("DELETE FROM news WHERE hasAny(symbols, ?)")
+        .bind(symbols)
+        .execute()
+        .await
+        .unwrap();
+}
+
+pub async fn cleanup(clickhouse_client: &Client) {
+    let assets = assets::select(clickhouse_client).await;
+
+    let symbols = assets
+        .into_iter()
+        .map(|asset| asset.abbreviation)
+        .collect::<Vec<_>>();
+
+    clickhouse_client
+        .query("DELETE FROM news WHERE NOT hasAny(symbols, ?)")
+        .bind(symbols)
+        .execute()
+        .await
+        .unwrap();
+}

src/init.rs

@@ -1,136 +0,0 @@
-use crate::{
-    config::{Config, ALPACA_API_BASE},
-    database,
-};
-use log::{info, warn};
-use qrust::{alpaca, types};
-use std::{collections::HashMap, sync::Arc};
-use time::OffsetDateTime;
-use tokio::join;
-
-pub async fn check_account(config: &Arc<Config>) {
-    let account = alpaca::account::get(
-        &config.alpaca_client,
-        &config.alpaca_rate_limiter,
-        None,
-        &ALPACA_API_BASE,
-    )
-    .await
-    .unwrap();
-
-    assert!(
-        !(account.status != types::alpaca::api::incoming::account::Status::Active),
-        "Account status is not active: {:?}.",
-        account.status
-    );
-    assert!(
-        !account.trade_suspend_by_user,
-        "Account trading is suspended by user."
-    );
-    assert!(!account.trading_blocked, "Account trading is blocked.");
-    assert!(!account.blocked, "Account is blocked.");
-
-    if account.cash == 0.0 {
-        warn!("Account cash is zero, qrust will not be able to trade.");
-    }
-
-    info!(
-        "qrust running on {} account with {} {}, avoid transferring funds without shutting down.",
-        *ALPACA_API_BASE, account.currency, account.cash
-    );
-}
-
-pub async fn rehydrate_orders(config: &Arc<Config>) {
-    let mut orders = vec![];
-    let mut after = OffsetDateTime::UNIX_EPOCH;
-
-    loop {
-        let message = alpaca::orders::get(
-            &config.alpaca_client,
-            &config.alpaca_rate_limiter,
-            &types::alpaca::api::outgoing::order::Order {
-                status: Some(types::alpaca::api::outgoing::order::Status::All),
-                after: Some(after),
-                ..Default::default()
-            },
-            None,
-            &ALPACA_API_BASE,
-        )
-        .await
-        .unwrap();
-
-        if message.is_empty() {
-            break;
-        }
-
-        orders.extend(message);
-        after = orders.last().unwrap().submitted_at;
-    }
-
-    let orders = orders
-        .into_iter()
-        .flat_map(&types::alpaca::api::incoming::order::Order::normalize)
-        .collect::<Vec<_>>();
-
-    database::orders::upsert_batch(
-        &config.clickhouse_client,
-        &config.clickhouse_concurrency_limiter,
-        &orders,
-    )
-    .await
-    .unwrap();
-}
-
-pub async fn rehydrate_positions(config: &Arc<Config>) {
-    let positions_future = async {
-        alpaca::positions::get(
-            &config.alpaca_client,
-            &config.alpaca_rate_limiter,
-            None,
-            &ALPACA_API_BASE,
-        )
-        .await
-        .unwrap()
-        .into_iter()
-        .map(|position| (position.symbol.clone(), position))
-        .collect::<HashMap<_, _>>()
-    };
-
-    let assets_future = async {
-        database::assets::select(
-            &config.clickhouse_client,
-            &config.clickhouse_concurrency_limiter,
-        )
-        .await
-        .unwrap()
-    };
-
-    let (mut positions, assets) = join!(positions_future, assets_future);
-
-    let assets = assets
-        .into_iter()
-        .map(|mut asset| {
-            if let Some(position) = positions.remove(&asset.symbol) {
-                asset.qty = position.qty_available;
-            } else {
-                asset.qty = 0.0;
-            }
-            asset
-        })
-        .collect::<Vec<_>>();
-
-    database::assets::upsert_batch(
-        &config.clickhouse_client,
-        &config.clickhouse_concurrency_limiter,
-        &assets,
-    )
-    .await
-    .unwrap();
-
-    for position in positions.values() {
-        warn!(
-            "Position for unmonitored asset: {}, {} shares.",
-            position.symbol, position.qty
-        );
-    }
-}

src/lib/alpaca/account.rs

@@ -1,39 +0,0 @@
-use super::error_to_backoff;
-use crate::types::alpaca::api::incoming::account::Account;
-use backoff::{future::retry_notify, ExponentialBackoff};
-use governor::DefaultDirectRateLimiter;
-use log::warn;
-use reqwest::{Client, Error};
-use std::time::Duration;
-
-pub async fn get(
-    client: &Client,
-    rate_limiter: &DefaultDirectRateLimiter,
-    backoff: Option<ExponentialBackoff>,
-    api_base: &str,
-) -> Result<Account, Error> {
-    retry_notify(
-        backoff.unwrap_or_default(),
-        || async {
-            rate_limiter.until_ready().await;
-            client
-                .get(&format!("https://{}.alpaca.markets/v2/account", api_base))
-                .send()
-                .await
-                .map_err(error_to_backoff)?
-                .error_for_status()
-                .map_err(error_to_backoff)?
-                .json::<Account>()
-                .await
-                .map_err(error_to_backoff)
-        },
-        |e, duration: Duration| {
-            warn!(
-                "Failed to get account, will retry in {} seconds: {}.",
-                duration.as_secs(),
-                e
-            );
-        },
-    )
-    .await
-}

src/lib/alpaca/assets.rs

@@ -1,132 +0,0 @@
-use super::error_to_backoff;
-use crate::types::alpaca::api::{
-    incoming::asset::{Asset, Class},
-    outgoing,
-};
-use backoff::{future::retry_notify, ExponentialBackoff};
-use governor::DefaultDirectRateLimiter;
-use itertools::Itertools;
-use log::warn;
-use reqwest::{Client, Error};
-use std::{collections::HashSet, time::Duration};
-use tokio::try_join;
-
-pub async fn get(
-    client: &Client,
-    rate_limiter: &DefaultDirectRateLimiter,
-    query: &outgoing::asset::Asset,
-    backoff: Option<ExponentialBackoff>,
-    api_base: &str,
-) -> Result<Vec<Asset>, Error> {
-    retry_notify(
-        backoff.unwrap_or_default(),
-        || async {
-            rate_limiter.until_ready().await;
-            client
-                .get(&format!("https://{}.alpaca.markets/v2/assets", api_base))
-                .query(query)
-                .send()
-                .await
-                .map_err(error_to_backoff)?
-                .error_for_status()
-                .map_err(error_to_backoff)?
-                .json::<Vec<Asset>>()
-                .await
-                .map_err(error_to_backoff)
-        },
-        |e, duration: Duration| {
-            warn!(
-                "Failed to get assets, will retry in {} seconds: {}.",
-                duration.as_secs(),
-                e
-            );
-        },
-    )
-    .await
-}
-
-pub async fn get_by_symbol(
-    client: &Client,
-    rate_limiter: &DefaultDirectRateLimiter,
-    symbol: &str,
-    backoff: Option<ExponentialBackoff>,
-    api_base: &str,
-) -> Result<Asset, Error> {
-    retry_notify(
-        backoff.unwrap_or_default(),
-        || async {
-            rate_limiter.until_ready().await;
-            client
-                .get(&format!(
-                    "https://{}.alpaca.markets/v2/assets/{}",
-                    api_base, symbol
-                ))
-                .send()
-                .await
-                .map_err(error_to_backoff)?
-                .error_for_status()
-                .map_err(error_to_backoff)?
-                .json::<Asset>()
-                .await
-                .map_err(error_to_backoff)
-        },
-        |e, duration: Duration| {
-            warn!(
-                "Failed to get asset, will retry in {} seconds: {}.",
-                duration.as_secs(),
-                e
-            );
-        },
-    )
-    .await
-}
-
-pub async fn get_by_symbols(
-    client: &Client,
-    rate_limiter: &DefaultDirectRateLimiter,
-    symbols: &[String],
-    backoff: Option<ExponentialBackoff>,
-    api_base: &str,
-) -> Result<Vec<Asset>, Error> {
-    if symbols.is_empty() {
-        return Ok(vec![]);
-    }
-
-    if symbols.len() == 1 {
-        let asset = get_by_symbol(client, rate_limiter, &symbols[0], backoff, api_base).await?;
-        return Ok(vec![asset]);
-    }
-
-    let symbols = symbols.iter().collect::<HashSet<_>>();
-
-    let backoff_clone = backoff.clone();
-
-    let us_equity_query = outgoing::asset::Asset {
-        class: Some(Class::UsEquity),
-        ..Default::default()
-    };
-
-    let us_equity_assets = get(
-        client,
-        rate_limiter,
-        &us_equity_query,
-        backoff_clone,
-        api_base,
-    );
-
-    let crypto_query = outgoing::asset::Asset {
-        class: Some(Class::Crypto),
-        ..Default::default()
-    };
-
-    let crypto_assets = get(client, rate_limiter, &crypto_query, backoff, api_base);
-
-    let (us_equity_assets, crypto_assets) = try_join!(us_equity_assets, crypto_assets)?;
-
-    Ok(crypto_assets
-        .into_iter()
-        .chain(us_equity_assets)
-        .dedup_by(|a, b| a.symbol == b.symbol)
-        .filter(|asset| symbols.contains(&asset.symbol))
-        .collect())
-}

src/lib/alpaca/bars.rs

@@ -1,50 +0,0 @@
-use super::error_to_backoff;
-use crate::types::alpaca::api::{incoming::bar::Bar, outgoing};
-use backoff::{future::retry_notify, ExponentialBackoff};
-use governor::DefaultDirectRateLimiter;
-use log::warn;
-use reqwest::{Client, Error};
-use serde::Deserialize;
-use std::{collections::HashMap, time::Duration};
-
-pub const MAX_LIMIT: i64 = 10_000;
-
-#[derive(Deserialize)]
-pub struct Message {
-    pub bars: HashMap<String, Vec<Bar>>,
-    pub next_page_token: Option<String>,
-}
-
-pub async fn get(
-    client: &Client,
-    rate_limiter: &DefaultDirectRateLimiter,
-    data_url: &str,
-    query: &outgoing::bar::Bar,
-    backoff: Option<ExponentialBackoff>,
-) -> Result<Message, Error> {
-    retry_notify(
-        backoff.unwrap_or_default(),
-        || async {
-            rate_limiter.until_ready().await;
-            client
-                .get(data_url)
-                .query(query)
-                .send()
-                .await
-                .map_err(error_to_backoff)?
-                .error_for_status()
-                .map_err(error_to_backoff)?
-                .json::<Message>()
-                .await
-                .map_err(error_to_backoff)
-        },
-        |e, duration: Duration| {
-            warn!(
-                "Failed to get historical bars, will retry in {} seconds: {}.",
-                duration.as_secs(),
-                e
-            );
-        },
-    )
-    .await
-}

src/lib/alpaca/calendar.rs

@@ -1,41 +0,0 @@
-use super::error_to_backoff;
-use crate::types::alpaca::api::{incoming::calendar::Calendar, outgoing};
-use backoff::{future::retry_notify, ExponentialBackoff};
-use governor::DefaultDirectRateLimiter;
-use log::warn;
-use reqwest::{Client, Error};
-use std::time::Duration;
-
-pub async fn get(
-    client: &Client,
-    rate_limiter: &DefaultDirectRateLimiter,
-    query: &outgoing::calendar::Calendar,
-    backoff: Option<ExponentialBackoff>,
-    api_base: &str,
-) -> Result<Vec<Calendar>, Error> {
-    retry_notify(
-        backoff.unwrap_or_default(),
-        || async {
-            rate_limiter.until_ready().await;
-            client
-                .get(&format!("https://{}.alpaca.markets/v2/calendar", api_base))
-                .query(query)
-                .send()
-                .await
-                .map_err(error_to_backoff)?
-                .error_for_status()
-                .map_err(error_to_backoff)?
-                .json::<Vec<Calendar>>()
-                .await
-                .map_err(error_to_backoff)
-        },
-        |e, duration: Duration| {
-            warn!(
-                "Failed to get calendar, will retry in {} seconds: {}.",
-                duration.as_secs(),
-                e
-            );
-        },
-    )
-    .await
-}

src/lib/alpaca/clock.rs

@@ -1,39 +0,0 @@
-use super::error_to_backoff;
-use crate::types::alpaca::api::incoming::clock::Clock;
-use backoff::{future::retry_notify, ExponentialBackoff};
-use governor::DefaultDirectRateLimiter;
-use log::warn;
-use reqwest::{Client, Error};
-use std::time::Duration;
-
-pub async fn get(
-    client: &Client,
-    rate_limiter: &DefaultDirectRateLimiter,
-    backoff: Option<ExponentialBackoff>,
-    api_base: &str,
-) -> Result<Clock, Error> {
-    retry_notify(
-        backoff.unwrap_or_default(),
-        || async {
-            rate_limiter.until_ready().await;
-            client
-                .get(&format!("https://{}.alpaca.markets/v2/clock", api_base))
-                .send()
-                .await
-                .map_err(error_to_backoff)?
-                .error_for_status()
-                .map_err(error_to_backoff)?
-                .json::<Clock>()
-                .await
-                .map_err(error_to_backoff)
-        },
-        |e, duration: Duration| {
-            warn!(
-                "Failed to get clock, will retry in {} seconds: {}.",
-                duration.as_secs(),
-                e
-            );
-        },
-    )
-    .await
-}

src/lib/alpaca/mod.rs

@@ -1,27 +0,0 @@
-pub mod account;
-pub mod assets;
-pub mod bars;
-pub mod calendar;
-pub mod clock;
-pub mod news;
-pub mod orders;
-pub mod positions;
-
-use reqwest::StatusCode;
-
-pub fn error_to_backoff(err: reqwest::Error) -> backoff::Error<reqwest::Error> {
-    if err.is_status() {
-        return match err.status() {
-            Some(StatusCode::BAD_REQUEST | StatusCode::FORBIDDEN | StatusCode::NOT_FOUND)
-            | None => backoff::Error::Permanent(err),
-            _ => err.into(),
-        };
-    }
-
-    if err.is_builder() || err.is_request() || err.is_redirect() || err.is_decode() || err.is_body()
-    {
-        return backoff::Error::Permanent(err);
-    }
-
-    err.into()
-}

src/lib/alpaca/news.rs

@@ -1,49 +0,0 @@
-use super::error_to_backoff;
-use crate::types::alpaca::api::{incoming::news::News, outgoing, ALPACA_NEWS_DATA_API_URL};
-use backoff::{future::retry_notify, ExponentialBackoff};
-use governor::DefaultDirectRateLimiter;
-use log::warn;
-use reqwest::{Client, Error};
-use serde::Deserialize;
-use std::time::Duration;
-
-pub const MAX_LIMIT: i64 = 50;
-
-#[derive(Deserialize)]
-pub struct Message {
-    pub news: Vec<News>,
-    pub next_page_token: Option<String>,
-}
-
-pub async fn get(
-    client: &Client,
-    rate_limiter: &DefaultDirectRateLimiter,
-    query: &outgoing::news::News,
-    backoff: Option<ExponentialBackoff>,
-) -> Result<Message, Error> {
-    retry_notify(
-        backoff.unwrap_or_default(),
-        || async {
-            rate_limiter.until_ready().await;
-            client
-                .get(ALPACA_NEWS_DATA_API_URL)
-                .query(query)
-                .send()
-                .await
-                .map_err(error_to_backoff)?
-                .error_for_status()
-                .map_err(error_to_backoff)?
-                .json::<Message>()
-                .await
-                .map_err(error_to_backoff)
-        },
-        |e, duration: Duration| {
-            warn!(
-                "Failed to get historical news, will retry in {} seconds: {}.",
-                duration.as_secs(),
-                e
-            );
-        },
-    )
-    .await
-}

src/lib/alpaca/orders.rs

@@ -1,43 +0,0 @@
-use super::error_to_backoff;
-use crate::types::alpaca::{api::outgoing, shared::order};
-use backoff::{future::retry_notify, ExponentialBackoff};
-use governor::DefaultDirectRateLimiter;
-use log::warn;
-use reqwest::{Client, Error};
-use std::time::Duration;
-
-pub use order::Order;
-
-pub async fn get(
-    client: &Client,
-    rate_limiter: &DefaultDirectRateLimiter,
-    query: &outgoing::order::Order,
-    backoff: Option<ExponentialBackoff>,
-    api_base: &str,
-) -> Result<Vec<Order>, Error> {
-    retry_notify(
-        backoff.unwrap_or_default(),
-        || async {
-            rate_limiter.until_ready().await;
-            client
-                .get(&format!("https://{}.alpaca.markets/v2/orders", api_base))
-                .query(query)
-                .send()
-                .await
-                .map_err(error_to_backoff)?
-                .error_for_status()
-                .map_err(error_to_backoff)?
-                .json::<Vec<Order>>()
-                .await
-                .map_err(error_to_backoff)
-        },
-        |e, duration: Duration| {
-            warn!(
-                "Failed to get orders, will retry in {} seconds: {}.",
-                duration.as_secs(),
-                e
-            );
-        },
-    )
-    .await
-}

src/lib/alpaca/positions.rs

@@ -1,108 +0,0 @@
-use super::error_to_backoff;
-use crate::types::alpaca::api::incoming::position::Position;
-use backoff::{future::retry_notify, ExponentialBackoff};
-use governor::DefaultDirectRateLimiter;
-use log::warn;
-use reqwest::Client;
-use std::{collections::HashSet, time::Duration};
-
-pub async fn get(
-    client: &Client,
-    rate_limiter: &DefaultDirectRateLimiter,
-    backoff: Option<ExponentialBackoff>,
-    api_base: &str,
-) -> Result<Vec<Position>, reqwest::Error> {
-    retry_notify(
-        backoff.unwrap_or_default(),
-        || async {
-            rate_limiter.until_ready().await;
-            client
-                .get(&format!("https://{}.alpaca.markets/v2/positions", api_base))
-                .send()
-                .await
-                .map_err(error_to_backoff)?
-                .error_for_status()
-                .map_err(error_to_backoff)?
-                .json::<Vec<Position>>()
-                .await
-                .map_err(error_to_backoff)
-        },
-        |e, duration: Duration| {
-            warn!(
-                "Failed to get positions, will retry in {} seconds: {}.",
-                duration.as_secs(),
-                e
-            );
-        },
-    )
-    .await
-}
-
-pub async fn get_by_symbol(
-    client: &Client,
-    rate_limiter: &DefaultDirectRateLimiter,
-    symbol: &str,
-    backoff: Option<ExponentialBackoff>,
-    api_base: &str,
-) -> Result<Option<Position>, reqwest::Error> {
-    retry_notify(
-        backoff.unwrap_or_default(),
-        || async {
-            rate_limiter.until_ready().await;
-            let response = client
-                .get(&format!(
-                    "https://{}.alpaca.markets/v2/positions/{}",
-                    api_base, symbol
-                ))
-                .send()
-                .await
-                .map_err(error_to_backoff)?;
-
-            if response.status() == reqwest::StatusCode::NOT_FOUND {
-                return Ok(None);
-            }
-
-            response
-                .error_for_status()
-                .map_err(error_to_backoff)?
-                .json::<Position>()
-                .await
-                .map_err(error_to_backoff)
-                .map(Some)
-        },
-        |e, duration: Duration| {
-            warn!(
-                "Failed to get position, will retry in {} seconds: {}.",
-                duration.as_secs(),
-                e
-            );
-        },
-    )
-    .await
-}
-
-pub async fn get_by_symbols(
-    client: &Client,
-    rate_limiter: &DefaultDirectRateLimiter,
-    symbols: &[String],
-    backoff: Option<ExponentialBackoff>,
-    api_base: &str,
-) -> Result<Vec<Position>, reqwest::Error> {
-    if symbols.is_empty() {
-        return Ok(vec![]);
-    }
-
-    if symbols.len() == 1 {
-        let position = get_by_symbol(client, rate_limiter, &symbols[0], backoff, api_base).await?;
-        return Ok(position.into_iter().collect());
-    }
-
-    let symbols = symbols.iter().collect::<HashSet<_>>();
-
-    let positions = get(client, rate_limiter, backoff, api_base).await?;
-
-    Ok(positions
-        .into_iter()
-        .filter(|position| symbols.contains(&position.symbol))
-        .collect())
-}

src/lib/database/assets.rs

@@ -1,50 +0,0 @@
-use std::sync::Arc;
-
-use crate::{
-    delete_where_symbols, optimize, select, select_where_symbol, types::Asset, upsert_batch,
-};
-use clickhouse::{error::Error, Client};
-use serde::Serialize;
-use tokio::sync::Semaphore;
-
-select!(Asset, "assets");
-select_where_symbol!(Asset, "assets");
-upsert_batch!(Asset, "assets");
-delete_where_symbols!("assets");
-optimize!("assets");
-
-pub async fn update_status_where_symbol<T>(
-    client: &Client,
-    concurrency_limiter: &Arc<Semaphore>,
-    symbol: &T,
-    status: bool,
-) -> Result<(), Error>
-where
-    T: AsRef<str> + Serialize + Send + Sync,
-{
-    let _ = concurrency_limiter.acquire().await.unwrap();
-    client
-        .query("ALTER TABLE assets UPDATE status = ? WHERE symbol = ?")
-        .bind(status)
-        .bind(symbol)
-        .execute()
-        .await
-}
-
-pub async fn update_qty_where_symbol<T>(
-    client: &Client,
-    concurrency_limiter: &Arc<Semaphore>,
-    symbol: &T,
-    qty: f64,
-) -> Result<(), Error>
-where
-    T: AsRef<str> + Serialize + Send + Sync,
-{
-    let _ = concurrency_limiter.acquire().await.unwrap();
-    client
-        .query("ALTER TABLE assets UPDATE qty = ? WHERE symbol = ?")
-        .bind(qty)
-        .bind(symbol)
-        .execute()
-        .await
-}

src/lib/database/backfills_bars.rs

@@ -1,11 +0,0 @@
-use crate::{
-    cleanup, delete_where_symbols, optimize, select_where_symbols, set_fresh_where_symbols,
-    types::Backfill, upsert_batch,
-};
-
-select_where_symbols!(Backfill, "backfills_bars");
-upsert_batch!(Backfill, "backfills_bars");
-delete_where_symbols!("backfills_bars");
-cleanup!("backfills_bars");
-optimize!("backfills_bars");
-set_fresh_where_symbols!("backfills_bars");

src/lib/database/backfills_news.rs

@@ -1,11 +0,0 @@
-use crate::{
-    cleanup, delete_where_symbols, optimize, select_where_symbols, set_fresh_where_symbols,
-    types::Backfill, upsert_batch,
-};
-
-select_where_symbols!(Backfill, "backfills_news");
-upsert_batch!(Backfill, "backfills_news");
-delete_where_symbols!("backfills_news");
-cleanup!("backfills_news");
-optimize!("backfills_news");
-set_fresh_where_symbols!("backfills_news");

src/lib/database/bars.rs

@@ -1,21 +0,0 @@
-use std::sync::Arc;
-
-use crate::{delete_where_symbols, optimize, types::Bar, upsert, upsert_batch};
-use clickhouse::Client;
-use tokio::sync::Semaphore;
-
-upsert!(Bar, "bars");
-upsert_batch!(Bar, "bars");
-delete_where_symbols!("bars");
-optimize!("bars");
-
-pub async fn cleanup(
-    client: &Client,
-    concurrency_limiter: &Arc<Semaphore>,
-) -> Result<(), clickhouse::error::Error> {
-    let _ = concurrency_limiter.acquire().await.unwrap();
-    client
-        .query("DELETE FROM bars WHERE symbol NOT IN (SELECT symbol FROM assets) OR symbol NOT IN (SELECT symbol FROM backfills_bars)")
-        .execute()
-        .await
-}

src/lib/database/calendar.rs

@@ -1,42 +0,0 @@
-use std::sync::Arc;
-
-use crate::{optimize, types::Calendar};
-use clickhouse::{error::Error, Client};
-use tokio::{sync::Semaphore, try_join};
-
-optimize!("calendar");
-
-pub async fn upsert_batch_and_delete<'a, I>(
-    client: &Client,
-    concurrency_limiter: &Arc<Semaphore>,
-    records: I,
-) -> Result<(), Error>
-where
-    I: IntoIterator<Item = &'a Calendar> + Send + Sync + Clone,
-    I::IntoIter: Send,
-{
-    let upsert_future = async {
-        let mut insert = client.insert("calendar")?;
-        for record in records.clone() {
-            insert.write(record).await?;
-        }
-        insert.end().await
-    };
-
-    let delete_future = async {
-        let dates = records
-            .clone()
-            .into_iter()
-            .map(|r| r.date)
-            .collect::<Vec<_>>();
-
-        client
-            .query("DELETE FROM calendar WHERE date NOT IN ?")
-            .bind(dates)
-            .execute()
-            .await
-    };
-
-    let _ = concurrency_limiter.acquire_many(2).await.unwrap();
-    try_join!(upsert_future, delete_future).map(|_| ())
-}

src/lib/database/mod.rs

@@ -1,223 +0,0 @@
-pub mod assets;
-pub mod backfills_bars;
-pub mod backfills_news;
-pub mod bars;
-pub mod calendar;
-pub mod news;
-pub mod orders;
-
-use clickhouse::{error::Error, Client};
-use tokio::try_join;
-
-#[macro_export]
-macro_rules! select {
-    ($record:ty, $table_name:expr) => {
-        pub async fn select(
-            client: &clickhouse::Client,
-            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
-        ) -> Result<Vec<$record>, clickhouse::error::Error> {
-            let _ = concurrency_limiter.acquire().await.unwrap();
-            client
-                .query(&format!("SELECT ?fields FROM {} FINAL", $table_name))
-                .fetch_all::<$record>()
-                .await
-        }
-    };
-}
-
-#[macro_export]
-macro_rules! select_where_symbol {
-    ($record:ty, $table_name:expr) => {
-        pub async fn select_where_symbol<T>(
-            client: &clickhouse::Client,
-            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
-            symbol: &T,
-        ) -> Result<Option<$record>, clickhouse::error::Error>
-        where
-            T: AsRef<str> + serde::Serialize + Send + Sync,
-        {
-            let _ = concurrency_limiter.acquire().await.unwrap();
-            client
-                .query(&format!(
-                    "SELECT ?fields FROM {} FINAL WHERE symbol = ?",
-                    $table_name
-                ))
-                .bind(symbol)
-                .fetch_optional::<$record>()
-                .await
-        }
-    };
-}
-
-#[macro_export]
-macro_rules! select_where_symbols {
-    ($record:ty, $table_name:expr) => {
-        pub async fn select_where_symbols<T>(
-            client: &clickhouse::Client,
-            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
-            symbols: &[T],
-        ) -> Result<Vec<$record>, clickhouse::error::Error>
-        where
-            T: AsRef<str> + serde::Serialize + Send + Sync,
-        {
-            let _ = concurrency_limiter.acquire().await.unwrap();
-            client
-                .query(&format!(
-                    "SELECT ?fields FROM {} FINAL WHERE symbol IN ?",
-                    $table_name
-                ))
-                .bind(symbols)
-                .fetch_all::<$record>()
-                .await
-        }
-    };
-}
-
-#[macro_export]
-macro_rules! upsert {
-    ($record:ty, $table_name:expr) => {
-        pub async fn upsert(
-            client: &clickhouse::Client,
-            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
-            record: &$record,
-        ) -> Result<(), clickhouse::error::Error> {
-            let _ = concurrency_limiter.acquire().await.unwrap();
-            let mut insert = client.insert($table_name)?;
-            insert.write(record).await?;
-            insert.end().await
-        }
-    };
-}
-
-#[macro_export]
-macro_rules! upsert_batch {
-    ($record:ty, $table_name:expr) => {
-        pub async fn upsert_batch<'a, I>(
-            client: &clickhouse::Client,
-            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
-            records: I,
-        ) -> Result<(), clickhouse::error::Error>
-        where
-            I: IntoIterator<Item = &'a $record> + Send + Sync,
-            I::IntoIter: Send,
-        {
-            let _ = concurrency_limiter.acquire().await.unwrap();
-            let mut insert = client.insert($table_name)?;
-            for record in records {
-                insert.write(record).await?;
-            }
-            insert.end().await
-        }
-    };
-}
-
-#[macro_export]
-macro_rules! delete_where_symbols {
-    ($table_name:expr) => {
-        pub async fn delete_where_symbols<T>(
-            client: &clickhouse::Client,
-            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
-            symbols: &[T],
-        ) -> Result<(), clickhouse::error::Error>
-        where
-            T: AsRef<str> + serde::Serialize + Send + Sync,
-        {
-            let _ = concurrency_limiter.acquire().await.unwrap();
-            client
-                .query(&format!("DELETE FROM {} WHERE symbol IN ?", $table_name))
-                .bind(symbols)
-                .execute()
-                .await
-        }
-    };
-}
-
-#[macro_export]
-macro_rules! cleanup {
-    ($table_name:expr) => {
-        pub async fn cleanup(
-            client: &clickhouse::Client,
-            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
-        ) -> Result<(), clickhouse::error::Error> {
-            let _ = concurrency_limiter.acquire().await.unwrap();
-            client
-                .query(&format!(
-                    "DELETE FROM {} WHERE symbol NOT IN (SELECT symbol FROM assets)",
-                    $table_name
-                ))
-                .execute()
-                .await
-        }
-    };
-}
-
-#[macro_export]
-macro_rules! optimize {
-    ($table_name:expr) => {
-        pub async fn optimize(
-            client: &clickhouse::Client,
-            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
-        ) -> Result<(), clickhouse::error::Error> {
-            let _ = concurrency_limiter.acquire().await.unwrap();
-            client
-                .query(&format!("OPTIMIZE TABLE {} FINAL", $table_name))
-                .execute()
-                .await
-        }
-    };
-}
-
-#[macro_export]
-macro_rules! set_fresh_where_symbols {
-    ($table_name:expr) => {
-        pub async fn set_fresh_where_symbols<T>(
-            client: &clickhouse::Client,
-            concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
-            fresh: bool,
-            symbols: &[T],
-        ) -> Result<(), clickhouse::error::Error>
-        where
-            T: AsRef<str> + serde::Serialize + Send + Sync,
-        {
-            let _ = concurrency_limiter.acquire().await.unwrap();
-            client
-                .query(&format!(
-                    "ALTER TABLE {} UPDATE fresh = ? WHERE symbol IN ?",
-                    $table_name
-                ))
-                .bind(fresh)
-                .bind(symbols)
-                .execute()
-                .await
-        }
-    };
-}
-
-pub async fn cleanup_all(
-    clickhouse_client: &Client,
-    concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
-) -> Result<(), Error> {
-    try_join!(
-        bars::cleanup(clickhouse_client, concurrency_limiter),
-        news::cleanup(clickhouse_client, concurrency_limiter),
-        backfills_bars::cleanup(clickhouse_client, concurrency_limiter),
-        backfills_news::cleanup(clickhouse_client, concurrency_limiter)
-    )
-    .map(|_| ())
-}
-
-pub async fn optimize_all(
-    clickhouse_client: &Client,
-    concurrency_limiter: &std::sync::Arc<tokio::sync::Semaphore>,
-) -> Result<(), Error> {
-    try_join!(
-        assets::optimize(clickhouse_client, concurrency_limiter),
-        bars::optimize(clickhouse_client, concurrency_limiter),
-        news::optimize(clickhouse_client, concurrency_limiter),
-        backfills_bars::optimize(clickhouse_client, concurrency_limiter),
-        backfills_news::optimize(clickhouse_client, concurrency_limiter),
-        orders::optimize(clickhouse_client, concurrency_limiter),
-        calendar::optimize(clickhouse_client, concurrency_limiter)
-    )
-    .map(|_| ())
-}

src/lib/database/news.rs

@@ -1,36 +0,0 @@
-use std::sync::Arc;
-
-use crate::{optimize, types::News, upsert, upsert_batch};
-use clickhouse::{error::Error, Client};
-use serde::Serialize;
-use tokio::sync::Semaphore;
-
-upsert!(News, "news");
-upsert_batch!(News, "news");
-optimize!("news");
-
-pub async fn delete_where_symbols<T>(
-    client: &Client,
-    concurrency_limiter: &Arc<Semaphore>,
-    symbols: &[T],
-) -> Result<(), Error>
-where
-    T: AsRef<str> + Serialize + Send + Sync,
-{
-    let _ = concurrency_limiter.acquire().await.unwrap();
-    client
-        .query("DELETE FROM news WHERE hasAny(symbols, ?) AND NOT hasAny(symbols, (SELECT groupArray(symbol) FROM assets))")
-        .bind(symbols)
-        .execute()
-        .await
-}
-
-pub async fn cleanup(client: &Client, concurrency_limiter: &Arc<Semaphore>) -> Result<(), Error> {
-    let _ = concurrency_limiter.acquire().await.unwrap();
-    client
-        .query(
-            "DELETE FROM news WHERE NOT hasAny(symbols, (SELECT groupArray(symbol) FROM assets))",
-        )
-        .execute()
-        .await
-}

src/lib/database/orders.rs

@@ -1,5 +0,0 @@
-use crate::{optimize, types::Order, upsert, upsert_batch};
-
-upsert!(Order, "orders");
-upsert_batch!(Order, "orders");
-optimize!("orders");

src/lib/mod.rs

@@ -1,4 +0,0 @@
-pub mod alpaca;
-pub mod database;
-pub mod types;
-pub mod utils;

src/lib/types/alpaca/api/incoming/account.rs

@@ -1,75 +0,0 @@
-use serde::Deserialize;
-use serde_aux::field_attributes::{
-    deserialize_number_from_string, deserialize_option_number_from_string,
-};
-use time::OffsetDateTime;
-use uuid::Uuid;
-
-#[derive(Deserialize, Debug, PartialEq, Eq)]
-#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
-pub enum Status {
-    Onboarding,
-    SubmissionFailed,
-    Submitted,
-    AccountUpdated,
-    ApprovalPending,
-    Active,
-    Rejected,
-}
-
-#[derive(Deserialize)]
-#[allow(clippy::struct_excessive_bools)]
-pub struct Account {
-    pub id: Uuid,
-    #[serde(rename = "account_number")]
-    pub number: String,
-    pub status: Status,
-    pub currency: String,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub cash: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub non_marginable_buying_power: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub accrued_fees: f64,
-    #[serde(default)]
-    #[serde(deserialize_with = "deserialize_option_number_from_string")]
-    pub pending_transfer_in: Option<f64>,
-    #[serde(default)]
-    #[serde(deserialize_with = "deserialize_option_number_from_string")]
-    pub pending_transfer_out: Option<f64>,
-    pub pattern_day_trader: bool,
-    #[serde(default)]
-    pub trade_suspend_by_user: bool,
-    pub trading_blocked: bool,
-    pub transfers_blocked: bool,
-    #[serde(rename = "account_blocked")]
-    pub blocked: bool,
-    #[serde(with = "time::serde::rfc3339")]
-    pub created_at: OffsetDateTime,
-    pub shorting_enabled: bool,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub long_market_value: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub short_market_value: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub equity: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub last_equity: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub multiplier: i8,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub buying_power: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub initial_margin: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub maintenance_margin: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub sma: f64,
-    pub daytrade_count: i64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub last_maintenance_margin: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub daytrading_buying_power: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub regt_buying_power: f64,
-}

src/lib/types/alpaca/api/incoming/asset.rs

@@ -1,39 +0,0 @@
-use super::position::Position;
-use crate::types::{self, alpaca::shared::asset};
-use serde::Deserialize;
-use serde_aux::field_attributes::deserialize_option_number_from_string;
-use uuid::Uuid;
-
-pub use asset::{Class, Exchange, Status};
-
-#[allow(clippy::struct_excessive_bools)]
-#[derive(Deserialize, Clone)]
-pub struct Asset {
-    pub id: Uuid,
-    pub class: Class,
-    pub exchange: Exchange,
-    pub symbol: String,
-    pub name: String,
-    pub status: Status,
-    pub tradable: bool,
-    pub marginable: bool,
-    pub shortable: bool,
-    pub easy_to_borrow: bool,
-    pub fractionable: bool,
-    #[serde(deserialize_with = "deserialize_option_number_from_string")]
-    pub maintenance_margin_requirement: Option<f32>,
-    pub attributes: Option<Vec<String>>,
-}
-
-impl From<(Asset, Option<Position>)> for types::Asset {
-    fn from((asset, position): (Asset, Option<Position>)) -> Self {
-        Self {
-            symbol: asset.symbol,
-            class: asset.class.into(),
-            exchange: asset.exchange.into(),
-            status: asset.status.into(),
-            time_added: time::OffsetDateTime::now_utc(),
-            qty: position.map(|position| position.qty).unwrap_or_default(),
-        }
-    }
-}

src/lib/types/alpaca/api/incoming/calendar.rs

@@ -1,26 +0,0 @@
-use crate::{
-    types,
-    utils::{de, time::EST_OFFSET},
-};
-use serde::Deserialize;
-use time::{Date, OffsetDateTime, Time};
-
-#[derive(Deserialize)]
-pub struct Calendar {
-    pub date: Date,
-    #[serde(deserialize_with = "de::human_time_hh_mm")]
-    pub open: Time,
-    #[serde(deserialize_with = "de::human_time_hh_mm")]
-    pub close: Time,
-    pub settlement_date: Date,
-}
-
-impl From<Calendar> for types::Calendar {
-    fn from(calendar: Calendar) -> Self {
-        Self {
-            date: calendar.date,
-            open: OffsetDateTime::new_in_offset(calendar.date, calendar.open, *EST_OFFSET),
-            close: OffsetDateTime::new_in_offset(calendar.date, calendar.close, *EST_OFFSET),
-        }
-    }
-}

src/lib/types/alpaca/api/incoming/mod.rs

@@ -1,8 +0,0 @@
-pub mod account;
-pub mod asset;
-pub mod bar;
-pub mod calendar;
-pub mod clock;
-pub mod news;
-pub mod order;
-pub mod position;

src/lib/types/alpaca/api/incoming/order.rs

@@ -1,3 +0,0 @@
-use crate::types::alpaca::shared::order;
-
-pub use order::{Order, Side};

src/lib/types/alpaca/api/incoming/position.rs

@@ -1,61 +0,0 @@
-use crate::{
-    types::alpaca::api::incoming::{
-        asset::{Class, Exchange},
-        order,
-    },
-    utils::de,
-};
-use serde::Deserialize;
-use serde_aux::field_attributes::deserialize_number_from_string;
-use uuid::Uuid;
-
-#[derive(Deserialize, Clone, Copy)]
-#[serde(rename_all = "snake_case")]
-pub enum Side {
-    Long,
-    Short,
-}
-
-impl From<Side> for order::Side {
-    fn from(side: Side) -> Self {
-        match side {
-            Side::Long => Self::Buy,
-            Side::Short => Self::Sell,
-        }
-    }
-}
-
-#[derive(Deserialize, Clone)]
-pub struct Position {
-    pub asset_id: Uuid,
-    #[serde(deserialize_with = "de::add_slash_to_symbol")]
-    pub symbol: String,
-    pub exchange: Exchange,
-    pub asset_class: Class,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub avg_entry_price: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub qty: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub qty_available: f64,
-    pub side: Side,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub market_value: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub cost_basis: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub unrealized_pl: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub unrealized_plpc: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub unrealized_intraday_pl: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub unrealized_intraday_plpc: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub current_price: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub lastday_price: f64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub change_today: f64,
-    pub asset_marginable: bool,
-}

src/lib/types/alpaca/api/mod.rs

@@ -1,6 +0,0 @@
-pub mod incoming;
-pub mod outgoing;
-
-pub const ALPACA_US_EQUITY_DATA_API_URL: &str = "https://data.alpaca.markets/v2/stocks/bars";
-pub const ALPACA_CRYPTO_DATA_API_URL: &str = "https://data.alpaca.markets/v1beta3/crypto/us/bars";
-pub const ALPACA_NEWS_DATA_API_URL: &str = "https://data.alpaca.markets/v1beta1/news";

src/lib/types/alpaca/api/outgoing/asset.rs

@@ -1,23 +0,0 @@
-use crate::types::alpaca::shared::asset;
-use serde::Serialize;
-
-pub use asset::{Class, Exchange, Status};
-
-#[derive(Serialize)]
-pub struct Asset {
-    pub status: Option<Status>,
-    pub class: Option<Class>,
-    pub exchange: Option<Exchange>,
-    pub attributes: Option<Vec<String>>,
-}
-
-impl Default for Asset {
-    fn default() -> Self {
-        Self {
-            status: None,
-            class: Some(Class::UsEquity),
-            exchange: None,
-            attributes: None,
-        }
-    }
-}

src/lib/types/alpaca/api/outgoing/bar.rs

@@ -1,108 +0,0 @@
-use crate::{
-    alpaca::bars::MAX_LIMIT,
-    types::alpaca::shared,
-    utils::{ser, ONE_MINUTE},
-};
-use serde::Serialize;
-use std::time::Duration;
-use time::OffsetDateTime;
-
-pub use shared::{Sort, Source};
-
-#[derive(Serialize)]
-#[serde(rename_all = "snake_case")]
-#[allow(dead_code)]
-pub enum Adjustment {
-    Raw,
-    Split,
-    Dividend,
-    All,
-}
-
-#[derive(Serialize)]
-pub struct UsEquity {
-    #[serde(serialize_with = "ser::join_symbols")]
-    pub symbols: Vec<String>,
-    #[serde(serialize_with = "ser::timeframe")]
-    pub timeframe: Duration,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub start: Option<OffsetDateTime>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub end: Option<OffsetDateTime>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub limit: Option<i64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub adjustment: Option<Adjustment>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub asof: Option<OffsetDateTime>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub feed: Option<Source>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub currency: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub page_token: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub sort: Option<Sort>,
-}
-
-impl Default for UsEquity {
-    fn default() -> Self {
-        Self {
-            symbols: vec![],
-            timeframe: ONE_MINUTE,
-            start: None,
-            end: None,
-            limit: Some(MAX_LIMIT),
-            adjustment: Some(Adjustment::All),
-            asof: None,
-            feed: Some(Source::Iex),
-            currency: None,
-            page_token: None,
-            sort: Some(Sort::Asc),
-        }
-    }
-}
-
-#[derive(Serialize)]
-pub struct Crypto {
-    #[serde(serialize_with = "ser::join_symbols")]
-    pub symbols: Vec<String>,
-    #[serde(serialize_with = "ser::timeframe")]
-    pub timeframe: Duration,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub start: Option<OffsetDateTime>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub end: Option<OffsetDateTime>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub limit: Option<i64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub page_token: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub sort: Option<Sort>,
-}
-
-impl Default for Crypto {
-    fn default() -> Self {
-        Self {
-            symbols: vec![],
-            timeframe: ONE_MINUTE,
-            start: None,
-            end: None,
-            limit: Some(MAX_LIMIT),
-            page_token: None,
-            sort: Some(Sort::Asc),
-        }
-    }
-}
-
-#[derive(Serialize)]
-#[serde(untagged)]
-pub enum Bar {
-    UsEquity(UsEquity),
-    Crypto(Crypto),
-}

src/lib/types/alpaca/api/outgoing/calendar.rs

@@ -1,31 +0,0 @@
-use crate::utils::time::MAX_TIMESTAMP;
-use serde::Serialize;
-use time::OffsetDateTime;
-
-#[derive(Serialize)]
-#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
-#[allow(dead_code)]
-pub enum DateType {
-    Trading,
-    Settlement,
-}
-
-#[derive(Serialize)]
-pub struct Calendar {
-    #[serde(with = "time::serde::rfc3339")]
-    pub start: OffsetDateTime,
-    #[serde(with = "time::serde::rfc3339")]
-    pub end: OffsetDateTime,
-    #[serde(rename = "date")]
-    pub date_type: DateType,
-}
-
-impl Default for Calendar {
-    fn default() -> Self {
-        Self {
-            start: OffsetDateTime::UNIX_EPOCH,
-            end: *MAX_TIMESTAMP,
-            date_type: DateType::Trading,
-        }
-    }
-}

src/lib/types/alpaca/api/outgoing/news.rs

@@ -1,40 +0,0 @@
-use crate::{alpaca::news::MAX_LIMIT, types::alpaca::shared::Sort, utils::ser};
-use serde::Serialize;
-use time::OffsetDateTime;
-
-#[derive(Serialize)]
-pub struct News {
-    #[serde(serialize_with = "ser::remove_slash_and_join_symbols")]
-    pub symbols: Vec<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub start: Option<OffsetDateTime>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub end: Option<OffsetDateTime>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub limit: Option<i64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub include_content: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub exclude_contentless: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub page_token: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub sort: Option<Sort>,
-}
-
-impl Default for News {
-    fn default() -> Self {
-        Self {
-            symbols: vec![],
-            start: None,
-            end: None,
-            limit: Some(MAX_LIMIT),
-            include_content: Some(true),
-            exclude_contentless: Some(false),
-            page_token: None,
-            sort: Some(Sort::Asc),
-        }
-    }
-}

src/lib/types/alpaca/api/outgoing/order.rs

@@ -1,55 +0,0 @@
-use crate::{
-    types::alpaca::shared::{order, Sort},
-    utils::ser,
-};
-use serde::Serialize;
-use time::OffsetDateTime;
-
-pub use order::Side;
-
-#[derive(Serialize)]
-#[serde(rename_all = "snake_case")]
-#[allow(dead_code)]
-pub enum Status {
-    Open,
-    Closed,
-    All,
-}
-
-#[derive(Serialize)]
-pub struct Order {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub status: Option<Status>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub limit: Option<i64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub after: Option<OffsetDateTime>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub until: Option<OffsetDateTime>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub direction: Option<Sort>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub nested: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(serialize_with = "ser::join_symbols_option")]
-    pub symbols: Option<Vec<String>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub side: Option<Side>,
-}
-
-impl Default for Order {
-    fn default() -> Self {
-        Self {
-            status: Some(Status::All),
-            limit: Some(500),
-            after: None,
-            until: None,
-            direction: Some(Sort::Asc),
-            nested: Some(true),
-            symbols: None,
-            side: None,
-        }
-    }
-}

src/lib/types/alpaca/mod.rs

@@ -1,3 +0,0 @@
-pub mod api;
-pub mod shared;
-pub mod websocket;

src/lib/types/alpaca/shared/asset.rs

@@ -1,53 +0,0 @@
-use crate::{impl_from_enum, types};
-use serde::{Deserialize, Serialize};
-
-#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
-#[serde(rename_all = "snake_case")]
-pub enum Class {
-    UsEquity,
-    Crypto,
-}
-
-impl_from_enum!(types::Class, Class, UsEquity, Crypto);
-
-#[derive(Serialize, Deserialize, Clone, Copy)]
-#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
-pub enum Exchange {
-    Amex,
-    Arca,
-    Bats,
-    Nyse,
-    Nasdaq,
-    Nysearca,
-    Otc,
-    Crypto,
-}
-
-impl_from_enum!(
-    types::Exchange,
-    Exchange,
-    Amex,
-    Arca,
-    Bats,
-    Nyse,
-    Nasdaq,
-    Nysearca,
-    Otc,
-    Crypto
-);
-
-#[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Copy)]
-#[serde(rename_all = "snake_case")]
-pub enum Status {
-    Active,
-    Inactive,
-}
-
-impl From<Status> for bool {
-    fn from(status: Status) -> Self {
-        match status {
-            Status::Active => true,
-            Status::Inactive => false,
-        }
-    }
-}

src/lib/types/alpaca/shared/mod.rs

@@ -1,10 +0,0 @@
-pub mod asset;
-pub mod mode;
-pub mod news;
-pub mod order;
-pub mod sort;
-pub mod source;
-
-pub use mode::Mode;
-pub use sort::Sort;
-pub use source::Source;

src/lib/types/alpaca/shared/mode.rs

@@ -1,33 +0,0 @@
-use serde::{Deserialize, Serialize};
-use std::{
-    fmt::{Display, Formatter},
-    str::FromStr,
-};
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub enum Mode {
-    Live,
-    Paper,
-}
-
-impl FromStr for Mode {
-    type Err = String;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s {
-            "live" => Ok(Self::Live),
-            "paper" => Ok(Self::Paper),
-            _ => Err(format!("Unknown mode: {s}")),
-        }
-    }
-}
-
-impl Display for Mode {
-    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> {
-        match self {
-            Self::Live => write!(f, "live"),
-            Self::Paper => write!(f, "paper"),
-        }
-    }
-}

src/lib/types/alpaca/shared/news.rs

@@ -1,15 +0,0 @@
-use lazy_static::lazy_static;
-use regex::Regex;
-
-lazy_static! {
-    static ref RE_TAGS: Regex = Regex::new("<[^>]+>").unwrap();
-    static ref RE_SPACES: Regex = Regex::new("[\\u00A0\\s]+").unwrap();
-}
-
-pub fn strip(content: &str) -> String {
-    let content = content.replace('\n', " ");
-    let content = RE_TAGS.replace_all(&content, "");
-    let content = RE_SPACES.replace_all(&content, " ");
-    let content = content.trim();
-    content.to_string()
-}

src/lib/types/alpaca/shared/order.rs

@@ -1,225 +0,0 @@
-use crate::{impl_from_enum, types};
-use serde::{Deserialize, Serialize};
-use serde_aux::field_attributes::{
-    deserialize_number_from_string, deserialize_option_number_from_string,
-};
-use time::OffsetDateTime;
-use uuid::Uuid;
-
-#[derive(Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
-#[serde(rename_all = "snake_case")]
-pub enum Class {
-    #[serde(alias = "")]
-    Simple,
-    Bracket,
-    Oco,
-    Oto,
-}
-
-impl_from_enum!(types::order::Class, Class, Simple, Bracket, Oco, Oto);
-
-#[derive(Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
-#[serde(rename_all = "snake_case")]
-pub enum Type {
-    Market,
-    Limit,
-    Stop,
-    StopLimit,
-    TrailingStop,
-}
-
-impl_from_enum!(
-    types::order::Type,
-    Type,
-    Market,
-    Limit,
-    Stop,
-    StopLimit,
-    TrailingStop
-);
-
-#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
-#[serde(rename_all = "snake_case")]
-pub enum Side {
-    Buy,
-    Sell,
-}
-
-impl_from_enum!(types::order::Side, Side, Buy, Sell);
-
-#[derive(Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
-#[serde(rename_all = "snake_case")]
-pub enum TimeInForce {
-    Day,
-    Gtc,
-    Opg,
-    Cls,
-    Ioc,
-    Fok,
-}
-
-impl_from_enum!(
-    types::order::TimeInForce,
-    TimeInForce,
-    Day,
-    Gtc,
-    Opg,
-    Cls,
-    Ioc,
-    Fok
-);
-
-#[derive(Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
-#[serde(rename_all = "snake_case")]
-pub enum Status {
-    New,
-    PartiallyFilled,
-    Filled,
-    DoneForDay,
-    Canceled,
-    Expired,
-    Replaced,
-    PendingCancel,
-    PendingReplace,
-    Accepted,
-    PendingNew,
-    AcceptedForBidding,
-    Stopped,
-    Rejected,
-    Suspended,
-    Calculated,
-}
-
-impl_from_enum!(
-    types::order::Status,
-    Status,
-    New,
-    PartiallyFilled,
-    Filled,
-    DoneForDay,
-    Canceled,
-    Expired,
-    Replaced,
-    PendingCancel,
-    PendingReplace,
-    Accepted,
-    PendingNew,
-    AcceptedForBidding,
-    Stopped,
-    Rejected,
-    Suspended,
-    Calculated
-);
-
-#[derive(Deserialize, Clone, Debug, PartialEq)]
-#[allow(clippy::struct_field_names)]
-pub struct Order {
-    pub id: Uuid,
-    pub client_order_id: Uuid,
-    #[serde(with = "time::serde::rfc3339")]
-    pub created_at: OffsetDateTime,
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub updated_at: Option<OffsetDateTime>,
-    #[serde(with = "time::serde::rfc3339")]
-    pub submitted_at: OffsetDateTime,
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub filled_at: Option<OffsetDateTime>,
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub expired_at: Option<OffsetDateTime>,
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub cancel_requested_at: Option<OffsetDateTime>,
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub canceled_at: Option<OffsetDateTime>,
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub failed_at: Option<OffsetDateTime>,
-    #[serde(with = "time::serde::rfc3339::option")]
-    pub replaced_at: Option<OffsetDateTime>,
-    pub replaced_by: Option<Uuid>,
-    pub replaces: Option<Uuid>,
-    pub asset_id: Uuid,
-    pub symbol: String,
-    pub asset_class: super::asset::Class,
-    #[serde(deserialize_with = "deserialize_option_number_from_string")]
-    pub notional: Option<f64>,
-    #[serde(deserialize_with = "deserialize_option_number_from_string")]
-    pub qty: Option<f64>,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub filled_qty: f64,
-    #[serde(deserialize_with = "deserialize_option_number_from_string")]
-    pub filled_avg_price: Option<f64>,
-    pub order_class: Class,
-    #[serde(rename = "type")]
-    pub order_type: Type,
-    pub side: Side,
-    pub time_in_force: TimeInForce,
-    #[serde(deserialize_with = "deserialize_option_number_from_string")]
-    pub limit_price: Option<f64>,
-    #[serde(deserialize_with = "deserialize_option_number_from_string")]
-    pub stop_price: Option<f64>,
-    pub status: Status,
-    pub extended_hours: bool,
-    pub legs: Option<Vec<Order>>,
-    #[serde(deserialize_with = "deserialize_option_number_from_string")]
-    pub trail_percent: Option<f64>,
-    #[serde(deserialize_with = "deserialize_option_number_from_string")]
-    pub trail_price: Option<f64>,
-    pub hwm: Option<f64>,
-}
-
-impl From<Order> for types::Order {
-    fn from(order: Order) -> Self {
-        Self {
-            id: order.id,
-            client_order_id: order.client_order_id,
-            time_submitted: order.submitted_at,
-            time_created: order.created_at,
-            time_updated: order.updated_at.unwrap_or(OffsetDateTime::UNIX_EPOCH),
-            time_filled: order.filled_at.unwrap_or(OffsetDateTime::UNIX_EPOCH),
-            time_expired: order.expired_at.unwrap_or(OffsetDateTime::UNIX_EPOCH),
-            time_cancel_requested: order
-                .cancel_requested_at
-                .unwrap_or(OffsetDateTime::UNIX_EPOCH),
-            time_canceled: order.canceled_at.unwrap_or(OffsetDateTime::UNIX_EPOCH),
-            time_failed: order.failed_at.unwrap_or(OffsetDateTime::UNIX_EPOCH),
-            time_replaced: order.replaced_at.unwrap_or(OffsetDateTime::UNIX_EPOCH),
-            replaced_by: order.replaced_by.unwrap_or_default(),
-            replaces: order.replaces.unwrap_or_default(),
-            symbol: order.symbol,
-            order_class: order.order_class.into(),
-            order_type: order.order_type.into(),
-            side: order.side.into(),
-            time_in_force: order.time_in_force.into(),
-            notional: order.notional.unwrap_or_default(),
-            qty: order.qty.unwrap_or_default(),
-            filled_qty: order.filled_qty,
-            filled_avg_price: order.filled_avg_price.unwrap_or_default(),
-            status: order.status.into(),
-            extended_hours: order.extended_hours,
-            limit_price: order.limit_price.unwrap_or_default(),
-            stop_price: order.stop_price.unwrap_or_default(),
-            trail_percent: order.trail_percent.unwrap_or_default(),
-            trail_price: order.trail_price.unwrap_or_default(),
-            hwm: order.hwm.unwrap_or_default(),
-            legs: order
-                .legs
-                .unwrap_or_default()
-                .into_iter()
-                .map(|order| order.id)
-                .collect(),
-        }
-    }
-}
-
-impl Order {
-    pub fn normalize(self) -> Vec<types::Order> {
-        let mut orders = vec![self.clone().into()];
-
-        if let Some(legs) = self.legs {
-            for leg in legs {
-                orders.extend(leg.normalize());
-            }
-        }
-
-        orders
-    }
-}

src/lib/types/alpaca/shared/sort.rs

@@ -1,9 +0,0 @@
-use serde::Serialize;
-
-#[derive(Serialize)]
-#[serde(rename_all = "snake_case")]
-#[allow(dead_code)]
-pub enum Sort {
-    Asc,
-    Desc,
-}

src/lib/types/alpaca/websocket/data/incoming/status.rs

@@ -1,154 +0,0 @@
-use serde::Deserialize;
-use serde_with::serde_as;
-use time::OffsetDateTime;
-
-#[derive(Deserialize, Debug, PartialEq, Eq)]
-#[serde(tag = "sc", content = "sm")]
-pub enum Status {
-    #[serde(rename = "2")]
-    #[serde(alias = "H")]
-    TradingHalt(String),
-    #[serde(rename = "3")]
-    Resume(String),
-    #[serde(rename = "5")]
-    PriceIndication(String),
-    #[serde(rename = "6")]
-    TradingRangeIndication(String),
-    #[serde(rename = "7")]
-    MarketImbalanceBuy(String),
-    #[serde(rename = "8")]
-    MarketImbalanceSell(String),
-    #[serde(rename = "9")]
-    MarketOnCloseImbalanceBuy(String),
-    #[serde(rename = "A")]
-    MarketOnCloseImbalanceSell(String),
-    #[serde(rename = "C")]
-    NoMarketImbalance(String),
-    #[serde(rename = "D")]
-    NoMarketOnCloseImbalance(String),
-    #[serde(rename = "E")]
-    ShortSaleRestriction(String),
-    #[serde(rename = "F")]
-    LimitUpLimitDown(String),
-    #[serde(rename = "Q")]
-    QuotationResumption(String),
-    #[serde(rename = "T")]
-    TradingResumption(String),
-    #[serde(rename = "P")]
-    VolatilityTradingPause(String),
-}
-
-#[derive(Deserialize, Debug, PartialEq, Eq)]
-#[serde(tag = "rc", content = "rm")]
-pub enum Reason {
-    #[serde(rename = "D")]
-    NewsReleased(String),
-    #[serde(rename = "I")]
-    OrderImbalance(String),
-    #[serde(rename = "M")]
-    LimitUpLimitDown(String),
-    #[serde(rename = "P")]
-    NewsPending(String),
-    #[serde(rename = "X")]
-    Operational(String),
-    #[serde(rename = "Y")]
-    SubPennyTrading(String),
-    #[serde(rename = "1")]
-    MarketWideCircuitBreakerL1Breached(String),
-    #[serde(rename = "2")]
-    MarketWideCircuitBreakerL2Breached(String),
-    #[serde(rename = "3")]
-    MarketWideCircuitBreakerL3Breached(String),
-    #[serde(rename = "T1")]
-    HaltNewsPending(String),
-    #[serde(rename = "T2")]
-    HaltNewsDissemination(String),
-    #[serde(rename = "T5")]
-    SingleStockTradingPauseInAffect(String),
-    #[serde(rename = "T6")]
-    RegulatoryHaltExtraordinaryMarketActivity(String),
-    #[serde(rename = "T8")]
-    HaltETF(String),
-    #[serde(rename = "T12")]
-    TradingHaltedForInformationRequestedByNASDAQ(String),
-    #[serde(rename = "H4")]
-    HaltNonCompliance(String),
-    #[serde(rename = "H9")]
-    HaltFilingsNotCurrent(String),
-    #[serde(rename = "H10")]
-    HaltSECTradingSuspension(String),
-    #[serde(rename = "H11")]
-    HaltRegulatoryConcern(String),
-    #[serde(rename = "01")]
-    OperationsHaltContactMarketOperations(String),
-    #[serde(rename = "IPO1")]
-    IPOIssueNotYetTrading(String),
-    #[serde(rename = "M1")]
-    CorporateAction(String),
-    #[serde(rename = "M2")]
-    QuotationNotAvailable(String),
-    #[serde(rename = "LUDP")]
-    VolatilityTradingPause(String),
-    #[serde(rename = "LUDS")]
-    VolatilityTradingPauseStraddleCondition(String),
-    #[serde(rename = "MWC1")]
-    MarketWideCircuitBreakerHaltL1(String),
-    #[serde(rename = "MWC2")]
-    MarketWideCircuitBreakerHaltL2(String),
-    #[serde(rename = "MWC3")]
-    MarketWideCircuitBreakerHaltL3(String),
-    #[serde(rename = "MWC0")]
-    MarketWideCircuitBreakerHaltCarryOverFromPreviousDay(String),
-    #[serde(rename = "T3")]
-    NewsAndResumptionTimes(String),
-    #[serde(rename = "T7")]
-    SingleStockTradingPauseQuotationOnlyPeriod(String),
-    #[serde(rename = "R4")]
-    QualificationsIssuesReviewedResolvedQuotationsTradingToResume(String),
-    #[serde(rename = "R9")]
-    FilingRequirementsSatisfiedResolvedQuotationsTradingToResume(String),
-    #[serde(rename = "C3")]
-    IssuerNewsNotForthcomingQuotationsTradingToResume(String),
-    #[serde(rename = "C4")]
-    QualificationsHaltEndedMaintReqMetResume(String),
-    #[serde(rename = "C9")]
-    QualificationsHaltConcludedFilingsMetQuotesTradesToResume(String),
-    #[serde(rename = "C11")]
-    TradeHaltConcludedByOtherRegulatoryAuthQuotesTradesResume(String),
-    #[serde(rename = "R1")]
-    NewIssueAvailable(String),
-    #[serde(rename = "R")]
-    IssueAvailable(String),
-    #[serde(rename = "IPOQ")]
-    IPOSecurityReleasedForQuotation(String),
-    #[serde(rename = "IPOE")]
-    IPOSecurityPositioningWindowExtension(String),
-    #[serde(rename = "MWCQ")]
-    MarketWideCircuitBreakerResumption(String),
-}
-
-#[derive(Deserialize, Debug, PartialEq, Eq)]
-pub enum Tape {
-    A,
-    B,
-    C,
-    O,
-}
-
-#[derive(Deserialize, Debug, PartialEq, Eq)]
-#[allow(clippy::struct_field_names)]
-#[serde_as]
-pub struct Message {
-    #[serde(rename = "t")]
-    #[serde(with = "time::serde::rfc3339")]
-    pub time: OffsetDateTime,
-    #[serde(rename = "S")]
-    pub symbol: String,
-    #[serde(flatten)]
-    pub status: Status,
-    #[serde(flatten)]
-    #[serde_as(as = "NoneAsEmptyString")]
-    pub reason: Option<Reason>,
-    #[serde(rename = "z")]
-    pub tape: Tape,
-}

src/lib/types/alpaca/websocket/data/incoming/subscription.rs

@@ -1,23 +0,0 @@
-use crate::utils::de;
-use serde::Deserialize;
-
-#[derive(Deserialize, Debug, PartialEq, Eq)]
-#[serde(untagged)]
-pub enum Message {
-    #[serde(rename_all = "camelCase")]
-    Market {
-        trades: Vec<String>,
-        quotes: Vec<String>,
-        bars: Vec<String>,
-        updated_bars: Vec<String>,
-        daily_bars: Vec<String>,
-        orderbooks: Option<Vec<String>>,
-        statuses: Option<Vec<String>>,
-        lulds: Option<Vec<String>>,
-        cancel_errors: Option<Vec<String>>,
-    },
-    News {
-        #[serde(deserialize_with = "de::add_slash_to_symbols")]
-        news: Vec<String>,
-    },
-}

src/lib/types/alpaca/websocket/data/mod.rs

@@ -1,53 +0,0 @@
-pub mod incoming;
-pub mod outgoing;
-
-use crate::types::alpaca::websocket;
-use core::panic;
-use futures_util::{
-    stream::{SplitSink, SplitStream},
-    SinkExt, StreamExt,
-};
-use serde_json::{from_str, to_string};
-use tokio::net::TcpStream;
-use tokio_tungstenite::{tungstenite::Message, MaybeTlsStream, WebSocketStream};
-
-pub async fn authenticate(
-    sink: &mut SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, Message>,
-    stream: &mut SplitStream<WebSocketStream<MaybeTlsStream<TcpStream>>>,
-    api_key: String,
-    api_secret: String,
-) {
-    match stream.next().await.unwrap().unwrap() {
-        Message::Text(data)
-            if from_str::<Vec<websocket::data::incoming::Message>>(&data)
-                .unwrap()
-                .first()
-                == Some(&websocket::data::incoming::Message::Success(
-                    websocket::data::incoming::success::Message::Connected,
-                )) => {}
-        _ => panic!("Failed to connect to Alpaca websocket."),
-    }
-
-    sink.send(Message::Text(
-        to_string(&websocket::data::outgoing::Message::Auth(
-            websocket::auth::Message {
-                key: api_key,
-                secret: api_secret,
-            },
-        ))
-        .unwrap(),
-    ))
-    .await
-    .unwrap();
-
-    match stream.next().await.unwrap().unwrap() {
-        Message::Text(data)
-            if from_str::<Vec<websocket::data::incoming::Message>>(&data)
-                .unwrap()
-                .first()
-                == Some(&websocket::data::incoming::Message::Success(
-                    websocket::data::incoming::success::Message::Authenticated,
-                )) => {}
-        _ => panic!("Failed to authenticate with Alpaca websocket."),
-    };
-}

src/lib/types/alpaca/websocket/data/outgoing/subscribe.rs

@@ -1,50 +0,0 @@
-use crate::utils::ser;
-use nonempty::NonEmpty;
-use serde::Serialize;
-
-#[derive(Serialize)]
-#[serde(untagged)]
-pub enum Market {
-    #[serde(rename_all = "camelCase")]
-    UsEquity {
-        bars: NonEmpty<String>,
-        updated_bars: NonEmpty<String>,
-        statuses: NonEmpty<String>,
-    },
-    #[serde(rename_all = "camelCase")]
-    Crypto {
-        bars: NonEmpty<String>,
-        updated_bars: NonEmpty<String>,
-    },
-}
-
-#[derive(Serialize)]
-#[serde(untagged)]
-pub enum Message {
-    Market(Market),
-    News {
-        #[serde(serialize_with = "ser::remove_slash_from_symbols")]
-        news: NonEmpty<String>,
-    },
-}
-
-impl Message {
-    pub fn new_market_us_equity(symbols: NonEmpty<String>) -> Self {
-        Self::Market(Market::UsEquity {
-            bars: symbols.clone(),
-            updated_bars: symbols.clone(),
-            statuses: symbols,
-        })
-    }
-
-    pub fn new_market_crypto(symbols: NonEmpty<String>) -> Self {
-        Self::Market(Market::Crypto {
-            bars: symbols.clone(),
-            updated_bars: symbols,
-        })
-    }
-
-    pub fn new_news(symbols: NonEmpty<String>) -> Self {
-        Self::News { news: symbols }
-    }
-}

src/lib/types/alpaca/websocket/mod.rs

@@ -1,8 +0,0 @@
-pub mod auth;
-pub mod data;
-pub mod trading;
-
-pub const ALPACA_US_EQUITY_DATA_WEBSOCKET_URL: &str = "wss://stream.data.alpaca.markets/v2";
-pub const ALPACA_CRYPTO_DATA_WEBSOCKET_URL: &str =
-    "wss://stream.data.alpaca.markets/v1beta3/crypto/us";
-pub const ALPACA_NEWS_DATA_WEBSOCKET_URL: &str = "wss://stream.data.alpaca.markets/v1beta1/news";

src/lib/types/alpaca/websocket/trading/incoming/auth.rs

@@ -1,22 +0,0 @@
-use serde::Deserialize;
-
-#[derive(Deserialize, Debug, PartialEq, Eq)]
-#[serde(rename_all = "snake_case")]
-pub enum Status {
-    Authorized,
-    Unauthorized,
-}
-
-#[derive(Deserialize, Debug, PartialEq, Eq)]
-pub enum Action {
-    #[serde(rename = "authenticate")]
-    Auth,
-    #[serde(rename = "listen")]
-    Subscribe,
-}
-
-#[derive(Deserialize, Debug, PartialEq, Eq)]
-pub struct Message {
-    pub status: Status,
-    pub action: Action,
-}

src/lib/types/alpaca/websocket/trading/incoming/mod.rs

@@ -1,16 +0,0 @@
-pub mod auth;
-pub mod order;
-pub mod subscription;
-
-use serde::Deserialize;
-
-#[derive(Deserialize, Debug, PartialEq)]
-#[serde(tag = "stream", content = "data")]
-pub enum Message {
-    #[serde(rename = "authorization")]
-    Auth(auth::Message),
-    #[serde(rename = "listening")]
-    Subscription(subscription::Message),
-    #[serde(rename = "trade_updates")]
-    Order(order::Message),
-}

src/lib/types/alpaca/websocket/trading/incoming/order.rs

@@ -1,57 +0,0 @@
-use crate::types::alpaca::shared::order;
-use serde::Deserialize;
-use serde_aux::prelude::deserialize_number_from_string;
-use time::OffsetDateTime;
-use uuid::Uuid;
-
-pub use order::Order;
-
-#[derive(Deserialize, Debug, PartialEq)]
-#[serde(rename_all = "snake_case")]
-#[serde(tag = "event")]
-pub enum Event {
-    New,
-    Fill {
-        timestamp: OffsetDateTime,
-        #[serde(deserialize_with = "deserialize_number_from_string")]
-        position_qty: f64,
-        #[serde(deserialize_with = "deserialize_number_from_string")]
-        price: f64,
-    },
-    PartialFill {
-        timestamp: OffsetDateTime,
-        #[serde(deserialize_with = "deserialize_number_from_string")]
-        position_qty: f64,
-        #[serde(deserialize_with = "deserialize_number_from_string")]
-        price: f64,
-    },
-    Canceled {
-        timestamp: OffsetDateTime,
-    },
-    Expired {
-        timestamp: OffsetDateTime,
-    },
-    DoneForDay,
-    Replaced {
-        timestamp: OffsetDateTime,
-    },
-    Rejected {
-        timestamp: OffsetDateTime,
-    },
-    PendingNew,
-    Stopped,
-    PendingCancel,
-    PendingReplace,
-    Calculated,
-    Suspended,
-    OrderReplaceRejected,
-    OrderCancelRejected,
-}
-
-#[derive(Deserialize, Debug, PartialEq)]
-pub struct Message {
-    pub execution_id: Uuid,
-    pub order: Order,
-    #[serde(flatten)]
-    pub event: Event,
-}

src/lib/types/alpaca/websocket/trading/incoming/subscription.rs

@@ -1,6 +0,0 @@
-use serde::Deserialize;
-
-#[derive(Deserialize, Debug, PartialEq, Eq)]
-pub struct Message {
-    pub streams: Vec<String>,
-}

src/lib/types/alpaca/websocket/trading/mod.rs

@@ -1,82 +0,0 @@
-pub mod incoming;
-pub mod outgoing;
-
-use crate::types::alpaca::websocket;
-use core::panic;
-use futures_util::{
-    stream::{SplitSink, SplitStream},
-    SinkExt, StreamExt,
-};
-use serde_json::{from_str, to_string};
-use tokio::net::TcpStream;
-use tokio_tungstenite::{tungstenite::Message, MaybeTlsStream, WebSocketStream};
-
-pub async fn authenticate(
-    sink: &mut SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, Message>,
-    stream: &mut SplitStream<WebSocketStream<MaybeTlsStream<TcpStream>>>,
-    api_key: String,
-    api_secret: String,
-) {
-    sink.send(Message::Text(
-        to_string(&websocket::trading::outgoing::Message::Auth(
-            websocket::auth::Message {
-                key: api_key,
-                secret: api_secret,
-            },
-        ))
-        .unwrap(),
-    ))
-    .await
-    .unwrap();
-
-    match stream.next().await.unwrap().unwrap() {
-        Message::Binary(data) => {
-            let data = String::from_utf8(data).unwrap();
-
-            if from_str::<websocket::trading::incoming::Message>(&data).unwrap()
-                != websocket::trading::incoming::Message::Auth(
-                    websocket::trading::incoming::auth::Message {
-                        status: websocket::trading::incoming::auth::Status::Authorized,
-                        action: websocket::trading::incoming::auth::Action::Auth,
-                    },
-                )
-            {
-                panic!("Failed to authenticate with Alpaca websocket.");
-            }
-        }
-        _ => panic!("Failed to authenticate with Alpaca websocket."),
-    };
-}
-
-pub async fn subscribe(
-    sink: &mut SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, Message>,
-    stream: &mut SplitStream<WebSocketStream<MaybeTlsStream<TcpStream>>>,
-) {
-    sink.send(Message::Text(
-        to_string(&websocket::trading::outgoing::Message::Subscribe {
-            data: websocket::trading::outgoing::subscribe::Message {
-                streams: vec![String::from("trade_updates")],
-            },
-        })
-        .unwrap(),
-    ))
-    .await
-    .unwrap();
-
-    match stream.next().await.unwrap().unwrap() {
-        Message::Binary(data) => {
-            let data = String::from_utf8(data).unwrap();
-
-            if from_str::<websocket::trading::incoming::Message>(&data).unwrap()
-                != websocket::trading::incoming::Message::Subscription(
-                    websocket::trading::incoming::subscription::Message {
-                        streams: vec![String::from("trade_updates")],
-                    },
-                )
-            {
-                panic!("Failed to subscribe to Alpaca websocket.");
-            }
-        }
-        _ => panic!("Failed to subscribe to Alpaca websocket."),
-    };
-}

src/lib/types/alpaca/websocket/trading/outgoing/mod.rs

@@ -1,15 +0,0 @@
-pub mod subscribe;
-
-use crate::types::alpaca::websocket::auth;
-use serde::Serialize;
-
-#[derive(Serialize)]
-#[serde(tag = "action")]
-#[serde(rename_all = "snake_case")]
-pub enum Message {
-    Auth(auth::Message),
-    #[serde(rename = "listen")]
-    Subscribe {
-        data: subscribe::Message,
-    },
-}

src/lib/types/alpaca/websocket/trading/outgoing/subscribe.rs

@@ -1,6 +0,0 @@
-use serde::Serialize;
-
-#[derive(Serialize)]
-pub struct Message {
-    pub streams: Vec<String>,
-}

src/lib/types/backfill.rs

@@ -1,11 +0,0 @@
-use clickhouse::Row;
-use serde::{Deserialize, Serialize};
-use time::OffsetDateTime;
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Row)]
-pub struct Backfill {
-    pub symbol: String,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub time: OffsetDateTime,
-    pub fresh: bool,
-}

src/lib/types/calendar.rs

@@ -1,13 +0,0 @@
-use clickhouse::Row;
-use serde::{Deserialize, Serialize};
-use time::{Date, OffsetDateTime};
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Row)]
-pub struct Calendar {
-    #[serde(with = "clickhouse::serde::time::date")]
-    pub date: Date,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub open: OffsetDateTime,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub close: OffsetDateTime,
-}

src/lib/types/news.rs

@@ -1,19 +0,0 @@
-use clickhouse::Row;
-use serde::{Deserialize, Serialize};
-use time::OffsetDateTime;
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Row)]
-pub struct News {
-    pub id: i64,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub time_created: OffsetDateTime,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub time_updated: OffsetDateTime,
-    pub symbols: Vec<String>,
-    pub headline: String,
-    pub author: String,
-    pub source: String,
-    pub summary: String,
-    pub content: String,
-    pub url: String,
-}

src/lib/types/order.rs

@@ -1,107 +0,0 @@
-use clickhouse::Row;
-use serde::{Deserialize, Serialize};
-use serde_repr::{Deserialize_repr, Serialize_repr};
-use time::OffsetDateTime;
-use uuid::Uuid;
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
-#[repr(i8)]
-pub enum Class {
-    Simple = 1,
-    Bracket = 2,
-    Oco = 3,
-    Oto = 4,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
-#[repr(i8)]
-pub enum Type {
-    Market = 1,
-    Limit = 2,
-    Stop = 3,
-    StopLimit = 4,
-    TrailingStop = 5,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
-#[repr(i8)]
-pub enum Side {
-    Buy = 1,
-    Sell = -1,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
-#[repr(i8)]
-pub enum TimeInForce {
-    Day = 1,
-    Gtc = 2,
-    Opg = 3,
-    Cls = 4,
-    Ioc = 5,
-    Fok = 6,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
-#[repr(i8)]
-pub enum Status {
-    New = 1,
-    PartiallyFilled = 2,
-    Filled = 3,
-    DoneForDay = 4,
-    Canceled = 5,
-    Expired = 6,
-    Replaced = 7,
-    PendingCancel = 8,
-    PendingReplace = 9,
-    Accepted = 10,
-    PendingNew = 11,
-    AcceptedForBidding = 12,
-    Stopped = 13,
-    Rejected = 14,
-    Suspended = 15,
-    Calculated = 16,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Row)]
-#[allow(clippy::struct_field_names)]
-pub struct Order {
-    pub id: Uuid,
-    pub client_order_id: Uuid,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub time_submitted: OffsetDateTime,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub time_created: OffsetDateTime,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub time_updated: OffsetDateTime,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub time_filled: OffsetDateTime,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub time_expired: OffsetDateTime,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub time_cancel_requested: OffsetDateTime,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub time_canceled: OffsetDateTime,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub time_failed: OffsetDateTime,
-    #[serde(with = "clickhouse::serde::time::datetime")]
-    pub time_replaced: OffsetDateTime,
-    pub replaced_by: Uuid,
-    pub replaces: Uuid,
-    pub symbol: String,
-    pub order_class: Class,
-    pub order_type: Type,
-    pub side: Side,
-    pub time_in_force: TimeInForce,
-    pub extended_hours: bool,
-    pub notional: f64,
-    pub qty: f64,
-    pub filled_qty: f64,
-    pub filled_avg_price: f64,
-    pub status: Status,
-    pub limit_price: f64,
-    pub stop_price: f64,
-    pub trail_percent: f64,
-    pub trail_price: f64,
-    pub hwm: f64,
-    pub legs: Vec<Uuid>,
-}

src/lib/utils/backoff.rs

@@ -1,8 +0,0 @@
-use backoff::ExponentialBackoff;
-
-pub fn infinite() -> ExponentialBackoff {
-    ExponentialBackoff {
-        max_elapsed_time: None,
-        ..ExponentialBackoff::default()
-    }
-}

src/lib/utils/de.rs

@@ -1,104 +0,0 @@
-use lazy_static::lazy_static;
-use regex::Regex;
-use serde::{
-    de::{self, SeqAccess, Visitor},
-    Deserializer,
-};
-use std::fmt;
-use time::{format_description::OwnedFormatItem, macros::format_description, Time};
-
-lazy_static! {
-    // This *will* break in the future if a crypto pair with one letter is added
-    static ref RE_SLASH: Regex = Regex::new(r"^(.{2,})(BTC|USD.?)$").unwrap();
-    static ref FMT_HH_MM: OwnedFormatItem = format_description!("[hour]:[minute]").into();
-}
-
-fn add_slash(pair: &str) -> String {
-    RE_SLASH.captures(pair).map_or_else(
-        || pair.to_string(),
-        |caps| format!("{}/{}", &caps[1], &caps[2]),
-    )
-}
-
-pub fn add_slash_to_symbol<'de, D>(deserializer: D) -> Result<String, D::Error>
-where
-    D: Deserializer<'de>,
-{
-    struct StringVisitor;
-
-    impl<'de> Visitor<'de> for StringVisitor {
-        type Value = String;
-
-        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
-            formatter.write_str("a string without a slash")
-        }
-
-        fn visit_str<E>(self, pair: &str) -> Result<Self::Value, E>
-        where
-            E: de::Error,
-        {
-            Ok(add_slash(pair))
-        }
-
-        fn visit_string<E>(self, pair: String) -> Result<Self::Value, E>
-        where
-            E: de::Error,
-        {
-            Ok(add_slash(&pair))
-        }
-    }
-
-    deserializer.deserialize_string(StringVisitor)
-}
-
-pub fn add_slash_to_symbols<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
-where
-    D: Deserializer<'de>,
-{
-    struct VecStringVisitor;
-
-    impl<'de> Visitor<'de> for VecStringVisitor {
-        type Value = Vec<String>;
-
-        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
-            formatter.write_str("a list of strings without a slash")
-        }
-
-        fn visit_seq<A>(self, mut seq: A) -> Result<Vec<String>, A::Error>
-        where
-            A: SeqAccess<'de>,
-        {
-            let mut vec = Vec::with_capacity(seq.size_hint().unwrap_or(25));
-            while let Some(value) = seq.next_element::<String>()? {
-                vec.push(add_slash(&value));
-            }
-            Ok(vec)
-        }
-    }
-
-    deserializer.deserialize_seq(VecStringVisitor)
-}
-
-pub fn human_time_hh_mm<'de, D>(deserializer: D) -> Result<Time, D::Error>
-where
-    D: Deserializer<'de>,
-{
-    struct TimeVisitor;
-
-    impl<'de> Visitor<'de> for TimeVisitor {
-        type Value = time::Time;
-
-        fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
-            formatter.write_str("a string in the format HH:MM")
-        }
-
-        fn visit_str<E>(self, time: &str) -> Result<Self::Value, E>
-        where
-            E: de::Error,
-        {
-            Time::parse(time, &FMT_HH_MM).map_err(|e| de::Error::custom(e.to_string()))
-        }
-    }
-
-    deserializer.deserialize_str(TimeVisitor)
-}

src/lib/utils/mod.rs

@@ -1,7 +0,0 @@
-pub mod backoff;
-pub mod de;
-pub mod macros;
-pub mod ser;
-pub mod time;
-
-pub use time::{duration_until, last_minute, FIFTEEN_MINUTES, ONE_MINUTE, ONE_SECOND};

src/lib/utils/ser.rs

@@ -1,89 +0,0 @@
-use serde::{ser::SerializeSeq, Serializer};
-use std::time::Duration;
-
-pub fn timeframe<S>(timeframe: &Duration, serializer: S) -> Result<S::Ok, S::Error>
-where
-    S: serde::Serializer,
-{
-    let mins = timeframe.as_secs() / 60;
-    if mins < 60 {
-        return serializer.serialize_str(&format!("{mins}Min"));
-    }
-
-    let hours = mins / 60;
-    if hours < 24 {
-        return serializer.serialize_str(&format!("{hours}Hour"));
-    }
-
-    let days = hours / 24;
-    if days == 1 {
-        return serializer.serialize_str("1Day");
-    }
-
-    let weeks = days / 7;
-    if weeks == 1 {
-        return serializer.serialize_str("1Week");
-    }
-
-    let months = days / 30;
-    if [1, 2, 3, 4, 6, 12].contains(&months) {
-        return serializer.serialize_str(&format!("{months}Month"));
-    };
-
-    Err(serde::ser::Error::custom("Invalid timeframe duration"))
-}
-
-fn remove_slash(pair: &str) -> String {
-    pair.replace('/', "")
-}
-
-pub fn join_symbols<S>(symbols: &[String], serializer: S) -> Result<S::Ok, S::Error>
-where
-    S: Serializer,
-{
-    let string = symbols.join(",");
-    serializer.serialize_str(&string)
-}
-
-pub fn join_symbols_option<S>(
-    symbols: &Option<Vec<String>>,
-    serializer: S,
-) -> Result<S::Ok, S::Error>
-where
-    S: Serializer,
-{
-    match symbols {
-        Some(symbols) => join_symbols(symbols, serializer),
-        None => serializer.serialize_none(),
-    }
-}
-
-pub fn remove_slash_from_symbols<'a, S, I>(symbols: I, serializer: S) -> Result<S::Ok, S::Error>
-where
-    S: Serializer,
-    I: IntoIterator<Item = &'a String>,
-{
-    let symbols = symbols
-        .into_iter()
-        .map(|pair| remove_slash(pair))
-        .collect::<Vec<_>>();
-
-    let mut seq = serializer.serialize_seq(Some(symbols.len()))?;
-    for symbol in symbols {
-        seq.serialize_element(&symbol)?;
-    }
-    seq.end()
-}
-
-pub fn remove_slash_and_join_symbols<'a, S, I>(symbols: I, serializer: S) -> Result<S::Ok, S::Error>
-where
-    S: Serializer,
-    I: IntoIterator<Item = &'a String>,
-{
-    let symbols = symbols
-        .into_iter()
-        .map(|symbol| remove_slash(symbol))
-        .collect::<Vec<_>>();
-
-    join_symbols(&symbols, serializer)
-}

src/main.rs

@@ -3,113 +3,48 @@
 #![feature(hash_extract_if)]
 
 mod config;
-mod init;
+mod database;
 mod routes;
 mod threads;
+mod types;
+mod utils;
 
-use config::{
-    Config, ALPACA_API_BASE, ALPACA_MODE, ALPACA_SOURCE, CLICKHOUSE_BATCH_BARS_SIZE,
-    CLICKHOUSE_BATCH_NEWS_SIZE, CLICKHOUSE_MAX_CONNECTIONS,
-};
+use config::Config;
 use dotenv::dotenv;
-use log::info;
 use log4rs::config::Deserializers;
-use nonempty::NonEmpty;
-use qrust::{create_send_await, database};
-use tokio::{join, spawn, sync::mpsc, try_join};
+use tokio::{spawn, sync::mpsc};
+use utils::{cleanup::cleanup, init};
 
 #[tokio::main]
 async fn main() {
     dotenv().ok();
     log4rs::init_file("log4rs.yaml", Deserializers::default()).unwrap();
-    let config = Config::arc_from_env();
+    let app_config = Config::arc_from_env();
 
-    let _ = *ALPACA_MODE;
-    let _ = *ALPACA_API_BASE;
-    let _ = *ALPACA_SOURCE;
-    let _ = *CLICKHOUSE_BATCH_BARS_SIZE;
-    let _ = *CLICKHOUSE_BATCH_NEWS_SIZE;
-    let _ = *CLICKHOUSE_MAX_CONNECTIONS;
+    cleanup(&app_config.clickhouse_client).await;
+    init::ollama(&app_config).await;
 
-    info!("Marking all assets as stale.");
-
-    let assets = database::assets::select(
-        &config.clickhouse_client,
-        &config.clickhouse_concurrency_limiter,
-    )
-    .await
-    .unwrap()
-    .into_iter()
-    .map(|asset| (asset.symbol, asset.class))
-    .collect::<Vec<_>>();
-
-    let symbols = assets.iter().map(|(symbol, _)| symbol).collect::<Vec<_>>();
-
-    try_join!(
-        database::backfills_bars::set_fresh_where_symbols(
-            &config.clickhouse_client,
-            &config.clickhouse_concurrency_limiter,
-            false,
-            &symbols
-        ),
-        database::backfills_news::set_fresh_where_symbols(
-            &config.clickhouse_client,
-            &config.clickhouse_concurrency_limiter,
-            false,
-            &symbols
-        )
-    )
-    .unwrap();
-
-    info!("Cleaning up database.");
-
-    database::cleanup_all(
-        &config.clickhouse_client,
-        &config.clickhouse_concurrency_limiter,
-    )
-    .await
-    .unwrap();
-
-    info!("Optimizing database.");
-
-    database::optimize_all(
-        &config.clickhouse_client,
-        &config.clickhouse_concurrency_limiter,
-    )
-    .await
-    .unwrap();
-
-    info!("Rehydrating account data.");
-
-    init::check_account(&config).await;
-    join!(
-        init::rehydrate_orders(&config),
-        init::rehydrate_positions(&config)
-    );
-
-    info!("Starting threads.");
-
-    spawn(threads::trading::run(config.clone()));
-
-    let (data_sender, data_receiver) = mpsc::channel::<threads::data::Message>(100);
+    let (asset_status_sender, asset_status_receiver) =
+        mpsc::channel::<threads::data::asset_status::Message>(10);
     let (clock_sender, clock_receiver) = mpsc::channel::<threads::clock::Message>(1);
 
     spawn(threads::data::run(
-        config.clone(),
-        data_receiver,
+        app_config.clone(),
+        asset_status_receiver,
         clock_receiver,
     ));
 
-    spawn(threads::clock::run(config.clone(), clock_sender));
+    spawn(threads::clock::run(app_config.clone(), clock_sender));
 
-    if let Some(assets) = NonEmpty::from_vec(assets) {
-        create_send_await!(
-            data_sender,
-            threads::data::Message::new,
-            threads::data::Action::Enable,
-            assets
-        );
-    }
+    let assets = database::assets::select(&app_config.clickhouse_client).await;
 
-    routes::run(config, data_sender).await;
+    let (asset_status_message, asset_status_receiver) =
+        threads::data::asset_status::Message::new(threads::data::asset_status::Action::Add, assets);
+    asset_status_sender
+        .send(asset_status_message)
+        .await
+        .unwrap();
+    asset_status_receiver.await.unwrap();
+
+    routes::run(app_config, asset_status_sender).await;
 }

src/routes/assets.rs

@@ -1,207 +1,111 @@
 use crate::{
-    config::{Config, ALPACA_API_BASE},
-    create_send_await, database, threads,
+    config::{Config, ALPACA_ASSET_API_URL},
+    database, threads,
+    types::{
+        alpaca::api::incoming::{self, asset::Status},
+        Asset,
+    },
 };
 use axum::{extract::Path, Extension, Json};
+use backoff::{future::retry, ExponentialBackoff};
+use core::panic;
 use http::StatusCode;
-use nonempty::{nonempty, NonEmpty};
-use qrust::{
-    alpaca,
-    types::{self, Asset},
-};
-use serde::{Deserialize, Serialize};
-use std::{
-    collections::{HashMap, HashSet},
-    sync::Arc,
-};
+use serde::Deserialize;
+use std::sync::Arc;
 use tokio::sync::mpsc;
 
 pub async fn get(
-    Extension(config): Extension<Arc<Config>>,
+    Extension(app_config): Extension<Arc<Config>>,
 ) -> Result<(StatusCode, Json<Vec<Asset>>), StatusCode> {
-    let assets = database::assets::select(
-        &config.clickhouse_client,
-        &config.clickhouse_concurrency_limiter,
-    )
-    .await
-    .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
-
+    let assets = database::assets::select(&app_config.clickhouse_client).await;
     Ok((StatusCode::OK, Json(assets)))
 }
 
 pub async fn get_where_symbol(
-    Extension(config): Extension<Arc<Config>>,
+    Extension(app_config): Extension<Arc<Config>>,
     Path(symbol): Path<String>,
 ) -> Result<(StatusCode, Json<Asset>), StatusCode> {
-    let asset = database::assets::select_where_symbol(
-        &config.clickhouse_client,
-        &config.clickhouse_concurrency_limiter,
-        &symbol,
-    )
-    .await
-    .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
-
+    let asset = database::assets::select_where_symbol(&app_config.clickhouse_client, &symbol).await;
     asset.map_or(Err(StatusCode::NOT_FOUND), |asset| {
         Ok((StatusCode::OK, Json(asset)))
     })
 }
 
 #[derive(Deserialize)]
-pub struct AddAssetsRequest {
-    symbols: Vec<String>,
-}
-
-#[derive(Serialize)]
-pub struct AddAssetsResponse {
-    added: Vec<String>,
-    skipped: Vec<String>,
-    failed: Vec<String>,
+pub struct AddAssetRequest {
+    symbol: String,
 }
 
 pub async fn add(
-    Extension(config): Extension<Arc<Config>>,
-    Extension(data_sender): Extension<mpsc::Sender<threads::data::Message>>,
-    Json(request): Json<AddAssetsRequest>,
-) -> Result<(StatusCode, Json<AddAssetsResponse>), StatusCode> {
-    let database_symbols = database::assets::select(
-        &config.clickhouse_client,
-        &config.clickhouse_concurrency_limiter,
-    )
-    .await
-    .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
-    .into_iter()
-    .map(|asset| asset.symbol)
-    .collect::<HashSet<_>>();
-
-    let mut alpaca_assets = alpaca::assets::get_by_symbols(
-        &config.alpaca_client,
-        &config.alpaca_rate_limiter,
-        &request.symbols,
-        None,
-        &ALPACA_API_BASE,
-    )
-    .await
-    .map_err(|e| {
-        e.status()
-            .map_or(StatusCode::INTERNAL_SERVER_ERROR, |status| {
-                StatusCode::from_u16(status.as_u16()).unwrap()
-            })
-    })?
-    .into_iter()
-    .map(|asset| (asset.symbol.clone(), asset))
-    .collect::<HashMap<_, _>>();
-
-    let num_symbols = request.symbols.len();
-    let (assets, skipped, failed) = request.symbols.into_iter().fold(
-        (Vec::with_capacity(num_symbols), vec![], vec![]),
-        |(mut assets, mut skipped, mut failed), symbol| {
-            if database_symbols.contains(&symbol) {
-                skipped.push(symbol);
-            } else if let Some(asset) = alpaca_assets.remove(&symbol) {
-                if asset.status == types::alpaca::api::incoming::asset::Status::Active
-                    && asset.tradable
-                    && asset.fractionable
-                {
-                    assets.push((asset.symbol, asset.class.into()));
-                } else {
-                    failed.push(asset.symbol);
-                }
-            } else {
-                failed.push(symbol);
-            }
-
-            (assets, skipped, failed)
-        },
-    );
-
-    if let Some(assets) = NonEmpty::from_vec(assets.clone()) {
-        create_send_await!(
-            data_sender,
-            threads::data::Message::new,
-            threads::data::Action::Add,
-            assets
-        );
-    }
-
-    Ok((
-        StatusCode::OK,
-        Json(AddAssetsResponse {
-            added: assets.into_iter().map(|asset| asset.0).collect(),
-            skipped,
-            failed,
-        }),
-    ))
-}
-
-pub async fn add_symbol(
-    Extension(config): Extension<Arc<Config>>,
-    Extension(data_sender): Extension<mpsc::Sender<threads::data::Message>>,
-    Path(symbol): Path<String>,
-) -> Result<StatusCode, StatusCode> {
-    if database::assets::select_where_symbol(
-        &config.clickhouse_client,
-        &config.clickhouse_concurrency_limiter,
-        &symbol,
-    )
-    .await
-    .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
-    .is_some()
+    Extension(app_config): Extension<Arc<Config>>,
+    Extension(asset_status_sender): Extension<mpsc::Sender<threads::data::asset_status::Message>>,
+    Json(request): Json<AddAssetRequest>,
+) -> Result<(StatusCode, Json<Asset>), StatusCode> {
+    if database::assets::select_where_symbol(&app_config.clickhouse_client, &request.symbol)
+        .await
+        .is_some()
     {
         return Err(StatusCode::CONFLICT);
     }
 
-    let asset = alpaca::assets::get_by_symbol(
-        &config.alpaca_client,
-        &config.alpaca_rate_limiter,
-        &symbol,
-        None,
-        &ALPACA_API_BASE,
-    )
+    let asset = retry(ExponentialBackoff::default(), || async {
+        app_config.alpaca_rate_limit.until_ready().await;
+        app_config
+            .alpaca_client
+            .get(&format!("{}/{}", ALPACA_ASSET_API_URL, request.symbol))
+            .send()
+            .await?
+            .error_for_status()
+            .map_err(backoff::Error::Permanent)?
+            .json::<incoming::asset::Asset>()
+            .await
+            .map_err(backoff::Error::Permanent)
+    })
     .await
-    .map_err(|e| {
-        e.status()
-            .map_or(StatusCode::INTERNAL_SERVER_ERROR, |status| {
-                StatusCode::from_u16(status.as_u16()).unwrap()
-            })
+    .map_err(|e| match e.status() {
+        Some(reqwest::StatusCode::NOT_FOUND) => StatusCode::NOT_FOUND,
+        _ => panic!("Unexpected error: {}.", e),
     })?;
 
-    if asset.status != types::alpaca::api::incoming::asset::Status::Active
-        || !asset.tradable
-        || !asset.fractionable
-    {
+    if asset.status != Status::Active || !asset.tradable || !asset.fractionable {
         return Err(StatusCode::FORBIDDEN);
     }
 
-    create_send_await!(
-        data_sender,
-        threads::data::Message::new,
-        threads::data::Action::Add,
-        nonempty![(asset.symbol, asset.class.into())]
+    let asset = Asset::from(asset);
+
+    let (asset_status_message, asset_status_response) = threads::data::asset_status::Message::new(
+        threads::data::asset_status::Action::Add,
+        vec![asset.clone()],
     );
 
-    Ok(StatusCode::CREATED)
+    asset_status_sender
+        .send(asset_status_message)
+        .await
+        .unwrap();
+    asset_status_response.await.unwrap();
+
+    Ok((StatusCode::CREATED, Json(asset)))
 }
 
 pub async fn delete(
-    Extension(config): Extension<Arc<Config>>,
-    Extension(data_sender): Extension<mpsc::Sender<threads::data::Message>>,
+    Extension(app_config): Extension<Arc<Config>>,
+    Extension(asset_status_sender): Extension<mpsc::Sender<threads::data::asset_status::Message>>,
     Path(symbol): Path<String>,
 ) -> Result<StatusCode, StatusCode> {
-    let asset = database::assets::select_where_symbol(
-        &config.clickhouse_client,
-        &config.clickhouse_concurrency_limiter,
-        &symbol,
-    )
-    .await
-    .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
-    .ok_or(StatusCode::NOT_FOUND)?;
+    let asset = database::assets::select_where_symbol(&app_config.clickhouse_client, &symbol)
+        .await
+        .ok_or(StatusCode::NOT_FOUND)?;
 
-    create_send_await!(
-        data_sender,
-        threads::data::Message::new,
-        threads::data::Action::Remove,
-        nonempty![(asset.symbol, asset.class)]
+    let (asset_status_message, asset_status_response) = threads::data::asset_status::Message::new(
+        threads::data::asset_status::Action::Remove,
+        vec![asset],
     );
 
+    asset_status_sender
+        .send(asset_status_message)
+        .await
+        .unwrap();
+    asset_status_response.await.unwrap();
+
     Ok(StatusCode::NO_CONTENT)
 }

src/routes/mod.rs

@@ -1,5 +1,5 @@
-mod assets;
-mod health;
+pub mod assets;
+pub mod health;
 
 use crate::{config::Config, threads};
 use axum::{
@@ -10,16 +10,18 @@ use log::info;
 use std::{net::SocketAddr, sync::Arc};
 use tokio::{net::TcpListener, sync::mpsc};
 
-pub async fn run(config: Arc<Config>, data_sender: mpsc::Sender<threads::data::Message>) {
+pub async fn run(
+    app_config: Arc<Config>,
+    asset_status_sender: mpsc::Sender<threads::data::asset_status::Message>,
+) {
     let app = Router::new()
         .route("/health", get(health::get))
         .route("/assets", get(assets::get))
         .route("/assets/:symbol", get(assets::get_where_symbol))
         .route("/assets", post(assets::add))
-        .route("/assets/:symbol", post(assets::add_symbol))
         .route("/assets/:symbol", delete(assets::delete))
-        .layer(Extension(config))
-        .layer(Extension(data_sender));
+        .layer(Extension(app_config))
+        .layer(Extension(asset_status_sender));
 
     let addr = SocketAddr::from(([0, 0, 0, 0], 7878));
     let listener = TcpListener::bind(addr).await.unwrap();

src/threads/clock.rs

@@ -1,16 +1,13 @@
 use crate::{
-    config::{Config, ALPACA_API_BASE},
-    database,
+    config::{Config, ALPACA_CLOCK_API_URL},
+    types::alpaca,
+    utils::duration_until,
 };
+use backoff::{future::retry, ExponentialBackoff};
 use log::info;
-use qrust::{
-    alpaca,
-    types::{self, Calendar},
-    utils::{backoff, duration_until},
-};
 use std::sync::Arc;
 use time::OffsetDateTime;
-use tokio::{join, sync::mpsc, time::sleep};
+use tokio::{sync::mpsc, time::sleep};
 
 pub enum Status {
     Open,
@@ -22,8 +19,8 @@ pub struct Message {
     pub next_switch: OffsetDateTime,
 }
 
-impl From<types::alpaca::api::incoming::clock::Clock> for Message {
-    fn from(clock: types::alpaca::api::incoming::clock::Clock) -> Self {
+impl From<alpaca::api::incoming::clock::Clock> for Message {
+    fn from(clock: alpaca::api::incoming::clock::Clock) -> Self {
         if clock.is_open {
             Self {
                 status: Status::Open,
@@ -38,35 +35,23 @@ impl From<types::alpaca::api::incoming::clock::Clock> for Message {
     }
 }
 
-pub async fn run(config: Arc<Config>, sender: mpsc::Sender<Message>) {
+pub async fn run(app_config: Arc<Config>, clock_sender: mpsc::Sender<Message>) {
     loop {
-        let clock_future = async {
-            alpaca::clock::get(
-                &config.alpaca_client,
-                &config.alpaca_rate_limiter,
-                Some(backoff::infinite()),
-                &ALPACA_API_BASE,
-            )
-            .await
-            .unwrap()
-        };
-
-        let calendar_future = async {
-            alpaca::calendar::get(
-                &config.alpaca_client,
-                &config.alpaca_rate_limiter,
-                &types::alpaca::api::outgoing::calendar::Calendar::default(),
-                Some(backoff::infinite()),
-                &ALPACA_API_BASE,
-            )
-            .await
-            .unwrap()
-            .into_iter()
-            .map(Calendar::from)
-            .collect::<Vec<_>>()
-        };
-
-        let (clock, calendar) = join!(clock_future, calendar_future);
+        let clock = retry(ExponentialBackoff::default(), || async {
+            app_config.alpaca_rate_limit.until_ready().await;
+            app_config
+                .alpaca_client
+                .get(ALPACA_CLOCK_API_URL)
+                .send()
+                .await?
+                .error_for_status()
+                .map_err(backoff::Error::Permanent)?
+                .json::<alpaca::api::incoming::clock::Clock>()
+                .await
+                .map_err(backoff::Error::Permanent)
+        })
+        .await
+        .unwrap();
 
         let sleep_until = duration_until(if clock.is_open {
             info!("Market is open, will close at {}.", clock.next_close);
@@ -76,19 +61,7 @@ pub async fn run(config: Arc<Config>, sender: mpsc::Sender<Message>) {
             clock.next_open
         });
 
-        let sleep_future = sleep(sleep_until);
-
-        let calendar_future = async {
-            database::calendar::upsert_batch_and_delete(
-                &config.clickhouse_client,
-                &config.clickhouse_concurrency_limiter,
-                &calendar,
-            )
-            .await
-            .unwrap();
-        };
-
-        join!(sleep_future, calendar_future);
-        sender.send(clock.into()).await.unwrap();
+        sleep(sleep_until).await;
+        clock_sender.send(clock.into()).await.unwrap();
     }
 }

src/threads/data/asset_status.rs

@@ -0,0 +1,170 @@
+use super::{Guard, ThreadType};
+use crate::{
+    config::Config,
+    database,
+    types::{alpaca::websocket, Asset},
+};
+use futures_util::{stream::SplitSink, SinkExt};
+use log::info;
+use serde_json::to_string;
+use std::sync::Arc;
+use tokio::{
+    join,
+    net::TcpStream,
+    spawn,
+    sync::{mpsc, oneshot, Mutex, RwLock},
+};
+use tokio_tungstenite::{tungstenite, MaybeTlsStream, WebSocketStream};
+
+#[derive(Clone)]
+pub enum Action {
+    Add,
+    Remove,
+}
+
+pub struct Message {
+    pub action: Action,
+    pub assets: Vec<Asset>,
+    pub response: oneshot::Sender<()>,
+}
+
+impl Message {
+    pub fn new(action: Action, assets: Vec<Asset>) -> (Self, oneshot::Receiver<()>) {
+        let (sender, receiver) = oneshot::channel::<()>();
+        (
+            Self {
+                action,
+                assets,
+                response: sender,
+            },
+            receiver,
+        )
+    }
+}
+
+pub async fn run(
+    app_config: Arc<Config>,
+    thread_type: ThreadType,
+    guard: Arc<RwLock<Guard>>,
+    mut asset_status_receiver: mpsc::Receiver<Message>,
+    websocket_sender: Arc<
+        Mutex<SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, tungstenite::Message>>,
+    >,
+) {
+    loop {
+        let message = asset_status_receiver.recv().await.unwrap();
+
+        spawn(handle_asset_status_message(
+            app_config.clone(),
+            thread_type,
+            guard.clone(),
+            websocket_sender.clone(),
+            message,
+        ));
+    }
+}
+
+#[allow(clippy::significant_drop_tightening)]
+async fn handle_asset_status_message(
+    app_config: Arc<Config>,
+    thread_type: ThreadType,
+    guard: Arc<RwLock<Guard>>,
+    websocket_sender: Arc<
+        Mutex<SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, tungstenite::Message>>,
+    >,
+    message: Message,
+) {
+    let symbols = message
+        .assets
+        .clone()
+        .into_iter()
+        .map(|asset| match thread_type {
+            ThreadType::Bars(_) => asset.symbol,
+            ThreadType::News => asset.abbreviation,
+        })
+        .collect::<Vec<_>>();
+
+    match message.action {
+        Action::Add => {
+            let mut guard = guard.write().await;
+
+            guard.symbols.extend(symbols.clone());
+            guard
+                .pending_subscriptions
+                .extend(symbols.clone().into_iter().zip(message.assets.clone()));
+
+            info!("{:?} - Added {:?}.", thread_type, symbols);
+
+            let database_future = async {
+                if matches!(thread_type, ThreadType::Bars(_)) {
+                    database::assets::upsert_batch(&app_config.clickhouse_client, message.assets)
+                        .await;
+                }
+            };
+
+            let websocket_future = async move {
+                websocket_sender
+                    .lock()
+                    .await
+                    .send(tungstenite::Message::Text(
+                        to_string(&websocket::outgoing::Message::Subscribe(
+                            websocket_market_message_factory(thread_type, symbols),
+                        ))
+                        .unwrap(),
+                    ))
+                    .await
+                    .unwrap();
+            };
+
+            join!(database_future, websocket_future);
+        }
+        Action::Remove => {
+            let mut guard = guard.write().await;
+
+            guard.symbols.retain(|symbol| !symbols.contains(symbol));
+            guard
+                .pending_unsubscriptions
+                .extend(symbols.clone().into_iter().zip(message.assets.clone()));
+
+            info!("{:?} - Removed {:?}.", thread_type, symbols);
+
+            let sybols_clone = symbols.clone();
+            let database_future = database::assets::delete_where_symbols(
+                &app_config.clickhouse_client,
+                &sybols_clone,
+            );
+
+            let websocket_future = async move {
+                websocket_sender
+                    .lock()
+                    .await
+                    .send(tungstenite::Message::Text(
+                        to_string(&websocket::outgoing::Message::Unsubscribe(
+                            websocket_market_message_factory(thread_type, symbols),
+                        ))
+                        .unwrap(),
+                    ))
+                    .await
+                    .unwrap();
+            };
+
+            join!(database_future, websocket_future);
+        }
+    }
+
+    message.response.send(()).unwrap();
+}
+
+fn websocket_market_message_factory(
+    thread_type: ThreadType,
+    symbols: Vec<String>,
+) -> websocket::outgoing::subscribe::Message {
+    match thread_type {
+        ThreadType::Bars(_) => websocket::outgoing::subscribe::Message::Market(
+            websocket::outgoing::subscribe::MarketMessage::new(symbols),
+        ),
+        ThreadType::News => websocket::outgoing::subscribe::Message::News(
+            websocket::outgoing::subscribe::NewsMessage::new(symbols),
+        ),
+    }
+}

src/threads/data/backfill.rs

@@ -0,0 +1,419 @@
+use super::{Guard, ThreadType};
+use crate::{
+    config::{Config, ALPACA_CRYPTO_DATA_URL, ALPACA_NEWS_DATA_URL, ALPACA_STOCK_DATA_URL},
+    database,
+    types::{
+        alpaca::{self, Source},
+        ollama, Asset, Bar, Class, News, Subset,
+    },
+    utils::{duration_until, last_minute, FIFTEEN_MINUTES, ONE_MINUTE},
+};
+use backoff::{future::retry, ExponentialBackoff};
+use log::{error, info};
+use serde_json::{from_str, to_string};
+use std::{collections::HashMap, sync::Arc};
+use time::OffsetDateTime;
+use tokio::{
+    join, spawn,
+    sync::{mpsc, oneshot, Mutex, RwLock},
+    task::JoinHandle,
+    time::sleep,
+};
+
+pub enum Action {
+    Backfill,
+    Purge,
+}
+
+pub struct Message {
+    pub action: Action,
+    pub assets: Subset<Asset>,
+    pub response: oneshot::Sender<()>,
+}
+
+impl Message {
+    pub fn new(action: Action, assets: Subset<Asset>) -> (Self, oneshot::Receiver<()>) {
+        let (sender, receiver) = oneshot::channel::<()>();
+        (
+            Self {
+                action,
+                assets,
+                response: sender,
+            },
+            receiver,
+        )
+    }
+}
+
+pub async fn run(
+    app_config: Arc<Config>,
+    thread_type: ThreadType,
+    guard: Arc<RwLock<Guard>>,
+    mut backfill_receiver: mpsc::Receiver<Message>,
+) {
+    let backfill_jobs = Arc::new(Mutex::new(HashMap::new()));
+
+    let data_url = match thread_type {
+        ThreadType::Bars(Class::UsEquity) => ALPACA_STOCK_DATA_URL.to_string(),
+        ThreadType::Bars(Class::Crypto) => ALPACA_CRYPTO_DATA_URL.to_string(),
+        ThreadType::News => ALPACA_NEWS_DATA_URL.to_string(),
+    };
+
+    loop {
+        let message = backfill_receiver.recv().await.unwrap();
+
+        spawn(handle_backfill_message(
+            app_config.clone(),
+            thread_type,
+            guard.clone(),
+            data_url.clone(),
+            backfill_jobs.clone(),
+            message,
+        ));
+    }
+}
+
+#[allow(clippy::significant_drop_tightening)]
+#[allow(clippy::too_many_lines)]
+async fn handle_backfill_message(
+    app_config: Arc<Config>,
+    thread_type: ThreadType,
+    guard: Arc<RwLock<Guard>>,
+    data_url: String,
+    backfill_jobs: Arc<Mutex<HashMap<String, JoinHandle<()>>>>,
+    message: Message,
+) {
+    let guard = guard.read().await;
+    let mut backfill_jobs = backfill_jobs.lock().await;
+
+    let symbols = match message.assets {
+        Subset::All => guard.symbols.clone().into_iter().collect::<Vec<_>>(),
+        Subset::Some(assets) => assets
+            .into_iter()
+            .map(|asset| match thread_type {
+                ThreadType::Bars(_) => asset.symbol,
+                ThreadType::News => asset.abbreviation,
+            })
+            .filter(|symbol| match message.action {
+                Action::Backfill => guard.symbols.contains(symbol),
+                Action::Purge => !guard.symbols.contains(symbol),
+            })
+            .collect::<Vec<_>>(),
+    };
+
+    match message.action {
+        Action::Backfill => {
+            for symbol in symbols {
+                if let Some(job) = backfill_jobs.remove(&symbol) {
+                    if !job.is_finished() {
+                        job.abort();
+                    }
+                    let _ = job.await;
+                }
+
+                let app_config = app_config.clone();
+                let data_url = data_url.clone();
+
+                backfill_jobs.insert(
+                    symbol.clone(),
+                    spawn(async move {
+                        let (fetch_from, fetch_to) =
+                            queue_backfill(&app_config, thread_type, &symbol).await;
+
+                        match thread_type {
+                            ThreadType::Bars(_) => {
+                                execute_backfill_bars(
+                                    app_config,
+                                    thread_type,
+                                    data_url,
+                                    symbol,
+                                    fetch_from,
+                                    fetch_to,
+                                )
+                                .await;
+                            }
+                            ThreadType::News => {
+                                execute_backfill_news(
+                                    app_config,
+                                    thread_type,
+                                    data_url,
+                                    symbol,
+                                    fetch_from,
+                                    fetch_to,
+                                )
+                                .await;
+                            }
+                        }
+                    }),
+                );
+            }
+        }
+        Action::Purge => {
+            for symbol in &symbols {
+                if let Some(job) = backfill_jobs.remove(symbol) {
+                    if !job.is_finished() {
+                        job.abort();
+                    }
+                    let _ = job.await;
+                }
+            }
+
+            let backfills_future = database::backfills::delete_where_symbols(
+                &app_config.clickhouse_client,
+                &thread_type,
+                &symbols,
+            );
+
+            let data_future = async {
+                match thread_type {
+                    ThreadType::Bars(_) => {
+                        database::bars::delete_where_symbols(
+                            &app_config.clickhouse_client,
+                            &symbols,
+                        )
+                        .await;
+                    }
+                    ThreadType::News => {
+                        database::news::delete_where_symbols(
+                            &app_config.clickhouse_client,
+                            &symbols,
+                        )
+                        .await;
+                    }
+                }
+            };
+
+            join!(backfills_future, data_future);
+        }
+    }
+
+    message.response.send(()).unwrap();
+}
+
+async fn queue_backfill(
+    app_config: &Arc<Config>,
+    thread_type: ThreadType,
+    symbol: &String,
+) -> (OffsetDateTime, OffsetDateTime) {
+    let latest_backfill = database::backfills::select_latest_where_symbol(
+        &app_config.clickhouse_client,
+        &thread_type,
+        &symbol,
+    )
+    .await;
+
+    let fetch_from = latest_backfill
+        .as_ref()
+        .map_or(OffsetDateTime::UNIX_EPOCH, |backfill| {
+            backfill.time + ONE_MINUTE
+        });
+
+    let fetch_to = last_minute();
+
+    if app_config.alpaca_source == Source::Iex {
+        let run_delay = duration_until(fetch_to + FIFTEEN_MINUTES + ONE_MINUTE);
+        info!(
+            "{:?} - Queing backfill for {} in {:?}.",
+            thread_type, symbol, run_delay
+        );
+        sleep(run_delay).await;
+    }
+
+    (fetch_from, fetch_to)
+}
+
+async fn execute_backfill_bars(
+    app_config: Arc<Config>,
+    thread_type: ThreadType,
+    data_url: String,
+    symbol: String,
+    fetch_from: OffsetDateTime,
+    fetch_to: OffsetDateTime,
+) {
+    if fetch_from > fetch_to {
+        return;
+    }
+
+    info!("{:?} - Backfilling data for {}.", thread_type, symbol);
+
+    let mut bars = Vec::new();
+    let mut next_page_token = None;
+
+    loop {
+        let message = retry(ExponentialBackoff::default(), || async {
+            app_config.alpaca_rate_limit.until_ready().await;
+            app_config
+                .alpaca_client
+                .get(&data_url)
+                .query(&alpaca::api::outgoing::bar::Bar {
+                    symbols: vec![symbol.clone()],
+                    timeframe: ONE_MINUTE,
+                    start: fetch_from,
+                    end: fetch_to,
+                    limit: 10000,
+                    page_token: next_page_token.clone(),
+                })
+                .send()
+                .await?
+                .error_for_status()
+                .map_err(backoff::Error::Permanent)?
+                .json::<alpaca::api::incoming::bar::Message>()
+                .await
+                .map_err(backoff::Error::Permanent)
+        })
+        .await;
+
+        let message = match message {
+            Ok(message) => message,
+            Err(e) => {
+                error!(
+                    "{:?} - Failed to backfill data for {}: {}.",
+                    thread_type, symbol, e
+                );
+                return;
+            }
+        };
+
+        message.bars.into_iter().for_each(|(symbol, bar_vec)| {
+            for bar in bar_vec {
+                bars.push(Bar::from((bar, symbol.clone())));
+            }
+        });
+
+        if message.next_page_token.is_none() {
+            break;
+        }
+        next_page_token = message.next_page_token;
+    }
+
+    if bars.is_empty() {
+        return;
+    }
+
+    let backfill = bars.last().unwrap().clone().into();
+    database::bars::upsert_batch(&app_config.clickhouse_client, bars).await;
+    database::backfills::upsert(&app_config.clickhouse_client, &thread_type, &backfill).await;
+
+    info!("{:?} - Backfilled data for {}.", thread_type, symbol);
+}
+
+async fn execute_backfill_news(
+    app_config: Arc<Config>,
+    thread_type: ThreadType,
+    data_url: String,
+    symbol: String,
+    fetch_from: OffsetDateTime,
+    fetch_to: OffsetDateTime,
+) {
+    if fetch_from > fetch_to {
+        return;
+    }
+
+    info!("{:?} - Backfilling data for {}.", thread_type, symbol);
+
+    let mut news = Vec::new();
+    let mut next_page_token = None;
+
+    loop {
+        let message = retry(ExponentialBackoff::default(), || async {
+            app_config.alpaca_rate_limit.until_ready().await;
+            app_config
+                .alpaca_client
+                .get(&data_url)
+                .query(&alpaca::api::outgoing::news::News {
+                    symbols: vec![symbol.clone()],
+                    start: fetch_from,
+                    end: fetch_to,
+                    limit: 50,
+                    include_content: true,
+                    exclude_contentless: false,
+                    page_token: next_page_token.clone(),
+                })
+                .send()
+                .await?
+                .error_for_status()
+                .map_err(backoff::Error::Permanent)?
+                .json::<alpaca::api::incoming::news::Message>()
+                .await
+                .map_err(backoff::Error::Permanent)
+        })
+        .await;
+
+        let message = match message {
+            Ok(message) => message,
+            Err(e) => {
+                error!(
+                    "{:?} - Failed to backfill data for {}: {}.",
+                    thread_type, symbol, e
+                );
+                return;
+            }
+        };
+
+        message.news.into_iter().for_each(|news_item| {
+            news.push(News::from(news_item));
+        });
+
+        if message.next_page_token.is_none() {
+            break;
+        }
+        next_page_token = message.next_page_token;
+    }
+
+    if news.is_empty() {
+        return;
+    }
+
+    for news in &mut news {
+        info!(
+            "{:?} - Getting sentiment for news: {}.",
+            thread_type, news.headline
+        );
+
+        let prediction = retry(ExponentialBackoff::default(), || async {
+            let response = app_config
+                .ollama_client
+                .post(format!("{}/api/chat", app_config.ollama_url))
+                .body(
+                    to_string(&ollama::outgoing::sentiment::Sentiment::new(
+                        app_config.ollama_model.clone(),
+                        &news.clone().into(),
+                    ))
+                    .unwrap(),
+                )
+                .send()
+                .await
+                .unwrap()
+                .json::<ollama::incoming::sentiment::Response>()
+                .await
+                .unwrap();
+
+            from_str::<ollama::incoming::sentiment::Prediction>(&response.message.content)
+                .map_err(Into::into)
+        })
+        .await;
+
+        match prediction {
+            Ok(prediction) => {
+                info!(
+                    "{:?} - Received sentiment for news: {:?}.",
+                    thread_type, prediction
+                );
+                news.sentiment = prediction.sentiment.into();
+                news.confidence = prediction.confidence.into();
+            }
+            Err(e) => {
+                error!(
+                    "{:?} - Failed to get sentiment for news: {:?}.",
+                    thread_type, e
+                );
+            }
+        }
+    }
+
+    let backfill = (news.last().unwrap().clone(), symbol.clone()).into();
+    database::news::upsert_batch(&app_config.clickhouse_client, news).await;
+    database::backfills::upsert(&app_config.clickhouse_client, &thread_type, &backfill).await;
+
+    info!("{:?} - Backfilled data for {}.", thread_type, symbol);
+}

src/threads/data/backfill/bars.rs

@@ -1,238 +0,0 @@
-use super::Job;
-use crate::{
-    config::{Config, ALPACA_SOURCE, CLICKHOUSE_BATCH_BARS_SIZE},
-    database,
-    threads::data::ThreadType,
-};
-use async_trait::async_trait;
-use log::{error, info};
-use nonempty::NonEmpty;
-use qrust::{
-    alpaca,
-    types::{
-        self,
-        alpaca::{
-            api::{ALPACA_CRYPTO_DATA_API_URL, ALPACA_US_EQUITY_DATA_API_URL},
-            shared::{Sort, Source},
-        },
-        Backfill, Bar, Class,
-    },
-    utils::{duration_until, FIFTEEN_MINUTES, ONE_MINUTE},
-};
-use std::{collections::HashMap, sync::Arc};
-use time::OffsetDateTime;
-use tokio::time::sleep;
-
-pub struct Handler {
-    pub config: Arc<Config>,
-    pub data_url: &'static str,
-    pub api_query_constructor: fn(
-        symbols: Vec<String>,
-        fetch_from: OffsetDateTime,
-        fetch_to: OffsetDateTime,
-        next_page_token: Option<String>,
-    ) -> types::alpaca::api::outgoing::bar::Bar,
-}
-
-pub fn us_equity_query_constructor(
-    symbols: Vec<String>,
-    fetch_from: OffsetDateTime,
-    fetch_to: OffsetDateTime,
-    next_page_token: Option<String>,
-) -> types::alpaca::api::outgoing::bar::Bar {
-    types::alpaca::api::outgoing::bar::Bar::UsEquity(types::alpaca::api::outgoing::bar::UsEquity {
-        symbols,
-        start: Some(fetch_from),
-        end: Some(fetch_to),
-        page_token: next_page_token,
-        sort: Some(Sort::Asc),
-        feed: Some(*ALPACA_SOURCE),
-        ..Default::default()
-    })
-}
-
-pub fn crypto_query_constructor(
-    symbols: Vec<String>,
-    fetch_from: OffsetDateTime,
-    fetch_to: OffsetDateTime,
-    next_page_token: Option<String>,
-) -> types::alpaca::api::outgoing::bar::Bar {
-    types::alpaca::api::outgoing::bar::Bar::Crypto(types::alpaca::api::outgoing::bar::Crypto {
-        symbols,
-        start: Some(fetch_from),
-        end: Some(fetch_to),
-        page_token: next_page_token,
-        sort: Some(Sort::Asc),
-        ..Default::default()
-    })
-}
-
-#[async_trait]
-impl super::Handler for Handler {
-    async fn select_latest_backfills(
-        &self,
-        symbols: &[String],
-    ) -> Result<Vec<Backfill>, clickhouse::error::Error> {
-        database::backfills_bars::select_where_symbols(
-            &self.config.clickhouse_client,
-            &self.config.clickhouse_concurrency_limiter,
-            symbols,
-        )
-        .await
-    }
-
-    async fn delete_backfills(&self, symbols: &[String]) -> Result<(), clickhouse::error::Error> {
-        database::backfills_bars::delete_where_symbols(
-            &self.config.clickhouse_client,
-            &self.config.clickhouse_concurrency_limiter,
-            symbols,
-        )
-        .await
-    }
-
-    async fn delete_data(&self, symbols: &[String]) -> Result<(), clickhouse::error::Error> {
-        database::bars::delete_where_symbols(
-            &self.config.clickhouse_client,
-            &self.config.clickhouse_concurrency_limiter,
-            symbols,
-        )
-        .await
-    }
-
-    async fn queue_backfill(&self, jobs: &NonEmpty<Job>) {
-        if *ALPACA_SOURCE == Source::Sip {
-            return;
-        }
-
-        let fetch_to = jobs.maximum_by_key(|job| job.fetch_to).fetch_to;
-        let run_delay = duration_until(fetch_to + FIFTEEN_MINUTES + ONE_MINUTE);
-        let symbols = jobs.iter().map(|job| &job.symbol).collect::<Vec<_>>();
-
-        info!("Queing bar backfill for {:?} in {:?}.", symbols, run_delay);
-        sleep(run_delay).await;
-    }
-
-    async fn backfill(&self, jobs: NonEmpty<Job>) {
-        let symbols = Vec::from(jobs.clone().map(|job| job.symbol));
-        let fetch_from = jobs.minimum_by_key(|job| job.fetch_from).fetch_from;
-        let fetch_to = jobs.maximum_by_key(|job| job.fetch_to).fetch_to;
-        let freshness = jobs
-            .into_iter()
-            .map(|job| (job.symbol, job.fresh))
-            .collect::<HashMap<_, _>>();
-
-        let mut bars = Vec::with_capacity(*CLICKHOUSE_BATCH_BARS_SIZE);
-        let mut last_times = HashMap::new();
-        let mut next_page_token = None;
-
-        info!("Backfilling bars for {:?}.", symbols);
-
-        loop {
-            let message = alpaca::bars::get(
-                &self.config.alpaca_client,
-                &self.config.alpaca_rate_limiter,
-                self.data_url,
-                &(self.api_query_constructor)(
-                    symbols.clone(),
-                    fetch_from,
-                    fetch_to,
-                    next_page_token.clone(),
-                ),
-                None,
-            )
-            .await;
-
-            if let Err(err) = message {
-                error!("Failed to backfill bars for {:?}: {:?}.", symbols, err);
-                return;
-            }
-
-            let message = message.unwrap();
-
-            for (symbol, bars_vec) in message.bars {
-                if let Some(last) = bars_vec.last() {
-                    last_times.insert(symbol.clone(), last.time);
-                }
-
-                for bar in bars_vec {
-                    bars.push(Bar::from((bar, symbol.clone())));
-                }
-            }
-
-            if bars.len() < *CLICKHOUSE_BATCH_BARS_SIZE && message.next_page_token.is_some() {
-                continue;
-            }
-
-            database::bars::upsert_batch(
-                &self.config.clickhouse_client,
-                &self.config.clickhouse_concurrency_limiter,
-                &bars,
-            )
-            .await
-            .unwrap();
-
-            let backfilled = last_times
-                .drain()
-                .map(|(symbol, time)| Backfill {
-                    fresh: freshness[&symbol],
-                    symbol,
-                    time,
-                })
-                .collect::<Vec<_>>();
-
-            database::backfills_bars::upsert_batch(
-                &self.config.clickhouse_client,
-                &self.config.clickhouse_concurrency_limiter,
-                &backfilled,
-            )
-            .await
-            .unwrap();
-
-            if message.next_page_token.is_none() {
-                break;
-            }
-
-            next_page_token = message.next_page_token;
-            bars.clear();
-        }
-
-        database::backfills_bars::set_fresh_where_symbols(
-            &self.config.clickhouse_client,
-            &self.config.clickhouse_concurrency_limiter,
-            true,
-            &symbols,
-        )
-        .await
-        .unwrap();
-
-        info!("Backfilled bars for {:?}.", symbols);
-    }
-
-    fn max_limit(&self) -> i64 {
-        alpaca::bars::MAX_LIMIT
-    }
-
-    fn log_string(&self) -> &'static str {
-        "bars"
-    }
-}
-
-pub fn create_handler(config: Arc<Config>, thread_type: ThreadType) -> Box<dyn super::Handler> {
-    let data_url = match thread_type {
-        ThreadType::Bars(Class::UsEquity) => ALPACA_US_EQUITY_DATA_API_URL,
-        ThreadType::Bars(Class::Crypto) => ALPACA_CRYPTO_DATA_API_URL,
-        _ => unreachable!(),
-    };
-
-    let api_query_constructor = match thread_type {
-        ThreadType::Bars(Class::UsEquity) => us_equity_query_constructor,
-        ThreadType::Bars(Class::Crypto) => crypto_query_constructor,
-        _ => unreachable!(),
-    };
-
-    Box::new(Handler {
-        config,
-        data_url,
-        api_query_constructor,
-    })
-}

src/threads/data/backfill/mod.rs

@@ -1,244 +0,0 @@
-pub mod bars;
-pub mod news;
-
-use async_trait::async_trait;
-use itertools::Itertools;
-use log::{info, warn};
-use nonempty::{nonempty, NonEmpty};
-use qrust::{
-    types::Backfill,
-    utils::{last_minute, ONE_SECOND},
-};
-use std::{collections::HashMap, hash::Hash, sync::Arc};
-use time::OffsetDateTime;
-use tokio::{
-    spawn,
-    sync::{mpsc, oneshot, Mutex},
-    task::JoinHandle,
-    try_join,
-};
-use uuid::Uuid;
-
-pub enum Action {
-    Backfill,
-    Purge,
-}
-
-pub struct Message {
-    pub action: Action,
-    pub symbols: NonEmpty<String>,
-    pub response: oneshot::Sender<()>,
-}
-
-impl Message {
-    pub fn new(action: Action, symbols: NonEmpty<String>) -> (Self, oneshot::Receiver<()>) {
-        let (sender, receiver) = oneshot::channel::<()>();
-        (
-            Self {
-                action,
-                symbols,
-                response: sender,
-            },
-            receiver,
-        )
-    }
-}
-
-#[derive(Clone)]
-pub struct Job {
-    pub symbol: String,
-    pub fetch_from: OffsetDateTime,
-    pub fetch_to: OffsetDateTime,
-    pub fresh: bool,
-}
-
-#[async_trait]
-pub trait Handler: Send + Sync {
-    async fn select_latest_backfills(
-        &self,
-        symbols: &[String],
-    ) -> Result<Vec<Backfill>, clickhouse::error::Error>;
-    async fn delete_backfills(&self, symbol: &[String]) -> Result<(), clickhouse::error::Error>;
-    async fn delete_data(&self, symbol: &[String]) -> Result<(), clickhouse::error::Error>;
-    async fn queue_backfill(&self, jobs: &NonEmpty<Job>);
-    async fn backfill(&self, jobs: NonEmpty<Job>);
-    fn max_limit(&self) -> i64;
-    fn log_string(&self) -> &'static str;
-}
-
-pub struct Jobs {
-    pub symbol_to_uuid: HashMap<String, Uuid>,
-    pub uuid_to_job: HashMap<Uuid, JoinHandle<()>>,
-}
-
-impl Jobs {
-    pub fn insert(&mut self, jobs: Vec<String>, fut: JoinHandle<()>) {
-        let uuid = Uuid::new_v4();
-        for symbol in jobs {
-            self.symbol_to_uuid.insert(symbol.clone(), uuid);
-        }
-        self.uuid_to_job.insert(uuid, fut);
-    }
-
-    pub fn contains_key(&self, symbol: &str) -> bool {
-        self.symbol_to_uuid.contains_key(symbol)
-    }
-
-    pub fn remove(&mut self, symbol: &str) -> Option<JoinHandle<()>> {
-        self.symbol_to_uuid
-            .remove(symbol)
-            .and_then(|uuid| self.uuid_to_job.remove(&uuid))
-    }
-
-    pub fn remove_many<T>(&mut self, symbols: &[T])
-    where
-        T: AsRef<str> + Hash + Eq,
-    {
-        for symbol in symbols {
-            self.symbol_to_uuid
-                .remove(symbol.as_ref())
-                .and_then(|uuid| self.uuid_to_job.remove(&uuid));
-        }
-    }
-
-    pub fn len(&self) -> usize {
-        self.symbol_to_uuid.len()
-    }
-}
-
-pub async fn run(handler: Arc<Box<dyn Handler>>, mut receiver: mpsc::Receiver<Message>) {
-    let backfill_jobs = Arc::new(Mutex::new(Jobs {
-        symbol_to_uuid: HashMap::new(),
-        uuid_to_job: HashMap::new(),
-    }));
-
-    loop {
-        let message = receiver.recv().await.unwrap();
-        spawn(handle_message(
-            handler.clone(),
-            backfill_jobs.clone(),
-            message,
-        ));
-    }
-}
-
-async fn handle_message(
-    handler: Arc<Box<dyn Handler>>,
-    backfill_jobs: Arc<Mutex<Jobs>>,
-    message: Message,
-) {
-    let backfill_jobs_clone = backfill_jobs.clone();
-    let mut backfill_jobs = backfill_jobs.lock().await;
-    let symbols = Vec::from(message.symbols);
-
-    match message.action {
-        Action::Backfill => {
-            let log_string = handler.log_string();
-            let max_limit = handler.max_limit();
-
-            let backfills = handler
-                .select_latest_backfills(&symbols)
-                .await
-                .unwrap()
-                .into_iter()
-                .map(|backfill| (backfill.symbol.clone(), backfill))
-                .collect::<HashMap<_, _>>();
-
-            let mut jobs = Vec::with_capacity(symbols.len());
-
-            for symbol in symbols {
-                if backfill_jobs.contains_key(&symbol) {
-                    warn!(
-                        "Backfill for {} {} is already running, skipping.",
-                        symbol, log_string
-                    );
-                    continue;
-                }
-
-                let backfill = backfills.get(&symbol);
-
-                let fetch_from = backfill.map_or(OffsetDateTime::UNIX_EPOCH, |backfill| {
-                    backfill.time + ONE_SECOND
-                });
-
-                let fetch_to = last_minute();
-
-                if fetch_from > fetch_to {
-                    info!("No need to backfill {} {}.", symbol, log_string,);
-                    return;
-                }
-
-                let fresh = backfill.map_or(false, |backfill| backfill.fresh);
-
-                jobs.push(Job {
-                    symbol,
-                    fetch_from,
-                    fetch_to,
-                    fresh,
-                });
-            }
-
-            let jobs = jobs
-                .into_iter()
-                .sorted_unstable_by_key(|job| job.fetch_from)
-                .collect::<Vec<_>>();
-
-            let mut job_groups: Vec<NonEmpty<Job>> = vec![];
-            let mut current_minutes = 0;
-
-            for job in jobs {
-                let minutes = (job.fetch_to - job.fetch_from).whole_minutes();
-
-                if job_groups.last().is_some() && current_minutes + minutes <= max_limit {
-                    let job_group = job_groups.last_mut().unwrap();
-                    job_group.push(job);
-                    current_minutes += minutes;
-                } else {
-                    job_groups.push(nonempty![job]);
-                    current_minutes = minutes;
-                }
-            }
-
-            for job_group in job_groups {
-                let symbols = job_group
-                    .iter()
-                    .map(|job| job.symbol.clone())
-                    .collect::<Vec<_>>();
-
-                let handler = handler.clone();
-                let symbols_clone = symbols.clone();
-                let backfill_jobs_clone = backfill_jobs_clone.clone();
-
-                let fut = spawn(async move {
-                    handler.queue_backfill(&job_group).await;
-                    handler.backfill(job_group).await;
-
-                    let mut backfill_jobs = backfill_jobs_clone.lock().await;
-                    backfill_jobs.remove_many(&symbols_clone);
-                    let remaining = backfill_jobs.len();
-                    drop(backfill_jobs);
-
-                    info!("{} {} backfills remaining.", remaining, log_string);
-                });
-
-                backfill_jobs.insert(symbols, fut);
-            }
-        }
-        Action::Purge => {
-            for symbol in &symbols {
-                if let Some(job) = backfill_jobs.remove(symbol) {
-                    job.abort();
-                    let _ = job.await;
-                }
-            }
-
-            try_join!(
-                handler.delete_backfills(&symbols),
-                handler.delete_data(&symbols)
-            )
-            .unwrap();
-        }
-    }
-
-    message.response.send(()).unwrap();
-}

src/threads/data/backfill/news.rs

@@ -1,186 +0,0 @@
-use super::Job;
-use crate::{
-    config::{Config, ALPACA_SOURCE, CLICKHOUSE_BATCH_NEWS_SIZE},
-    database,
-};
-use async_trait::async_trait;
-use log::{error, info};
-use nonempty::NonEmpty;
-use qrust::{
-    alpaca,
-    types::{
-        self,
-        alpaca::shared::{Sort, Source},
-        Backfill, News,
-    },
-    utils::{duration_until, FIFTEEN_MINUTES, ONE_MINUTE},
-};
-use std::{
-    collections::{HashMap, HashSet},
-    sync::Arc,
-};
-use tokio::time::sleep;
-
-pub struct Handler {
-    pub config: Arc<Config>,
-}
-
-#[async_trait]
-impl super::Handler for Handler {
-    async fn select_latest_backfills(
-        &self,
-        symbols: &[String],
-    ) -> Result<Vec<Backfill>, clickhouse::error::Error> {
-        database::backfills_news::select_where_symbols(
-            &self.config.clickhouse_client,
-            &self.config.clickhouse_concurrency_limiter,
-            symbols,
-        )
-        .await
-    }
-
-    async fn delete_backfills(&self, symbols: &[String]) -> Result<(), clickhouse::error::Error> {
-        database::backfills_news::delete_where_symbols(
-            &self.config.clickhouse_client,
-            &self.config.clickhouse_concurrency_limiter,
-            symbols,
-        )
-        .await
-    }
-
-    async fn delete_data(&self, symbols: &[String]) -> Result<(), clickhouse::error::Error> {
-        database::news::delete_where_symbols(
-            &self.config.clickhouse_client,
-            &self.config.clickhouse_concurrency_limiter,
-            symbols,
-        )
-        .await
-    }
-
-    async fn queue_backfill(&self, jobs: &NonEmpty<Job>) {
-        if *ALPACA_SOURCE == Source::Sip {
-            return;
-        }
-
-        let fetch_to = jobs.maximum_by_key(|job| job.fetch_to).fetch_to;
-        let run_delay = duration_until(fetch_to + FIFTEEN_MINUTES + ONE_MINUTE);
-        let symbols = jobs.iter().map(|job| &job.symbol).collect::<Vec<_>>();
-
-        info!("Queing news backfill for {:?} in {:?}.", symbols, run_delay);
-        sleep(run_delay).await;
-    }
-
-    #[allow(clippy::too_many_lines)]
-    #[allow(clippy::iter_with_drain)]
-    async fn backfill(&self, jobs: NonEmpty<Job>) {
-        let symbols = Vec::from(jobs.clone().map(|job| job.symbol));
-        let symbols_set = symbols.clone().into_iter().collect::<HashSet<_>>();
-        let fetch_from = jobs.minimum_by_key(|job| job.fetch_from).fetch_from;
-        let fetch_to = jobs.maximum_by_key(|job| job.fetch_to).fetch_to;
-        let freshness = jobs
-            .into_iter()
-            .map(|job| (job.symbol, job.fresh))
-            .collect::<HashMap<_, _>>();
-
-        let mut news = Vec::with_capacity(*CLICKHOUSE_BATCH_NEWS_SIZE);
-        let mut last_times = HashMap::new();
-        let mut next_page_token = None;
-
-        info!("Backfilling news for {:?}.", symbols);
-
-        loop {
-            let message = alpaca::news::get(
-                &self.config.alpaca_client,
-                &self.config.alpaca_rate_limiter,
-                &types::alpaca::api::outgoing::news::News {
-                    symbols: symbols.clone(),
-                    start: Some(fetch_from),
-                    end: Some(fetch_to),
-                    page_token: next_page_token.clone(),
-                    sort: Some(Sort::Asc),
-                    ..Default::default()
-                },
-                None,
-            )
-            .await;
-
-            if let Err(err) = message {
-                error!("Failed to backfill news for {:?}: {:?}.", symbols, err);
-                return;
-            }
-
-            let message = message.unwrap();
-
-            for news_item in message.news {
-                let news_item = News::from(news_item);
-
-                for symbol in &news_item.symbols {
-                    if symbols_set.contains(symbol) {
-                        last_times.insert(symbol.clone(), news_item.time_created);
-                    }
-                }
-
-                news.push(news_item);
-            }
-
-            if news.len() < *CLICKHOUSE_BATCH_NEWS_SIZE && message.next_page_token.is_some() {
-                continue;
-            }
-
-            database::news::upsert_batch(
-                &self.config.clickhouse_client,
-                &self.config.clickhouse_concurrency_limiter,
-                &news,
-            )
-            .await
-            .unwrap();
-
-            let backfilled = last_times
-                .drain()
-                .map(|(symbol, time)| Backfill {
-                    fresh: freshness[&symbol],
-                    symbol,
-                    time,
-                })
-                .collect::<Vec<_>>();
-
-            database::backfills_news::upsert_batch(
-                &self.config.clickhouse_client,
-                &self.config.clickhouse_concurrency_limiter,
-                &backfilled,
-            )
-            .await
-            .unwrap();
-
-            if message.next_page_token.is_none() {
-                break;
-            }
-
-            next_page_token = message.next_page_token;
-            news.clear();
-        }
-
-        database::backfills_news::set_fresh_where_symbols(
-            &self.config.clickhouse_client,
-            &self.config.clickhouse_concurrency_limiter,
-            true,
-            &symbols,
-        )
-        .await
-        .unwrap();
-
-        info!("Backfilled news for {:?}.", symbols);
-    }
-
-    fn max_limit(&self) -> i64 {
-        alpaca::news::MAX_LIMIT
-    }
-
-    fn log_string(&self) -> &'static str {
-        "news"
-    }
-}
-
-pub fn create_handler(config: Arc<Config>) -> Box<dyn super::Handler> {
-    Box::new(Handler { config })
-}

src/threads/data/mod.rs

@@ -1,384 +1,218 @@
-mod backfill;
-mod websocket;
+pub mod asset_status;
+pub mod backfill;
+pub mod websocket;
 
 use super::clock;
 use crate::{
-    config::{Config, ALPACA_API_BASE, ALPACA_SOURCE},
-    create_send_await, database,
-};
-use itertools::{Either, Itertools};
-use log::error;
-use nonempty::NonEmpty;
-use qrust::{
-    alpaca,
-    types::{
-        alpaca::websocket::{
-            ALPACA_CRYPTO_DATA_WEBSOCKET_URL, ALPACA_NEWS_DATA_WEBSOCKET_URL,
-            ALPACA_US_EQUITY_DATA_WEBSOCKET_URL,
-        },
-        Asset, Class,
+    config::{
+        Config, ALPACA_CRYPTO_WEBSOCKET_URL, ALPACA_NEWS_WEBSOCKET_URL, ALPACA_STOCK_WEBSOCKET_URL,
     },
+    types::{Asset, Class, Subset},
+    utils::authenticate,
+};
+use futures_util::StreamExt;
+use std::{
+    collections::{HashMap, HashSet},
+    sync::Arc,
 };
-use std::{collections::HashMap, sync::Arc};
 use tokio::{
     join, select, spawn,
-    sync::{mpsc, oneshot},
+    sync::{mpsc, Mutex, RwLock},
 };
+use tokio_tungstenite::connect_async;
 
-#[derive(Clone, Copy, PartialEq, Eq)]
-#[allow(dead_code)]
-pub enum Action {
-    Add,
-    Enable,
-    Remove,
-    Disable,
+pub struct Guard {
+    pub symbols: HashSet<String>,
+    pub pending_subscriptions: HashMap<String, Asset>,
+    pub pending_unsubscriptions: HashMap<String, Asset>,
 }
 
-pub struct Message {
-    pub action: Action,
-    pub assets: NonEmpty<(String, Class)>,
-    pub response: oneshot::Sender<()>,
-}
-
-impl Message {
-    pub fn new(action: Action, assets: NonEmpty<(String, Class)>) -> (Self, oneshot::Receiver<()>) {
-        let (sender, receiver) = oneshot::channel();
-        (
-            Self {
-                action,
-                assets,
-                response: sender,
-            },
-            receiver,
-        )
-    }
-}
-
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Debug)]
 pub enum ThreadType {
     Bars(Class),
     News,
 }
 
 pub async fn run(
-    config: Arc<Config>,
-    mut receiver: mpsc::Receiver<Message>,
+    app_config: Arc<Config>,
+    mut asset_receiver: mpsc::Receiver<asset_status::Message>,
     mut clock_receiver: mpsc::Receiver<clock::Message>,
 ) {
-    let (bars_us_equity_websocket_sender, bars_us_equity_backfill_sender) =
-        init_thread(config.clone(), ThreadType::Bars(Class::UsEquity));
-    let (bars_crypto_websocket_sender, bars_crypto_backfill_sender) =
-        init_thread(config.clone(), ThreadType::Bars(Class::Crypto));
-    let (news_websocket_sender, news_backfill_sender) =
-        init_thread(config.clone(), ThreadType::News);
+    let (bars_us_equity_asset_status_sender, bars_us_equity_backfill_sender) =
+        init_thread(app_config.clone(), ThreadType::Bars(Class::UsEquity)).await;
+    let (bars_crypto_asset_status_sender, bars_crypto_backfill_sender) =
+        init_thread(app_config.clone(), ThreadType::Bars(Class::Crypto)).await;
+    let (news_asset_status_sender, news_backfill_sender) =
+        init_thread(app_config.clone(), ThreadType::News).await;
 
     loop {
         select! {
-            Some(message) = receiver.recv() => {
-                spawn(handle_message(
-                    config.clone(),
-                    bars_us_equity_websocket_sender.clone(),
-                    bars_us_equity_backfill_sender.clone(),
-                    bars_crypto_websocket_sender.clone(),
-                    bars_crypto_backfill_sender.clone(),
-                    news_websocket_sender.clone(),
-                    news_backfill_sender.clone(),
-                    message,
+            Some(asset_message) = asset_receiver.recv() => {
+                spawn(handle_asset_message(
+                    bars_us_equity_asset_status_sender.clone(),
+                    bars_crypto_asset_status_sender.clone(),
+                    news_asset_status_sender.clone(),
+                    asset_message,
                 ));
             }
             Some(_) = clock_receiver.recv() => {
                 spawn(handle_clock_message(
-                    config.clone(),
                     bars_us_equity_backfill_sender.clone(),
                     bars_crypto_backfill_sender.clone(),
                     news_backfill_sender.clone(),
                 ));
             }
-            else => panic!("Communication channel unexpectedly closed.")
+            else => {
+                panic!("Communication channel unexpectedly closed.")
+            }
         }
     }
 }
 
-fn init_thread(
-    config: Arc<Config>,
+async fn init_thread(
+    app_config: Arc<Config>,
     thread_type: ThreadType,
 ) -> (
-    mpsc::Sender<websocket::Message>,
+    mpsc::Sender<asset_status::Message>,
     mpsc::Sender<backfill::Message>,
 ) {
+    let guard = Arc::new(RwLock::new(Guard {
+        symbols: HashSet::new(),
+        pending_subscriptions: HashMap::new(),
+        pending_unsubscriptions: HashMap::new(),
+    }));
+
     let websocket_url = match thread_type {
-        ThreadType::Bars(Class::UsEquity) => {
-            format!("{}/{}", ALPACA_US_EQUITY_DATA_WEBSOCKET_URL, *ALPACA_SOURCE)
-        }
-        ThreadType::Bars(Class::Crypto) => ALPACA_CRYPTO_DATA_WEBSOCKET_URL.into(),
-        ThreadType::News => ALPACA_NEWS_DATA_WEBSOCKET_URL.into(),
+        ThreadType::Bars(Class::UsEquity) => format!(
+            "{}/{}",
+            ALPACA_STOCK_WEBSOCKET_URL, &app_config.alpaca_source
+        ),
+        ThreadType::Bars(Class::Crypto) => ALPACA_CRYPTO_WEBSOCKET_URL.into(),
+        ThreadType::News => ALPACA_NEWS_WEBSOCKET_URL.into(),
     };
 
-    let backfill_handler = match thread_type {
-        ThreadType::Bars(_) => backfill::bars::create_handler(config.clone(), thread_type),
-        ThreadType::News => backfill::news::create_handler(config.clone()),
-    };
+    let (websocket, _) = connect_async(websocket_url).await.unwrap();
+    let (mut websocket_sender, mut websocket_receiver) = websocket.split();
+    authenticate(&app_config, &mut websocket_sender, &mut websocket_receiver).await;
+    let websocket_sender = Arc::new(Mutex::new(websocket_sender));
 
-    let (backfill_sender, backfill_receiver) = mpsc::channel(100);
-
-    spawn(backfill::run(backfill_handler.into(), backfill_receiver));
-
-    let websocket_handler = match thread_type {
-        ThreadType::Bars(_) => websocket::bars::create_handler(config, thread_type),
-        ThreadType::News => websocket::news::create_handler(config),
-    };
-
-    let (websocket_sender, websocket_receiver) = mpsc::channel(100);
-
-    spawn(websocket::run(
-        websocket_handler.into(),
-        websocket_receiver,
-        websocket_url,
+    let (asset_status_sender, asset_status_receiver) = mpsc::channel(10);
+    spawn(asset_status::run(
+        app_config.clone(),
+        thread_type,
+        guard.clone(),
+        asset_status_receiver,
+        websocket_sender.clone(),
     ));
 
-    (websocket_sender, backfill_sender)
+    let (backfill_sender, backfill_receiver) = mpsc::channel(10);
+    spawn(backfill::run(
+        app_config.clone(),
+        thread_type,
+        guard.clone(),
+        backfill_receiver,
+    ));
+
+    spawn(websocket::run(
+        app_config.clone(),
+        thread_type,
+        guard.clone(),
+        websocket_sender,
+        websocket_receiver,
+        backfill_sender.clone(),
+    ));
+
+    (asset_status_sender, backfill_sender)
 }
 
-#[allow(clippy::too_many_arguments)]
-#[allow(clippy::too_many_lines)]
-async fn handle_message(
-    config: Arc<Config>,
-    bars_us_equity_websocket_sender: mpsc::Sender<websocket::Message>,
-    bars_us_equity_backfill_sender: mpsc::Sender<backfill::Message>,
-    bars_crypto_websocket_sender: mpsc::Sender<websocket::Message>,
-    bars_crypto_backfill_sender: mpsc::Sender<backfill::Message>,
-    news_websocket_sender: mpsc::Sender<websocket::Message>,
-    news_backfill_sender: mpsc::Sender<backfill::Message>,
-    message: Message,
+async fn handle_asset_message(
+    bars_us_equity_asset_status_sender: mpsc::Sender<asset_status::Message>,
+    bars_crypto_asset_status_sender: mpsc::Sender<asset_status::Message>,
+    news_asset_status_sender: mpsc::Sender<asset_status::Message>,
+    asset_status_message: asset_status::Message,
 ) {
-    let (us_equity_symbols, crypto_symbols): (Vec<_>, Vec<_>) = message
+    let (us_equity_assets, crypto_assets): (Vec<_>, Vec<_>) = asset_status_message
         .assets
         .clone()
         .into_iter()
-        .partition_map(|asset| match asset.1 {
-            Class::UsEquity => Either::Left(asset.0),
-            Class::Crypto => Either::Right(asset.0),
-        });
-
-    let symbols = message.assets.map(|(symbol, _)| symbol);
+        .partition(|asset| asset.class == Class::UsEquity);
 
     let bars_us_equity_future = async {
-        if let Some(us_equity_symbols) = NonEmpty::from_vec(us_equity_symbols.clone()) {
-            create_send_await!(
-                bars_us_equity_websocket_sender,
-                websocket::Message::new,
-                message.action.into(),
-                us_equity_symbols
-            );
+        if !us_equity_assets.is_empty() {
+            let (bars_us_equity_asset_status_message, bars_us_equity_asset_status_receiver) =
+                asset_status::Message::new(asset_status_message.action.clone(), us_equity_assets);
+            bars_us_equity_asset_status_sender
+                .send(bars_us_equity_asset_status_message)
+                .await
+                .unwrap();
+            bars_us_equity_asset_status_receiver.await.unwrap();
         }
     };
 
     let bars_crypto_future = async {
-        if let Some(crypto_symbols) = NonEmpty::from_vec(crypto_symbols.clone()) {
-            create_send_await!(
-                bars_crypto_websocket_sender,
-                websocket::Message::new,
-                message.action.into(),
-                crypto_symbols
-            );
+        if !crypto_assets.is_empty() {
+            let (crypto_asset_status_message, crypto_asset_status_receiver) =
+                asset_status::Message::new(asset_status_message.action.clone(), crypto_assets);
+            bars_crypto_asset_status_sender
+                .send(crypto_asset_status_message)
+                .await
+                .unwrap();
+            crypto_asset_status_receiver.await.unwrap();
         }
     };
 
     let news_future = async {
-        create_send_await!(
-            news_websocket_sender,
-            websocket::Message::new,
-            message.action.into(),
-            symbols.clone()
-        );
+        if !asset_status_message.assets.is_empty() {
+            let (news_asset_status_message, news_asset_status_receiver) =
+                asset_status::Message::new(
+                    asset_status_message.action.clone(),
+                    asset_status_message.assets,
+                );
+            news_asset_status_sender
+                .send(news_asset_status_message)
+                .await
+                .unwrap();
+            news_asset_status_receiver.await.unwrap();
+        }
     };
 
     join!(bars_us_equity_future, bars_crypto_future, news_future);
-
-    if message.action == Action::Disable {
-        message.response.send(()).unwrap();
-        return;
-    }
-
-    match message.action {
-        Action::Add | Action::Enable => {
-            let symbols = Vec::from(symbols.clone());
-
-            let assets = async {
-                alpaca::assets::get_by_symbols(
-                    &config.alpaca_client,
-                    &config.alpaca_rate_limiter,
-                    &symbols,
-                    None,
-                    &ALPACA_API_BASE,
-                )
-                .await
-                .unwrap()
-                .into_iter()
-                .map(|asset| (asset.symbol.clone(), asset))
-                .collect::<HashMap<_, _>>()
-            };
-
-            let positions = async {
-                alpaca::positions::get_by_symbols(
-                    &config.alpaca_client,
-                    &config.alpaca_rate_limiter,
-                    &symbols,
-                    None,
-                    &ALPACA_API_BASE,
-                )
-                .await
-                .unwrap()
-                .into_iter()
-                .map(|position| (position.symbol.clone(), position))
-                .collect::<HashMap<_, _>>()
-            };
-
-            let (mut assets, mut positions) = join!(assets, positions);
-
-            let mut batch = Vec::with_capacity(symbols.len());
-
-            for symbol in &symbols {
-                if let Some(asset) = assets.remove(symbol) {
-                    let position = positions.remove(symbol);
-                    batch.push(Asset::from((asset, position)));
-                } else {
-                    error!("Failed to find asset for symbol: {}.", symbol);
-                }
-            }
-
-            database::assets::upsert_batch(
-                &config.clickhouse_client,
-                &config.clickhouse_concurrency_limiter,
-                &batch,
-            )
-            .await
-            .unwrap();
-        }
-        Action::Remove => {
-            database::assets::delete_where_symbols(
-                &config.clickhouse_client,
-                &config.clickhouse_concurrency_limiter,
-                &Vec::from(symbols.clone()),
-            )
-            .await
-            .unwrap();
-        }
-        Action::Disable => unreachable!(),
-    }
-
-    let bars_us_equity_future = async {
-        if let Some(us_equity_symbols) = NonEmpty::from_vec(us_equity_symbols) {
-            create_send_await!(
-                bars_us_equity_backfill_sender,
-                backfill::Message::new,
-                match message.action {
-                    Action::Add | Action::Enable => backfill::Action::Backfill,
-                    Action::Remove => backfill::Action::Purge,
-                    Action::Disable => unreachable!(),
-                },
-                us_equity_symbols
-            );
-        }
-    };
-
-    let bars_crypto_future = async {
-        if let Some(crypto_symbols) = NonEmpty::from_vec(crypto_symbols) {
-            create_send_await!(
-                bars_crypto_backfill_sender,
-                backfill::Message::new,
-                match message.action {
-                    Action::Add | Action::Enable => backfill::Action::Backfill,
-                    Action::Remove => backfill::Action::Purge,
-                    Action::Disable => unreachable!(),
-                },
-                crypto_symbols
-            );
-        }
-    };
-
-    let news_future = async {
-        create_send_await!(
-            news_backfill_sender,
-            backfill::Message::new,
-            match message.action {
-                Action::Add | Action::Enable => backfill::Action::Backfill,
-                Action::Remove => backfill::Action::Purge,
-                Action::Disable => unreachable!(),
-            },
-            symbols
-        );
-    };
-
-    join!(bars_us_equity_future, bars_crypto_future, news_future);
-
-    message.response.send(()).unwrap();
+    asset_status_message.response.send(()).unwrap();
 }
 
 async fn handle_clock_message(
-    config: Arc<Config>,
     bars_us_equity_backfill_sender: mpsc::Sender<backfill::Message>,
     bars_crypto_backfill_sender: mpsc::Sender<backfill::Message>,
     news_backfill_sender: mpsc::Sender<backfill::Message>,
 ) {
-    database::cleanup_all(
-        &config.clickhouse_client,
-        &config.clickhouse_concurrency_limiter,
-    )
-    .await
-    .unwrap();
-
-    let assets = database::assets::select(
-        &config.clickhouse_client,
-        &config.clickhouse_concurrency_limiter,
-    )
-    .await
-    .unwrap();
-
-    let (us_equity_symbols, crypto_symbols): (Vec<_>, Vec<_>) = assets
-        .clone()
-        .into_iter()
-        .partition_map(|asset| match asset.class {
-            Class::UsEquity => Either::Left(asset.symbol),
-            Class::Crypto => Either::Right(asset.symbol),
-        });
-
-    let symbols = assets
-        .into_iter()
-        .map(|asset| asset.symbol)
-        .collect::<Vec<_>>();
-
     let bars_us_equity_future = async {
-        if let Some(us_equity_symbols) = NonEmpty::from_vec(us_equity_symbols) {
-            create_send_await!(
-                bars_us_equity_backfill_sender,
-                backfill::Message::new,
-                backfill::Action::Backfill,
-                us_equity_symbols
-            );
-        }
+        let (bars_us_equity_backfill_message, bars_us_equity_backfill_receiver) =
+            backfill::Message::new(backfill::Action::Backfill, Subset::All);
+        bars_us_equity_backfill_sender
+            .send(bars_us_equity_backfill_message)
+            .await
+            .unwrap();
+        bars_us_equity_backfill_receiver.await.unwrap();
     };
 
     let bars_crypto_future = async {
-        if let Some(crypto_symbols) = NonEmpty::from_vec(crypto_symbols) {
-            create_send_await!(
-                bars_crypto_backfill_sender,
-                backfill::Message::new,
-                backfill::Action::Backfill,
-                crypto_symbols
-            );
-        }
+        let (bars_crypto_backfill_message, bars_crypto_backfill_receiver) =
+            backfill::Message::new(backfill::Action::Backfill, Subset::All);
+        bars_crypto_backfill_sender
+            .send(bars_crypto_backfill_message)
+            .await
+            .unwrap();
+        bars_crypto_backfill_receiver.await.unwrap();
     };
 
     let news_future = async {
-        if let Some(symbols) = NonEmpty::from_vec(symbols) {
-            create_send_await!(
-                news_backfill_sender,
-                backfill::Message::new,
-                backfill::Action::Backfill,
-                symbols
-            );
-        }
+        let (news_backfill_message, news_backfill_receiver) =
+            backfill::Message::new(backfill::Action::Backfill, Subset::All);
+        news_backfill_sender
+            .send(news_backfill_message)
+            .await
+            .unwrap();
+        news_backfill_receiver.await.unwrap();
     };
 
     join!(bars_us_equity_future, bars_crypto_future, news_future);

src/threads/data/websocket.rs

@@ -0,0 +1,256 @@
+use super::{backfill, Guard, ThreadType};
+use crate::{
+    config::Config,
+    database,
+    types::{alpaca::websocket, ollama, Bar, News, Subset},
+};
+use backoff::{future::retry, ExponentialBackoff};
+use futures_util::{
+    stream::{SplitSink, SplitStream},
+    SinkExt, StreamExt,
+};
+use log::{error, info, warn};
+use serde_json::{from_str, to_string};
+use std::{
+    collections::{HashMap, HashSet},
+    sync::Arc,
+};
+use tokio::{
+    join,
+    net::TcpStream,
+    spawn,
+    sync::{mpsc, Mutex, RwLock},
+};
+use tokio_tungstenite::{tungstenite, MaybeTlsStream, WebSocketStream};
+
+pub async fn run(
+    app_config: Arc<Config>,
+    thread_type: ThreadType,
+    guard: Arc<RwLock<Guard>>,
+    websocket_sender: Arc<
+        Mutex<SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, tungstenite::Message>>,
+    >,
+    mut websocket_receiver: SplitStream<WebSocketStream<MaybeTlsStream<TcpStream>>>,
+    backfill_sender: mpsc::Sender<backfill::Message>,
+) {
+    loop {
+        let message = websocket_receiver.next().await.unwrap().unwrap();
+
+        spawn(handle_websocket_message(
+            app_config.clone(),
+            thread_type,
+            guard.clone(),
+            websocket_sender.clone(),
+            backfill_sender.clone(),
+            message,
+        ));
+    }
+}
+
+async fn handle_websocket_message(
+    app_config: Arc<Config>,
+    thread_type: ThreadType,
+    guard: Arc<RwLock<Guard>>,
+    websocket_sender: Arc<
+        Mutex<SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, tungstenite::Message>>,
+    >,
+    backfill_sender: mpsc::Sender<backfill::Message>,
+    message: tungstenite::Message,
+) {
+    match message {
+        tungstenite::Message::Text(message) => {
+            let message = from_str::<Vec<websocket::incoming::Message>>(&message);
+
+            if let Ok(message) = message {
+                for message in message {
+                    spawn(handle_parsed_websocket_message(
+                        app_config.clone(),
+                        thread_type,
+                        guard.clone(),
+                        backfill_sender.clone(),
+                        message,
+                    ));
+                }
+            } else {
+                error!(
+                    "{:?} - Failed to deserialize websocket message: {:?}",
+                    thread_type, message
+                );
+            }
+        }
+        tungstenite::Message::Ping(_) => {
+            websocket_sender
+                .lock()
+                .await
+                .send(tungstenite::Message::Pong(vec![]))
+                .await
+                .unwrap();
+        }
+        _ => error!(
+            "{:?} - Unexpected websocket message: {:?}",
+            thread_type, message
+        ),
+    }
+}
+
+#[allow(clippy::significant_drop_tightening)]
+#[allow(clippy::too_many_lines)]
+async fn handle_parsed_websocket_message(
+    app_config: Arc<Config>,
+    thread_type: ThreadType,
+    guard: Arc<RwLock<Guard>>,
+    backfill_sender: mpsc::Sender<backfill::Message>,
+    message: websocket::incoming::Message,
+) {
+    match message {
+        websocket::incoming::Message::Subscription(message) => {
+            let symbols = match message {
+                websocket::incoming::subscription::Message::Market(message) => message.bars,
+                websocket::incoming::subscription::Message::News(message) => message.news,
+            };
+
+            let mut guard = guard.write().await;
+
+            let newly_subscribed = guard
+                .pending_subscriptions
+                .extract_if(|symbol, _| symbols.contains(symbol))
+                .collect::<HashMap<_, _>>();
+
+            let newly_unsubscribed = guard
+                .pending_unsubscriptions
+                .extract_if(|symbol, _| !symbols.contains(symbol))
+                .collect::<HashMap<_, _>>();
+
+            drop(guard);
+
+            let newly_subscribed_future = async {
+                if !newly_subscribed.is_empty() {
+                    info!(
+                        "{:?} - Subscribed to {:?}.",
+                        thread_type,
+                        newly_subscribed.keys().collect::<Vec<_>>()
+                    );
+
+                    let (backfill_message, backfill_receiver) = backfill::Message::new(
+                        backfill::Action::Backfill,
+                        Subset::Some(newly_subscribed.into_values().collect::<Vec<_>>()),
+                    );
+
+                    backfill_sender.send(backfill_message).await.unwrap();
+                    backfill_receiver.await.unwrap();
+                }
+            };
+
+            let newly_unsubscribed_future = async {
+                if !newly_unsubscribed.is_empty() {
+                    info!(
+                        "{:?} - Unsubscribed from {:?}.",
+                        thread_type,
+                        newly_unsubscribed.keys().collect::<Vec<_>>()
+                    );
+
+                    let (purge_message, purge_receiver) = backfill::Message::new(
+                        backfill::Action::Purge,
+                        Subset::Some(newly_unsubscribed.into_values().collect::<Vec<_>>()),
+                    );
+
+                    backfill_sender.send(purge_message).await.unwrap();
+                    purge_receiver.await.unwrap();
+                }
+            };
+
+            join!(newly_subscribed_future, newly_unsubscribed_future);
+        }
+        websocket::incoming::Message::Bar(message)
+        | websocket::incoming::Message::UpdatedBar(message) => {
+            let bar = Bar::from(message);
+
+            let guard = guard.read().await;
+            if guard.symbols.get(&bar.symbol).is_none() {
+                warn!(
+                    "{:?} - Race condition: received bar for unsubscribed symbol: {:?}.",
+                    thread_type, bar.symbol
+                );
+                return;
+            }
+
+            info!(
+                "{:?} - Received bar for {}: {}.",
+                thread_type, bar.symbol, bar.time
+            );
+            database::bars::upsert(&app_config.clickhouse_client, &bar).await;
+        }
+        websocket::incoming::Message::News(message) => {
+            let mut news = News::from(message);
+            let symbols = news.symbols.clone().into_iter().collect::<HashSet<_>>();
+
+            let guard = guard.read().await;
+            if !guard.symbols.iter().any(|symbol| symbols.contains(symbol)) {
+                warn!(
+                    "{:?} - Race condition: received news for unsubscribed symbols: {:?}.",
+                    thread_type, news.symbols
+                );
+                return;
+            }
+
+            info!(
+                "{:?} - Received news for {:?}: {}.",
+                thread_type, news.symbols, news.time_created
+            );
+
+            info!(
+                "{:?} - Getting sentiment for news: {}.",
+                thread_type, news.headline
+            );
+
+            let prediction = retry(ExponentialBackoff::default(), || async {
+                let response = app_config
+                    .ollama_client
+                    .post(format!("{}/api/chat", app_config.ollama_url))
+                    .body(
+                        to_string(&ollama::outgoing::sentiment::Sentiment::new(
+                            app_config.ollama_model.clone(),
+                            &news.clone().into(),
+                        ))
+                        .unwrap(),
+                    )
+                    .send()
+                    .await
+                    .unwrap()
+                    .json::<ollama::incoming::sentiment::Response>()
+                    .await
+                    .unwrap();
+
+                from_str::<ollama::incoming::sentiment::Prediction>(&response.message.content)
+                    .map_err(Into::into)
+            })
+            .await;
+
+            match prediction {
+                Ok(prediction) => {
+                    info!(
+                        "{:?} - Received sentiment for news: {:?}.",
+                        thread_type, prediction
+                    );
+                    news.sentiment = prediction.sentiment.into();
+                    news.confidence = prediction.confidence.into();
+                }
+                Err(e) => {
+                    error!(
+                        "{:?} - Failed to get sentiment for news: {:?}.",
+                        thread_type, e
+                    );
+                }
+            }
+
+            database::news::upsert(&app_config.clickhouse_client, &news).await;
+        }
+        websocket::incoming::Message::Success(_) => {}
+        websocket::incoming::Message::Error(message) => {
+            error!(
+                "{:?} - Received error message: {}.",
+                thread_type, message.message
+            );
+        }
+    }
+}

src/threads/data/websocket/bars.rs

@@ -1,171 +0,0 @@
-use super::State;
-use crate::{
-    config::{Config, CLICKHOUSE_BATCH_BARS_SIZE},
-    database,
-    threads::data::ThreadType,
-};
-use async_trait::async_trait;
-use clickhouse::inserter::Inserter;
-use log::{debug, error, info};
-use nonempty::NonEmpty;
-use qrust::{
-    types::{alpaca::websocket, Bar, Class},
-    utils::ONE_SECOND,
-};
-use std::{
-    collections::{HashMap, HashSet},
-    sync::Arc,
-};
-use tokio::sync::{Mutex, RwLock};
-
-pub struct Handler {
-    pub config: Arc<Config>,
-    pub inserter: Arc<Mutex<Inserter<Bar>>>,
-    pub subscription_message_constructor:
-        fn(NonEmpty<String>) -> websocket::data::outgoing::subscribe::Message,
-}
-
-#[async_trait]
-impl super::Handler for Handler {
-    fn create_subscription_message(
-        &self,
-        symbols: NonEmpty<String>,
-    ) -> websocket::data::outgoing::subscribe::Message {
-        (self.subscription_message_constructor)(symbols)
-    }
-
-    async fn handle_websocket_message(
-        &self,
-        state: Arc<RwLock<State>>,
-        message: websocket::data::incoming::Message,
-    ) {
-        match message {
-            websocket::data::incoming::Message::Subscription(message) => {
-                let websocket::data::incoming::subscription::Message::Market {
-                    bars: symbols, ..
-                } = message
-                else {
-                    unreachable!()
-                };
-
-                let symbols = symbols.into_iter().collect::<HashSet<_>>();
-                let mut state = state.write().await;
-
-                let newly_subscribed = state
-                    .pending_subscriptions
-                    .extract_if(|symbol, _| symbols.contains(symbol))
-                    .collect::<HashMap<_, _>>();
-
-                let newly_unsubscribed = state
-                    .pending_unsubscriptions
-                    .extract_if(|symbol, _| !symbols.contains(symbol))
-                    .collect::<HashMap<_, _>>();
-
-                state
-                    .active_subscriptions
-                    .extend(newly_subscribed.keys().cloned());
-
-                drop(state);
-
-                if !newly_subscribed.is_empty() {
-                    info!(
-                        "Subscribed to bars for {:?}.",
-                        newly_subscribed.keys().collect::<Vec<_>>()
-                    );
-
-                    for sender in newly_subscribed.into_values() {
-                        sender.send(()).unwrap();
-                    }
-                }
-
-                if !newly_unsubscribed.is_empty() {
-                    info!(
-                        "Unsubscribed from bars for {:?}.",
-                        newly_unsubscribed.keys().collect::<Vec<_>>()
-                    );
-
-                    for sender in newly_unsubscribed.into_values() {
-                        sender.send(()).unwrap();
-                    }
-                }
-            }
-            websocket::data::incoming::Message::Bar(message)
-            | websocket::data::incoming::Message::UpdatedBar(message) => {
-                let bar = Bar::from(message);
-                debug!("Received bar for {}: {}.", bar.symbol, bar.time);
-                self.inserter.lock().await.write(&bar).await.unwrap();
-            }
-            websocket::data::incoming::Message::Status(message) => {
-                debug!(
-                    "Received status message for {}: {:?}.",
-                    message.symbol, message.status
-                );
-
-                match message.status {
-                    websocket::data::incoming::status::Status::TradingHalt(_)
-                    | websocket::data::incoming::status::Status::VolatilityTradingPause(_) => {
-                        database::assets::update_status_where_symbol(
-                            &self.config.clickhouse_client,
-                            &self.config.clickhouse_concurrency_limiter,
-                            &message.symbol,
-                            false,
-                        )
-                        .await
-                        .unwrap();
-                    }
-                    websocket::data::incoming::status::Status::Resume(_)
-                    | websocket::data::incoming::status::Status::TradingResumption(_) => {
-                        database::assets::update_status_where_symbol(
-                            &self.config.clickhouse_client,
-                            &self.config.clickhouse_concurrency_limiter,
-                            &message.symbol,
-                            true,
-                        )
-                        .await
-                        .unwrap();
-                    }
-                    _ => {}
-                }
-            }
-            websocket::data::incoming::Message::Error(message) => {
-                error!("Received error message: {}.", message.message);
-            }
-            _ => unreachable!(),
-        }
-    }
-
-    fn log_string(&self) -> &'static str {
-        "bars"
-    }
-
-    async fn run_inserter(&self) {
-        super::run_inserter(self.inserter.clone()).await;
-    }
-}
-
-pub fn create_handler(config: Arc<Config>, thread_type: ThreadType) -> Box<dyn super::Handler> {
-    let inserter = Arc::new(Mutex::new(
-        config
-            .clickhouse_client
-            .inserter("bars")
-            .unwrap()
-            .with_period(Some(ONE_SECOND))
-            .with_max_entries((*CLICKHOUSE_BATCH_BARS_SIZE).try_into().unwrap()),
-    ));
-
-    let subscription_message_constructor = match thread_type {
-        ThreadType::Bars(Class::UsEquity) => {
-            websocket::data::outgoing::subscribe::Message::new_market_us_equity
-        }
-        ThreadType::Bars(Class::Crypto) => {
-            websocket::data::outgoing::subscribe::Message::new_market_crypto
-        }
-        _ => unreachable!(),
-    };
-
-    Box::new(Handler {
-        config,
-        inserter,
-        subscription_message_constructor,
-    })
-}

src/threads/data/websocket/mod.rs

@@ -1,353 +0,0 @@
-pub mod bars;
-pub mod news;
-
-use crate::config::{ALPACA_API_KEY, ALPACA_API_SECRET};
-use async_trait::async_trait;
-use backoff::{future::retry_notify, ExponentialBackoff};
-use clickhouse::{inserter::Inserter, Row};
-use futures_util::{future::join_all, SinkExt, StreamExt};
-use log::error;
-use nonempty::NonEmpty;
-use qrust::types::alpaca::{self, websocket};
-use serde::Serialize;
-use serde_json::{from_str, to_string};
-use std::{
-    collections::{HashMap, HashSet},
-    sync::Arc,
-    time::Duration,
-};
-use tokio::{
-    net::TcpStream,
-    select, spawn,
-    sync::{mpsc, oneshot, Mutex, RwLock},
-};
-use tokio_tungstenite::{connect_async, tungstenite, MaybeTlsStream, WebSocketStream};
-
-pub enum Action {
-    Subscribe,
-    Unsubscribe,
-}
-
-impl From<super::Action> for Option<Action> {
-    fn from(action: super::Action) -> Self {
-        match action {
-            super::Action::Add | super::Action::Enable => Some(Action::Subscribe),
-            super::Action::Remove | super::Action::Disable => Some(Action::Unsubscribe),
-        }
-    }
-}
-
-pub struct Message {
-    pub action: Option<Action>,
-    pub symbols: NonEmpty<String>,
-    pub response: oneshot::Sender<()>,
-}
-
-impl Message {
-    pub fn new(action: Option<Action>, symbols: NonEmpty<String>) -> (Self, oneshot::Receiver<()>) {
-        let (sender, receiver) = oneshot::channel();
-        (
-            Self {
-                action,
-                symbols,
-                response: sender,
-            },
-            receiver,
-        )
-    }
-}
-
-pub struct State {
-    pub active_subscriptions: HashSet<String>,
-    pub pending_subscriptions: HashMap<String, oneshot::Sender<()>>,
-    pub pending_unsubscriptions: HashMap<String, oneshot::Sender<()>>,
-}
-
-#[async_trait]
-pub trait Handler: Send + Sync + 'static {
-    fn create_subscription_message(
-        &self,
-        symbols: NonEmpty<String>,
-    ) -> websocket::data::outgoing::subscribe::Message;
-    async fn handle_websocket_message(
-        &self,
-        state: Arc<RwLock<State>>,
-        message: websocket::data::incoming::Message,
-    );
-    fn log_string(&self) -> &'static str;
-    async fn run_inserter(&self);
-}
-
-pub async fn run(
-    handler: Arc<Box<dyn Handler>>,
-    mut receiver: mpsc::Receiver<Message>,
-    websocket_url: String,
-) {
-    let state = Arc::new(RwLock::new(State {
-        active_subscriptions: HashSet::new(),
-        pending_subscriptions: HashMap::new(),
-        pending_unsubscriptions: HashMap::new(),
-    }));
-
-    let handler_clone = handler.clone();
-    spawn(async move { handler_clone.run_inserter().await });
-
-    let (sink_sender, sink_receiver) = mpsc::channel(100);
-    let (stream_sender, mut stream_receiver) = mpsc::channel(10_000);
-
-    spawn(run_connection(
-        handler.clone(),
-        sink_receiver,
-        stream_sender,
-        websocket_url.clone(),
-        state.clone(),
-    ));
-
-    loop {
-        select! {
-            Some(message) = receiver.recv() => {
-                spawn(handle_message(
-                    handler.clone(),
-                    state.clone(),
-                    sink_sender.clone(),
-                    message,
-                ));
-            }
-            Some(message) = stream_receiver.recv() => {
-                match message {
-                    tungstenite::Message::Text(message) => {
-                        let parsed_message = from_str::<Vec<websocket::data::incoming::Message>>(&message);
-
-                        if parsed_message.is_err() {
-                            error!("Failed to deserialize websocket message: {:?}.", message);
-                            continue;
-                        }
-
-                        for message in parsed_message.unwrap() {
-                            let handler = handler.clone();
-                            let state = state.clone();
-                            spawn(async move {
-                                handler.handle_websocket_message(state, message).await;
-                            });
-                        }
-                    }
-                    tungstenite::Message::Ping(_) => {}
-                    _ => error!("Unexpected websocket message: {:?}.", message),
-                }
-            }
-            else => panic!("Communication channel unexpectedly closed.")
-        }
-    }
-}
-
-#[allow(clippy::too_many_lines)]
-async fn run_connection(
-    handler: Arc<Box<dyn Handler>>,
-    mut sink_receiver: mpsc::Receiver<tungstenite::Message>,
-    stream_sender: mpsc::Sender<tungstenite::Message>,
-    websocket_url: String,
-    state: Arc<RwLock<State>>,
-) {
-    let mut peek = None;
-
-    'connection: loop {
-        let (websocket, _): (WebSocketStream<MaybeTlsStream<TcpStream>>, _) = retry_notify(
-            ExponentialBackoff::default(),
-            || async {
-                connect_async(websocket_url.clone())
-                    .await
-                    .map_err(Into::into)
-            },
-            |e, duration: Duration| {
-                error!(
-                    "Failed to connect to {} websocket, will retry in {} seconds: {}.",
-                    handler.log_string(),
-                    duration.as_secs(),
-                    e
-                );
-            },
-        )
-        .await
-        .unwrap();
-
-        let (mut sink, mut stream) = websocket.split();
-        alpaca::websocket::data::authenticate(
-            &mut sink,
-            &mut stream,
-            (*ALPACA_API_KEY).to_string(),
-            (*ALPACA_API_SECRET).to_string(),
-        )
-        .await;
-
-        let mut state = state.write().await;
-
-        state
-            .pending_unsubscriptions
-            .drain()
-            .for_each(|(_, sender)| {
-                sender.send(()).unwrap();
-            });
-
-        let (recovered_subscriptions, receivers) = state
-            .active_subscriptions
-            .iter()
-            .map(|symbol| {
-                let (sender, receiver) = oneshot::channel();
-                ((symbol.clone(), sender), receiver)
-            })
-            .unzip::<_, _, Vec<_>, Vec<_>>();
-
-        state.pending_subscriptions.extend(recovered_subscriptions);
-
-        let pending_subscriptions = state
-            .pending_subscriptions
-            .keys()
-            .cloned()
-            .collect::<Vec<_>>();
-
-        drop(state);
-
-        if let Some(pending_subscriptions) = NonEmpty::from_vec(pending_subscriptions) {
-            if let Err(err) = sink
-                .send(tungstenite::Message::Text(
-                    to_string(&websocket::data::outgoing::Message::Subscribe(
-                        handler.create_subscription_message(pending_subscriptions),
-                    ))
-                    .unwrap(),
-                ))
-                .await
-            {
-                error!("Failed to send websocket message: {:?}.", err);
-                continue;
-            }
-        }
-
-        join_all(receivers).await;
-
-        if peek.is_some() {
-            if let Err(err) = sink.send(peek.clone().unwrap()).await {
-                error!("Failed to send websocket message: {:?}.", err);
-                continue;
-            }
-            peek = None;
-        }
-
-        loop {
-            select! {
-                Some(message) = sink_receiver.recv() => {
-                    peek = Some(message.clone());
-
-                    if let Err(err) = sink.send(message).await {
-                        error!("Failed to send websocket message: {:?}.", err);
-                        continue 'connection;
-                    };
-
-                    peek = None;
-                }
-                message = stream.next() => {
-                    if message.is_none() {
-                        error!("Websocket stream unexpectedly closed.");
-                        continue 'connection;
-                    }
-
-                    let message = message.unwrap();
-
-                    if let Err(err) = message {
-                        error!("Failed to receive websocket message: {:?}.", err);
-                        continue 'connection;
-                    }
-
-                    let message = message.unwrap();
-
-                    if message.is_close() {
-                        error!("Websocket connection closed.");
-                        continue 'connection;
-                    }
-
-                    stream_sender.send(message).await.unwrap();
-                }
-                else => error!("Communication channel unexpectedly closed.")
-            }
-        }
-    }
-}
-
-async fn handle_message(
-    handler: Arc<Box<dyn Handler>>,
-    pending: Arc<RwLock<State>>,
-    sink_sender: mpsc::Sender<tungstenite::Message>,
-    message: Message,
-) {
-    match message.action {
-        Some(Action::Subscribe) => {
-            let (pending_subscriptions, receivers) = message
-                .symbols
-                .iter()
-                .map(|symbol| {
-                    let (sender, receiver) = oneshot::channel();
-                    ((symbol.clone(), sender), receiver)
-                })
-                .unzip::<_, _, Vec<_>, Vec<_>>();
-
-            pending
-                .write()
-                .await
-                .pending_subscriptions
-                .extend(pending_subscriptions);
-
-            sink_sender
-                .send(tungstenite::Message::Text(
-                    to_string(&websocket::data::outgoing::Message::Subscribe(
-                        handler.create_subscription_message(message.symbols),
-                    ))
-                    .unwrap(),
-                ))
-                .await
-                .unwrap();
-
-            join_all(receivers).await;
-        }
-        Some(Action::Unsubscribe) => {
-            let (pending_unsubscriptions, receivers): (Vec<_>, Vec<_>) = message
-                .symbols
-                .iter()
-                .map(|symbol| {
-                    let (sender, receiver) = oneshot::channel();
-                    ((symbol.clone(), sender), receiver)
-                })
-                .unzip();
-
-            pending
-                .write()
-                .await
-                .pending_unsubscriptions
-                .extend(pending_unsubscriptions);
-
-            sink_sender
-                .send(tungstenite::Message::Text(
-                    to_string(&websocket::data::outgoing::Message::Unsubscribe(
-                        handler.create_subscription_message(message.symbols.clone()),
-                    ))
-                    .unwrap(),
-                ))
-                .await
-                .unwrap();
-
-            join_all(receivers).await;
-        }
-        None => {}
-    }
-
-    message.response.send(()).unwrap();
-}
-
-async fn run_inserter<T>(inserter: Arc<Mutex<Inserter<T>>>)
-where
-    T: Row + Serialize,
-{
-    loop {
-        let time_left = inserter.lock().await.time_left().unwrap();
-        tokio::time::sleep(time_left).await;
-        inserter.lock().await.commit().await.unwrap();
-    }
-}

src/threads/data/websocket/news.rs

@@ -1,116 +0,0 @@
-use super::State;
-use crate::config::{Config, CLICKHOUSE_BATCH_NEWS_SIZE};
-use async_trait::async_trait;
-use clickhouse::inserter::Inserter;
-use log::{debug, error, info};
-use nonempty::NonEmpty;
-use qrust::{
-    types::{alpaca::websocket, News},
-    utils::ONE_SECOND,
-};
-use std::{collections::HashMap, sync::Arc};
-use tokio::sync::{Mutex, RwLock};
-
-pub struct Handler {
-    pub config: Arc<Config>,
-    pub inserter: Arc<Mutex<Inserter<News>>>,
-}
-
-#[async_trait]
-impl super::Handler for Handler {
-    fn create_subscription_message(
-        &self,
-        symbols: NonEmpty<String>,
-    ) -> websocket::data::outgoing::subscribe::Message {
-        websocket::data::outgoing::subscribe::Message::new_news(symbols)
-    }
-
-    async fn handle_websocket_message(
-        &self,
-        state: Arc<RwLock<State>>,
-        message: websocket::data::incoming::Message,
-    ) {
-        match message {
-            websocket::data::incoming::Message::Subscription(message) => {
-                let websocket::data::incoming::subscription::Message::News { news: symbols } =
-                    message
-                else {
-                    unreachable!()
-                };
-
-                let mut state = state.write().await;
-
-                let newly_subscribed = state
-                    .pending_subscriptions
-                    .extract_if(|symbol, _| symbols.contains(symbol))
-                    .collect::<HashMap<_, _>>();
-
-                let newly_unsubscribed = state
-                    .pending_unsubscriptions
-                    .extract_if(|symbol, _| !symbols.contains(symbol))
-                    .collect::<HashMap<_, _>>();
-
-                state
-                    .active_subscriptions
-                    .extend(newly_subscribed.keys().cloned());
-
-                drop(state);
-
-                if !newly_subscribed.is_empty() {
-                    info!(
-                        "Subscribed to news for {:?}.",
-                        newly_subscribed.keys().collect::<Vec<_>>()
-                    );
-
-                    for sender in newly_subscribed.into_values() {
-                        sender.send(()).unwrap();
-                    }
-                }
-
-                if !newly_unsubscribed.is_empty() {
-                    info!(
-                        "Unsubscribed from news for {:?}.",
-                        newly_unsubscribed.keys().collect::<Vec<_>>()
-                    );
-
-                    for sender in newly_unsubscribed.into_values() {
-                        sender.send(()).unwrap();
-                    }
-                }
-            }
-            websocket::data::incoming::Message::News(message) => {
-                let news = News::from(message);
-                debug!(
-                    "Received news for {:?}: {}.",
-                    news.symbols, news.time_created
-                );
-                self.inserter.lock().await.write(&news).await.unwrap();
-            }
-            websocket::data::incoming::Message::Error(message) => {
-                error!("Received error message: {}.", message.message);
-            }
-            _ => unreachable!(),
-        }
-    }
-
-    fn log_string(&self) -> &'static str {
-        "news"
-    }
-
-    async fn run_inserter(&self) {
-        super::run_inserter(self.inserter.clone()).await;
-    }
-}
-
-pub fn create_handler(config: Arc<Config>) -> Box<dyn super::Handler> {
-    let inserter = Arc::new(Mutex::new(
-        config
-            .clickhouse_client
-            .inserter("news")
-            .unwrap()
-            .with_period(Some(ONE_SECOND))
-            .with_max_entries((*CLICKHOUSE_BATCH_NEWS_SIZE).try_into().unwrap()),
-    ));
-
-    Box::new(Handler { config, inserter })
-}

src/threads/mod.rs

@@ -1,3 +1,2 @@
 pub mod clock;
 pub mod data;
-pub mod trading;

src/threads/trading/mod.rs

@@ -1,27 +0,0 @@
-mod websocket;
-
-use crate::config::{Config, ALPACA_API_BASE, ALPACA_API_KEY, ALPACA_API_SECRET};
-use futures_util::StreamExt;
-use qrust::types::alpaca;
-use std::sync::Arc;
-use tokio::spawn;
-use tokio_tungstenite::connect_async;
-
-pub async fn run(config: Arc<Config>) {
-    let (websocket, _) =
-        connect_async(&format!("wss://{}.alpaca.markets/stream", *ALPACA_API_BASE))
-            .await
-            .unwrap();
-    let (mut websocket_sink, mut websocket_stream) = websocket.split();
-
-    alpaca::websocket::trading::authenticate(
-        &mut websocket_sink,
-        &mut websocket_stream,
-        (*ALPACA_API_KEY).to_string(),
-        (*ALPACA_API_SECRET).to_string(),
-    )
-    .await;
-    alpaca::websocket::trading::subscribe(&mut websocket_sink, &mut websocket_stream).await;
-
-    spawn(websocket::run(config, websocket_stream));
-}

src/threads/trading/websocket.rs

@@ -1,79 +0,0 @@
-use crate::{config::Config, database};
-use futures_util::{stream::SplitStream, StreamExt};
-use log::{debug, error};
-use qrust::types::{alpaca::websocket, Order};
-use serde_json::from_str;
-use std::sync::Arc;
-use tokio::{net::TcpStream, spawn};
-use tokio_tungstenite::{tungstenite, MaybeTlsStream, WebSocketStream};
-
-pub async fn run(
-    config: Arc<Config>,
-    mut websocket_stream: SplitStream<WebSocketStream<MaybeTlsStream<TcpStream>>>,
-) {
-    loop {
-        let message = websocket_stream.next().await.unwrap().unwrap();
-
-        match message {
-            tungstenite::Message::Binary(message) => {
-                let parsed_message = from_str::<websocket::trading::incoming::Message>(
-                    &String::from_utf8_lossy(&message),
-                );
-
-                if parsed_message.is_err() {
-                    error!("Failed to deserialize websocket message: {:?}.", message);
-                    continue;
-                }
-
-                spawn(handle_websocket_message(
-                    config.clone(),
-                    parsed_message.unwrap(),
-                ));
-            }
-            tungstenite::Message::Ping(_) => {}
-            _ => error!("Unexpected websocket message: {:?}.", message),
-        }
-    }
-}
-
-async fn handle_websocket_message(
-    config: Arc<Config>,
-    message: websocket::trading::incoming::Message,
-) {
-    match message {
-        websocket::trading::incoming::Message::Order(message) => {
-            debug!(
-                "Received order message for {}: {:?}.",
-                message.order.symbol, message.event
-            );
-
-            let order = Order::from(message.order);
-
-            database::orders::upsert(
-                &config.clickhouse_client,
-                &config.clickhouse_concurrency_limiter,
-                &order,
-            )
-            .await
-            .unwrap();
-
-            match message.event {
-                websocket::trading::incoming::order::Event::Fill { position_qty, .. }
-                | websocket::trading::incoming::order::Event::PartialFill {
-                    position_qty, ..
-                } => {
-                    database::assets::update_qty_where_symbol(
-                        &config.clickhouse_client,
-                        &config.clickhouse_concurrency_limiter,
-                        &order.symbol,
-                        position_qty,
-                    )
-                    .await
-                    .unwrap();
-                }
-                _ => (),
-            }
-        }
-        _ => unreachable!(),
-    }
-}

src/types/algebraic/mod.rs

@@ -0,0 +1,3 @@
+pub mod subset;
+
+pub use subset::Subset;

src/types/algebraic/subset.rs

@@ -0,0 +1,5 @@
+#[derive(Clone, Debug)]
+pub enum Subset<T> {
+    Some(Vec<T>),
+    All,
+}

src/types/alpaca/api/incoming/asset.rs

@@ -0,0 +1,74 @@
+use crate::types::{self, alpaca::api::impl_from_enum};
+use serde::Deserialize;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Class {
+    UsEquity,
+    Crypto,
+}
+
+impl_from_enum!(types::Class, Class, UsEquity, Crypto);
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)]
+#[serde(rename_all = "UPPERCASE")]
+pub enum Exchange {
+    Amex,
+    Arca,
+    Bats,
+    Nyse,
+    Nasdaq,
+    Nysearca,
+    Otc,
+    Crypto,
+}
+
+impl_from_enum!(
+    types::Exchange,
+    Exchange,
+    Amex,
+    Arca,
+    Bats,
+    Nyse,
+    Nasdaq,
+    Nysearca,
+    Otc,
+    Crypto
+);
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum Status {
+    Active,
+    Inactive,
+}
+
+#[allow(clippy::struct_excessive_bools)]
+#[derive(Clone, Debug, PartialEq, Deserialize)]
+pub struct Asset {
+    pub id: String,
+    pub class: Class,
+    pub exchange: Exchange,
+    pub symbol: String,
+    pub name: String,
+    pub status: Status,
+    pub tradable: bool,
+    pub marginable: bool,
+    pub shortable: bool,
+    pub easy_to_borrow: bool,
+    pub fractionable: bool,
+    pub maintenance_margin_requirement: Option<f32>,
+    pub attributes: Option<Vec<String>>,
+}
+
+impl From<Asset> for types::Asset {
+    fn from(item: Asset) -> Self {
+        Self {
+            symbol: item.symbol.clone(),
+            abbreviation: item.symbol.replace('/', ""),
+            class: item.class.into(),
+            exchange: item.exchange.into(),
+            time_added: time::OffsetDateTime::now_utc(),
+        }
+    }
+}

src/types/alpaca/api/incoming/bar.rs

@@ -1,8 +1,9 @@
 use crate::types;
 use serde::Deserialize;
+use std::collections::HashMap;
 use time::OffsetDateTime;
 
-#[derive(Deserialize)]
+#[derive(Clone, Debug, PartialEq, Deserialize)]
 pub struct Bar {
     #[serde(rename = "t")]
     #[serde(with = "time::serde::rfc3339")]
@@ -38,3 +39,9 @@ impl From<(Bar, String)> for types::Bar {
         }
     }
 }
+
+#[derive(Clone, Debug, PartialEq, Deserialize)]
+pub struct Message {
+    pub bars: HashMap<String, Vec<Bar>>,
+    pub next_page_token: Option<String>,
+}

src/types/alpaca/api/incoming/clock.rs

@@ -1,7 +1,7 @@
 use serde::Deserialize;
 use time::OffsetDateTime;
 
-#[derive(Deserialize)]
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
 pub struct Clock {
     #[serde(with = "time::serde::rfc3339")]
     pub timestamp: OffsetDateTime,

src/types/alpaca/api/incoming/mod.rs

@@ -1,5 +1,4 @@
 pub mod asset;
 pub mod bar;
-pub mod calendar;
+pub mod clock;
 pub mod news;
-pub mod order;

src/types/alpaca/api/incoming/news.rs

@@ -1,25 +1,28 @@
 use crate::{
-    types::{self, alpaca::shared::news::strip},
-    utils::de,
+    types::{
+        self,
+        news::{Confidence, Sentiment},
+    },
+    utils::news,
 };
 use serde::Deserialize;
 use time::OffsetDateTime;
 
-#[derive(Deserialize)]
-#[serde(rename_all = "snake_case")]
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub enum ImageSize {
     Thumb,
     Small,
     Large,
 }
 
-#[derive(Deserialize)]
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
 pub struct Image {
     pub size: ImageSize,
     pub url: String,
 }
 
-#[derive(Deserialize)]
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
 pub struct News {
     pub id: i64,
     #[serde(with = "time::serde::rfc3339")]
@@ -28,7 +31,6 @@ pub struct News {
     #[serde(with = "time::serde::rfc3339")]
     #[serde(rename = "updated_at")]
     pub time_updated: OffsetDateTime,
-    #[serde(deserialize_with = "de::add_slash_to_symbols")]
     pub symbols: Vec<String>,
     pub headline: String,
     pub author: String,
@@ -46,12 +48,17 @@ impl From<News> for types::News {
             time_created: news.time_created,
             time_updated: news.time_updated,
             symbols: news.symbols,
-            headline: strip(&news.headline),
-            author: strip(&news.author),
-            source: strip(&news.source),
-            summary: news.summary,
-            content: news.content,
-            url: news.url.unwrap_or_default(),
+            headline: news::normalize(&news.headline),
+            author: news::normalize(&news.author),
+            content: news::normalize(&news.content),
+            sentiment: Sentiment::Neutral,
+            confidence: Confidence::VeryUncertain,
         }
     }
 }
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
+pub struct Message {
+    pub news: Vec<News>,
+    pub next_page_token: Option<String>,
+}

src/types/alpaca/api/mod.rs

@@ -1,4 +1,6 @@
-#[macro_export]
+pub mod incoming;
+pub mod outgoing;
+
 macro_rules! impl_from_enum {
     ($source:ty, $target:ty, $( $variant:ident ),* ) => {
         impl From<$source> for $target {
@@ -19,11 +21,4 @@ macro_rules! impl_from_enum {
     };
 }
 
-#[macro_export]
-macro_rules! create_send_await {
-    ($sender:expr, $action:expr, $($contents:expr),*) => {
-        let (message, receiver) = $action($($contents),*);
-        $sender.send(message).await.unwrap();
-        receiver.await.unwrap()
-    };
-}
+use impl_from_enum;

src/types/alpaca/api/outgoing/bar.rs

@@ -0,0 +1,51 @@
+use super::serialize_symbols;
+use serde::Serialize;
+use std::time::Duration;
+use time::OffsetDateTime;
+
+fn serialize_timeframe<S>(timeframe: &Duration, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+{
+    let mins = timeframe.as_secs() / 60;
+    if mins < 60 {
+        return serializer.serialize_str(&format!("{mins}Min"));
+    }
+
+    let hours = mins / 60;
+    if hours < 24 {
+        return serializer.serialize_str(&format!("{hours}Hour"));
+    }
+
+    let days = hours / 24;
+    if days == 1 {
+        return serializer.serialize_str("1Day");
+    }
+
+    let weeks = days / 7;
+    if weeks == 1 {
+        return serializer.serialize_str("1Week");
+    }
+
+    let months = days / 30;
+    if [1, 2, 3, 4, 6, 12].contains(&months) {
+        return serializer.serialize_str(&format!("{months}Month"));
+    };
+
+    Err(serde::ser::Error::custom("Invalid timeframe duration"))
+}
+
+#[derive(Serialize)]
+pub struct Bar {
+    #[serde(serialize_with = "serialize_symbols")]
+    pub symbols: Vec<String>,
+    #[serde(serialize_with = "serialize_timeframe")]
+    pub timeframe: Duration,
+    #[serde(with = "time::serde::rfc3339")]
+    pub start: OffsetDateTime,
+    #[serde(with = "time::serde::rfc3339")]
+    pub end: OffsetDateTime,
+    pub limit: i64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub page_token: Option<String>,
+}

src/types/alpaca/api/outgoing/mod.rs

@@ -0,0 +1,12 @@
+pub mod bar;
+pub mod news;
+
+use serde::Serializer;
+
+fn serialize_symbols<S>(symbols: &[String], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let string = symbols.join(",");
+    serializer.serialize_str(&string)
+}

src/types/alpaca/api/outgoing/news.rs

@@ -0,0 +1,18 @@
+use super::serialize_symbols;
+use serde::Serialize;
+use time::OffsetDateTime;
+
+#[derive(Serialize)]
+pub struct News {
+    #[serde(serialize_with = "serialize_symbols")]
+    pub symbols: Vec<String>,
+    #[serde(with = "time::serde::rfc3339")]
+    pub start: OffsetDateTime,
+    #[serde(with = "time::serde::rfc3339")]
+    pub end: OffsetDateTime,
+    pub limit: i64,
+    pub include_content: bool,
+    pub exclude_contentless: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub page_token: Option<String>,
+}

src/types/alpaca/mod.rs

@@ -0,0 +1,5 @@
+pub mod api;
+pub mod source;
+pub mod websocket;
+
+pub use source::Source;