From de3989ec35622454c6a54e66a40cba4c93fc410e Mon Sep 17 00:00:00 2001 From: Nikolaos Karaolidis Date: Mon, 15 Jan 2024 23:51:53 +0000 Subject: [PATCH] Fix the Sin of Man - Migrate to ClickHouse - Simplify serde renaming - Simplify backfill logic - Compartmentalize database columns Signed-off-by: Nikolaos Karaolidis --- ...f4ce3a9318721bed46666cd4cd74542bc24ba.json | 71 - ...0712731ae3449ac49d1f278ca4a4b8a2c9497.json | 116 -- ...59a1a32f0fbd8802cfec980e05706fb697b58.json | 71 - ...94b9515cedf85d4b7432142a859638772aaf7.json | 84 - ...715642e95b6d0a2982792c6ee6853d33c6c5a.json | 84 - ...1554ca40f35d2c87f3acf67c0d284c693cc8b.json | 85 - ...857e73ae69df9152ece55a8569698a5b13f8b.json | 94 - ...8a16634fe25ac2d8ef86084fe5b554e7ce832.json | 82 - ...b163c3beb6709fbefd8039b9b82f343d5a6c2.json | 22 - ...01ac5ca7ccfc1414fc39b28f66beff87e328e.json | 85 - ...e71bf99e5d92a7ee1e5c13090706afde9147c.json | 78 - .vscode/settings.json | 12 +- Cargo.lock | 1519 +++++++---------- Cargo.toml | 41 +- Dockerfile | 4 +- docker-compose.yml | 12 +- src/config.rs | 31 +- src/data/market.rs | 276 +-- src/data/mod.rs | 12 +- src/database/assets.rs | 115 +- src/database/bars.rs | 199 +-- src/main.rs | 17 +- src/routes/assets.rs | 157 +- src/routes/mod.rs | 20 +- src/time.rs | 13 +- src/types/api/incoming/asset.rs | 59 +- src/types/api/incoming/bar.rs | 30 +- src/types/api/incoming/calendar_date.rs | 29 - src/types/api/incoming/mod.rs | 5 - src/types/api/mod.rs | 22 + src/types/asset.rs | 87 +- src/types/bar.rs | 84 +- src/types/mod.rs | 6 +- src/types/source.rs | 2 +- src/types/websocket/data/incoming/bar.rs | 28 +- src/types/websocket/data/incoming/mod.rs | 4 +- .../websocket/data/incoming/subscription.rs | 8 +- src/types/websocket/data/incoming/success.rs | 15 +- src/types/websocket/data/outgoing/mod.rs | 4 +- .../websocket/data/outgoing/subscribe.rs | 2 +- support/clickhouse/docker-compose.yml | 29 + .../docker-entrypoint-initdb.d/0000_init.sql | 38 + support/clickhouse/users.d/default-user.xml | 18 + support/timescaledb/999_init.sh | 48 - support/timescaledb/docker-compose.yml | 20 - 45 files changed, 1120 insertions(+), 2718 deletions(-) delete mode 100644 .sqlx/query-073ee42ebcc5a5dffd34abaf3e1f4ce3a9318721bed46666cd4cd74542bc24ba.json delete mode 100644 .sqlx/query-503ed46c4f7f8bb7d418a101ed80712731ae3449ac49d1f278ca4a4b8a2c9497.json delete mode 100644 .sqlx/query-6d9509cd482fbc022bfd157af8e59a1a32f0fbd8802cfec980e05706fb697b58.json delete mode 100644 .sqlx/query-742fef7dab68fe792675866c57394b9515cedf85d4b7432142a859638772aaf7.json delete mode 100644 .sqlx/query-7adf5172b6a3c8641f2a9fee848715642e95b6d0a2982792c6ee6853d33c6c5a.json delete mode 100644 .sqlx/query-8e8173b6e769fe9299a33a6e62a1554ca40f35d2c87f3acf67c0d284c693cc8b.json delete mode 100644 .sqlx/query-9d1121766d12528f51b3352d2fe857e73ae69df9152ece55a8569698a5b13f8b.json delete mode 100644 .sqlx/query-cb8a317dff39b7624abc7e78d6a8a16634fe25ac2d8ef86084fe5b554e7ce832.json delete mode 100644 .sqlx/query-e594f833a1e3435039c6e28e2c6b163c3beb6709fbefd8039b9b82f343d5a6c2.json delete mode 100644 .sqlx/query-e7d8b69f3f4eede80c1ce1451e301ac5ca7ccfc1414fc39b28f66beff87e328e.json delete mode 100644 .sqlx/query-ece42c3a72569b95f1b0d77faffe71bf99e5d92a7ee1e5c13090706afde9147c.json delete mode 100644 src/types/api/incoming/calendar_date.rs create mode 100644 support/clickhouse/docker-compose.yml create mode 100644 support/clickhouse/docker-entrypoint-initdb.d/0000_init.sql create mode 100644 support/clickhouse/users.d/default-user.xml delete mode 100644 support/timescaledb/999_init.sh delete mode 100644 support/timescaledb/docker-compose.yml diff --git a/.sqlx/query-073ee42ebcc5a5dffd34abaf3e1f4ce3a9318721bed46666cd4cd74542bc24ba.json b/.sqlx/query-073ee42ebcc5a5dffd34abaf3e1f4ce3a9318721bed46666cd4cd74542bc24ba.json deleted file mode 100644 index 7d4f6d4..0000000 --- a/.sqlx/query-073ee42ebcc5a5dffd34abaf3e1f4ce3a9318721bed46666cd4cd74542bc24ba.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT * FROM bars WHERE asset_symbol = $1 AND timestamp > $2 ORDER BY timestamp ASC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "timestamp", - "type_info": "Timestamptz" - }, - { - "ordinal": 1, - "name": "asset_symbol", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "open", - "type_info": "Float8" - }, - { - "ordinal": 3, - "name": "high", - "type_info": "Float8" - }, - { - "ordinal": 4, - "name": "low", - "type_info": "Float8" - }, - { - "ordinal": 5, - "name": "close", - "type_info": "Float8" - }, - { - "ordinal": 6, - "name": "volume", - "type_info": "Float8" - }, - { - "ordinal": 7, - "name": "num_trades", - "type_info": "Int8" - }, - { - "ordinal": 8, - "name": "volume_weighted", - "type_info": "Float8" - } - ], - "parameters": { - "Left": [ - "Text", - "Timestamptz" - ] - }, - "nullable": [ - false, - false, - true, - true, - true, - true, - false, - false, - false - ] - }, - "hash": "073ee42ebcc5a5dffd34abaf3e1f4ce3a9318721bed46666cd4cd74542bc24ba" -} diff --git a/.sqlx/query-503ed46c4f7f8bb7d418a101ed80712731ae3449ac49d1f278ca4a4b8a2c9497.json b/.sqlx/query-503ed46c4f7f8bb7d418a101ed80712731ae3449ac49d1f278ca4a4b8a2c9497.json deleted file mode 100644 index 13beaf7..0000000 --- a/.sqlx/query-503ed46c4f7f8bb7d418a101ed80712731ae3449ac49d1f278ca4a4b8a2c9497.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO assets (symbol, class, exchange, trading, timestamp_added, timestamp_first, timestamp_last) VALUES ($1, $2::CLASS, $3::EXCHANGE, $4, $5, $6, $7)\n RETURNING symbol, class as \"class: Class\", exchange as \"exchange: Exchange\", trading, timestamp_added, timestamp_first, timestamp_last", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "symbol", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "class: Class", - "type_info": { - "Custom": { - "name": "class", - "kind": { - "Enum": [ - "us_equity", - "crypto" - ] - } - } - } - }, - { - "ordinal": 2, - "name": "exchange: Exchange", - "type_info": { - "Custom": { - "name": "exchange", - "kind": { - "Enum": [ - "AMEX", - "ARCA", - "BATS", - "NASDAQ", - "NYSE", - "NYSEARCA", - "OTC", - "CRYPTO" - ] - } - } - } - }, - { - "ordinal": 3, - "name": "trading", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "timestamp_added", - "type_info": "Timestamptz" - }, - { - "ordinal": 5, - "name": "timestamp_first", - "type_info": "Timestamptz" - }, - { - "ordinal": 6, - "name": "timestamp_last", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - { - "Custom": { - "name": "class", - "kind": { - "Enum": [ - "us_equity", - "crypto" - ] - } - } - }, - { - "Custom": { - "name": "exchange", - "kind": { - "Enum": [ - "AMEX", - "ARCA", - "BATS", - "NASDAQ", - "NYSE", - "NYSEARCA", - "OTC", - "CRYPTO" - ] - } - } - }, - "Bool", - "Timestamptz", - "Timestamptz", - "Timestamptz" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false - ] - }, - "hash": "503ed46c4f7f8bb7d418a101ed80712731ae3449ac49d1f278ca4a4b8a2c9497" -} diff --git a/.sqlx/query-6d9509cd482fbc022bfd157af8e59a1a32f0fbd8802cfec980e05706fb697b58.json b/.sqlx/query-6d9509cd482fbc022bfd157af8e59a1a32f0fbd8802cfec980e05706fb697b58.json deleted file mode 100644 index 24a317d..0000000 --- a/.sqlx/query-6d9509cd482fbc022bfd157af8e59a1a32f0fbd8802cfec980e05706fb697b58.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT * FROM bars WHERE asset_symbol = $1 AND timestamp < $2 AND open IS NOT NULL AND high IS NOT NULL AND low IS NOT NULL AND close IS NOT NULL ORDER BY timestamp DESC LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "timestamp", - "type_info": "Timestamptz" - }, - { - "ordinal": 1, - "name": "asset_symbol", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "open", - "type_info": "Float8" - }, - { - "ordinal": 3, - "name": "high", - "type_info": "Float8" - }, - { - "ordinal": 4, - "name": "low", - "type_info": "Float8" - }, - { - "ordinal": 5, - "name": "close", - "type_info": "Float8" - }, - { - "ordinal": 6, - "name": "volume", - "type_info": "Float8" - }, - { - "ordinal": 7, - "name": "num_trades", - "type_info": "Int8" - }, - { - "ordinal": 8, - "name": "volume_weighted", - "type_info": "Float8" - } - ], - "parameters": { - "Left": [ - "Text", - "Timestamptz" - ] - }, - "nullable": [ - false, - false, - true, - true, - true, - true, - false, - false, - false - ] - }, - "hash": "6d9509cd482fbc022bfd157af8e59a1a32f0fbd8802cfec980e05706fb697b58" -} diff --git a/.sqlx/query-742fef7dab68fe792675866c57394b9515cedf85d4b7432142a859638772aaf7.json b/.sqlx/query-742fef7dab68fe792675866c57394b9515cedf85d4b7432142a859638772aaf7.json deleted file mode 100644 index d299ddc..0000000 --- a/.sqlx/query-742fef7dab68fe792675866c57394b9515cedf85d4b7432142a859638772aaf7.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM assets WHERE symbol = $1\n RETURNING symbol, class as \"class: Class\", exchange as \"exchange: Exchange\", trading, timestamp_added, timestamp_first, timestamp_last", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "symbol", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "class: Class", - "type_info": { - "Custom": { - "name": "class", - "kind": { - "Enum": [ - "us_equity", - "crypto" - ] - } - } - } - }, - { - "ordinal": 2, - "name": "exchange: Exchange", - "type_info": { - "Custom": { - "name": "exchange", - "kind": { - "Enum": [ - "AMEX", - "ARCA", - "BATS", - "NASDAQ", - "NYSE", - "NYSEARCA", - "OTC", - "CRYPTO" - ] - } - } - } - }, - { - "ordinal": 3, - "name": "trading", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "timestamp_added", - "type_info": "Timestamptz" - }, - { - "ordinal": 5, - "name": "timestamp_first", - "type_info": "Timestamptz" - }, - { - "ordinal": 6, - "name": "timestamp_last", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false - ] - }, - "hash": "742fef7dab68fe792675866c57394b9515cedf85d4b7432142a859638772aaf7" -} diff --git a/.sqlx/query-7adf5172b6a3c8641f2a9fee848715642e95b6d0a2982792c6ee6853d33c6c5a.json b/.sqlx/query-7adf5172b6a3c8641f2a9fee848715642e95b6d0a2982792c6ee6853d33c6c5a.json deleted file mode 100644 index 3475bad..0000000 --- a/.sqlx/query-7adf5172b6a3c8641f2a9fee848715642e95b6d0a2982792c6ee6853d33c6c5a.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT symbol, class as \"class: Class\", exchange as \"exchange: Exchange\", trading, timestamp_added, timestamp_first, timestamp_last FROM assets WHERE symbol = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "symbol", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "class: Class", - "type_info": { - "Custom": { - "name": "class", - "kind": { - "Enum": [ - "us_equity", - "crypto" - ] - } - } - } - }, - { - "ordinal": 2, - "name": "exchange: Exchange", - "type_info": { - "Custom": { - "name": "exchange", - "kind": { - "Enum": [ - "AMEX", - "ARCA", - "BATS", - "NASDAQ", - "NYSE", - "NYSEARCA", - "OTC", - "CRYPTO" - ] - } - } - } - }, - { - "ordinal": 3, - "name": "trading", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "timestamp_added", - "type_info": "Timestamptz" - }, - { - "ordinal": 5, - "name": "timestamp_first", - "type_info": "Timestamptz" - }, - { - "ordinal": 6, - "name": "timestamp_last", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false - ] - }, - "hash": "7adf5172b6a3c8641f2a9fee848715642e95b6d0a2982792c6ee6853d33c6c5a" -} diff --git a/.sqlx/query-8e8173b6e769fe9299a33a6e62a1554ca40f35d2c87f3acf67c0d284c693cc8b.json b/.sqlx/query-8e8173b6e769fe9299a33a6e62a1554ca40f35d2c87f3acf67c0d284c693cc8b.json deleted file mode 100644 index 4304086..0000000 --- a/.sqlx/query-8e8173b6e769fe9299a33a6e62a1554ca40f35d2c87f3acf67c0d284c693cc8b.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE assets SET trading = $1 WHERE symbol = $2\n RETURNING symbol, class as \"class: Class\", exchange as \"exchange: Exchange\", trading, timestamp_added, timestamp_first, timestamp_last", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "symbol", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "class: Class", - "type_info": { - "Custom": { - "name": "class", - "kind": { - "Enum": [ - "us_equity", - "crypto" - ] - } - } - } - }, - { - "ordinal": 2, - "name": "exchange: Exchange", - "type_info": { - "Custom": { - "name": "exchange", - "kind": { - "Enum": [ - "AMEX", - "ARCA", - "BATS", - "NASDAQ", - "NYSE", - "NYSEARCA", - "OTC", - "CRYPTO" - ] - } - } - } - }, - { - "ordinal": 3, - "name": "trading", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "timestamp_added", - "type_info": "Timestamptz" - }, - { - "ordinal": 5, - "name": "timestamp_first", - "type_info": "Timestamptz" - }, - { - "ordinal": 6, - "name": "timestamp_last", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Bool", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false - ] - }, - "hash": "8e8173b6e769fe9299a33a6e62a1554ca40f35d2c87f3acf67c0d284c693cc8b" -} diff --git a/.sqlx/query-9d1121766d12528f51b3352d2fe857e73ae69df9152ece55a8569698a5b13f8b.json b/.sqlx/query-9d1121766d12528f51b3352d2fe857e73ae69df9152ece55a8569698a5b13f8b.json deleted file mode 100644 index 97438d3..0000000 --- a/.sqlx/query-9d1121766d12528f51b3352d2fe857e73ae69df9152ece55a8569698a5b13f8b.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT symbol, class as \"class: Class\", exchange as \"exchange: Exchange\", trading, timestamp_added, timestamp_first, timestamp_last FROM assets WHERE class = $1::CLASS", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "symbol", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "class: Class", - "type_info": { - "Custom": { - "name": "class", - "kind": { - "Enum": [ - "us_equity", - "crypto" - ] - } - } - } - }, - { - "ordinal": 2, - "name": "exchange: Exchange", - "type_info": { - "Custom": { - "name": "exchange", - "kind": { - "Enum": [ - "AMEX", - "ARCA", - "BATS", - "NASDAQ", - "NYSE", - "NYSEARCA", - "OTC", - "CRYPTO" - ] - } - } - } - }, - { - "ordinal": 3, - "name": "trading", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "timestamp_added", - "type_info": "Timestamptz" - }, - { - "ordinal": 5, - "name": "timestamp_first", - "type_info": "Timestamptz" - }, - { - "ordinal": 6, - "name": "timestamp_last", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - { - "Custom": { - "name": "class", - "kind": { - "Enum": [ - "us_equity", - "crypto" - ] - } - } - } - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false - ] - }, - "hash": "9d1121766d12528f51b3352d2fe857e73ae69df9152ece55a8569698a5b13f8b" -} diff --git a/.sqlx/query-cb8a317dff39b7624abc7e78d6a8a16634fe25ac2d8ef86084fe5b554e7ce832.json b/.sqlx/query-cb8a317dff39b7624abc7e78d6a8a16634fe25ac2d8ef86084fe5b554e7ce832.json deleted file mode 100644 index 6cce28b..0000000 --- a/.sqlx/query-cb8a317dff39b7624abc7e78d6a8a16634fe25ac2d8ef86084fe5b554e7ce832.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT symbol, class as \"class: Class\", exchange as \"exchange: Exchange\", trading, timestamp_added, timestamp_first, timestamp_last FROM assets", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "symbol", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "class: Class", - "type_info": { - "Custom": { - "name": "class", - "kind": { - "Enum": [ - "us_equity", - "crypto" - ] - } - } - } - }, - { - "ordinal": 2, - "name": "exchange: Exchange", - "type_info": { - "Custom": { - "name": "exchange", - "kind": { - "Enum": [ - "AMEX", - "ARCA", - "BATS", - "NASDAQ", - "NYSE", - "NYSEARCA", - "OTC", - "CRYPTO" - ] - } - } - } - }, - { - "ordinal": 3, - "name": "trading", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "timestamp_added", - "type_info": "Timestamptz" - }, - { - "ordinal": 5, - "name": "timestamp_first", - "type_info": "Timestamptz" - }, - { - "ordinal": 6, - "name": "timestamp_last", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false - ] - }, - "hash": "cb8a317dff39b7624abc7e78d6a8a16634fe25ac2d8ef86084fe5b554e7ce832" -} diff --git a/.sqlx/query-e594f833a1e3435039c6e28e2c6b163c3beb6709fbefd8039b9b82f343d5a6c2.json b/.sqlx/query-e594f833a1e3435039c6e28e2c6b163c3beb6709fbefd8039b9b82f343d5a6c2.json deleted file mode 100644 index 3ce0d60..0000000 --- a/.sqlx/query-e594f833a1e3435039c6e28e2c6b163c3beb6709fbefd8039b9b82f343d5a6c2.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO bars (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ON CONFLICT (timestamp, asset_symbol) DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Timestamptz", - "Text", - "Float8", - "Float8", - "Float8", - "Float8", - "Float8", - "Int8", - "Float8" - ] - }, - "nullable": [] - }, - "hash": "e594f833a1e3435039c6e28e2c6b163c3beb6709fbefd8039b9b82f343d5a6c2" -} diff --git a/.sqlx/query-e7d8b69f3f4eede80c1ce1451e301ac5ca7ccfc1414fc39b28f66beff87e328e.json b/.sqlx/query-e7d8b69f3f4eede80c1ce1451e301ac5ca7ccfc1414fc39b28f66beff87e328e.json deleted file mode 100644 index 1e6544c..0000000 --- a/.sqlx/query-e7d8b69f3f4eede80c1ce1451e301ac5ca7ccfc1414fc39b28f66beff87e328e.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE assets SET timestamp_last = $1 WHERE symbol = $2\n RETURNING symbol, class as \"class: Class\", exchange as \"exchange: Exchange\", trading, timestamp_added, timestamp_first, timestamp_last", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "symbol", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "class: Class", - "type_info": { - "Custom": { - "name": "class", - "kind": { - "Enum": [ - "us_equity", - "crypto" - ] - } - } - } - }, - { - "ordinal": 2, - "name": "exchange: Exchange", - "type_info": { - "Custom": { - "name": "exchange", - "kind": { - "Enum": [ - "AMEX", - "ARCA", - "BATS", - "NASDAQ", - "NYSE", - "NYSEARCA", - "OTC", - "CRYPTO" - ] - } - } - } - }, - { - "ordinal": 3, - "name": "trading", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "timestamp_added", - "type_info": "Timestamptz" - }, - { - "ordinal": 5, - "name": "timestamp_first", - "type_info": "Timestamptz" - }, - { - "ordinal": 6, - "name": "timestamp_last", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Timestamptz", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false - ] - }, - "hash": "e7d8b69f3f4eede80c1ce1451e301ac5ca7ccfc1414fc39b28f66beff87e328e" -} diff --git a/.sqlx/query-ece42c3a72569b95f1b0d77faffe71bf99e5d92a7ee1e5c13090706afde9147c.json b/.sqlx/query-ece42c3a72569b95f1b0d77faffe71bf99e5d92a7ee1e5c13090706afde9147c.json deleted file mode 100644 index 7be0803..0000000 --- a/.sqlx/query-ece42c3a72569b95f1b0d77faffe71bf99e5d92a7ee1e5c13090706afde9147c.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO bars (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ON CONFLICT (timestamp, asset_symbol) DO UPDATE SET open = $3, high = $4, low = $5, close = $6, volume = $7, num_trades = $8, volume_weighted = $9\n RETURNING timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "timestamp", - "type_info": "Timestamptz" - }, - { - "ordinal": 1, - "name": "asset_symbol", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "open", - "type_info": "Float8" - }, - { - "ordinal": 3, - "name": "high", - "type_info": "Float8" - }, - { - "ordinal": 4, - "name": "low", - "type_info": "Float8" - }, - { - "ordinal": 5, - "name": "close", - "type_info": "Float8" - }, - { - "ordinal": 6, - "name": "volume", - "type_info": "Float8" - }, - { - "ordinal": 7, - "name": "num_trades", - "type_info": "Int8" - }, - { - "ordinal": 8, - "name": "volume_weighted", - "type_info": "Float8" - } - ], - "parameters": { - "Left": [ - "Timestamptz", - "Text", - "Float8", - "Float8", - "Float8", - "Float8", - "Float8", - "Int8", - "Float8" - ] - }, - "nullable": [ - false, - false, - true, - true, - true, - true, - false, - false, - false - ] - }, - "hash": "ece42c3a72569b95f1b0d77faffe71bf99e5d92a7ee1e5c13090706afde9147c" -} diff --git a/.vscode/settings.json b/.vscode/settings.json index fa5b14a..af2971c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,14 +1,16 @@ { "sqltools.connections": [ { - "previewLimit": 50, "server": "localhost", - "port": 5432, - "driver": "PostgreSQL", - "name": "QRust", + "port": 8123, + "useHTTPS": false, "database": "qrust", "username": "qrust", - "password": "qrust" + "enableTls": false, + "previewLimit": 50, + "password": "qrust", + "driver": "ClickHouse", + "name": "qrust" } ] } diff --git a/Cargo.lock b/Cargo.lock index 39629c7..a85c95c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,24 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "ahash" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" -dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "version_check", -] - -[[package]] -name = "allocator-api2" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" - [[package]] name = "android-tzdata" version = "0.1.1" @@ -52,9 +34,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "arc-swap" @@ -64,22 +46,13 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", -] - -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", + "syn 2.0.48", ] [[package]] @@ -90,18 +63,19 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.20" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +checksum = "1236b4b292f6c4d6dc34604bb5120d85c3fe1d1aa596bd5cc52ca054d13e7b9e" dependencies = [ "async-trait", "axum-core", - "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.1.0", + "hyper-util", "itoa", "matchit", "memchr", @@ -118,23 +92,28 @@ dependencies = [ "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] name = "axum-core" -version = "0.3.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", "mime", + "pin-project-lite", "rustversion", + "sync_wrapper", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -154,15 +133,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.3" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" - -[[package]] -name = "base64ct" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bitflags" @@ -172,12 +145,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" -dependencies = [ - "serde", -] +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "block-buffer" @@ -189,16 +159,25 @@ dependencies = [ ] [[package]] -name = "bumpalo" -version = "3.13.0" +name = "bstr" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" +dependencies = [ + "memchr", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -223,29 +202,70 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.30" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "wasm-bindgen", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] -name = "const-oid" -version = "0.9.5" +name = "clickhouse" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +checksum = "a0875e527e299fc5f4faba42870bf199a39ab0bb2dbba1b8aef0a2151451130f" +dependencies = [ + "bstr", + "bytes", + "clickhouse-derive", + "clickhouse-rs-cityhash-sys", + "futures", + "hyper 0.14.28", + "hyper-tls", + "lz4", + "sealed", + "serde", + "serde_json", + "sha-1", + "static_assertions", + "thiserror", + "time", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "clickhouse-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18af5425854858c507eec70f7deb4d5d8cec4216fcb086283a78872387281ea5" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + +[[package]] +name = "clickhouse-rs-cityhash-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4baf9d4700a28d6cb600e17ed6ae2b43298a5245f1f76b4eab63027ebfd592b9" +dependencies = [ + "cc", +] [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -253,52 +273,24 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" - -[[package]] -name = "crossbeam-queue" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crypto-common" @@ -317,7 +309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "lock_api", "once_cell", "parking_lot_core", @@ -325,27 +317,17 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" - -[[package]] -name = "der" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" -dependencies = [ - "const-oid", - "pem-rfc7468", - "zeroize", -] +checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ + "powerfmt", "serde", ] @@ -373,9 +355,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", - "const-oid", "crypto-common", - "subtle", ] [[package]] @@ -384,21 +364,6 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" -[[package]] -name = "dotenvy" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" - -[[package]] -name = "either" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" -dependencies = [ - "serde", -] - [[package]] name = "encoding_rs" version = "0.8.33" @@ -416,65 +381,19 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.3" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "errno-dragonfly", "libc", - "windows-sys", + "windows-sys 0.52.0", ] -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "etcetera" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" -dependencies = [ - "cfg-if", - "home", - "windows-sys", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - [[package]] name = "fastrand" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" - -[[package]] -name = "finl_unicode" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" - -[[package]] -name = "flume" -version = "0.10.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" -dependencies = [ - "futures-core", - "futures-sink", - "pin-project", - "spin 0.9.8", -] +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fnv" @@ -499,18 +418,18 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -523,9 +442,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -533,60 +452,49 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", "futures-util", ] -[[package]] -name = "futures-intrusive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" -dependencies = [ - "futures-core", - "lock_api", - "parking_lot", -] - [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.48", ] [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -596,9 +504,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -624,9 +532,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -635,9 +543,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "governor" @@ -659,17 +567,36 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "b553656127a00601c8ae5590fcfdc118e4083a7924b6cf4ffc1ea4b99dc429d7" dependencies = [ "bytes", "fnv", "futures-core", "futures-sink", "futures-util", - "http", - "indexmap 1.9.3", + "http 0.2.11", + "indexmap 2.1.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "991910e35c615d8cab86b5ab04be67e6ad24d2bf5f4f11fdbbed26da999bbeab" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.0.0", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", @@ -684,76 +611,41 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" -dependencies = [ - "ahash", - "allocator-api2", -] - -[[package]] -name = "hashlink" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" -dependencies = [ - "hashbrown 0.14.0", -] +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "heck" -version = "0.4.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" dependencies = [ "unicode-segmentation", ] [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] -name = "hex" -version = "0.4.3" +name = "http" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hkdf" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ - "hmac", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys", + "bytes", + "fnv", + "itoa", ] [[package]] name = "http" -version = "0.2.9" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" dependencies = [ "bytes", "fnv", @@ -762,12 +654,35 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.0.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -791,28 +706,47 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.23", + "http 0.2.11", + "http-body 0.4.6", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2", "tokio", "tower-service", "tracing", "want", ] +[[package]] +name = "hyper" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.1", + "http 1.0.0", + "http-body 1.0.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "tokio", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -820,24 +754,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.28", "native-tls", "tokio", "tokio-native-tls", ] [[package]] -name = "iana-time-zone" -version = "0.1.57" +name = "hyper-util" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.1.0", + "pin-project-lite", + "socket2", + "tokio", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -851,9 +803,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -871,40 +823,31 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.3", ] [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" - -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -914,32 +857,12 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -dependencies = [ - "spin 0.5.2", -] [[package]] name = "libc" -version = "0.2.147" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" - -[[package]] -name = "libm" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" - -[[package]] -name = "libsqlite3-sys" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "linked-hash-map" @@ -949,15 +872,15 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -1005,34 +928,45 @@ dependencies = [ ] [[package]] -name = "mach2" -version = "0.4.1" +name = "lz4" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" +dependencies = [ + "libc", + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "mach2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" dependencies = [ "libc", ] [[package]] name = "matchit" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" - -[[package]] -name = "md-5" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" -dependencies = [ - "digest", -] +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.6.3" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "mime" @@ -1040,12 +974,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.7.1" @@ -1057,13 +985,13 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1090,68 +1018,19 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "nonzero_ext" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" -[[package]] -name = "num-bigint-dig" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" -dependencies = [ - "byteorder", - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand", - "smallvec", - "zeroize", -] - -[[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -1166,26 +1045,26 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", @@ -1202,7 +1081,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.48", ] [[package]] @@ -1213,9 +1092,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" dependencies = [ "cc", "libc", @@ -1225,9 +1104,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" dependencies = [ "num-traits", ] @@ -1244,37 +1123,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall", "smallvec", - "windows-targets", -] - -[[package]] -name = "paste" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", + "windows-targets 0.48.5", ] [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project" @@ -1293,7 +1157,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.48", ] [[package]] @@ -1308,32 +1172,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der", - "pkcs8", - "spki", -] - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" @@ -1343,9 +1192,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -1355,20 +1204,21 @@ name = "qrust" version = "0.1.0" dependencies = [ "axum", + "clickhouse", "dotenv", "futures-util", "governor", - "http", - "indexmap 2.0.0", + "http 1.0.0", "log", "log4rs", "reqwest", "serde", "serde_json", - "sqlx", + "serde_repr", "time", "tokio", "tokio-tungstenite", + "uuid", ] [[package]] @@ -1389,9 +1239,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -1437,37 +1287,28 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "base64", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.23", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-tls", "ipnet", "js-sys", @@ -1480,6 +1321,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-native-tls", "tower-service", @@ -1490,28 +1332,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "rsa" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8" -dependencies = [ - "byteorder", - "const-oid", - "digest", - "num-bigint-dig", - "num-integer", - "num-iter", - "num-traits", - "pkcs1", - "pkcs8", - "rand_core", - "signature", - "spki", - "subtle", - "zeroize", -] - [[package]] name = "rustc-demangle" version = "0.1.23" @@ -1520,15 +1340,15 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.11" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1539,17 +1359,17 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1558,6 +1378,18 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sealed" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b5e421024b5e5edfbaa8e60ecf90bda9dbffc602dbb230e6028763f85f0c68c" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "security-framework" version = "2.9.2" @@ -1583,9 +1415,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.188" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] @@ -1602,20 +1434,31 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.48", +] + +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -1624,14 +1467,25 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c" dependencies = [ "itoa", "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b2e6b945e9d3df726b65d6ee24060aff8e3533d431f677a9695db04eff9dfdb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1656,38 +1510,28 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "sha-1" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", "digest", ] -[[package]] -name = "sha2" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "signature" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" -dependencies = [ - "digest", - "rand_core", -] - [[package]] name = "slab" version = "0.4.9" @@ -1699,285 +1543,25 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "2593d31f82ead8df961d8bd23a64c2ccf2eb5dd34b0a34bfb4dd54011c72009e" [[package]] name = "socket2" -version = "0.4.9" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", - "winapi", + "windows-sys 0.48.0", ] [[package]] -name = "socket2" -version = "0.5.3" +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - -[[package]] -name = "spki" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "sqlformat" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" -dependencies = [ - "itertools", - "nom", - "unicode_categories", -] - -[[package]] -name = "sqlx" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e58421b6bc416714d5115a2ca953718f6c621a51b68e4f4922aea5a4391a721" -dependencies = [ - "sqlx-core", - "sqlx-macros", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", -] - -[[package]] -name = "sqlx-core" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" -dependencies = [ - "ahash", - "atoi", - "byteorder", - "bytes", - "crc", - "crossbeam-queue", - "dotenvy", - "either", - "event-listener", - "futures-channel", - "futures-core", - "futures-intrusive", - "futures-io", - "futures-util", - "hashlink", - "hex", - "indexmap 2.0.0", - "log", - "memchr", - "once_cell", - "paste", - "percent-encoding", - "serde", - "serde_json", - "sha2", - "smallvec", - "sqlformat", - "thiserror", - "time", - "tokio", - "tokio-stream", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "sqlx-macros" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2" -dependencies = [ - "proc-macro2", - "quote", - "sqlx-core", - "sqlx-macros-core", - "syn 1.0.109", -] - -[[package]] -name = "sqlx-macros-core" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a4a8336d278c62231d87f24e8a7a74898156e34c1c18942857be2acb29c7dfc" -dependencies = [ - "dotenvy", - "either", - "heck", - "hex", - "once_cell", - "proc-macro2", - "quote", - "serde", - "serde_json", - "sha2", - "sqlx-core", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", - "syn 1.0.109", - "tempfile", - "tokio", - "url", -] - -[[package]] -name = "sqlx-mysql" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" -dependencies = [ - "atoi", - "base64", - "bitflags 2.4.0", - "byteorder", - "bytes", - "crc", - "digest", - "dotenvy", - "either", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "generic-array", - "hex", - "hkdf", - "hmac", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "percent-encoding", - "rand", - "rsa", - "serde", - "sha1", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "time", - "tracing", - "uuid", - "whoami", -] - -[[package]] -name = "sqlx-postgres" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" -dependencies = [ - "atoi", - "base64", - "bitflags 2.4.0", - "byteorder", - "crc", - "dotenvy", - "etcetera", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "hex", - "hkdf", - "hmac", - "home", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "rand", - "serde", - "serde_json", - "sha1", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "time", - "tracing", - "uuid", - "whoami", -] - -[[package]] -name = "sqlx-sqlite" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4c21bf34c7cae5b283efb3ac1bcc7670df7561124dc2f8bdc0b59be40f79a2" -dependencies = [ - "atoi", - "flume", - "futures-channel", - "futures-core", - "futures-executor", - "futures-intrusive", - "futures-util", - "libsqlite3-sys", - "log", - "percent-encoding", - "serde", - "sqlx-core", - "time", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "stringprep" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" -dependencies = [ - "finl_unicode", - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "subtle" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" @@ -1992,9 +1576,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -2008,57 +1592,78 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] -name = "tempfile" -version = "3.8.0" +name = "system-configuration" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.3.5", + "redox_syscall", "rustix", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.48", ] [[package]] name = "thread-id" -version = "4.2.0" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79474f573561cdc4871a0de34a51c92f7f5a56039113fbb5b9c9f96bdb756669" +checksum = "f0ec81c46e9eb50deaa257be2f148adf052d1fb7701cfd55ccfab2525280b70b" dependencies = [ "libc", - "redox_syscall 0.2.16", "winapi", ] [[package]] name = "time" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" dependencies = [ "deranged", "itoa", + "powerfmt", "serde", "time-core", "time-macros", @@ -2066,15 +1671,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" dependencies = [ "time-core", ] @@ -2096,9 +1701,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", @@ -2106,20 +1711,20 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", - "socket2 0.5.3", + "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.48", ] [[package]] @@ -2132,22 +1737,11 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-tungstenite" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2dbec703c26b00d74844519606ef15d09a7d6857860f84ad223dec002ddea2" +checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" dependencies = [ "futures-util", "log", @@ -2159,9 +1753,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -2201,53 +1795,40 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", - "tracing-attributes", "tracing-core", ] -[[package]] -name = "tracing-attributes" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.31", -] - [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e862a1c4128df0112ab625f55cd5c934bcb4312ba80b39ae4b4835a3fd58e649" +checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" dependencies = [ "byteorder", "bytes", "data-encoding", - "http", + "http 1.0.0", "httparse", "log", "native-tls", @@ -2269,21 +1850,21 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -2300,12 +1881,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - [[package]] name = "unsafe-any-ors" version = "1.0.0" @@ -2317,9 +1892,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -2334,9 +1909,9 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "uuid" -version = "1.4.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" [[package]] name = "vcpkg" @@ -2367,9 +1942,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2377,24 +1952,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.48", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if", "js-sys", @@ -2404,9 +1979,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2414,39 +1989,33 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", ] -[[package]] -name = "whoami" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" - [[package]] name = "winapi" version = "0.3.9" @@ -2470,12 +2039,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.0", ] [[package]] @@ -2484,7 +2053,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", ] [[package]] @@ -2493,13 +2071,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", ] [[package]] @@ -2508,42 +2101,84 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winreg" version = "0.50.0" @@ -2551,7 +2186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2562,9 +2197,3 @@ checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" dependencies = [ "linked-hash-map", ] - -[[package]] -name = "zeroize" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" diff --git a/Cargo.toml b/Cargo.toml index f6bdc58..645c521 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,34 +12,37 @@ codegen-units = 1 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -axum = "0.6.20" +axum = "0.7.4" dotenv = "0.15.0" -sqlx = { version = "0.7.1", features = [ - "uuid", - "time", - "postgres", - "runtime-tokio", -] } tokio = { version = "1.32.0", features = [ "macros", "rt-multi-thread", ] } -serde = "1.0.188" -log = "0.4.20" -serde_json = "1.0.105" -log4rs = "1.2.0" -time = { version = "0.3.27", features = [ - "serde", +tokio-tungstenite = { version = "0.21.0", features = [ + "tokio-native-tls", + "native-tls", ] } +log = "0.4.20" +log4rs = "1.2.0" +serde = "1.0.188" +serde_json = "1.0.105" +serde_repr = "0.1.18" futures-util = "0.3.28" reqwest = { version = "0.11.20", features = [ "json", "serde_json", ] } -tokio-tungstenite = { version = "0.20.0", features = [ - "tokio-native-tls", - "native-tls", -] } -http = "0.2.9" +http = "1.0.0" governor = "0.6.0" -indexmap = "2.0.0" +clickhouse = { version = "0.11.6", features = [ + "watch", + "time", + "uuid", +] } +uuid = "1.6.1" +time = { version = "0.3.31", features = [ + "serde", + "formatting", + "macros", + "serde-well-known", +] } diff --git a/Dockerfile b/Dockerfile index 7680e7c..a651dba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,8 @@ RUN apk add --no-cache pkgconf musl-dev openssl-dev WORKDIR /usr/src/qrust -ENV SQLX_OFFLINE=true - RUN mkdir src && echo "fn main() {}" > src/main.rs -COPY Cargo.toml .sqlx ./ +COPY Cargo.toml ./ RUN cargo build --release RUN rm -rf src diff --git a/docker-compose.yml b/docker-compose.yml index eba2485..6346062 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ services: - timescaledb: + clickhouse: extends: - file: support/timescaledb/docker-compose.yml - service: timescaledb + file: support/clickhouse/docker-compose.yml + service: clickhouse qrust: build: @@ -13,10 +13,10 @@ services: ports: - 7878:7878 depends_on: - - timescaledb + - clickhouse env_file: - .env.docker volumes: - timescaledb-data: - timescaledb-logs: + clickhouse-lib: + clickhouse-log: diff --git a/src/config.rs b/src/config.rs index 4cba9b7..fb90c93 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,8 +1,6 @@ use crate::types::Source; use governor::{DefaultDirectRateLimiter, Quota, RateLimiter}; -use http::HeaderMap; -use reqwest::Client; -use sqlx::{postgres::PgPoolOptions, PgPool}; +use reqwest::{header::HeaderMap, Client}; use std::{env, num::NonZeroU32, sync::Arc}; use time::{format_description::FormatItem, macros::format_description}; use tokio::time::Duration; @@ -15,8 +13,6 @@ pub const ALPACA_CRYPTO_WEBSOCKET_URL: &str = "wss://stream.data.alpaca.markets/ pub const ALPACA_TIMESTAMP_FORMAT: &[FormatItem] = format_description!("[year]-[month]-[day]T[hour]:[minute]:[second]Z"); -const NUM_CLIENTS: u32 = 10; - pub struct Config { pub alpaca_api_key: String, pub alpaca_api_secret: String, @@ -24,12 +20,11 @@ pub struct Config { pub alpaca_rate_limit: DefaultDirectRateLimiter, pub alpaca_historical_offset: Duration, pub alpaca_source: Source, - pub postgres_pool: PgPool, + pub clickhouse_client: clickhouse::Client, } impl Config { - pub async fn from_env() -> Self { - let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set."); + pub fn from_env() -> Self { let alpaca_api_key = env::var("ALPACA_API_KEY").expect("ALPACA_API_KEY must be set."); let alpaca_api_secret = env::var("ALPACA_API_SECRET").expect("ALPACA_API_SECRET must be set."); @@ -38,6 +33,12 @@ impl Config { .parse() .expect("ALPACA_SOURCE must be a either 'iex' or 'sip'."); + let clickhouse_url = env::var("CLICKHOUSE_URL").expect("CLICKHOUSE_URL must be set."); + let clickhouse_user = env::var("CLICKHOUSE_USER").expect("CLICKHOUSE_USER must be set."); + let clickhouse_password = + env::var("CLICKHOUSE_PASSWORD").expect("CLICKHOUSE_PASSWORD must be set."); + let clickhouse_db = env::var("CLICKHOUSE_DB").expect("CLICKHOUSE_DB must be set."); + Self { alpaca_api_key: alpaca_api_key.clone(), alpaca_api_secret: alpaca_api_secret.clone(), @@ -59,15 +60,15 @@ impl Config { Source::Sip => 0, }), alpaca_source, - postgres_pool: PgPoolOptions::new() - .max_connections(NUM_CLIENTS) - .connect(&database_url) - .await - .unwrap(), + clickhouse_client: clickhouse::Client::default() + .with_url(clickhouse_url) + .with_user(clickhouse_user) + .with_password(clickhouse_password) + .with_database(clickhouse_db), } } - pub async fn arc_from_env() -> Arc { - Arc::new(Self::from_env().await) + pub fn arc_from_env() -> Arc { + Arc::new(Self::from_env()) } } diff --git a/src/data/market.rs b/src/data/market.rs index a7e3295..185cb4e 100644 --- a/src/data/market.rs +++ b/src/data/market.rs @@ -5,11 +5,11 @@ use crate::{ }, data::authenticate_websocket, database, - time::{duration_until, last_minute, next_30s, next_minute, ONE_MINUTE, THIRTY_SECONDS}, + time::{duration_until, last_minute, next_minute, ONE_MINUTE}, types::{ - api, + api::incoming, asset::{self, Asset}, - websocket, Bar, BroadcastMessage, Class, + websocket, Bar, BarValidity, BroadcastMessage, Class, }, }; use core::panic; @@ -17,16 +17,12 @@ use futures_util::{ stream::{SplitSink, SplitStream}, SinkExt, StreamExt, }; -use http::StatusCode; -use indexmap::IndexMap; use log::{error, info, warn}; use serde_json::from_str; use std::{ collections::{HashMap, HashSet}, sync::Arc, - time::Instant, }; -use time::OffsetDateTime; use tokio::{ net::TcpStream, spawn, @@ -34,15 +30,14 @@ use tokio::{ broadcast::{Receiver, Sender}, RwLock, }, - task::spawn_blocking, - time::{interval_at, sleep}, + time::sleep, }; use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; pub async fn run( app_config: Arc, class: Class, - asset_broadcast_sender: Sender, + broadcast_sender: Sender, ) { info!("Running live data threads for {:?}.", class); @@ -59,38 +54,35 @@ pub async fn run( authenticate_websocket(&app_config, &mut stream, &mut sink).await; let sink = Arc::new(RwLock::new(sink)); - let backfilled = Arc::new(RwLock::new(HashMap::new())); - - spawn(websocket_broadcast_handler( + spawn(broadcast_handler( class, sink.clone(), - asset_broadcast_sender.subscribe(), + broadcast_sender.subscribe(), )); - database::assets::select_where_class(&app_config.postgres_pool, class) + database::assets::select_where_class(&app_config.clickhouse_client, class) .await .into_iter() .for_each(|asset| { - asset_broadcast_sender + broadcast_sender .send(BroadcastMessage::Asset(asset::BroadcastMessage::Added( asset, ))) .unwrap(); }); - spawn(null_handler(app_config.clone(), backfilled.clone())); - websocket_message_handler(app_config, class, stream, sink, backfilled).await; + websocket_handler(app_config, class, stream, sink).await; unreachable!() } -async fn websocket_broadcast_handler( +async fn broadcast_handler( class: Class, sink: Arc>, Message>>>, - mut asset_broadcast_receiver: Receiver, + mut broadcast_receiver: Receiver, ) { loop { - match asset_broadcast_receiver.recv().await.unwrap() { + match broadcast_receiver.recv().await.unwrap() { BroadcastMessage::Asset(asset::BroadcastMessage::Added(asset)) if asset.class == class => { @@ -128,13 +120,14 @@ async fn websocket_broadcast_handler( } } -async fn websocket_message_handler( +async fn websocket_handler( app_config: Arc, class: Class, mut stream: SplitStream>>, sink: Arc>, Message>>>, - backfilled: Arc>>, ) { + let backfilled = Arc::new(RwLock::new(HashMap::new())); + loop { match stream.next().await { Some(Ok(Message::Text(data))) => { @@ -147,7 +140,7 @@ async fn websocket_message_handler( } for message in parsed_data.unwrap_or_default() { - websocket_handle_text_message(&app_config, class, message, &backfilled).await; + websocket_handle_message(&app_config, class, &backfilled, message).await; } } Some(Ok(Message::Ping(_))) => sink @@ -162,11 +155,11 @@ async fn websocket_message_handler( } } -async fn websocket_handle_text_message( +async fn websocket_handle_message( app_config: &Arc, class: Class, - message: websocket::data::incoming::Message, backfilled: &Arc>>, + message: websocket::data::incoming::Message, ) { match message { websocket::data::incoming::Message::Subscription(subscription_message) => { @@ -185,20 +178,35 @@ async fn websocket_handle_text_message( let deleted_assets = old_assets.difference(&new_assets).collect::>(); for asset_symbol in &added_assets { - let asset = - database::assets::select_where_symbol(&app_config.postgres_pool, asset_symbol) - .await - .unwrap(); + let asset = database::assets::select_where_symbol( + &app_config.clickhouse_client, + asset_symbol, + ) + .await + .unwrap(); backfilled.write().await.insert(asset.symbol.clone(), false); + + let bar_validity = BarValidity::none(asset.symbol.clone()); + database::bars::upsert_validity(&app_config.clickhouse_client, &bar_validity).await; + spawn(backfill( app_config.clone(), - asset.clone(), backfilled.clone(), + asset.clone(), )); } for asset_symbol in &deleted_assets { + database::bars::delete_validity_where_symbol( + &app_config.clickhouse_client, + asset_symbol, + ) + .await; + + database::bars::delete_where_symbol(&app_config.clickhouse_client, asset_symbol) + .await; + backfilled.write().await.remove(*asset_symbol); } @@ -207,156 +215,45 @@ async fn websocket_handle_text_message( class, added_assets, deleted_assets ); } - websocket::data::incoming::Message::Bars(bar_message) => { + websocket::data::incoming::Message::Bars(bar_message) + | websocket::data::incoming::Message::UpdatedBars(bar_message) => { let bar = Bar::from(bar_message); - info!( - "websocket::Incoming bar for {}: {}", - bar.asset_symbol, bar.timestamp - ); - database::bars::upsert( - &app_config.postgres_pool, - &bar, - backfilled.read().await[&bar.asset_symbol], - ) - .await; - } - websocket::data::incoming::Message::UpdatedBars(bar_message) => { - let bar = Bar::from(bar_message); - info!( - "websocket::Incoming bar for {}: {}", - bar.asset_symbol, bar.timestamp - ); + info!("websocket::Incoming bar for {}: {}", bar.symbol, bar.time); - let transaction = app_config.postgres_pool.begin().await.unwrap(); - let backfilled_asset_symbol = backfilled.read().await[&bar.asset_symbol]; - database::bars::upsert(&app_config.postgres_pool, &bar, backfilled_asset_symbol).await; - if backfilled_asset_symbol { - database::assets::update_timestamp_last_where_symbol( - &app_config.postgres_pool, - &bar.asset_symbol, - &bar.timestamp, - ) - .await; + database::bars::upsert(&app_config.clickhouse_client, &bar).await; + if backfilled.read().await[&bar.symbol] { + database::bars::upsert_validity(&app_config.clickhouse_client, &bar.into()).await; } - transaction.commit().await.unwrap(); } - _ => {} - } -} - -#[allow(clippy::significant_drop_in_scrutinee)] -async fn null_handler(app_config: Arc, backfilled: Arc>>) { - #[derive(PartialEq)] - enum NullHandlerState { - Bars, - UpdatedBars, - } - - let next_30s = next_30s(); - let mut state = if next_30s.unix_timestamp() % 30 == 0 { - NullHandlerState::Bars - } else { - NullHandlerState::UpdatedBars - }; - let mut interval = interval_at( - (Instant::now() + duration_until(next_30s)).into(), - THIRTY_SECONDS, - ); - - loop { - interval.tick().await; - let timestamp = last_minute() - ONE_MINUTE; - - let backfilled = backfilled.read().await; - for asset_symbol in backfilled.keys().cloned() { - let bar = Bar::empty(timestamp, asset_symbol); - - let transaction = app_config.postgres_pool.begin().await.unwrap(); - database::bars::insert_or_skip( - &app_config.postgres_pool, - &bar, - backfilled[&bar.asset_symbol], - ) - .await; - if backfilled[&bar.asset_symbol] && state == NullHandlerState::Bars { - database::assets::update_timestamp_last_where_symbol( - &app_config.postgres_pool, - &bar.asset_symbol, - &bar.timestamp, - ) - .await; - } - transaction.commit().await.unwrap(); - } - - state = match state { - NullHandlerState::Bars => NullHandlerState::UpdatedBars, - NullHandlerState::UpdatedBars => NullHandlerState::Bars, - }; + websocket::data::incoming::Message::Success(_) => {} } } pub async fn backfill( app_config: Arc, - asset: Asset, backfilled: Arc>>, + asset: Asset, ) { - info!("Backfilling historical data for {}...", asset.symbol); + let bar_validity = + database::bars::select_validity_where_symbol(&app_config.clickhouse_client, &asset.symbol) + .await + .unwrap(); - let task_run_offsetdatetime = next_minute() + app_config.alpaca_historical_offset; - let fetch_from = asset.timestamp_last + ONE_MINUTE; - let fetch_until = task_run_offsetdatetime - app_config.alpaca_historical_offset - ONE_MINUTE; + let fetch_from = bar_validity.time_last + ONE_MINUTE; + let fetch_until = last_minute(); if fetch_from > fetch_until { return; } - let wait_duration = task_run_offsetdatetime - OffsetDateTime::now_utc(); - if wait_duration.is_positive() { - sleep(wait_duration.unsigned_abs()).await; - } + info!("Queing historical data backfill for {}...", asset.symbol); + let task_run_offsetdatetime = next_minute() + app_config.alpaca_historical_offset; + sleep(duration_until(task_run_offsetdatetime)).await; - let bars = backfill_bars_from_api(&app_config, &asset, fetch_from, fetch_until).await; - - let transaction = app_config.postgres_pool.begin().await.unwrap(); - database::bars::upsert_batch(&app_config.postgres_pool, &bars, true).await; - database::assets::update_timestamp_last_where_symbol( - &app_config.postgres_pool, - &asset.symbol, - &fetch_until, - ) - .await; - derive_recent_nulls(&app_config, &asset, &fetch_until, &backfilled).await; - transaction.commit().await.unwrap(); - - info!("Backfilled historical data for {}.", asset.symbol); -} - -fn generate_per_minute_bars( - from: OffsetDateTime, - until: OffsetDateTime, - asset: &Asset, -) -> IndexMap { - let mut bars = IndexMap::new(); - let mut current_time = from; - while current_time <= until { - bars.insert(current_time, Bar::empty(current_time, asset.symbol.clone())); - current_time += ONE_MINUTE; - } - bars -} - -async fn backfill_bars_from_api( - app_config: &Arc, - asset: &Asset, - from: OffsetDateTime, - until: OffsetDateTime, -) -> Vec { - let asset_clone = asset.clone(); - let mut bars = spawn_blocking(move || generate_per_minute_bars(from, until, &asset_clone)) - .await - .unwrap(); + info!("Running historical data backfill for {}...", asset.symbol); + let mut bars = Vec::new(); let mut next_page_token = None; + loop { let request = app_config .alpaca_client @@ -369,11 +266,17 @@ async fn backfill_bars_from_api( ("timeframe", &String::from("1Min")), ( "start", - &from.format(ALPACA_TIMESTAMP_FORMAT).unwrap().to_string(), + &fetch_from + .format(ALPACA_TIMESTAMP_FORMAT) + .unwrap() + .to_string(), ), ( "end", - &until.format(ALPACA_TIMESTAMP_FORMAT).unwrap().to_string(), + &fetch_until + .format(ALPACA_TIMESTAMP_FORMAT) + .unwrap() + .to_string(), ), ("limit", &String::from("10000")), ("page_token", &next_page_token.clone().unwrap_or_default()), @@ -381,17 +284,14 @@ async fn backfill_bars_from_api( app_config.alpaca_rate_limit.until_ready().await; let response = request.send().await.unwrap(); - let mut response = if response.status() == StatusCode::OK { - response - .json::() - .await - .unwrap() + let mut response = if response.status() == reqwest::StatusCode::OK { + response.json::().await.unwrap() } else { error!( "Failed to backfill historical data for {} from {} to {}: {}", asset.symbol, - from, - until, + fetch_from, + fetch_until, response.text().await.unwrap() ); break; @@ -403,7 +303,7 @@ async fn backfill_bars_from_api( .unwrap_or_default() .unwrap_or_default() { - bars.insert(bar.timestamp, Bar::from((bar, asset.symbol.clone()))); + bars.push(Bar::from((bar, asset.symbol.clone()))); } if response.next_page_token.is_none() { @@ -412,29 +312,13 @@ async fn backfill_bars_from_api( next_page_token = response.next_page_token; } - bars.into_values().collect::>() -} + database::bars::upsert_batch(&app_config.clickhouse_client, &bars).await; + if let Some(last_bar) = bars.last() { + database::bars::upsert_validity(&app_config.clickhouse_client, &last_bar.clone().into()) + .await; + } -#[allow(clippy::significant_drop_tightening)] -async fn derive_recent_nulls( - app_config: &Arc, - asset: &Asset, - from: &OffsetDateTime, - backfilled: &Arc>>, -) { - let mut backfilled = backfilled.write().await; - let bars = database::bars::select_where_symbol_where_timestamp_larger_than( - &app_config.postgres_pool, - &asset.symbol, - from, - ) - .await; - database::bars::upsert_batch(&app_config.postgres_pool, &bars, true).await; - database::assets::update_timestamp_last_where_symbol( - &app_config.postgres_pool, - &asset.symbol, - &bars.last().unwrap().timestamp, - ) - .await; - backfilled.insert(asset.symbol.clone(), true); + backfilled.write().await.insert(asset.symbol.clone(), true); + + info!("Backfilled historical data for {}.", asset.symbol); } diff --git a/src/data/mod.rs b/src/data/mod.rs index 1d63dff..b676882 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -20,11 +20,9 @@ async fn authenticate_websocket( Some(Ok(Message::Text(data))) if from_str::>(&data) .unwrap() - .get(0) + .first() == Some(&websocket::data::incoming::Message::Success( - websocket::data::incoming::success::Message { - msg: websocket::data::incoming::success::MessageType::Connected, - }, + websocket::data::incoming::success::Message::Connected, )) => {} _ => panic!(), } @@ -45,11 +43,9 @@ async fn authenticate_websocket( Some(Ok(Message::Text(data))) if from_str::>(&data) .unwrap() - .get(0) + .first() == Some(&websocket::data::incoming::Message::Success( - websocket::data::incoming::success::Message { - msg: websocket::data::incoming::success::MessageType::Authenticated, - }, + websocket::data::incoming::success::Message::Authenticated, )) => {} _ => panic!(), }; diff --git a/src/database/assets.rs b/src/database/assets.rs index dc5e36a..73609e9 100644 --- a/src/database/assets.rs +++ b/src/database/assets.rs @@ -1,92 +1,43 @@ -use crate::types::{Asset, Class, Exchange}; -use sqlx::{query_as, PgPool}; -use std::convert::Into; -use time::OffsetDateTime; +use crate::types::{Asset, Class}; +use clickhouse::Client; -pub async fn select(postgres_pool: &PgPool) -> Vec { - query_as!( - Asset, - r#"SELECT symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last FROM assets"# - ) - .fetch_all(postgres_pool) - .await - .unwrap() +pub async fn select(clickhouse_client: &Client) -> Vec { + clickhouse_client + .query("SELECT ?fields FROM assets") + .fetch_all::() + .await + .unwrap() } -pub async fn select_where_class(postgres_pool: &PgPool, class: Class) -> Vec { - query_as!( - Asset, - r#"SELECT symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last FROM assets WHERE class = $1::CLASS"#, - class as Class - ) - .fetch_all(postgres_pool) - .await - .unwrap() +pub async fn select_where_class(clickhouse_client: &Client, class: Class) -> Vec { + clickhouse_client + .query("SELECT ?fields FROM assets WHERE class = ?") + .bind(class) + .fetch_all::() + .await + .unwrap() } -pub async fn select_where_symbol(postgres_pool: &PgPool, symbol: &str) -> Option { - query_as!( - Asset, - r#"SELECT symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last FROM assets WHERE symbol = $1"#, - symbol - ) - .fetch_optional(postgres_pool) - .await - .unwrap() +pub async fn select_where_symbol(clickhouse_client: &Client, symbol: &str) -> Option { + clickhouse_client + .query("SELECT ?fields FROM assets WHERE symbol = ?") + .bind(symbol) + .fetch_optional::() + .await + .unwrap() } -pub async fn insert(postgres_pool: &PgPool, asset: &Asset) -> Asset { - query_as!( - Asset, - r#"INSERT INTO assets (symbol, class, exchange, trading, timestamp_added, timestamp_first, timestamp_last) VALUES ($1, $2::CLASS, $3::EXCHANGE, $4, $5, $6, $7) - RETURNING symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last"#, - asset.symbol, asset.class as Class, asset.exchange as Exchange, asset.trading, asset.timestamp_added, asset.timestamp_first, asset.timestamp_last - ) - .fetch_one(postgres_pool) - .await - .unwrap() +pub async fn insert(clickhouse_client: &Client, asset: &Asset) { + let mut insert = clickhouse_client.insert("assets").unwrap(); + insert.write(asset).await.unwrap(); + insert.end().await.unwrap(); } -pub async fn update_trading_where_symbol( - postgres_pool: &PgPool, - symbol: &str, - trading: &bool, -) -> Option { - query_as!( - Asset, - r#"UPDATE assets SET trading = $1 WHERE symbol = $2 - RETURNING symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last"#, - trading, symbol - ) - .fetch_optional(postgres_pool) - .await - .unwrap() -} - -pub async fn update_timestamp_last_where_symbol( - postgres_pool: &PgPool, - symbol: &str, - timestamp_last: &OffsetDateTime, -) -> Option { - query_as!( - Asset, - r#"UPDATE assets SET timestamp_last = $1 WHERE symbol = $2 - RETURNING symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last"#, - timestamp_last, symbol - ) - .fetch_optional(postgres_pool) - .await - .unwrap() -} - -pub async fn delete_where_symbol(postgres_pool: &PgPool, symbol: &str) -> Option { - query_as!( - Asset, - r#"DELETE FROM assets WHERE symbol = $1 - RETURNING symbol, class as "class: Class", exchange as "exchange: Exchange", trading, timestamp_added, timestamp_first, timestamp_last"#, - symbol - ) - .fetch_optional(postgres_pool) - .await - .unwrap() +pub async fn delete_where_symbol(clickhouse_client: &Client, symbol: &str) { + clickhouse_client + .query("DELETE FROM assets WHERE symbol = ?") + .bind(symbol) + .execute() + .await + .unwrap(); } diff --git a/src/database/bars.rs b/src/database/bars.rs index 25c6e77..f2f8afc 100644 --- a/src/database/bars.rs +++ b/src/database/bars.rs @@ -1,161 +1,52 @@ -use crate::types::Bar; -use sqlx::{query_as, PgPool, Postgres}; -use time::OffsetDateTime; +use crate::types::{Bar, BarValidity}; +use clickhouse::Client; -pub async fn select_not_null_where_symbol_where_timestamp_smaller_than_order_by_timestamp_desc_limit_one( - postgres_pool: &PgPool, +pub async fn upsert(clickhouse_client: &Client, bar: &Bar) { + let mut insert = clickhouse_client.insert("bars").unwrap(); + insert.write(bar).await.unwrap(); + insert.end().await.unwrap(); +} + +pub async fn upsert_batch(clickhouse_client: &Client, bars: &[Bar]) { + let mut insert = clickhouse_client.insert("bars").unwrap(); + for bar in bars { + insert.write(bar).await.unwrap(); + } + insert.end().await.unwrap(); +} + +pub async fn delete_where_symbol(clickhouse_client: &Client, symbol: &str) { + clickhouse_client + .query("DELETE FROM bars WHERE symbol = ?") + .bind(symbol) + .execute() + .await + .unwrap(); +} + +pub async fn select_validity_where_symbol( + clickhouse_client: &Client, symbol: &str, - timestamp: &OffsetDateTime, -) -> Bar { - query_as!( - Bar, - r#"SELECT * FROM bars WHERE asset_symbol = $1 AND timestamp < $2 AND open IS NOT NULL AND high IS NOT NULL AND low IS NOT NULL AND close IS NOT NULL ORDER BY timestamp DESC LIMIT 1"#, - symbol, - timestamp - ) - .fetch_one(postgres_pool) - .await - .unwrap() +) -> Option { + clickhouse_client + .query("SELECT ?fields FROM bars_validity FINAL WHERE symbol = ?") + .bind(symbol) + .fetch_optional::() + .await + .unwrap() } -pub async fn select_where_symbol_where_timestamp_larger_than( - postgres_pool: &PgPool, - symbol: &str, - timestamp: &OffsetDateTime, -) -> Vec { - query_as!( - Bar, - r#"SELECT * FROM bars WHERE asset_symbol = $1 AND timestamp > $2 ORDER BY timestamp ASC"#, - symbol, - timestamp - ) - .fetch_all(postgres_pool) - .await - .unwrap() +pub async fn upsert_validity(clickhouse_client: &Client, bar_validity: &BarValidity) { + let mut insert = clickhouse_client.insert("bars_validity").unwrap(); + insert.write(bar_validity).await.unwrap(); + insert.end().await.unwrap(); } -pub async fn upsert(postgres_pool: &PgPool, bar: &Bar, backfill: bool) -> Bar { - let mut bar = bar.clone(); - - if backfill - && (bar.open.is_none() || bar.high.is_none() || bar.low.is_none() || bar.close.is_none()) - { - let filled_bar = select_not_null_where_symbol_where_timestamp_smaller_than_order_by_timestamp_desc_limit_one( - postgres_pool, - &bar.asset_symbol, - &bar.timestamp, - ).await; - bar.merge_empty(&filled_bar); - } - - query_as!( - Bar, - r#"INSERT INTO bars (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (timestamp, asset_symbol) DO UPDATE SET open = $3, high = $4, low = $5, close = $6, volume = $7, num_trades = $8, volume_weighted = $9 - RETURNING timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted"#, - bar.timestamp, bar.asset_symbol, bar.open, bar.high, bar.low, bar.close, bar.volume, bar.num_trades, bar.volume_weighted - ) - .fetch_one(postgres_pool) - .await - .unwrap() -} - -pub async fn insert_or_skip(postgres_pool: &PgPool, bar: &Bar, backfill: bool) { - let mut bar = bar.clone(); - - if backfill - && (bar.open.is_none() || bar.high.is_none() || bar.low.is_none() || bar.close.is_none()) - { - let filled_bar = select_not_null_where_symbol_where_timestamp_smaller_than_order_by_timestamp_desc_limit_one( - postgres_pool, - &bar.asset_symbol, - &bar.timestamp, - ).await; - bar.merge_empty(&filled_bar); - } - - query_as!( - Bar, - r#"INSERT INTO bars (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (timestamp, asset_symbol) DO NOTHING"#, - bar.timestamp, bar.asset_symbol, bar.open, bar.high, bar.low, bar.close, bar.volume, bar.num_trades, bar.volume_weighted - ) - .execute(postgres_pool) - .await - .unwrap(); -} - -pub async fn upsert_batch(postgres_pool: &PgPool, bars: &[Bar], backfill: bool) -> Vec { - let mut bars = bars.to_vec(); - - if bars.is_empty() { - return bars; - } - - if backfill - && (bars[0].open.is_none() - || bars[0].high.is_none() - || bars[0].low.is_none() - || bars[0].close.is_none()) - { - let filled_bar = select_not_null_where_symbol_where_timestamp_smaller_than_order_by_timestamp_desc_limit_one( - postgres_pool, - &bars[0].asset_symbol, - &bars[0].timestamp, - ).await; - bars[0].merge_empty(&filled_bar); - } - - let mut timestamp = Vec::with_capacity(bars.len()); - let mut asset_symbol = Vec::with_capacity(bars.len()); - let mut open = Vec::with_capacity(bars.len()); - let mut high = Vec::with_capacity(bars.len()); - let mut low = Vec::with_capacity(bars.len()); - let mut close = Vec::with_capacity(bars.len()); - let mut volume = Vec::with_capacity(bars.len()); - let mut num_trades = Vec::with_capacity(bars.len()); - let mut volume_weighted = Vec::with_capacity(bars.len()); - - let mut last_filled_bar = bars[0].clone(); - - for mut bar in bars { - if backfill { - if bar.open.is_none() || bar.high.is_none() || bar.low.is_none() || bar.close.is_none() - { - bar.merge_empty(&last_filled_bar); - } else { - last_filled_bar = bar.clone(); - } - } - - timestamp.push(bar.timestamp); - asset_symbol.push(bar.asset_symbol.clone()); - open.push(bar.open); - high.push(bar.high); - low.push(bar.low); - close.push(bar.close); - volume.push(bar.volume); - num_trades.push(bar.num_trades); - volume_weighted.push(bar.volume_weighted); - } - - // No type-safety here because of NULLABLE bulk insert - query_as::( - r#"INSERT INTO bars (timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted) - SELECT * FROM UNNEST($1::timestamptz[], $2::text[], $3::float8[], $4::float8[], $5::float8[], $6::float8[], $7::float8[], $8::int8[], $9::float8[]) - ON CONFLICT (timestamp, asset_symbol) DO UPDATE SET open = EXCLUDED.open, high = EXCLUDED.high, low = EXCLUDED.low, close = EXCLUDED.close, volume = EXCLUDED.volume, num_trades = EXCLUDED.num_trades, volume_weighted = EXCLUDED.volume_weighted - RETURNING timestamp, asset_symbol, open, high, low, close, volume, num_trades, volume_weighted"#, - ) - .bind(timestamp) - .bind(asset_symbol) - .bind(open) - .bind(high) - .bind(low) - .bind(close) - .bind(volume) - .bind(num_trades) - .bind(volume_weighted) - .fetch_all(postgres_pool) - .await - .unwrap() +pub async fn delete_validity_where_symbol(clickhouse_client: &Client, symbol: &str) { + clickhouse_client + .query("DELETE FROM bars_validity WHERE symbol = ?") + .bind(symbol) + .execute() + .await + .unwrap(); } diff --git a/src/main.rs b/src/main.rs index 8dae5ab..91a193a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,35 +11,32 @@ mod types; use config::Config; use dotenv::dotenv; use log4rs::config::Deserializers; -use sqlx::error::BoxDynError; +use std::error::Error; use tokio::{spawn, sync::broadcast}; use types::{BroadcastMessage, Class}; #[tokio::main] -async fn main() -> Result<(), BoxDynError> { +async fn main() -> Result<(), Box> { dotenv().ok(); log4rs::init_file("log4rs.yaml", Deserializers::default())?; - let app_config = Config::arc_from_env().await; + let app_config = Config::arc_from_env(); let mut threads = Vec::new(); - let (asset_broadcast_sender, _) = broadcast::channel::(100); + let (broadcast_sender, _) = broadcast::channel::(100); threads.push(spawn(data::market::run( app_config.clone(), Class::UsEquity, - asset_broadcast_sender.clone(), + broadcast_sender.clone(), ))); threads.push(spawn(data::market::run( app_config.clone(), Class::Crypto, - asset_broadcast_sender.clone(), + broadcast_sender.clone(), ))); - threads.push(spawn(routes::run( - app_config.clone(), - asset_broadcast_sender, - ))); + threads.push(spawn(routes::run(app_config.clone(), broadcast_sender))); for thread in threads { thread.await?; diff --git a/src/routes/assets.rs b/src/routes/assets.rs index b3b56aa..e21cc0a 100644 --- a/src/routes/assets.rs +++ b/src/routes/assets.rs @@ -1,29 +1,28 @@ -use crate::config::{ - Config, ALPACA_ASSET_API_URL, ALPACA_CRYPTO_DATA_URL, ALPACA_STOCK_DATA_URL, - ALPACA_TIMESTAMP_FORMAT, -}; +use crate::config::{Config, ALPACA_ASSET_API_URL}; use crate::database; -use crate::types::Class; -use crate::types::{api::incoming, asset, Asset, BroadcastMessage, Status}; -use axum::{extract::Path, http::StatusCode, Extension, Json}; +use crate::types::{ + api::incoming::{self, asset::Status}, + asset, Asset, BroadcastMessage, +}; +use axum::{extract::Path, Extension, Json}; +use http::StatusCode; use log::info; use serde::Deserialize; use std::sync::Arc; -use time::OffsetDateTime; use tokio::sync::broadcast::Sender; -pub async fn get_all( - Extension(app_config): Extension>, -) -> Result<(StatusCode, Json>), StatusCode> { - let assets = database::assets::select(&app_config.postgres_pool).await; - Ok((StatusCode::OK, Json(assets))) -} - pub async fn get( Extension(app_config): Extension>, +) -> Result<(StatusCode, Json>), StatusCode> { + let assets = database::assets::select(&app_config.clickhouse_client).await; + Ok((StatusCode::OK, Json(assets))) +} + +pub async fn get_where_symbol( + Extension(app_config): Extension>, Path(symbol): Path, ) -> Result<(StatusCode, Json), StatusCode> { - let asset = database::assets::select_where_symbol(&app_config.postgres_pool, &symbol).await; + let asset = database::assets::select_where_symbol(&app_config.clickhouse_client, &symbol).await; asset.map_or(Err(StatusCode::NOT_FOUND), |asset| { Ok((StatusCode::OK, Json(asset))) }) @@ -32,15 +31,14 @@ pub async fn get( #[derive(Deserialize)] pub struct AddAssetRequest { symbol: String, - trading: Option, } pub async fn add( Extension(app_config): Extension>, - Extension(asset_broadcast_sender): Extension>, + Extension(broadcast_sender): Extension>, Json(request): Json, ) -> Result<(StatusCode, Json), StatusCode> { - if database::assets::select_where_symbol(&app_config.postgres_pool, &request.symbol) + if database::assets::select_where_symbol(&app_config.clickhouse_client, &request.symbol) .await .is_some() { @@ -53,66 +51,25 @@ pub async fn add( .get(&format!("{}/{}", ALPACA_ASSET_API_URL, request.symbol)) .send() .await - .map_err(|e| match e.status() { - Some(StatusCode::NOT_FOUND) => StatusCode::NOT_FOUND, - _ => panic!(), - })?; + .map_err(|e| { + if e.status() == Some(reqwest::StatusCode::NOT_FOUND) { + StatusCode::NOT_FOUND + } else { + panic!() + } + }) + .unwrap(); - let asset = asset.json::().await.unwrap(); + let asset = asset.json::().await.unwrap(); if asset.status != Status::Active || !asset.tradable || !asset.fractionable { return Err(StatusCode::FORBIDDEN); } - let mut earliest_bar_request = app_config - .alpaca_client - .get(match asset.class { - Class::UsEquity => ALPACA_STOCK_DATA_URL, - Class::Crypto => ALPACA_CRYPTO_DATA_URL, - }) - .query(&[ - ("symbols", &asset.symbol), - ("timeframe", &String::from("1Min")), - ( - "start", - &OffsetDateTime::UNIX_EPOCH - .format(ALPACA_TIMESTAMP_FORMAT) - .unwrap(), - ), - ("limit", &String::from("1")), - ]); + let asset = Asset::from(asset); + database::assets::insert(&app_config.clickhouse_client, &asset).await; - if asset.class == Class::UsEquity { - earliest_bar_request = - earliest_bar_request.query(&[("feed", &app_config.alpaca_source.to_string())]); - } - - let earliest_bar = earliest_bar_request - .send() - .await - .unwrap() - .json::() - .await - .unwrap(); - - let earliest_bar = earliest_bar - .bars - .get(&asset.symbol) - .ok_or(StatusCode::NOT_FOUND)? - .as_ref() - .ok_or(StatusCode::NOT_FOUND)? - .first() - .ok_or(StatusCode::NOT_FOUND)?; - - let asset = Asset::from(( - asset, - request.trading.unwrap_or(false), - earliest_bar.timestamp, - )); - - database::assets::insert(&app_config.postgres_pool, &asset).await; - - asset_broadcast_sender + broadcast_sender .send(BroadcastMessage::Asset(asset::BroadcastMessage::Added( asset.clone(), ))) @@ -122,50 +79,24 @@ pub async fn add( Ok((StatusCode::CREATED, Json(asset))) } -#[allow(dead_code)] -#[derive(Deserialize)] -pub struct UpdateAssetRequest { - trading: bool, -} - -pub async fn update( - Extension(app_config): Extension>, - Extension(asset_broadcast_sender): Extension>, - Path(symbol): Path, - Json(request): Json, -) -> Result<(StatusCode, Json), StatusCode> { - let asset = database::assets::update_trading_where_symbol( - &app_config.postgres_pool, - &symbol, - &request.trading, - ) - .await; - - asset.map_or(Err(StatusCode::NOT_FOUND), |asset| { - asset_broadcast_sender - .send(BroadcastMessage::Asset(asset::BroadcastMessage::Updated( - asset.clone(), - ))) - .unwrap(); - info!("Updated asset {}.", symbol); - Ok((StatusCode::OK, Json(asset))) - }) -} - pub async fn delete( Extension(app_config): Extension>, - Extension(asset_broadcast_sender): Extension>, + Extension(broadcast_sender): Extension>, Path(symbol): Path, ) -> Result { - let asset = database::assets::delete_where_symbol(&app_config.postgres_pool, &symbol).await; + let asset = database::assets::select_where_symbol(&app_config.clickhouse_client, &symbol) + .await + .ok_or(StatusCode::NOT_FOUND) + .unwrap(); - asset.map_or(Err(StatusCode::NOT_FOUND), |asset| { - asset_broadcast_sender - .send(BroadcastMessage::Asset(asset::BroadcastMessage::Deleted( - asset, - ))) - .unwrap(); - info!("Deleted asset {}.", symbol); - Ok(StatusCode::NO_CONTENT) - }) + broadcast_sender + .send(BroadcastMessage::Asset(asset::BroadcastMessage::Deleted( + asset, + ))) + .unwrap(); + + database::assets::delete_where_symbol(&app_config.clickhouse_client, &symbol).await; + + info!("Deleted asset {}.", symbol); + Ok(StatusCode::NO_CONTENT) } diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 0925256..c53e4c1 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -1,30 +1,26 @@ use crate::{config::Config, types::BroadcastMessage}; use axum::{ routing::{delete, get, post}, - Extension, Router, Server, + serve, Extension, Router, }; use log::info; use std::{net::SocketAddr, sync::Arc}; -use tokio::sync::broadcast::Sender; +use tokio::{net::TcpListener, sync::broadcast::Sender}; pub mod assets; -pub async fn run(app_config: Arc, asset_broadcast_sender: Sender) { +pub async fn run(app_config: Arc, broadcast_sender: Sender) { let app = Router::new() - .route("/assets", get(assets::get_all)) - .route("/assets/:symbol", get(assets::get)) + .route("/assets", get(assets::get)) + .route("/assets/:symbol", get(assets::get_where_symbol)) .route("/assets", post(assets::add)) - .route("/assets/:symbol", post(assets::update)) .route("/assets/:symbol", delete(assets::delete)) .layer(Extension(app_config)) - .layer(Extension(asset_broadcast_sender)); + .layer(Extension(broadcast_sender)); let addr = SocketAddr::from(([0, 0, 0, 0], 7878)); + let listener = TcpListener::bind(addr).await.unwrap(); info!("Listening on {}.", addr); - Server::bind(&addr) - .serve(app.into_make_service()) - .await - .unwrap(); - + serve(listener, app).await.unwrap(); unreachable!() } diff --git a/src/time.rs b/src/time.rs index d2e4733..9813731 100644 --- a/src/time.rs +++ b/src/time.rs @@ -1,7 +1,6 @@ use std::time::Duration; use time::OffsetDateTime; -pub const THIRTY_SECONDS: Duration = Duration::from_secs(30); pub const ONE_MINUTE: Duration = Duration::from_secs(60); pub fn last_minute() -> OffsetDateTime { @@ -13,18 +12,8 @@ pub fn next_minute() -> OffsetDateTime { last_minute() + ONE_MINUTE } -pub fn last_30s() -> OffsetDateTime { - let now_timestamp = OffsetDateTime::now_utc().unix_timestamp(); - OffsetDateTime::from_unix_timestamp(now_timestamp - now_timestamp % 30).unwrap() -} - -pub fn next_30s() -> OffsetDateTime { - last_30s() + THIRTY_SECONDS -} - pub fn duration_until(time: OffsetDateTime) -> Duration { - let now = OffsetDateTime::now_utc(); - let duration = time - now; + let duration = time - OffsetDateTime::now_utc(); if duration.is_positive() { duration.unsigned_abs() diff --git a/src/types/api/incoming/asset.rs b/src/types/api/incoming/asset.rs index b137715..972d79c 100644 --- a/src/types/api/incoming/asset.rs +++ b/src/types/api/incoming/asset.rs @@ -1,9 +1,51 @@ #![allow(clippy::struct_excessive_bools)] -use crate::types::{Class, Exchange, Status}; -use serde::Deserialize; +use crate::types::api::impl_from_enum; +use serde::{Deserialize, Serialize}; -#[derive(Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Class { + UsEquity, + Crypto, +} + +impl_from_enum!(crate::types::Class, Class, UsEquity, Crypto); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum Exchange { + Amex, + Arca, + Bats, + Nyse, + Nasdaq, + Nysearca, + Otc, + Crypto, +} + +impl_from_enum!( + crate::types::Exchange, + Exchange, + Amex, + Arca, + Bats, + Nyse, + Nasdaq, + Nysearca, + Otc, + Crypto +); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Status { + Active, + Inactive, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct Asset { pub id: String, pub class: Class, @@ -19,3 +61,14 @@ pub struct Asset { pub maintenance_margin_requirement: Option, pub attributes: Option>, } + +impl From for crate::types::Asset { + fn from(item: Asset) -> Self { + Self { + symbol: item.symbol, + class: item.class.into(), + exchange: item.exchange.into(), + time_added: time::OffsetDateTime::now_utc(), + } + } +} diff --git a/src/types/api/incoming/bar.rs b/src/types/api/incoming/bar.rs index 4b23c47..78045b4 100644 --- a/src/types/api/incoming/bar.rs +++ b/src/types/api/incoming/bar.rs @@ -1,12 +1,12 @@ -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; use time::OffsetDateTime; -#[derive(Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct Bar { #[serde(rename = "t")] #[serde(with = "time::serde::rfc3339")] - pub timestamp: OffsetDateTime, + pub time: OffsetDateTime, #[serde(rename = "o")] pub open: f64, #[serde(rename = "h")] @@ -16,14 +16,30 @@ pub struct Bar { #[serde(rename = "c")] pub close: f64, #[serde(rename = "v")] - pub volume: f64, + pub volume: i64, #[serde(rename = "n")] - pub num_trades: i64, + pub trades: i64, #[serde(rename = "vw")] - pub volume_weighted: f64, + pub vwap: f64, } -#[derive(Deserialize)] +impl From<(Bar, String)> for crate::types::Bar { + fn from((bar, symbol): (Bar, String)) -> Self { + Self { + time: bar.time, + symbol, + open: bar.open, + high: bar.high, + low: bar.low, + close: bar.close, + volume: bar.volume, + trades: bar.trades, + vwap: bar.vwap, + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct Message { pub bars: HashMap>>, pub next_page_token: Option, diff --git a/src/types/api/incoming/calendar_date.rs b/src/types/api/incoming/calendar_date.rs deleted file mode 100644 index f399d56..0000000 --- a/src/types/api/incoming/calendar_date.rs +++ /dev/null @@ -1,29 +0,0 @@ -use serde::{Deserialize, Deserializer}; -use time::{macros::format_description, Date, Time}; - -#[derive(Deserialize)] -pub struct CalendarDate { - #[serde(deserialize_with = "deserialize_date")] - pub date: Date, - #[serde(deserialize_with = "deserialize_time")] - pub open: Time, - #[serde(deserialize_with = "deserialize_time")] - pub close: Time, -} - -fn deserialize_date<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let date_str = String::deserialize(deserializer)?; - Date::parse(&date_str, format_description!("[year]-[month]-[day]")) - .map_err(serde::de::Error::custom) -} - -fn deserialize_time<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let time_str = String::deserialize(deserializer)?; - Time::parse(&time_str, format_description!("[hour]:[minute]")).map_err(serde::de::Error::custom) -} diff --git a/src/types/api/incoming/mod.rs b/src/types/api/incoming/mod.rs index 7609673..c01abb8 100644 --- a/src/types/api/incoming/mod.rs +++ b/src/types/api/incoming/mod.rs @@ -1,7 +1,2 @@ pub mod asset; pub mod bar; -pub mod calendar_date; - -pub use asset::Asset; -pub use bar::Bar; -pub use calendar_date::CalendarDate; diff --git a/src/types/api/mod.rs b/src/types/api/mod.rs index 28f2603..1b61618 100644 --- a/src/types/api/mod.rs +++ b/src/types/api/mod.rs @@ -1 +1,23 @@ pub mod incoming; + +macro_rules! impl_from_enum { + ($source:ty, $target:ty, $( $variant:ident ),* ) => { + impl From<$source> for $target { + fn from(item: $source) -> Self { + match item { + $( <$source>::$variant => <$target>::$variant, )* + } + } + } + + impl From<$target> for $source { + fn from(item: $target) -> Self { + match item { + $( <$target>::$variant => <$source>::$variant, )* + } + } + } + }; +} + +use impl_from_enum; diff --git a/src/types/asset.rs b/src/types/asset.rs index af16534..de80381 100644 --- a/src/types/asset.rs +++ b/src/types/asset.rs @@ -1,86 +1,39 @@ -use super::api; +use clickhouse::Row; use serde::{Deserialize, Serialize}; -use sqlx::{FromRow, Type}; +use serde_repr::{Deserialize_repr, Serialize_repr}; use time::OffsetDateTime; -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize_repr, Deserialize_repr)] +#[repr(u8)] pub enum Class { - #[sqlx(rename = "us_equity")] - #[serde(rename = "us_equity")] - UsEquity, - #[sqlx(rename = "crypto")] - #[serde(rename = "crypto")] - Crypto, + UsEquity = 1, + Crypto = 2, } -#[derive(Clone, Copy, Debug, Serialize, Deserialize, Type)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize_repr, Deserialize_repr)] +#[repr(u8)] pub enum Exchange { - #[sqlx(rename = "AMEX")] - #[serde(rename = "AMEX")] - Amex, - #[sqlx(rename = "ARCA")] - #[serde(rename = "ARCA")] - Arca, - #[sqlx(rename = "BATS")] - #[serde(rename = "BATS")] - Bats, - #[sqlx(rename = "NYSE")] - #[serde(rename = "NYSE")] - Nyse, - #[sqlx(rename = "NASDAQ")] - #[serde(rename = "NASDAQ")] - Nasdaq, - #[sqlx(rename = "NYSEARCA")] - #[serde(rename = "NYSEARCA")] - Nysearca, - #[sqlx(rename = "OTC")] - #[serde(rename = "OTC")] - Otc, - #[sqlx(rename = "CRYPTO")] - #[serde(rename = "CRYPTO")] - Crypto, + Amex = 1, + Arca = 2, + Bats = 3, + Nyse = 4, + Nasdaq = 5, + Nysearca = 6, + Otc = 7, + Crypto = 8, } -#[derive(PartialEq, Eq, Deserialize, Type)] -pub enum Status { - #[sqlx(rename = "active")] - #[serde(rename = "active")] - Active, - #[sqlx(rename = "inactive")] - #[serde(rename = "inactive")] - Inactive, -} - -#[derive(Clone, Debug, FromRow, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Row)] pub struct Asset { pub symbol: String, pub class: Class, pub exchange: Exchange, - pub trading: bool, - pub timestamp_added: OffsetDateTime, - pub timestamp_first: OffsetDateTime, - pub timestamp_last: OffsetDateTime, + #[serde(with = "clickhouse::serde::time::datetime")] + pub time_added: OffsetDateTime, } -impl From<(api::incoming::Asset, bool, OffsetDateTime)> for Asset { - fn from( - (asset, trading, timestamp_first): (api::incoming::Asset, bool, OffsetDateTime), - ) -> Self { - Self { - symbol: asset.symbol, - class: asset.class, - exchange: asset.exchange, - trading, - timestamp_added: OffsetDateTime::now_utc(), - timestamp_first, - timestamp_last: timestamp_first, - } - } -} - -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum BroadcastMessage { Added(Asset), - Updated(Asset), Deleted(Asset), } diff --git a/src/types/bar.rs b/src/types/bar.rs index 471ec0e..d7f5616 100644 --- a/src/types/bar.rs +++ b/src/types/bar.rs @@ -1,72 +1,44 @@ -use super::{api, websocket}; -use serde::Serialize; -use sqlx::FromRow; +#![allow(clippy::module_name_repetitions)] + +use clickhouse::Row; +use serde::{Deserialize, Serialize}; use time::OffsetDateTime; -#[derive(Clone, Debug, FromRow, Serialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Row)] pub struct Bar { - pub timestamp: OffsetDateTime, - pub asset_symbol: String, - pub open: Option, - pub high: Option, - pub low: Option, - pub close: Option, - pub volume: f64, - pub num_trades: i64, - pub volume_weighted: f64, + #[serde(with = "clickhouse::serde::time::datetime")] + pub time: OffsetDateTime, + pub symbol: String, + pub open: f64, + pub high: f64, + pub low: f64, + pub close: f64, + pub volume: i64, + pub trades: i64, + pub vwap: f64, } -impl Bar { - pub const fn empty(timestamp: OffsetDateTime, asset_symbol: String) -> Self { - Self { - timestamp, - asset_symbol, - open: None, - high: None, - low: None, - close: None, - volume: 0.0, - num_trades: 0, - volume_weighted: 0.0, - } - } - - pub fn merge_empty(&mut self, other: &Self) { - self.open = other.open; - self.high = other.high; - self.low = other.low; - self.close = other.close; - } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Row)] +pub struct BarValidity { + pub symbol: String, + #[serde(with = "clickhouse::serde::time::datetime")] + pub time_last: OffsetDateTime, } -impl From for Bar { - fn from(bar_message: websocket::data::incoming::bar::Message) -> Self { +impl BarValidity { + pub const fn none(symbol: String) -> Self { Self { - timestamp: bar_message.timestamp, - asset_symbol: bar_message.symbol, - open: Some(bar_message.open), - high: Some(bar_message.high), - low: Some(bar_message.low), - close: Some(bar_message.close), - volume: bar_message.volume, - num_trades: bar_message.num_trades, - volume_weighted: bar_message.volume_weighted, + symbol, + time_last: OffsetDateTime::UNIX_EPOCH, } } } -impl From<(api::incoming::Bar, String)> for Bar { - fn from((bar, asset_symbol): (api::incoming::Bar, String)) -> Self { +impl From for BarValidity { + fn from(bar: Bar) -> Self { Self { - timestamp: bar.timestamp, - asset_symbol, - open: Some(bar.open), - high: Some(bar.high), - low: Some(bar.low), - close: Some(bar.close), - volume: bar.volume, - num_trades: bar.num_trades, - volume_weighted: bar.volume_weighted, + symbol: bar.symbol, + time_last: bar.time, } } } diff --git a/src/types/mod.rs b/src/types/mod.rs index 2b6196a..323d5ae 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -4,11 +4,11 @@ pub mod bar; pub mod source; pub mod websocket; -pub use asset::{Asset, Class, Exchange, Status}; -pub use bar::Bar; +pub use asset::{Asset, Class, Exchange}; +pub use bar::{Bar, BarValidity}; pub use source::Source; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum BroadcastMessage { Asset(asset::BroadcastMessage), } diff --git a/src/types/source.rs b/src/types/source.rs index cf56619..f734d3e 100644 --- a/src/types/source.rs +++ b/src/types/source.rs @@ -3,7 +3,7 @@ use std::{ str::FromStr, }; -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Source { Iex, Sip, diff --git a/src/types/websocket/data/incoming/bar.rs b/src/types/websocket/data/incoming/bar.rs index 5db1c39..cea2023 100644 --- a/src/types/websocket/data/incoming/bar.rs +++ b/src/types/websocket/data/incoming/bar.rs @@ -1,11 +1,11 @@ -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use time::OffsetDateTime; -#[derive(PartialEq, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct Message { #[serde(rename = "t")] #[serde(with = "time::serde::rfc3339")] - pub timestamp: OffsetDateTime, + pub time: OffsetDateTime, #[serde(rename = "S")] pub symbol: String, #[serde(rename = "o")] @@ -17,9 +17,25 @@ pub struct Message { #[serde(rename = "c")] pub close: f64, #[serde(rename = "v")] - pub volume: f64, + pub volume: i64, #[serde(rename = "n")] - pub num_trades: i64, + pub trades: i64, #[serde(rename = "vw")] - pub volume_weighted: f64, + pub vwap: f64, +} + +impl From for crate::types::Bar { + fn from(bar: Message) -> Self { + Self { + time: bar.time, + symbol: bar.symbol, + open: bar.open, + high: bar.high, + low: bar.low, + close: bar.close, + volume: bar.volume, + trades: bar.trades, + vwap: bar.vwap, + } + } } diff --git a/src/types/websocket/data/incoming/mod.rs b/src/types/websocket/data/incoming/mod.rs index 18ec825..1a2f413 100644 --- a/src/types/websocket/data/incoming/mod.rs +++ b/src/types/websocket/data/incoming/mod.rs @@ -2,9 +2,9 @@ pub mod bar; pub mod subscription; pub mod success; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; -#[derive(PartialEq, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "T")] pub enum Message { #[serde(rename = "success")] diff --git a/src/types/websocket/data/incoming/subscription.rs b/src/types/websocket/data/incoming/subscription.rs index 731f7ab..8531f88 100644 --- a/src/types/websocket/data/incoming/subscription.rs +++ b/src/types/websocket/data/incoming/subscription.rs @@ -1,17 +1,15 @@ -use serde::Deserialize; +use serde::{Deserialize, Serialize}; -#[derive(PartialEq, Eq, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct Message { pub trades: Vec, pub quotes: Vec, pub bars: Vec, - #[serde(rename = "updatedBars")] pub updated_bars: Vec, - #[serde(rename = "dailyBars")] pub daily_bars: Vec, pub orderbooks: Option>, pub statuses: Option>, pub lulds: Option>, - #[serde(rename = "cancelErrors")] pub cancel_errors: Option>, } diff --git a/src/types/websocket/data/incoming/success.rs b/src/types/websocket/data/incoming/success.rs index 37ac155..c130169 100644 --- a/src/types/websocket/data/incoming/success.rs +++ b/src/types/websocket/data/incoming/success.rs @@ -1,14 +1,9 @@ -use serde::Deserialize; +use serde::{Deserialize, Serialize}; -#[derive(PartialEq, Eq, Deserialize)] -pub enum MessageType { - #[serde(rename = "connected")] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "msg")] +#[serde(rename_all = "camelCase")] +pub enum Message { Connected, - #[serde(rename = "authenticated")] Authenticated, } - -#[derive(PartialEq, Eq, Deserialize)] -pub struct Message { - pub msg: MessageType, -} diff --git a/src/types/websocket/data/outgoing/mod.rs b/src/types/websocket/data/outgoing/mod.rs index 4bf2d01..bebf22b 100644 --- a/src/types/websocket/data/outgoing/mod.rs +++ b/src/types/websocket/data/outgoing/mod.rs @@ -5,11 +5,9 @@ use serde::Serialize; #[derive(Serialize)] #[serde(tag = "action")] +#[serde(rename_all = "camelCase")] pub enum Message { - #[serde(rename = "auth")] Auth(auth::Message), - #[serde(rename = "subscribe")] Subscribe(subscribe::Message), - #[serde(rename = "unsubscribe")] Unsubscribe(subscribe::Message), } diff --git a/src/types/websocket/data/outgoing/subscribe.rs b/src/types/websocket/data/outgoing/subscribe.rs index f0ab6c7..5250dd9 100644 --- a/src/types/websocket/data/outgoing/subscribe.rs +++ b/src/types/websocket/data/outgoing/subscribe.rs @@ -1,9 +1,9 @@ use serde::Serialize; #[derive(Serialize)] +#[serde(rename_all = "camelCase")] pub struct Message { bars: Vec, - #[serde(rename = "updatedBars")] updated_bars: Vec, } diff --git a/support/clickhouse/docker-compose.yml b/support/clickhouse/docker-compose.yml new file mode 100644 index 0000000..a42af39 --- /dev/null +++ b/support/clickhouse/docker-compose.yml @@ -0,0 +1,29 @@ +services: + clickhouse: + image: clickhouse/clickhouse-server + hostname: clickhouse + restart: unless-stopped + volumes: + - clickhouse-lib:/var/lib/clickhouse + - clickhouse-log:/var/log/clickhouse-server + - ./config.d:/etc/clickhouse-server/config.d + - ./users.d:/etc/clickhouse-server/users.d + - ./docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + environment: + - CLICKHOUSE_USER=${CLICKHOUSE_USER} + - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD} + - CLICKHOUSE_DB=${CLICKHOUSE_DB} + - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 + network_mode: host + ulimits: + nofile: + soft: 262144 + hard: 262144 + cap_add: + - SYS_NICE + - NET_ADMIN + - IPC_LOCK + +volumes: + clickhouse-data: + clickhouse-logs: diff --git a/support/clickhouse/docker-entrypoint-initdb.d/0000_init.sql b/support/clickhouse/docker-entrypoint-initdb.d/0000_init.sql new file mode 100644 index 0000000..cf76efc --- /dev/null +++ b/support/clickhouse/docker-entrypoint-initdb.d/0000_init.sql @@ -0,0 +1,38 @@ +CREATE TABLE IF NOT EXISTS qrust.assets ( + symbol String, + class Enum('us_equity' = 1, 'crypto' = 2), + exchange Enum( + 'AMEX' = 1, + 'ARCA' = 2, + 'BATS' = 3, + 'NASDAQ' = 4, + 'NYSE' = 5, + 'NYSEARCA' = 6, + 'OTC' = 7, + 'CRYPTO' = 8 + ), + time_added DateTime DEFAULT now() +) +ENGINE = ReplacingMergeTree() +PRIMARY KEY symbol; + +CREATE TABLE IF NOT EXISTS qrust.bars ( + symbol String, + time DateTime, + open Float64, + high Float64, + low Float64, + close Float64, + volume Int64, + trades Int64, + vwap Float64 +) +ENGINE = ReplacingMergeTree() +PRIMARY KEY (symbol, time); + +CREATE TABLE IF NOT EXISTS qrust.bars_validity ( + symbol String, + time_last DateTime +) +ENGINE = ReplacingMergeTree() +PRIMARY KEY symbol; diff --git a/support/clickhouse/users.d/default-user.xml b/support/clickhouse/users.d/default-user.xml new file mode 100644 index 0000000..db46af6 --- /dev/null +++ b/support/clickhouse/users.d/default-user.xml @@ -0,0 +1,18 @@ + + + + + + + + + default + + ::/0 + + qrust + default + 1 + + + diff --git a/support/timescaledb/999_init.sh b/support/timescaledb/999_init.sh deleted file mode 100644 index 640c3bd..0000000 --- a/support/timescaledb/999_init.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - CREATE TYPE CLASS AS ENUM ('us_equity', 'crypto'); - - CREATE TYPE EXCHANGE AS ENUM ( - 'AMEX', - 'ARCA', - 'BATS', - 'NASDAQ', - 'NYSE', - 'NYSEARCA', - 'OTC', - 'CRYPTO' - ); - - CREATE TABLE assets ( - symbol TEXT PRIMARY KEY, - class CLASS NOT NULL, - exchange EXCHANGE NOT NULL, - trading BOOLEAN NOT NULL DEFAULT FALSE, - timestamp_added TIMESTAMPTZ NOT NULL DEFAULT NOW(), - timestamp_first TIMESTAMPTZ NOT NULL, - timestamp_last TIMESTAMPTZ NOT NULL - ); - - CREATE TABLE bars ( - timestamp TIMESTAMPTZ, - asset_symbol TEXT REFERENCES assets(symbol) ON DELETE CASCADE ON UPDATE CASCADE, - open DOUBLE PRECISION, - high DOUBLE PRECISION, - low DOUBLE PRECISION, - close DOUBLE PRECISION, - volume DOUBLE PRECISION NOT NULL, - num_trades BIGINT NOT NULL, - volume_weighted DOUBLE PRECISION NOT NULL, - PRIMARY KEY (asset_symbol, timestamp) - ); - - SELECT create_hypertable('bars', 'timestamp', 'asset_symbol', 15); - - ALTER TABLE bars SET ( - timescaledb.compress, - timescaledb.compress_segmentby = 'asset_symbol' - ); - - SELECT add_compression_policy('bars', INTERVAL '30 days'); -EOSQL diff --git a/support/timescaledb/docker-compose.yml b/support/timescaledb/docker-compose.yml deleted file mode 100644 index 5a79a59..0000000 --- a/support/timescaledb/docker-compose.yml +++ /dev/null @@ -1,20 +0,0 @@ -services: - timescaledb: - image: timescale/timescaledb-ha:pg15-all - hostname: timescaledb - restart: unless-stopped - ports: - - 5432:5432 - volumes: - - timescaledb-data:/home/postgres/pgdata/data - - timescaledb-logs:/home/postgres/pg_log - - ./999_init.sh:/docker-entrypoint-initdb.d/999_init.sh - environment: - - TIMESCALEDB_TELEMETRY=off - - POSTGRES_USER=${POSTGRES_USER} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - POSTGRES_DB=${POSTGRES_DB} - -volumes: - timescaledb-data: - timescaledb-logs: