Fix the Sin of Man

- Migrate to ClickHouse
- Simplify serde renaming
- Simplify backfill logic
- Compartmentalize database columns

Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>

support/clickhouse/docker-compose.yml

@@ -0,0 +1,29 @@
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    hostname: clickhouse
+    restart: unless-stopped
+    volumes:
+      - clickhouse-lib:/var/lib/clickhouse
+      - clickhouse-log:/var/log/clickhouse-server
+      - ./config.d:/etc/clickhouse-server/config.d
+      - ./users.d:/etc/clickhouse-server/users.d
+      - ./docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+    environment:
+      - CLICKHOUSE_USER=${CLICKHOUSE_USER}
+      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}
+      - CLICKHOUSE_DB=${CLICKHOUSE_DB}
+      - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1
+    network_mode: host
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    cap_add:
+      - SYS_NICE
+      - NET_ADMIN
+      - IPC_LOCK
+
+volumes:
+  clickhouse-data:
+  clickhouse-logs:

support/clickhouse/docker-entrypoint-initdb.d/0000_init.sql

@@ -0,0 +1,38 @@
+CREATE TABLE IF NOT EXISTS qrust.assets (
+    symbol String,
+    class Enum('us_equity' = 1, 'crypto' = 2),
+    exchange Enum(
+        'AMEX' = 1,
+        'ARCA' = 2,
+        'BATS' = 3,
+        'NASDAQ' = 4,
+        'NYSE' = 5,
+        'NYSEARCA' = 6,
+        'OTC' = 7,
+        'CRYPTO' = 8
+    ),
+    time_added DateTime DEFAULT now()
+)
+ENGINE = ReplacingMergeTree()
+PRIMARY KEY symbol;
+
+CREATE TABLE IF NOT EXISTS qrust.bars (
+    symbol String,
+    time DateTime,
+    open Float64,
+    high Float64,
+    low Float64,
+    close Float64,
+    volume Int64,
+    trades Int64,
+    vwap Float64
+)
+ENGINE = ReplacingMergeTree()
+PRIMARY KEY (symbol, time);
+
+CREATE TABLE IF NOT EXISTS qrust.bars_validity (
+    symbol String,
+    time_last DateTime
+)
+ENGINE = ReplacingMergeTree()
+PRIMARY KEY symbol;

support/clickhouse/users.d/default-user.xml

@@ -0,0 +1,18 @@
+    <clickhouse>
+      <!-- Docs: <https://clickhouse.com/docs/en/operations/settings/settings_users/> -->
+      <users>
+        <!-- Remove default user -->
+        <default remove="remove">
+        </default>
+
+        <qrust>
+          <profile>default</profile>
+          <networks>
+            <ip>::/0</ip>
+          </networks>
+          <password>qrust</password>
+          <quota>default</quota>
+          <access_management>1</access_management>
+        </qrust>
+      </users>
+    </clickhouse>

support/timescaledb/999_init.sh

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
-    CREATE TYPE CLASS AS ENUM ('us_equity', 'crypto');
-
-    CREATE TYPE EXCHANGE AS ENUM (
-        'AMEX',
-        'ARCA',
-        'BATS',
-        'NASDAQ',
-        'NYSE',
-        'NYSEARCA',
-        'OTC',
-        'CRYPTO'
-    );
-
-    CREATE TABLE assets (
-        symbol TEXT PRIMARY KEY,
-        class CLASS NOT NULL,
-        exchange EXCHANGE NOT NULL,
-        trading BOOLEAN NOT NULL DEFAULT FALSE,
-        timestamp_added TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-        timestamp_first TIMESTAMPTZ NOT NULL,
-        timestamp_last TIMESTAMPTZ NOT NULL
-    );
-
-    CREATE TABLE bars (
-        timestamp TIMESTAMPTZ,
-        asset_symbol TEXT REFERENCES assets(symbol) ON DELETE CASCADE ON UPDATE CASCADE,
-        open DOUBLE PRECISION,
-        high DOUBLE PRECISION,
-        low DOUBLE PRECISION,
-        close DOUBLE PRECISION,
-        volume DOUBLE PRECISION NOT NULL,
-        num_trades BIGINT NOT NULL,
-        volume_weighted DOUBLE PRECISION NOT NULL,
-        PRIMARY KEY (asset_symbol, timestamp)
-    );
-
-    SELECT create_hypertable('bars', 'timestamp', 'asset_symbol', 15);
-
-    ALTER TABLE bars SET (
-        timescaledb.compress,
-        timescaledb.compress_segmentby = 'asset_symbol'
-    );
-
-    SELECT add_compression_policy('bars', INTERVAL '30 days');
-EOSQL

support/timescaledb/docker-compose.yml

@@ -1,20 +0,0 @@
-services:
-  timescaledb:
-    image: timescale/timescaledb-ha:pg15-all
-    hostname: timescaledb
-    restart: unless-stopped
-    ports:
-      - 5432:5432
-    volumes:
-      - timescaledb-data:/home/postgres/pgdata/data
-      - timescaledb-logs:/home/postgres/pg_log
-      - ./999_init.sh:/docker-entrypoint-initdb.d/999_init.sh
-    environment:
-      - TIMESCALEDB_TELEMETRY=off
-      - POSTGRES_USER=${POSTGRES_USER}
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-      - POSTGRES_DB=${POSTGRES_DB}
-
-volumes:
-  timescaledb-data:
-  timescaledb-logs: