1 Commits

Author SHA1 Message Date
5b8bfea0b2 Remove single-table mode
Signed-off-by: Nikolaos Karaolidis <nick@karaolidis.com>
2023-07-06 10:07:52 +03:00
2 changed files with 12 additions and 16 deletions

View File

@@ -18,8 +18,6 @@ build:
expire_in: 1d expire_in: 1d
reports: reports:
dotenv: job.env dotenv: job.env
rules:
- if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
release: release:
image: registry.gitlab.com/gitlab-org/release-cli image: registry.gitlab.com/gitlab-org/release-cli
@@ -40,5 +38,3 @@ release:
url: "https://git.karaolidis.com/karaolidis/telegraf-clickhouse-plugin/-/jobs/$JOB_ID/artifacts/raw/telegraf-clickhouse-plugin-linux-amd64" url: "https://git.karaolidis.com/karaolidis/telegraf-clickhouse-plugin/-/jobs/$JOB_ID/artifacts/raw/telegraf-clickhouse-plugin-linux-amd64"
- name: "telegraf-clickhouse-plugin-linux-arm64" - name: "telegraf-clickhouse-plugin-linux-arm64"
url: "https://git.karaolidis.com/karaolidis/telegraf-clickhouse-plugin/-/jobs/$JOB_ID/artifacts/raw/telegraf-clickhouse-plugin-linux-arm64" url: "https://git.karaolidis.com/karaolidis/telegraf-clickhouse-plugin/-/jobs/$JOB_ID/artifacts/raw/telegraf-clickhouse-plugin-linux-arm64"
rules:
- if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"

View File

@@ -158,12 +158,11 @@ func (ch *ClickHouse) toDatatype(value interface{}) string {
return datatype return datatype
} }
func (ch *ClickHouse) toNullableDatatype(key string, value interface{}) string { func (ch *ClickHouse) toNullable(pair *orderedmap.Pair[string, string]) string {
datatype := ch.toDatatype(value) if pair.Key != "host" && pair.Key != ch.TimestampColumn && pair.Key != "measurement" {
if key != "host" && key != ch.TimestampColumn && key != "measurement" { return fmt.Sprintf("Nullable(%s)", pair.Value)
return fmt.Sprintf("Nullable(%s)", datatype)
} }
return datatype return pair.Value
} }
func (ch *ClickHouse) pepareMetrics(metrics []telegraf.Metric, metricsData map[string][]map[string]interface{}, columns map[string]*orderedmap.OrderedMap[string, string]) { func (ch *ClickHouse) pepareMetrics(metrics []telegraf.Metric, metricsData map[string][]map[string]interface{}, columns map[string]*orderedmap.OrderedMap[string, string]) {
@@ -189,12 +188,12 @@ func (ch *ClickHouse) pepareMetrics(metrics []telegraf.Metric, metricsData map[s
for _, tag := range metric.TagList() { for _, tag := range metric.TagList() {
metricEntry[tag.Key] = tag.Value metricEntry[tag.Key] = tag.Value
columns[tablename].Set(tag.Key, ch.toNullableDatatype(tag.Key, tag.Value)) columns[tablename].Set(tag.Key, ch.toDatatype(tag.Value))
} }
for _, field := range metric.FieldList() { for _, field := range metric.FieldList() {
metricEntry[field.Key] = field.Value metricEntry[field.Key] = field.Value
columns[tablename].Set(field.Key, ch.toNullableDatatype(field.Key, field.Value)) columns[tablename].Set(field.Key, ch.toDatatype(field.Value))
} }
metricsData[tablename] = append(metricsData[tablename], metricEntry) metricsData[tablename] = append(metricsData[tablename], metricEntry)
@@ -204,10 +203,10 @@ func (ch *ClickHouse) pepareMetrics(metrics []telegraf.Metric, metricsData map[s
func (ch *ClickHouse) generateCreateTable(tablename string, columns *orderedmap.OrderedMap[string, string]) string { func (ch *ClickHouse) generateCreateTable(tablename string, columns *orderedmap.OrderedMap[string, string]) string {
columnDefs := make([]string, 0, columns.Len()) columnDefs := make([]string, 0, columns.Len())
for pair := columns.Oldest(); pair != nil; pair = pair.Next() { for pair := columns.Oldest(); pair != nil; pair = pair.Next() {
columnDefs = append(columnDefs, fmt.Sprintf("%s %s", quoteIdent(pair.Key), pair.Value)) columnDefs = append(columnDefs, fmt.Sprintf("%s %s", quoteIdent(pair.Key), ch.toNullable(pair)))
} }
orderBy := make([]string, 0, 2) orderBy := make([]string, 0, 3)
if _, ok := columns.Get("host"); ok { if _, ok := columns.Get("host"); ok {
orderBy = append(orderBy, "host") orderBy = append(orderBy, "host")
} }
@@ -231,14 +230,15 @@ func (ch *ClickHouse) generateAlterTable(tablename string, columns *orderedmap.O
modifyDefs := make([]string, 0, columns.Len()) modifyDefs := make([]string, 0, columns.Len())
for pair := columns.Oldest(); pair != nil; pair = pair.Next() { for pair := columns.Oldest(); pair != nil; pair = pair.Next() {
columnType := ch.toNullable(pair)
alterDefs = append(alterDefs, fmt.Sprintf("ADD COLUMN IF NOT EXISTS %s %s", alterDefs = append(alterDefs, fmt.Sprintf("ADD COLUMN IF NOT EXISTS %s %s",
quoteIdent(pair.Key), quoteIdent(pair.Key),
pair.Value)) columnType))
modifyDefs = append(modifyDefs, fmt.Sprintf("MODIFY COLUMN IF EXISTS %s %s", modifyDefs = append(modifyDefs, fmt.Sprintf("MODIFY COLUMN IF EXISTS %s %s",
quoteIdent(pair.Key), quoteIdent(pair.Key),
pair.Value)) columnType))
} }
return fmt.Sprintf("ALTER TABLE %s %s, %s", return fmt.Sprintf("ALTER TABLE %s %s, %s",
@@ -286,7 +286,7 @@ func (ch *ClickHouse) ensureTable(tablename string, columns *orderedmap.OrderedM
} }
for pair := columns.Oldest(); pair != nil; pair = pair.Next() { for pair := columns.Oldest(); pair != nil; pair = pair.Next() {
if _, ok := tableColumns[pair.Key]; !ok || tableColumns[pair.Key] != pair.Value { if _, ok := tableColumns[pair.Key]; !ok || tableColumns[pair.Key] != ch.toNullable(pair) {
_, err = ch.db.Exec(ch.generateAlterTable(tablename, columns)) _, err = ch.db.Exec(ch.generateAlterTable(tablename, columns))
if err != nil { if err != nil {
return err return err