fix: clickhouse sync table with order by changes

Author: Qiu Jian

backends/clickhouse/clickhouse.go

@@ -123,17 +123,17 @@ func (click *SClickhouseBackend) GetCreateSQLs(ts sqlchemy.ITableSpec) []string
 	var ttlCol IClickhouseColumnSpec
 	for _, c := range ts.Columns() {
 		cols = append(cols, c.DefinitionString())
-		if c.IsPrimary() {
-			primaries = append(primaries, fmt.Sprintf("`%s`", c.Name()))
-		}
 		if cc, ok := c.(IClickhouseColumnSpec); ok {
-			if cc.IsOrderBy() {
-				orderbys = append(orderbys, fmt.Sprintf("`%s`", c.Name()))
-			}
 			partition := cc.PartitionBy()
 			if len(partition) > 0 && !utils.IsInStringArray(partition, partitions) {
 				partitions = append(partitions, partition)
 			}
+			if c.IsPrimary() && len(partition) == 0 {
+				primaries = append(primaries, fmt.Sprintf("`%s`", c.Name()))
+			}
+			if cc.IsOrderBy() && len(partition) == 0 {
+				orderbys = append(orderbys, fmt.Sprintf("`%s`", c.Name()))
+			}
 			ttlC, ttlU := cc.GetTTL()
 			if ttlC > 0 && len(ttlU) > 0 {
 				ttlCol = cc
@@ -183,7 +183,7 @@ func (click *SClickhouseBackend) GetCreateSQLs(ts sqlchemy.ITableSpec) []string
 			createSql += fmt.Sprintf("\nTTL `%s` + INTERVAL %d %s", ttlCol.Name(), ttlCount, ttlUnit)
 		}
 		// set default time zone of table to UTC
-		createSql += "\nSETTINGS index_granularity=8192"
+		createSql += "\nSETTINGS index_granularity=8192, allow_nullable_key=1"
 	}
 	return []string{
 		createSql,

backends/clickhouse/sync.go

@@ -47,6 +47,18 @@ func findTtlColumn(cols []sqlchemy.IColumnSpec) sColumnTTL {
 	return ret
 }
 
+func findOrderByColumns(cols []sqlchemy.IColumnSpec) []string {
+	var ret []string
+	for _, col := range cols {
+		if clickCol, ok := col.(IClickhouseColumnSpec); ok {
+			if (clickCol.IsOrderBy() || clickCol.IsPrimary()) && len(clickCol.PartitionBy()) == 0 {
+				ret = append(ret, clickCol.Name())
+			}
+		}
+	}
+	return ret
+}
+
 func findPartitions(cols []sqlchemy.IColumnSpec) []string {
 	parts := make([]string, 0)
 	for i := range cols {
@@ -158,18 +170,41 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe
 	}*/
 
 	// check TTL
-	oldTtlSpec := findTtlColumn(changes.OldColumns)
-	newTtlSpec := findTtlColumn(ts.Columns())
-	log.Debugf("old: %s new: %s", jsonutils.Marshal(oldTtlSpec), jsonutils.Marshal(newTtlSpec))
-	if oldTtlSpec != newTtlSpec {
-		if oldTtlSpec.Count > 0 && newTtlSpec.Count == 0 {
-			// remove
-			sql := fmt.Sprintf("REMOVE TTL")
-			alters = append(alters, sql)
-		} else {
+	{
+		oldTtlSpec := findTtlColumn(changes.OldColumns)
+		newTtlSpec := findTtlColumn(ts.Columns())
+		log.Debugf("old: %s new: %s", jsonutils.Marshal(oldTtlSpec), jsonutils.Marshal(newTtlSpec))
+		if oldTtlSpec != newTtlSpec {
+			if oldTtlSpec.Count > 0 && newTtlSpec.Count == 0 {
+				// remove
+				sql := fmt.Sprintf("REMOVE TTL")
+				alters = append(alters, sql)
+			} else {
+				// alter
+				sql := fmt.Sprintf("MODIFY TTL `%s` + INTERVAL %d %s", newTtlSpec.ColName, newTtlSpec.Count, newTtlSpec.Unit)
+				alters = append(alters, sql)
+			}
+		}
+	}
+
+	// check order by
+	{
+		oldOrderBys := findOrderByColumns(changes.OldColumns)
+		newOrderBys := findOrderByColumns(ts.Columns())
+		log.Debugf("old: %s new: %s", jsonutils.Marshal(oldOrderBys), jsonutils.Marshal(newOrderBys))
+		if jsonutils.Marshal(oldOrderBys).String() != jsonutils.Marshal(newOrderBys).String() {
 			// alter
-			sql := fmt.Sprintf("MODIFY TTL `%s` + INTERVAL %d %s", newTtlSpec.ColName, newTtlSpec.Count, newTtlSpec.Unit)
-			alters = append(alters, sql)
+			altered := false
+			for i := range newOrderBys {
+				if !utils.IsInStringArray(newOrderBys[i], oldOrderBys) {
+					oldOrderBys = append(oldOrderBys, newOrderBys[i])
+					altered = true
+				}
+			}
+			if altered {
+				sql := fmt.Sprintf("MODIFY ORDER BY (%s)", strings.Join(oldOrderBys, ", "))
+				alters = append(alters, sql)
+			}
 		}
 	}