add external model cluster views

docker-compose.yml

@@ -100,7 +100,6 @@ services:
             },
             'models': {
                 'external': {
-                    'defaultCluster': '{remote_cluster}',
                     'defaultDatabase': 'default'
                 },
                 'transformations': {

example.env

@@ -1,5 +1,6 @@
 # Network configuration
 NETWORK=mainnet
+EXTERNAL_MODEL_CLUSTER={remote_cluster}
 
 # ClickHouse xatu configuration
 CLICKHOUSE_HOST=localhost

internal/config/config.go

@@ -18,6 +18,7 @@ type Config struct {
 	ClickhouseUsername       string
 	ClickhousePassword       string
 	ClickhouseCluster        string
+	ExternalModelCluster     string
 }
 
 // Load reads configuration from environment variables and .env file
@@ -31,11 +32,12 @@ func Load() (*Config, error) {
 	}
 
 	cfg := &Config{
-		Network:            getEnv("NETWORK", "mainnet"),
-		ClickhouseHost:     getEnv("CLICKHOUSE_HOST", "localhost"),
-		ClickhouseUsername: getEnv("CLICKHOUSE_USERNAME", "default"),
-		ClickhousePassword: getEnv("CLICKHOUSE_PASSWORD", ""),
-		ClickhouseCluster:  getEnv("CLICKHOUSE_CLUSTER", ""),
+		Network:              getEnv("NETWORK", "mainnet"),
+		ClickhouseHost:       getEnv("CLICKHOUSE_HOST", "localhost"),
+		ClickhouseUsername:   getEnv("CLICKHOUSE_USERNAME", "default"),
+		ClickhousePassword:   getEnv("CLICKHOUSE_PASSWORD", ""),
+		ClickhouseCluster:    getEnv("CLICKHOUSE_CLUSTER", ""),
+		ExternalModelCluster: getEnv("EXTERNAL_MODEL_CLUSTER", ""),
 	}
 
 	// Parse numeric values
@@ -73,19 +75,26 @@ func (c *Config) String() string {
 		clusterDisplay = "(single-node)"
 	}
 
+	externalClusterDisplay := c.ExternalModelCluster
+	if externalClusterDisplay == "" {
+		externalClusterDisplay = "(not set)"
+	}
+
 	return fmt.Sprintf(`Current Configuration:
 ======================
-Network:                %s
-ClickHouse Host:        %s
-ClickHouse Native Port: %d
-ClickHouse Username:    %s
-ClickHouse Password:    %s
-ClickHouse Cluster:     %s`,
+Network:                  %s
+ClickHouse Host:          %s
+ClickHouse Native Port:   %d
+ClickHouse Username:      %s
+ClickHouse Password:      %s
+ClickHouse Cluster:       %s
+External Model Cluster:   %s`,
 		c.Network,
 		c.ClickhouseHost,
 		c.ClickhouseNativePort,
 		c.ClickhouseUsername,
 		passwordDisplay,
 		clusterDisplay,
+		externalClusterDisplay,
 	)
 }

internal/migrations/migrations.go

@@ -29,7 +29,7 @@ func PrepareAndRun(cfg *config.Config) error {
 	migrationsDir := "migrations"
 
 	// Copy and process migration files
-	if procErr := processMigrationFiles(migrationsDir, tempDir, cfg.Network); procErr != nil {
+	if procErr := processMigrationFiles(migrationsDir, tempDir, cfg.Network, cfg.ExternalModelCluster); procErr != nil {
 		return fmt.Errorf("failed to process migration files: %w", procErr)
 	}
 
@@ -72,8 +72,8 @@ func PrepareAndRun(cfg *config.Config) error {
 	return nil
 }
 
-// processMigrationFiles copies migration files from source to dest, substituting ${NETWORK_NAME}
-func processMigrationFiles(sourceDir, destDir, network string) error {
+// processMigrationFiles copies migration files from source to dest, substituting ${NETWORK_NAME} and ${EXTERNAL_MODEL_CLUSTER}
+func processMigrationFiles(sourceDir, destDir, network, externalCluster string) error {
 	// Read all files from source directory
 	entries, err := os.ReadDir(sourceDir)
 	if err != nil {
@@ -102,6 +102,8 @@ func processMigrationFiles(sourceDir, destDir, network string) error {
 
 		// Replace ${NETWORK_NAME} with actual network
 		processedContent := strings.ReplaceAll(string(content), "${NETWORK_NAME}", network)
+		// Replace ${EXTERNAL_MODEL_CLUSTER} with actual external cluster
+		processedContent = strings.ReplaceAll(processedContent, "${EXTERNAL_MODEL_CLUSTER}", externalCluster)
 
 		// Write processed content to destination
 		if err := os.WriteFile(destPath, []byte(processedContent), 0o600); err != nil {
@@ -151,7 +153,7 @@ func GetMigrationStatus(cfg *config.Config) (version uint, dirty bool, err error
 	}()
 
 	// Copy migration files (needed for migrate instance)
-	if procErr := processMigrationFiles("migrations", tempDir, cfg.Network); procErr != nil {
+	if procErr := processMigrationFiles("migrations", tempDir, cfg.Network, cfg.ExternalModelCluster); procErr != nil {
 		return 0, false, fmt.Errorf("failed to process migration files: %w", procErr)
 	}
 
@@ -184,11 +186,11 @@ func GetMigrationStatus(cfg *config.Config) (version uint, dirty bool, err error
 
 // CopyMigrationsToDirectory copies and processes migration files to a specific directory
 // This is useful for debugging or manual inspection
-func CopyMigrationsToDirectory(sourceDir, destDir, network string) error {
+func CopyMigrationsToDirectory(sourceDir, destDir, network, externalCluster string) error {
 	// Create destination directory if it doesn't exist
 	if err := os.MkdirAll(destDir, 0o750); err != nil {
 		return fmt.Errorf("failed to create destination directory: %w", err)
 	}
 
-	return processMigrationFiles(sourceDir, destDir, network)
+	return processMigrationFiles(sourceDir, destDir, network, externalCluster)
 }