From 766ffd16d5158bf01415361d618af9c626714d5b Mon Sep 17 00:00:00 2001 From: claytongentry Date: Tue, 11 Mar 2025 09:42:50 -0400 Subject: [PATCH 1/6] Factor out SQL Generator --- lib/manifold/api/workspace.rb | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/lib/manifold/api/workspace.rb b/lib/manifold/api/workspace.rb index 24f555a..94cc4e2 100644 --- a/lib/manifold/api/workspace.rb +++ b/lib/manifold/api/workspace.rb @@ -25,29 +25,6 @@ def generate(path) end end - # Handles SQL generation for manifold workspaces - class SqlGenerator - def initialize(name, manifold_yaml) - @name = name - @manifold_yaml = manifold_yaml - end - - def generate_dimensions_merge_sql(source_sql) - return unless valid_dimensions_config? - - sql_builder = Terraform::SQLBuilder.new(@name, @manifold_yaml) - sql_builder.build_dimensions_merge_sql(source_sql) - end - - private - - def valid_dimensions_config? - return false unless @manifold_yaml - - !@manifold_yaml["dimensions"]&.dig("merge", "source").nil? - end - end - # Handles schema file generation for manifold workspaces class SchemaWriter def initialize(name, vectors, vector_service, manifold_yaml, logger) @@ -190,8 +167,17 @@ def dimensions_merge_source_exists? end def generate_dimensions_merge_sql + return unless valid_dimensions_config? + source_sql = File.read(Pathname.pwd.join(manifold_yaml["dimensions"]["merge"]["source"])) - SqlGenerator.new(name, manifold_yaml).generate_dimensions_merge_sql(source_sql) + sql_builder = Terraform::SQLBuilder.new(name, manifold_yaml) + sql_builder.build_dimensions_merge_sql(source_sql) + end + + def valid_dimensions_config? + return false unless manifold_yaml + + !manifold_yaml["dimensions"]&.dig("merge", "source").nil? end def write_dimensions_merge_sql_file(sql) From f9155b5516d693f0d30183b5f1c0a13347e11660 Mon Sep 17 00:00:00 2001 From: claytongentry Date: Tue, 11 Mar 2025 09:45:36 -0400 Subject: [PATCH 2/6] more factoring out --- lib/manifold/api/workspace.rb | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/manifold/api/workspace.rb b/lib/manifold/api/workspace.rb index 94cc4e2..f373986 100644 --- a/lib/manifold/api/workspace.rb +++ b/lib/manifold/api/workspace.rb @@ -154,9 +154,11 @@ def write_manifold_merge_sql end def write_dimensions_merge_sql - return unless dimensions_merge_source_exists? + return unless valid_dimensions_config? - sql = generate_dimensions_merge_sql + source_sql = File.read(Pathname.pwd.join(manifold_yaml["dimensions"]["merge"]["source"])) + sql_builder = Terraform::SQLBuilder.new(name, manifold_yaml) + sql = sql_builder.build_dimensions_merge_sql(source_sql) return unless sql write_dimensions_merge_sql_file(sql) @@ -166,14 +168,6 @@ def dimensions_merge_source_exists? manifold_yaml["dimensions"]&.dig("merge", "source") end - def generate_dimensions_merge_sql - return unless valid_dimensions_config? - - source_sql = File.read(Pathname.pwd.join(manifold_yaml["dimensions"]["merge"]["source"])) - sql_builder = Terraform::SQLBuilder.new(name, manifold_yaml) - sql_builder.build_dimensions_merge_sql(source_sql) - end - def valid_dimensions_config? return false unless manifold_yaml From beb3c4634da8b20395f751c22d4cd5320b371a8d Mon Sep 17 00:00:00 2001 From: claytongentry Date: Tue, 11 Mar 2025 09:47:00 -0400 Subject: [PATCH 3/6] debride --- lib/manifold/api/workspace.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/manifold/api/workspace.rb b/lib/manifold/api/workspace.rb index f373986..d1ce29d 100644 --- a/lib/manifold/api/workspace.rb +++ b/lib/manifold/api/workspace.rb @@ -164,10 +164,6 @@ def write_dimensions_merge_sql write_dimensions_merge_sql_file(sql) end - def dimensions_merge_source_exists? - manifold_yaml["dimensions"]&.dig("merge", "source") - end - def valid_dimensions_config? return false unless manifold_yaml From 5dd0e056116d39d1347cbe9ce93c9951ca7049b9 Mon Sep 17 00:00:00 2001 From: claytongentry Date: Tue, 11 Mar 2025 10:02:26 -0400 Subject: [PATCH 4/6] Consolidate Schema Generation --- ...{schema_generator.rb => schema_manager.rb} | 49 +++++++++++++-- lib/manifold/api/workspace.rb | 62 ++----------------- 2 files changed, 47 insertions(+), 64 deletions(-) rename lib/manifold/api/{schema_generator.rb => schema_manager.rb} (60%) diff --git a/lib/manifold/api/schema_generator.rb b/lib/manifold/api/schema_manager.rb similarity index 60% rename from lib/manifold/api/schema_generator.rb rename to lib/manifold/api/schema_manager.rb index 59916fb..6aa5797 100644 --- a/lib/manifold/api/schema_generator.rb +++ b/lib/manifold/api/schema_manager.rb @@ -2,27 +2,39 @@ module Manifold module API - # Handles schema generation for Manifold tables - class SchemaGenerator - def initialize(dimensions_fields, manifold_yaml) - @dimensions_fields = dimensions_fields + # Handles schema generation and writing for Manifold tables + class SchemaManager + def initialize(name, vectors, vector_service, manifold_yaml, logger) + @name = name + @vectors = vectors + @vector_service = vector_service @manifold_yaml = manifold_yaml + @logger = logger end + # Generates and writes schemas to the specified directory + def write_schemas(tables_directory) + tables_directory.mkpath + write_dimensions_schema(tables_directory) + write_manifold_schema(tables_directory) + end + + # Returns the dimensions schema structure def dimensions_schema [ { "type" => "STRING", "name" => "id", "mode" => "REQUIRED" }, { "type" => "RECORD", "name" => "dimensions", "mode" => "REQUIRED", - "fields" => @dimensions_fields } + "fields" => dimensions_fields } ] end + # Returns the manifold schema structure def manifold_schema [ { "type" => "STRING", "name" => "id", "mode" => "REQUIRED" }, { "type" => "TIMESTAMP", "name" => "timestamp", "mode" => "REQUIRED" }, { "type" => "RECORD", "name" => "dimensions", "mode" => "REQUIRED", - "fields" => @dimensions_fields }, + "fields" => dimensions_fields }, { "type" => "RECORD", "name" => "metrics", "mode" => "REQUIRED", "fields" => metrics_fields } ] @@ -30,6 +42,31 @@ def manifold_schema private + def write_dimensions_schema(tables_directory) + dimensions_path = tables_directory.join("dimensions.json") + dimensions_path.write(dimensions_schema_json.concat("\n")) + end + + def write_manifold_schema(tables_directory) + manifold_path = tables_directory.join("manifold.json") + manifold_path.write(manifold_schema_json.concat("\n")) + end + + def dimensions_fields + @dimensions_fields ||= @vectors.filter_map do |vector| + @logger.info("Loading vector schema for '#{vector}'.") + @vector_service.load_vector_schema(vector) + end + end + + def dimensions_schema_json + JSON.pretty_generate(dimensions_schema) + end + + def manifold_schema_json + JSON.pretty_generate(manifold_schema) + end + def metrics_fields return [] unless @manifold_yaml["metrics"] diff --git a/lib/manifold/api/workspace.rb b/lib/manifold/api/workspace.rb index d1ce29d..9050c8c 100644 --- a/lib/manifold/api/workspace.rb +++ b/lib/manifold/api/workspace.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative "schema_manager" + module Manifold module API # Handles terraform configuration generation @@ -25,62 +27,6 @@ def generate(path) end end - # Handles schema file generation for manifold workspaces - class SchemaWriter - def initialize(name, vectors, vector_service, manifold_yaml, logger) - @name = name - @vectors = vectors - @vector_service = vector_service - @manifold_yaml = manifold_yaml - @logger = logger - end - - def write_schemas(tables_directory) - tables_directory.mkpath - write_dimensions_schema(tables_directory) - write_manifold_schema(tables_directory) - end - - private - - def write_dimensions_schema(tables_directory) - dimensions_path = tables_directory.join("dimensions.json") - dimensions_path.write(dimensions_schema_json.concat("\n")) - end - - def write_manifold_schema(tables_directory) - manifold_path = tables_directory.join("manifold.json") - manifold_path.write(manifold_schema_json.concat("\n")) - end - - def schema_generator - @schema_generator ||= SchemaGenerator.new(dimensions_fields, @manifold_yaml) - end - - def manifold_schema - schema_generator.manifold_schema - end - - def dimensions_schema - schema_generator.dimensions_schema - end - - def dimensions_fields - @dimensions_fields ||= @vectors.filter_map do |vector| - @logger.info("Loading vector schema for '#{vector}'.") - @vector_service.load_vector_schema(vector) - end - end - - def dimensions_schema_json - JSON.pretty_generate(dimensions_schema) - end - - def manifold_schema_json - JSON.pretty_generate(manifold_schema) - end - end - # Encapsulates a single manifold. class Workspace attr_reader :name, :template_path, :logger @@ -190,8 +136,8 @@ def manifold_yaml end def generate_schemas - SchemaWriter.new(name, vectors, @vector_service, manifold_yaml, logger) - .write_schemas(tables_directory) + SchemaManager.new(name, vectors, @vector_service, manifold_yaml, logger) + .write_schemas(tables_directory) end def any_vectors? From 52404896fd9c7de9100118178fd2e5702d40d270 Mon Sep 17 00:00:00 2001 From: claytongentry Date: Tue, 11 Mar 2025 10:05:05 -0400 Subject: [PATCH 5/6] snip --- lib/manifold/api/workspace.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/manifold/api/workspace.rb b/lib/manifold/api/workspace.rb index 9050c8c..a47886d 100644 --- a/lib/manifold/api/workspace.rb +++ b/lib/manifold/api/workspace.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_relative "schema_manager" - module Manifold module API # Handles terraform configuration generation From 2bdb65117c6495a0cf36fdd88930c8b4165391ed Mon Sep 17 00:00:00 2001 From: claytongentry Date: Tue, 11 Mar 2025 10:06:20 -0400 Subject: [PATCH 6/6] rename --- lib/manifold/api/workspace.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/manifold/api/workspace.rb b/lib/manifold/api/workspace.rb index a47886d..5b49db3 100644 --- a/lib/manifold/api/workspace.rb +++ b/lib/manifold/api/workspace.rb @@ -52,7 +52,7 @@ def add def generate(with_terraform: false) return nil unless manifold_exists? && any_vectors? - generate_schemas + write_schemas logger.info("Generated BigQuery dimensions table schema for workspace '#{name}'.") return unless with_terraform @@ -133,7 +133,7 @@ def manifold_yaml @manifold_yaml ||= YAML.safe_load_file(manifold_path) end - def generate_schemas + def write_schemas SchemaManager.new(name, vectors, @vector_service, manifold_yaml, logger) .write_schemas(tables_directory) end