Implement Suggestions (Soft Constraints)

[obie]

Overview

Implement Suggestions as soft constraints that guide optimization without causing failures, complementing the existing hard Assertions.

Description

While Assertions enforce hard constraints that fail execution, Suggestions provide soft constraints used during optimization. They help guide the optimization process toward desired behaviors without breaking functionality.

Key Features to Implement

• Soft constraint definition
• Integration with optimization pipeline
• Scoring mechanism for suggestions
• Feedback collection without failures
• Suggestion aggregation
• Priority/weight support

Implementation Requirements

1. Suggestion Class

module Desiru
  class Suggestion
    attr_reader :description, :check, :weight, :feedback
    
    def initialize(description:, weight: 1.0, &check)
      @description = description
      @check = check
      @weight = weight
      @feedback = nil
    end
    
    # Evaluate suggestion (never raises)
    def evaluate(result)
      begin
        success = @check.call(result)
        @feedback = success ? nil : generate_feedback(result)
        
        SuggestionResult.new(
          suggestion: self,
          success: success,
          score: success ? @weight : 0.0,
          feedback: @feedback
        )
      rescue => e
        # Suggestions should never break execution
        SuggestionResult.new(
          suggestion: self,
          success: false,
          score: 0.0,
          feedback: "Suggestion evaluation failed: #{e.message}"
        )
      end
    end
    
    private
    
    def generate_feedback(result)
      "Suggestion not met: #{@description}"
    end
  end
  
  class SuggestionResult
    attr_reader :suggestion, :success, :score, :feedback
    
    def initialize(suggestion:, success:, score:, feedback:)
      @suggestion = suggestion
      @success = success
      @score = score
      @feedback = feedback
    end
  end
end

2. Module Integration

module Desiru
  module Suggestions
    def self.included(base)
      base.extend(ClassMethods)
      base.include(InstanceMethods)
    end
    
    module ClassMethods
      def suggest(description, weight: 1.0, &block)
        suggestions << Suggestion.new(
          description: description,
          weight: weight,
          &block
        )
      end
      
      def suggestions
        @suggestions ||= []
      end
    end
    
    module InstanceMethods
      def evaluate_suggestions(result)
        suggestion_results = self.class.suggestions.map do |suggestion|
          suggestion.evaluate(result)
        end
        
        SuggestionReport.new(suggestion_results)
      end
      
      # Override forward to track suggestions
      def forward(**inputs)
        result = super
        
        if optimization_mode?
          report = evaluate_suggestions(result)
          store_suggestion_feedback(report)
        end
        
        result
      end
    end
  end
end

3. Suggestion Report

class SuggestionReport
  attr_reader :results
  
  def initialize(results)
    @results = results
  end
  
  def total_score
    @results.sum(&:score)
  end
  
  def success_rate
    successful = @results.count(&:success)
    successful.to_f / @results.count
  end
  
  def feedback_messages
    @results
      .reject(&:success)
      .map(&:feedback)
      .compact
  end
  
  def to_optimization_metric
    # Convert to a metric optimizers can use
    {
      score: total_score,
      success_rate: success_rate,
      feedback: feedback_messages
    }
  end
end

4. Optimization Integration

module Desiru::Optimizers
  class Base
    def compile_with_suggestions(program, dataset, metric)
      # Standard compilation
      base_score = evaluate(program, dataset, metric)
      
      # Evaluate suggestions
      suggestion_scores = evaluate_suggestions(program, dataset)
      
      # Combined optimization objective
      combined_score = combine_scores(base_score, suggestion_scores)
      
      # Optimize considering both metric and suggestions
      optimize_with_objective(combined_score)
    end
    
    private
    
    def evaluate_suggestions(program, dataset)
      dataset.map do |example|
        result = program.forward(**example)
        report = program.evaluate_suggestions(result)
        report.to_optimization_metric
      end
    end
  end
end

5. Common Suggestion Patterns

module Desiru::Suggestions::Common
  # Length constraints
  def self.length_between(min, max, field: :answer)
    Suggestion.new(description: "Output length between #{min} and #{max}") do |result|
      length = result[field].to_s.length
      length >= min && length <= max
    end
  end
  
  # Format suggestions
  def self.matches_format(format, field: :answer)
    Suggestion.new(description: "Output matches format: #{format}") do |result|
      result[field].to_s.match?(format)
    end
  end
  
  # Content suggestions
  def self.includes_keywords(keywords, field: :answer)
    Suggestion.new(description: "Output includes keywords: #{keywords.join(', ')}") do |result|
      text = result[field].to_s.downcase
      keywords.all? { |kw| text.include?(kw.downcase) }
    end
  end
  
  # Tone suggestions
  def self.professional_tone(field: :answer)
    Suggestion.new(description: "Professional tone", weight: 2.0) do |result|
      # Could use sentiment analysis or keyword detection
      text = result[field].to_s
      \!text.match?(/\b(hey|gonna|wanna|lol)\b/i)
    end
  end
end

Example Usage

class EmailResponder < Desiru::Module
  include Desiru::Suggestions
  
  # Hard constraint - must include greeting
  assert "includes greeting" do |result|
    result.response.match?(/^(Hello|Hi|Dear)/)
  end
  
  # Soft constraints - preferences during optimization
  suggest "professional tone", weight: 2.0 do |result|
    \!result.response.match?(/\!{2,}/) && # No multiple exclamation marks
    \!result.response.match?(/\b(awesome|cool|hey)\b/i)
  end
  
  suggest "appropriate length" do |result|
    length = result.response.split.length
    length >= 50 && length <= 200
  end
  
  suggest "includes actionable next steps" do |result|
    result.response.match?(/next steps?|follow up|action items?/i)
  end
  
  def initialize
    super(signature: "customer_email -> response")
  end
end

# During optimization, suggestions guide the process
optimizer = BootstrapFewShot.new(
  program: EmailResponder.new,
  consider_suggestions: true
)

# The optimizer will prefer examples that meet suggestions
optimized = optimizer.compile(dataset, metric)

Testing Requirements

• Unit tests for suggestion evaluation
• Integration tests with optimizers
• Test that suggestions never break execution
• Performance impact testing
• Test suggestion scoring and weighting

Benefits

• Guide optimization without breaking functionality
• Express preferences vs requirements
• Gradual improvement during optimization
• Better aligned outputs without rigid constraints

Priority

Medium - Valuable for optimization but not critical for basic functionality