From a45c8ccb26dfe8bd382c41e63201451776edecaf Mon Sep 17 00:00:00 2001
From: Maksym Melnychok <maksym.melnychok@gmail.com>
Date: Mon, 9 Mar 2015 17:33:42 +0100
Subject: [PATCH 1/5] some cleanups and features

- make route table main
- propagate route table to vgw
---
 lib/puppet/provider/aws_iam_role/api.rb   |  6 +-
 lib/puppet/provider/aws_routetable/api.rb | 77 ++++++++++++++---------
 lib/puppet/provider/aws_vpc/api.rb        |  2 +-
 lib/puppet/type/aws_routetable.rb         |  2 +-
 lib/puppet_x/bobtfish/ec2_api.rb          | 51 +++++++--------
 5 files changed, 75 insertions(+), 63 deletions(-)

diff --git a/lib/puppet/provider/aws_iam_role/api.rb b/lib/puppet/provider/aws_iam_role/api.rb
index 7970fb5..36612d2 100644
--- a/lib/puppet/provider/aws_iam_role/api.rb
+++ b/lib/puppet/provider/aws_iam_role/api.rb
@@ -24,12 +24,12 @@ def service_principal
   end
 
   def service_principal=(service)
-    assume_role_policy_document ||= service_template(service)
+    assume_role_policy_document ||= service_tempalte(service)
     assume_role_policy_document['Statement']['Principal']['Service'] = service
   end
 
   def create
-    resource[:assume_role_policy_document] ||= service_template(resource[:service_principal])
+    resource[:assume_role_policy_document] ||= service_tempalte(resource[:service_principal])
     iam.client.create_role(
       :role_name => resource[:name],
       :assume_role_policy_document => JSON.dump(resource[:assume_role_policy_document]),
@@ -58,7 +58,7 @@ def destroy
 
   private
 
-  def service_template(service)
+  def service_tempalte(service)
     return {'Statement' => [
       {
         'Action' => 'sts:AssumeRole',
diff --git a/lib/puppet/provider/aws_routetable/api.rb b/lib/puppet/provider/aws_routetable/api.rb
index 2ee207e..435f0d2 100644
--- a/lib/puppet/provider/aws_routetable/api.rb
+++ b/lib/puppet/provider/aws_routetable/api.rb
@@ -16,55 +16,74 @@ def self.new_from_aws(region_name, item)
       :main             => item.main? ? 'true' : 'false',
       :vpc              => name_or_id(item.vpc),
       :subnets          => item.subnets.map { |subnet| subnet.tags.to_h['Name'] || subnet.id },
-      :routes           => item.routes.map { |route|
-        {
-          :destination_cidr_block => route.destination_cidr_block,
+      :routes           => item.routes.map do |route|
+        { :destination_cidr_block => route.destination_cidr_block,
           :state => route.state,
           :target => name_or_id(route.target),
           :origin => route.origin,
           :network_interface => name_or_id(route.network_interface),
-          :internet_gateway => name_or_id(route.internet_gateway)
-        }.reject { |k, v| v.nil? } },
-      :propagate_routes_from => []
-    )
+          :internet_gateway => name_or_id(route.internet_gateway) }.
+            reject { |k,v| v.nil? }
+      end,
+      :propagate_to => [])
   end
-  read_only(:vpc, :subnets, :routes, :main)
+
+  read_only(:vpc, :subnets, :routes, :main, :propagate_to)
+
   def self.instances
     regions.collect do |region_name|
       ec2.regions[region_name].route_tables.collect { |item| new_from_aws(region_name,item) }
     end.flatten
   end
+
   def exists?
     @property_hash[:ensure] == :present
   end
+
   def create
     vpc = find_vpc_item_by_name resource[:vpc]
-    if !vpc
-      fail("Could not find vpc #{resource[:vpc]}")
+    fail("Could not find vpc #{resource[:vpc]}") unless vpc
+
+    my_region   = find_region_name_for_vpc_name resource[:vpc]
+    region      = ec2.regions[my_region]
+    route_table = region.route_tables.create(:vpc => vpc.id)
+
+    tags = (resource[:tags] || {}).merge('Name' => resource[:name])
+    region.client.create_tags(
+      :resources => [route_table.id],
+      :tags => tags.map {|k,v| {:key => k, :value => v}})
+
+    if resource[:main].to_s == 'true' # fuck puppet
+      current_main = vpc.route_tables.map do |rt|
+        rt.associations.find{ |as| as.main }
+      end.compact.first
+
+      if current_main && current_main.route_table.id != route_table.id
+        region.client.replace_route_table_association(
+          :association_id => current_main.id,
+          :route_table_id => route_table.id)
+      end
     end
-    my_region = find_region_name_for_vpc_name resource[:vpc]
-    begin
-      route_table = ec2.regions[my_region].route_tables.create({:vpc => vpc.id})
-      tag_with_name route_table, resource[:name]
-      tags = resource[:tags] || {}
-      tags.each { |k,v| route.add_tag(k, :value => v) }
-      route_table
-    rescue Exception => e
-      fail e
+
+    if resource[:propagate_to]
+      gw_coll = region.vpn_gateways.map{|vgw| vgw} # force fetch
+      [resource[:propagate_to]].flatten.each do |gw_resource|
+        gateway = gw_coll.find{|gw| gw.tags['Name'] == gw_resource.title}
+        next unless gateway
+
+        region.client.enable_vgw_route_propagation(
+          :route_table_id => route_table.id,
+          :gateway_id     => gateway.id)
+      end
     end
+
+    route_table
+  rescue Exception => e
+    fail e
   end
+
   def destroy
     @property_hash[:aws_item].delete
     @property_hash[:ensure] = :absent
   end
-
-  def propagate_routes_from=(vgws)
-    Array(vgws).each do |vgw|
-      ec2.regions[my_region].enable_vgw_route_propagation(
-        :route_table_id => id,
-        :gateway_id     => vgw
-      )
-    end
-  end
 end
-
diff --git a/lib/puppet/provider/aws_vpc/api.rb b/lib/puppet/provider/aws_vpc/api.rb
index d3c47d3..874d3bc 100644
--- a/lib/puppet/provider/aws_vpc/api.rb
+++ b/lib/puppet/provider/aws_vpc/api.rb
@@ -56,7 +56,7 @@ def create
     tags = resource[:tags] || {}
     tags.each { |k,v| vpc.add_tag(k, :value => v) }
     # Tag-name the default SG for this VPC so we know we're managing it:
-    vpc.security_groups.find{|sg| sg.name == 'default'}.tags['Name'] = 'default'
+    vpc.security_groups.find{|sg| sg.name == 'default'}.tags['Name'] = resource[:name]
 
     if dhopts_name
       vpc.dhcp_options = dhopts_name
diff --git a/lib/puppet/type/aws_routetable.rb b/lib/puppet/type/aws_routetable.rb
index 03b4b24..0a745bb 100644
--- a/lib/puppet/type/aws_routetable.rb
+++ b/lib/puppet/type/aws_routetable.rb
@@ -10,6 +10,6 @@
     newvalue 'false'
   end
   newproperty(:tags)
-  newproperty(:propagate_routes_from)
+  newproperty(:propagate_to)
 end
 
diff --git a/lib/puppet_x/bobtfish/ec2_api.rb b/lib/puppet_x/bobtfish/ec2_api.rb
index 3bcc921..81f6f74 100644
--- a/lib/puppet_x/bobtfish/ec2_api.rb
+++ b/lib/puppet_x/bobtfish/ec2_api.rb
@@ -2,19 +2,19 @@
 require 'puppet'
 
 module Puppet_X
-  module Bobtfish
-    class DieLikeThePigDogYouAre < Exception
-      alias_method :old_to_s, :to_s
-      def to_s
-        STDERR.puts "EMERGENCY BAIL OUT - probably due to amazon API errors in prefetch (are you over the limits?): #{old_to_s}"
-        kill 15, $$
-      end
-    end
+module Bobtfish
+class Ec2_api < Puppet::Provider
+  HAVE_AWS_SDK = begin
+    require 'aws-sdk'
+    #AWS.config(
+    #  :logger        => Logger.new($stdout),
+    #  :log_formatter => AWS::Core::LogFormatter.colored,
+    #  :log_level     => :debug)
+    true
+  rescue LoadError
+    STDERR.puts "Coudln't load AWS SDK gem"
+    false
   end
-end
-
-class Puppet_X::Bobtfish::Ec2_api < Puppet::Provider
-  HAVE_AWS_SDK = begin; require 'aws-sdk'; true; rescue Exception; false; end
 
   confine :true => HAVE_AWS_SDK
 
@@ -26,22 +26,16 @@ def self.instances
   end
 
   def self.prefetch(resources)
-    begin
-      all_instances = if method(:instances).arity > 0
-        # This is so hacky, I am so sorry
-        instances(resources)
-      else
-        instances
+    instances.each do |provider|
+      if resource = resources[provider.name]
+        resource.provider = provider
       end
-      all_instances.each do |provider|
-        if resource = resources[provider.name] then
-          resource.provider = provider
-        end
-      end
-    rescue Exception => e
-      raise Puppet_X::Bobtfish::DieLikeThePigDogYouAre(e.to_s)
     end
-
+  rescue Exception => e
+    STDERR.puts "EMERGENCY BAIL OUT"
+    STDERR.puts "probably due to amazon API errors in prefetch (are you over the limits?)"
+    STDERR.puts e.to_s
+    Kernel.exit 1
   end
 
   def lookup(type, name)
@@ -236,7 +230,6 @@ def vpc=(vpc_name)
     @property_hash[:aws_item].attach(vpc)
     @property_hash[:vpc] = vpc_name
   end
-
-
 end
-
+end # module Bobtfish
+end # module Puppet_X

From 7278117a87761d94f283092100456d70ea1c4df3 Mon Sep 17 00:00:00 2001
From: Maksym Melnychok <maksym.melnychok@gmail.com>
Date: Tue, 10 Mar 2015 17:14:54 +0100
Subject: [PATCH 2/5] make puppet correctly recognize and resolve diffs in
 :main and :propagate_from params

---
 lib/puppet/provider/aws_routetable/api.rb | 94 +++++++++++++++--------
 lib/puppet/provider/aws_vpc/api.rb        | 80 ++++++++++---------
 lib/puppet/type/aws_routetable.rb         |  2 +-
 lib/puppet/type/aws_vpc.rb                |  1 -
 lib/puppet_x/bobtfish/ec2_api.rb          |  4 +-
 5 files changed, 106 insertions(+), 75 deletions(-)

diff --git a/lib/puppet/provider/aws_routetable/api.rb b/lib/puppet/provider/aws_routetable/api.rb
index 435f0d2..a8ff881 100644
--- a/lib/puppet/provider/aws_routetable/api.rb
+++ b/lib/puppet/provider/aws_routetable/api.rb
@@ -3,10 +3,34 @@
 Puppet::Type.type(:aws_routetable).provide(:api, :parent => Puppet_X::Bobtfish::Ec2_api) do
   mk_resource_methods
   remove_method :tags= # We want the method inherited from the parent
+  read_only :vpc, :subnets, :routes
+
+  def main=(value)
+    if value.to_s != 'true'
+      debug "Setting :main to false is a noop"
+    elsif @property_hash[:aws_item].main?
+      @property_hash[:main] = 'true'
+    else
+      set_as_main!(find_vpc_item_by_name(@property_hash[:vpc]), @property_hash[:aws_item])
+    end
+  end
+
+  def propagate_from=(value)
+    propagate_from!([value].flatten, @property_hash[:aws_item])
+  end
 
   def self.new_from_aws(region_name, item)
+    debug "#{self.inspect} #new_from_aws"
     tags = item.tags.to_h
     name = tags.delete('Name') || item.id
+
+    # lol this API
+    route_table_attrs = item.send(:describe_call)[:route_table_set][0]
+    gw_ids   = route_table_attrs[:propagating_vgw_set].map{|pvs| pvs[:gateway_id]}
+    gw_names = ec2.regions[region_name].vpn_gateways.
+      inject({}) {|h,vgw| h.merge!(vgw.id => vgw.tags['Name'] || vgw.id) }.
+      values_at(*gw_ids)
+
     new(
       :aws_item         => item,
       :name             => name,
@@ -25,11 +49,9 @@ def self.new_from_aws(region_name, item)
           :internet_gateway => name_or_id(route.internet_gateway) }.
             reject { |k,v| v.nil? }
       end,
-      :propagate_to => [])
+      :propagate_from => gw_names)
   end
 
-  read_only(:vpc, :subnets, :routes, :main, :propagate_to)
-
   def self.instances
     regions.collect do |region_name|
       ec2.regions[region_name].route_tables.collect { |item| new_from_aws(region_name,item) }
@@ -44,38 +66,12 @@ def create
     vpc = find_vpc_item_by_name resource[:vpc]
     fail("Could not find vpc #{resource[:vpc]}") unless vpc
 
-    my_region   = find_region_name_for_vpc_name resource[:vpc]
-    region      = ec2.regions[my_region]
-    route_table = region.route_tables.create(:vpc => vpc.id)
-
+    route_table = current_region.route_tables.create(:vpc => vpc.id)
     tags = (resource[:tags] || {}).merge('Name' => resource[:name])
-    region.client.create_tags(
-      :resources => [route_table.id],
-      :tags => tags.map {|k,v| {:key => k, :value => v}})
-
-    if resource[:main].to_s == 'true' # fuck puppet
-      current_main = vpc.route_tables.map do |rt|
-        rt.associations.find{ |as| as.main }
-      end.compact.first
-
-      if current_main && current_main.route_table.id != route_table.id
-        region.client.replace_route_table_association(
-          :association_id => current_main.id,
-          :route_table_id => route_table.id)
-      end
-    end
-
-    if resource[:propagate_to]
-      gw_coll = region.vpn_gateways.map{|vgw| vgw} # force fetch
-      [resource[:propagate_to]].flatten.each do |gw_resource|
-        gateway = gw_coll.find{|gw| gw.tags['Name'] == gw_resource.title}
-        next unless gateway
+    route_table.tags.set(tags)
 
-        region.client.enable_vgw_route_propagation(
-          :route_table_id => route_table.id,
-          :gateway_id     => gateway.id)
-      end
-    end
+    set_as_main!(vpc, route_table) if resource[:main].to_s == 'true'
+    propagate_from!([resource[:propagate_from]].flatten, route_table) if resource[:propagate_from]
 
     route_table
   rescue Exception => e
@@ -86,4 +82,36 @@ def destroy
     @property_hash[:aws_item].delete
     @property_hash[:ensure] = :absent
   end
+
+  private
+
+  def set_as_main!(vpc, route_table, region=current_region)
+    current_main = vpc.route_tables.map do |rt|
+      rt.associations.find{ |as| as.main }
+    end.compact.first
+
+    if current_main && current_main.route_table.id != route_table.id
+      region.client.replace_route_table_association(
+        :association_id => current_main.id,
+        :route_table_id => route_table.id)
+    end
+
+    @property_hash[:main] = 'true'
+  end
+
+  def propagate_from!(vgws, route_table, region=current_region)
+    gw_coll = region.vpn_gateways.map{|vgw| vgw} # force fetch
+    vgws.each do |vgw|
+      gateway = gw_coll.find{|gw| gw.tags['Name'] == vgw}
+      next unless gateway # should we fail here?
+
+      region.client.enable_vgw_route_propagation(
+        :route_table_id => route_table.id,
+        :gateway_id     => gateway.id)
+    end
+  end
+
+  def current_region
+    @current_region ||= ec2.regions[find_region_name_for_vpc_name(resource[:vpc])]
+  end
 end
diff --git a/lib/puppet/provider/aws_vpc/api.rb b/lib/puppet/provider/aws_vpc/api.rb
index 874d3bc..a7a65d8 100644
--- a/lib/puppet/provider/aws_vpc/api.rb
+++ b/lib/puppet/provider/aws_vpc/api.rb
@@ -3,21 +3,21 @@
 Puppet::Type.type(:aws_vpc).provide(:api, :parent => Puppet_X::Bobtfish::Ec2_api) do
   mk_resource_methods
   remove_method :tags= # We want the method inherited from the parent
+  read_only :cidr, :id, :region, :instance_tenancy # can't set ID can we?
 
-  def self.vpcs_for_region(region)
-    ec2.regions[region].vpcs
-  end
-  def vpcs_for_region(region)
-    self.class.vpcs_for_region region
+  def dhcp_options=(value)
+    dopts = find_dhopts_item_by_name(value)
+    fail("Could not find dhcp options named '#{value}'") unless dopts
+    @property_hash[:aws_item].dhcp_options = dopts.id
+    @property_hash[:dhcp_options] = value
   end
+
   def self.new_from_aws(region_name, item)
     tags = item.tags.to_h
     name = tags.delete('Name') || item.id
-    dopts_item = find_dhopts_item_by_name item.dhcp_options_id
-    dopts_name = nil
-    if dopts_item
-      dopts_name = name_or_id dopts_item
-    end
+    dopts_item = find_dhopts_item_by_name(item.dhcp_options_id)
+    dopts_name = name_or_id(dopts_item) if dopts_item
+
     new(
       :aws_item         => item,
       :name             => name,
@@ -27,47 +27,38 @@ def self.new_from_aws(region_name, item)
       :dhcp_options     => dopts_name,
       :instance_tenancy => item.instance_tenancy.to_s,
       :region           => region_name,
-      :tags             => tags
-    )
+      :tags             => tags)
   end
+
   def self.instances
-    regions.collect do |region_name|
-      vpcs_for_region(region_name).collect { |item| new_from_aws(region_name, item) }
+    regions.map do |region_name|
+      vpcs_for_region(region_name).map { |item| new_from_aws(region_name, item) }
     end.flatten
   end
-  read_only(:cidr, :id,  :region, :aws_dops, :instance_tenancy) # can't set ID can we?
-  def dhcp_options=(value)
-    dopts = find_dhopts_item_by_name(value)
-    fail("Could not find dhcp options named '#{value}'") unless dopts
-    @property_hash[:aws_item].dhcp_options = dopts.id
-    @property_hash[:dhcp_options] = value
-  end
+
   def create
-    dhopts_name = nil
+    fail("CIDR must be provided") unless resource[:cidr]
+
+    vpc = ec2.regions[resource[:region]].vpcs.create(resource[:cidr])
+    wait_until_state vpc, :available
+
+    vpc.tags.set((resource[:tags] || {}).merge({'Name' => resource[:name]}))
+    vpc.security_groups.first.tags['Name'] = resource[:name]
+
     if resource[:dhcp_options]
       dhopts = find_dhopts_item_by_name(resource[:dhcp_options])
       fail("Cannot find dhcp options named '#{resource[:dhcp_options]}'") unless dhopts
-      dhopts_name = dhopts.id
+      vpc.dhcp_options = dhopts.id
     end
 
-    vpc = ec2.regions[resource[:region]].vpcs.create(resource[:cidr])
-    wait_until_state vpc, :available
-    tag_with_name vpc, resource[:name]
-    tags = resource[:tags] || {}
-    tags.each { |k,v| vpc.add_tag(k, :value => v) }
-    # Tag-name the default SG for this VPC so we know we're managing it:
-    vpc.security_groups.find{|sg| sg.name == 'default'}.tags['Name'] = resource[:name]
-
-    if dhopts_name
-      vpc.dhcp_options = dhopts_name
-    end
     vpc
-
   end
+
   def destroy
     @property_hash[:aws_item].delete
     @property_hash[:ensure] = :absent
   end
+
   def purge
     vpc = @property_hash[:aws_item]
     subnets = vpc.subnets
@@ -76,6 +67,7 @@ def purge
     regions = subnets.collect do |sn|
       sn.availability_zone.region_name
     end
+
     regions.each do |region|
       elb(region).load_balancers.each do |lb|
         if lb.subnets.all?{|sn| subnets.include?(sn)}
@@ -85,6 +77,7 @@ def purge
         end
       end
     end
+
     # Freeze the current set
     instances = vpc.instances.to_a
 
@@ -93,6 +86,7 @@ def purge
       debug "Stopping instance #{node.tags['Name']}"
       node.stop
     end
+
     instances.each do |node|
       wait_until_status(node, :stopped)
       # Dispose of EIPs
@@ -129,8 +123,7 @@ def purge
     end
 
     # Gateways
-    igw = vpc.internet_gateway
-    if igw
+    if igw = vpc.internet_gateway
       debug "Detach Internet Gateway: #{igw.tags['Name']}"
       igw.detach(vpc)
       debug "Disposing of #{igw.tags['Name']}"
@@ -151,7 +144,18 @@ def purge
     # Finally, the VPC itself
     debug "Purging VPC #{vpc.tags['Name']}"
     vpc.delete
+
+    AWS.reset_memoization
     @property_hash[:ensure] = :purged
   end
-end
 
+private
+
+  def self.vpcs_for_region(region)
+    ec2.regions[region].vpcs
+  end
+
+  def vpcs_for_region(region)
+    self.class.vpcs_for_region region
+  end
+end
diff --git a/lib/puppet/type/aws_routetable.rb b/lib/puppet/type/aws_routetable.rb
index 0a745bb..d2c34aa 100644
--- a/lib/puppet/type/aws_routetable.rb
+++ b/lib/puppet/type/aws_routetable.rb
@@ -10,6 +10,6 @@
     newvalue 'false'
   end
   newproperty(:tags)
-  newproperty(:propagate_to)
+  newproperty(:propagate_from, :array_matching => :all)
 end
 
diff --git a/lib/puppet/type/aws_vpc.rb b/lib/puppet/type/aws_vpc.rb
index b2ac46d..d6f41db 100644
--- a/lib/puppet/type/aws_vpc.rb
+++ b/lib/puppet/type/aws_vpc.rb
@@ -18,4 +18,3 @@
   newproperty(:instance_tenancy)
   newproperty(:tags)
 end
-
diff --git a/lib/puppet_x/bobtfish/ec2_api.rb b/lib/puppet_x/bobtfish/ec2_api.rb
index 81f6f74..744e90e 100644
--- a/lib/puppet_x/bobtfish/ec2_api.rb
+++ b/lib/puppet_x/bobtfish/ec2_api.rb
@@ -6,7 +6,7 @@ module Bobtfish
 class Ec2_api < Puppet::Provider
   HAVE_AWS_SDK = begin
     require 'aws-sdk'
-    #AWS.config(
+    # AWS.config(
     #  :logger        => Logger.new($stdout),
     #  :log_formatter => AWS::Core::LogFormatter.colored,
     #  :log_level     => :debug)
@@ -156,7 +156,7 @@ def regions
   end
 
   def tags=(newtags)
-    newtags.each { |k,v| @property_hash[:aws_item].add_tag(k, :value => v) }
+    @property_hash[:aws_item].tags.set(newtags)
     @property_hash[:tags] = newtags
   end
 

From bd30fef8588fcc180043cbda3f2bbe1639e3a6d8 Mon Sep 17 00:00:00 2001
From: Maksym Melnychok <maksym.melnychok@gmail.com>
Date: Thu, 12 Mar 2015 13:04:44 +0100
Subject: [PATCH 3/5] more aggressive caching, more emergency bailouts (on
 duplicates)

---
 lib/puppet/provider/aws_cgw/api.rb            |  35 ++--
 lib/puppet/provider/aws_dopts/api.rb          |  12 +-
 lib/puppet/provider/aws_igw/api.rb            |  19 +-
 lib/puppet/provider/aws_routetable/api.rb     |  44 +++--
 lib/puppet/provider/aws_security_group/api.rb |  17 +-
 lib/puppet/provider/aws_subnet/api.rb         |  33 ++--
 lib/puppet/provider/aws_vgw/api.rb            |  11 +-
 lib/puppet/provider/aws_vpc/api.rb            |  32 ++--
 lib/puppet_x/bobtfish/ec2_api.rb              | 163 ++++++++----------
 9 files changed, 163 insertions(+), 203 deletions(-)

diff --git a/lib/puppet/provider/aws_cgw/api.rb b/lib/puppet/provider/aws_cgw/api.rb
index 8b0420a..ebc5152 100644
--- a/lib/puppet/provider/aws_cgw/api.rb
+++ b/lib/puppet/provider/aws_cgw/api.rb
@@ -3,8 +3,9 @@
 Puppet::Type.type(:aws_cgw).provide(:api, :parent => Puppet_X::Bobtfish::Ec2_api) do
   mk_resource_methods
   remove_method :tags= # We want the method inherited from the parent
-  def self.new_from_aws(region_name, item)
-    tags = item.tags.to_h
+
+  def self.new_from_aws(region_name, item, tags=nil)
+    tags ||= item.tags.to_h
     name = tags.delete('Name') || item.id
     new(
       :aws_item   => item,
@@ -18,28 +19,24 @@ def self.new_from_aws(region_name, item)
       :tags       => tags
     )
   end
-  def self.instances()
-    regions.collect do |region_name|
-      ec2.regions[region_name].customer_gateways.reject { |item| item.state == :deleting or item.state == :deleted }.collect { |item| new_from_aws(region_name,item) }
-    end.flatten
-  end
+
+  def self.instances_class; AWS::EC2::CustomerGateway; end
 
   read_only(:ip_address, :bgp_asn, :region, :type)
 
   def create
-    begin
-      fail "Cannot create aws_cgw #{resource[:title]} without a region" unless resource[:region]
-      region = ec2.regions[resource[:region]]
-      fail "Cannot find region '#{resource[:region]} for resource #{resource[:title]}" unless region
-      cgw = region.customer_gateways.create(resource[:bgp_asn].to_i, resource[:ip_address])
-      tag_with_name cgw, resource[:name]
-      tags = resource[:tags] || {}
-      tags.each { |k,v| cgw.add_tag(k, :value => v) }
-      cgw
-    rescue Exception => e
-      fail e
-    end
+    fail "Cannot create aws_cgw #{resource[:title]} without a region" unless resource[:region]
+    region = ec2.regions[resource[:region]]
+    fail "Cannot find region '#{resource[:region]} for resource #{resource[:title]}" unless region
+    cgw = region.customer_gateways.create(resource[:bgp_asn].to_i, resource[:ip_address])
+    tag_with_name cgw, resource[:name]
+    tags = resource[:tags] || {}
+    tags.each { |k,v| cgw.add_tag(k, :value => v) }
+    cgw
+  rescue Exception => e
+    fail e
   end
+
   def destroy
     @property_hash[:aws_item].delete
     @property_hash[:ensure] = :absent
diff --git a/lib/puppet/provider/aws_dopts/api.rb b/lib/puppet/provider/aws_dopts/api.rb
index 057ffbd..22d87bd 100644
--- a/lib/puppet/provider/aws_dopts/api.rb
+++ b/lib/puppet/provider/aws_dopts/api.rb
@@ -3,8 +3,9 @@
 Puppet::Type.type(:aws_dopts).provide(:api, :parent => Puppet_X::Bobtfish::Ec2_api) do
   mk_resource_methods
   remove_method :tags= # We want the method inherited from the parent
-  def self.new_from_aws(region_name, item)
-    tags = item.tags.to_h
+
+  def self.new_from_aws(region_name, item, tags=nil)
+    tags ||= item.tags.to_h
     name = tags.delete('Name') || item.id
     c = item.configuration
     new(
@@ -21,11 +22,8 @@ def self.new_from_aws(region_name, item)
       :netbios_node_type    => c[:netbios_node_type].to_s
     )
   end
-  def self.instances
-    regions.collect do |region_name|
-      ec2.regions[region_name].dhcp_options.collect { |item| new_from_aws(region_name,item) }
-    end.flatten
-  end
+
+  def self.instances_class; AWS::EC2::DHCPOptions; end
 
   read_only(:domain_name, :ntp_servers, :netbios_name_servers, :netbios_node_type)
 
diff --git a/lib/puppet/provider/aws_igw/api.rb b/lib/puppet/provider/aws_igw/api.rb
index b5c1bcf..c716089 100644
--- a/lib/puppet/provider/aws_igw/api.rb
+++ b/lib/puppet/provider/aws_igw/api.rb
@@ -4,13 +4,13 @@
   mk_resource_methods
   remove_method :tags= # We want the method inherited from the parent
 
-  def self.new_from_aws(item)
-    tags = item.tags.to_h
+  def self.new_from_aws(region_name, item, tags=nil)
+    tags ||= item.tags.to_h
     name = tags.delete('Name') || item.id
-    vpc_name = nil
-    if item.vpc
-      vpc_name = name_or_id item.vpc
-    end
+
+    vpc_id   = item.pre_attachment_set.map{|as| as[:vpc_id]}.first
+    vpc_name = name_or_id(find_vpc_item_by_name(name)) if vpc_id
+
     new(
       :aws_item         => item,
       :name             => name,
@@ -20,11 +20,8 @@ def self.new_from_aws(item)
       :tags             => tags
     )
   end
-  def self.instances
-    regions.collect do |region_name|
-      ec2.regions[region_name].internet_gateways.collect { |item| new_from_aws(item) }
-    end.flatten
-  end
+
+  def self.instances_class; AWS::EC2::InternetGateway; end
 
   read_only(:vpc)
 
diff --git a/lib/puppet/provider/aws_routetable/api.rb b/lib/puppet/provider/aws_routetable/api.rb
index a8ff881..142ad62 100644
--- a/lib/puppet/provider/aws_routetable/api.rb
+++ b/lib/puppet/provider/aws_routetable/api.rb
@@ -19,17 +19,20 @@ def propagate_from=(value)
     propagate_from!([value].flatten, @property_hash[:aws_item])
   end
 
-  def self.new_from_aws(region_name, item)
-    debug "#{self.inspect} #new_from_aws"
-    tags = item.tags.to_h
+  def self.new_from_aws(region_name, item, tags=nil)
+    tags ||= item.tags.to_h
     name = tags.delete('Name') || item.id
 
-    # lol this API
-    route_table_attrs = item.send(:describe_call)[:route_table_set][0]
-    gw_ids   = route_table_attrs[:propagating_vgw_set].map{|pvs| pvs[:gateway_id]}
-    gw_names = ec2.regions[region_name].vpn_gateways.
-      inject({}) {|h,vgw| h.merge!(vgw.id => vgw.tags['Name'] || vgw.id) }.
-      values_at(*gw_ids)
+    cached_assocs = item.pre_association_set.map do |assoc|
+      AWS::EC2::RouteTable::Association.new(item,
+        assoc[:route_table_association_id],
+        assoc[:subnet_id])
+    end
+    item.define_singleton_method(:associations) { cached_assocs }
+
+    gw_ids   = item.pre_propagating_vgw_set.map{|pvs| pvs[:gateway_id]}
+    gw_names = Puppet::Type.type(:aws_vgw).provider(:api).instances.
+      select{|vgw| gw_ids.include? vgw.aws_item.id }.map{|vgw| vgw.name}
 
     new(
       :aws_item         => item,
@@ -37,26 +40,27 @@ def self.new_from_aws(region_name, item)
       :id               => item.id,
       :ensure           => :present,
       :tags             => tags,
-      :main             => item.main? ? 'true' : 'false',
-      :vpc              => name_or_id(item.vpc),
+      :main             => item.pre_association_set.find{|a| a[:main] } ? 'true' : 'false',
+      :vpc              => name_or_id(find_vpc_item_by_name(item.pre_vpc_id)),
       :subnets          => item.subnets.map { |subnet| subnet.tags.to_h['Name'] || subnet.id },
-      :routes           => item.routes.map do |route|
+      :routes           => item.pre_route_set.map do |route_details|
+        route = AWS::EC2::RouteTable::Route.new(item, route_details)
+        igw   = Puppet::Type.type(:aws_igw).provider(:api).instances.
+          find {|igw| igw.aws_item.id == route.internet_gateway.id} if route.internet_gateway
+        igw   = igw.aws_item if igw
+
         { :destination_cidr_block => route.destination_cidr_block,
           :state => route.state,
-          :target => name_or_id(route.target),
+          :target => route.target.id == 'local' ? 'local' : name_or_id(route.target),
           :origin => route.origin,
           :network_interface => name_or_id(route.network_interface),
-          :internet_gateway => name_or_id(route.internet_gateway) }.
+          :internet_gateway => name_or_id(igw) }.
             reject { |k,v| v.nil? }
       end,
       :propagate_from => gw_names)
   end
 
-  def self.instances
-    regions.collect do |region_name|
-      ec2.regions[region_name].route_tables.collect { |item| new_from_aws(region_name,item) }
-    end.flatten
-  end
+  def self.instances_class; AWS::EC2::RouteTable; end
 
   def exists?
     @property_hash[:ensure] == :present
@@ -73,6 +77,8 @@ def create
     set_as_main!(vpc, route_table) if resource[:main].to_s == 'true'
     propagate_from!([resource[:propagate_from]].flatten, route_table) if resource[:propagate_from]
 
+    self.class.instance_variable_set('@instances', nil)
+
     route_table
   rescue Exception => e
     fail e
diff --git a/lib/puppet/provider/aws_security_group/api.rb b/lib/puppet/provider/aws_security_group/api.rb
index 766a2cc..5752496 100644
--- a/lib/puppet/provider/aws_security_group/api.rb
+++ b/lib/puppet/provider/aws_security_group/api.rb
@@ -5,14 +5,8 @@
   mk_resource_methods
   remove_method :tags= # We want the method inherited from the parent
 
-  def self.instances_for_region(region)
-    ec2.regions[region].security_groups
-  end
-  def instances_for_region(region)
-    self.class.instances_for_region region
-  end
-  def self.new_from_aws(region_name, item)
-    tags = item.tags.to_h
+  def self.new_from_aws(region_name, item, tags=nil)
+    tags ||= item.tags.to_h
     name = tags.delete('Name') || item.id
     if item.vpc_id
       vpc = ec2.regions[region_name].vpcs[item.vpc_id].tags['Name']
@@ -48,11 +42,8 @@ def self.new_from_aws(region_name, item)
       :authorize_egress  => egress
     )
   end
-  def self.instances
-    regions.collect do |region_name|
-      instances_for_region(region_name).collect { |item| new_from_aws(region_name, item) }
-    end.flatten
-  end
+
+  def self.instances_class; AWS::EC2::SecurityGroup; end
 
   read_only(:description, :vpc, :authorize_ingress, :authorize_egress)
 
diff --git a/lib/puppet/provider/aws_subnet/api.rb b/lib/puppet/provider/aws_subnet/api.rb
index 6867878..2370d17 100644
--- a/lib/puppet/provider/aws_subnet/api.rb
+++ b/lib/puppet/provider/aws_subnet/api.rb
@@ -17,38 +17,25 @@ def create cidr_block, options = {}
 Puppet::Type.type(:aws_subnet).provide(:api, :parent => Puppet_X::Bobtfish::Ec2_api) do
   mk_resource_methods
   remove_method :tags= # We want the method inherited from the parent
+  read_only(:vpc, :cidr, :az, :route_table)
 
-  def self.vpcs_for_region(region)
-    ec2.regions[region].vpcs
-  end
-  def vpcs_for_region(region)
-    self.class.vpcs_for_region region
-  end
-  def self.new_from_aws(vpc_id, item)
-    tags = item.tags.to_h
+  def self.new_from_aws(region_name, item, tags=nil)
+    tags ||= item.tags.to_h
     name = tags.delete('Name') || item.id
+
     new(
       :aws_item => item,
       :name     => name,
       :id       => item.id,
       :ensure   => :present,
-      :vpc      => vpc_id,
+      :vpc      => name_or_id(find_vpc_item_by_name(item.pre_vpc_id)),
       :cidr     => item.cidr_block,
       :az       => item.availability_zone_name,
-      :tags     => tags.to_hash
-    )
-  end
-  def self.instances
-    regions.collect do |region_name|
-      vpcs_for_region(region_name).collect do |vpc|
-        vpc_name = name_or_id vpc
-        vpc.subnets.collect do |item|
-          new_from_aws(vpc_name, item)
-        end
-      end.flatten
-    end.flatten
+      :tags     => tags)
   end
-  read_only(:vpc, :cidr, :az, :route_table)
+
+  def self.instances_class; AWS::EC2::Subnet; end
+
   def create
     vpc = find_vpc_item_by_name(resource[:vpc])
 
@@ -73,8 +60,8 @@ def create
     tags = resource[:tags] || {}
     tags.each { |k,v| subnet.add_tag(k, :value => v) }
     subnet
-
   end
+
   def destroy
     @property_hash[:aws_item].delete
     @property_hash[:ensure] = :absent
diff --git a/lib/puppet/provider/aws_vgw/api.rb b/lib/puppet/provider/aws_vgw/api.rb
index 5f7906c..09acca0 100644
--- a/lib/puppet/provider/aws_vgw/api.rb
+++ b/lib/puppet/provider/aws_vgw/api.rb
@@ -4,8 +4,8 @@
   mk_resource_methods
   remove_method :tags= # We want the method inherited from the parent
 
-  def self.new_from_aws(item, region_name)
-    tags = item.tags.to_h
+  def self.new_from_aws(region_name, item, tags=nil)
+    tags ||= item.tags.to_h
     name = tags.delete('Name') || item.id
     vpc_name = nil
     if item.vpc
@@ -21,11 +21,8 @@ def self.new_from_aws(item, region_name)
       :region_name      => region_name
     )
   end
-  def self.instances
-    regions.collect do |region_name|
-      ec2.regions[region_name].vpn_gateways.reject { |item| item.state == :deleting or item.state == :deleted }.collect { |item| new_from_aws(item, region_name) }
-    end.flatten
-  end
+
+  def self.instances_class; AWS::EC2::VPNGateway; end
 
   read_only(:region, :vpn_type, :region_name, :availability_zone)
 
diff --git a/lib/puppet/provider/aws_vpc/api.rb b/lib/puppet/provider/aws_vpc/api.rb
index a7a65d8..ee7acd1 100644
--- a/lib/puppet/provider/aws_vpc/api.rb
+++ b/lib/puppet/provider/aws_vpc/api.rb
@@ -5,18 +5,26 @@
   remove_method :tags= # We want the method inherited from the parent
   read_only :cidr, :id, :region, :instance_tenancy # can't set ID can we?
 
+  def self.find_dopt(name_or_id)
+    Puppet::Type.type(:aws_dopts).instances.find do |dopt|
+      dopt.provider.name          == name_or_id ||
+      dopt.provider.aws_item.id   == name_or_id
+    end
+  end
+  def find_dopt(name_or_id); self.class.find_dopt(name_or_id); end
+
   def dhcp_options=(value)
-    dopts = find_dhopts_item_by_name(value)
+    dopts = find_dopt(value)
     fail("Could not find dhcp options named '#{value}'") unless dopts
-    @property_hash[:aws_item].dhcp_options = dopts.id
+    @property_hash[:aws_item].dhcp_options = dopts.provider.aws_item
     @property_hash[:dhcp_options] = value
   end
 
-  def self.new_from_aws(region_name, item)
-    tags = item.tags.to_h
+  def self.new_from_aws(region_name, item, tags=nil)
+    tags ||= item.tags.to_h
     name = tags.delete('Name') || item.id
-    dopts_item = find_dhopts_item_by_name(item.dhcp_options_id)
-    dopts_name = name_or_id(dopts_item) if dopts_item
+    dopts_item = find_dopt(item.pre_dhcp_options_id)
+    dopts_name = dopts_item.title if dopts_item
 
     new(
       :aws_item         => item,
@@ -30,11 +38,7 @@ def self.new_from_aws(region_name, item)
       :tags             => tags)
   end
 
-  def self.instances
-    regions.map do |region_name|
-      vpcs_for_region(region_name).map { |item| new_from_aws(region_name, item) }
-    end.flatten
-  end
+  def self.instances_class; AWS::EC2::VPC; end
 
   def create
     fail("CIDR must be provided") unless resource[:cidr]
@@ -46,11 +50,13 @@ def create
     vpc.security_groups.first.tags['Name'] = resource[:name]
 
     if resource[:dhcp_options]
-      dhopts = find_dhopts_item_by_name(resource[:dhcp_options])
+      dhopts = find_dopt(resource[:dhcp_options])
       fail("Cannot find dhcp options named '#{resource[:dhcp_options]}'") unless dhopts
-      vpc.dhcp_options = dhopts.id
+      vpc.dhcp_options = dhopts
     end
 
+    self.class.instance_variable_set('@instances', nil)
+
     vpc
   end
 
diff --git a/lib/puppet_x/bobtfish/ec2_api.rb b/lib/puppet_x/bobtfish/ec2_api.rb
index 744e90e..2b8bbf9 100644
--- a/lib/puppet_x/bobtfish/ec2_api.rb
+++ b/lib/puppet_x/bobtfish/ec2_api.rb
@@ -6,10 +6,10 @@ module Bobtfish
 class Ec2_api < Puppet::Provider
   HAVE_AWS_SDK = begin
     require 'aws-sdk'
-    # AWS.config(
-    #  :logger        => Logger.new($stdout),
-    #  :log_formatter => AWS::Core::LogFormatter.colored,
-    #  :log_level     => :debug)
+    AWS.config(
+     :logger        => Logger.new($stdout),
+     :log_formatter => AWS::Core::LogFormatter.colored,
+     :log_level     => :debug) if ENV['AWS_DEBUG'] == '1'
     true
   rescue LoadError
     STDERR.puts "Coudln't load AWS SDK gem"
@@ -21,8 +21,44 @@ class Ec2_api < Puppet::Provider
   desc "Helper for Providers which use the EC2 API"
   self.initvars
 
+  def self.instances_class
+    raise "#instances_method not implemented"
+  end
+
   def self.instances
-    raise NotImplementedError
+    @instance_names ||= []
+    @instances ||= regions.map do |region_name|
+      region        = ec2.regions[region_name]
+      describe_call = instances_class.send :describe_call_name
+      set_call      = :"#{instances_class.send :inflected_name}_set"
+      id_call       = :"#{instances_class.send :inflected_name}_id"
+      item_set      = region.client.send(describe_call).send(set_call)
+
+      item_set.map do |item_attrs|
+        item = instances_class.new_from(
+          describe_call, item_attrs, item_attrs.send(id_call), :config => region.config)
+
+        item_attrs.keys.each {|k| item.define_singleton_method(:"pre_#{k}") { item_attrs[k] } }
+
+        original_tags = item.tags
+        item.define_singleton_method(:set_tags) {|tags| original_tags.set(tags) }
+
+        item_tags = item.pre_tag_set.inject({}) {|tags, tag| tags.merge!(tag[:key] => tag[:value]) }
+        item.define_singleton_method(:tags) { item_tags }
+
+        name = [region_name, item.tags['Name']].join('/')
+        raise "#{item} : #{name} is a duplicate" if @instance_names.include? name
+        @instance_names << name
+
+        new_from_aws(region_name, item, item_tags.clone)
+      end
+    end.flatten.compact.sort_by{|i| i.aws_item.id}
+  rescue Exception => e
+    STDERR.puts "EMERGENCY BAIL OUT"
+    STDERR.puts "probably due to amazon API errors in prefetch (are you over the limits?)"
+    STDERR.puts e.to_s
+    STDERR.puts e.backtrace[0..5]
+    Kernel.exit 1
   end
 
   def self.prefetch(resources)
@@ -35,6 +71,7 @@ def self.prefetch(resources)
     STDERR.puts "EMERGENCY BAIL OUT"
     STDERR.puts "probably due to amazon API errors in prefetch (are you over the limits?)"
     STDERR.puts e.to_s
+    STDERR.puts e.backtrace[0..5]
     Kernel.exit 1
   end
 
@@ -85,119 +122,63 @@ def get_creds
   end
 
   def self.default_creds
-    {
-      :access_key_id => (ENV['AWS_ACCESS_KEY_ID']||ENV['AWS_ACCESS_KEY']),
-      :secret_access_key => (ENV['AWS_SECRET_ACCESS_KEY']||ENV['AWS_SECRET_KEY'])
-    }
-  end
-
-  def self.amazon_thing(which)
-    which.new
+    { :access_key_id => (ENV['AWS_ACCESS_KEY_ID']||ENV['AWS_ACCESS_KEY']),
+      :secret_access_key => (ENV['AWS_SECRET_ACCESS_KEY']||ENV['AWS_SECRET_KEY']) }
   end
 
-  def self.iam()
-    amazon_thing(AWS::IAM)
-  end
-  def iam
-    self.class.iam()
-  end
-
-  def self.ec2()
-    amazon_thing(AWS::EC2)
-  end
-  def ec2()
-    self.class.ec2()
-  end
+  def self.amazon_thing(which); which.new; end
 
-  def self.r53
-    amazon_thing(AWS::Route53)
-  end
+  def self.iam; @@iam ||= AWS::IAM.new; end
+  def iam; self.class.iam; end
 
-  def r53
-    self.class.r53
-  end
+  def self.ec2; @@ec2 ||= AWS::EC2.new; end
+  def ec2; self.class.ec2; end
 
-  def self.elb(region = None)
-    AWS::ELB.new(:region => region)
-  end
+  def self.r53; @@r53 ||= AWS::Route53.new; end
+  def r53; self.class.r53; end
 
-  def elb(region=None)
-    self.class.elb(region)
-  end
+  def self.elb(region=nil); (@@elb ||= {})[region] ||= AWS::ELB.new(:region => region); end
+  def elb(region=nil); self.class.elb(region); end
 
-  def self.rds(region=None)
-    AWS::RDS.new(:region => region)
-  end
-  def rds(region=None)
-    self.class.rds(region)
-  end
+  def self.rds(region=nil); (@@rds ||= {})[region] ||= AWS::RDS.new(:region => region); end
+  def rds(region=nil); self.class.rds(region); end
 
-  def self.elcc(region=None)
-    AWS::ElastiCache.new(:region => region)
-  end
-  def elcc(region=None)
-    self.class.elcc(region)
-  end
+  def self.elcc(region=nil); (@@elcc ||= {})[region] ||= AWS::ElastiCache.new(:region => region); end
+  def elcc(region=nil); self.class.elcc(region); end
 
   def self.regions
-    @@regions ||= begin
-      if ENV['AWS_REGION'] and not ENV['AWS_REGION'].empty?
-        [ENV['AWS_REGION']]
-      elsif HAVE_AWS_SDK
-        ec2.regions.collect { |r| r.name }
-      else
-        []
-      end
+    @@regions ||=  if ENV['AWS_REGION'] and not ENV['AWS_REGION'].empty?
+      [ENV['AWS_REGION']]
+    elsif HAVE_AWS_SDK
+      ec2.regions.collect { |r| r.name }
+    else
+      []
     end
   end
 
   def regions
-    self.class.regions()
+    self.class.regions
   end
 
   def tags=(newtags)
-    @property_hash[:aws_item].tags.set(newtags)
+    @property_hash[:aws_item].set_tags(newtags)
     @property_hash[:tags] = newtags
   end
 
-  def self.find_dhopts_item_by_name(name)
-    @@dhoptions ||= begin
-      regions.collect do |region_name|
-        ec2.regions[region_name].dhcp_options.to_a
-      end.flatten
-    end
-    @@dhoptions.find do |dopt|
-      dopt_name = dopt.tags.to_h['Name'] || dopt.id
-      dopt_name == name || dopt.id == name
-    end
-  end
-
-  def find_dhopts_item_by_name(name)
-    self.class.find_dhopts_item_by_name(name)
-  end
-
   def find_vpc_item_by_name(name)
-    @@vpcs ||= begin
-      regions.collect do |region_name|
-        ec2.regions[region_name].vpcs.to_a
-      end.flatten
-    end
-    @@vpcs.find do |vpc|
-      vpc_name = vpc.tags.to_h['Name'] || vpc.vpc_id
-      vpc_name == name
-    end
+    self.class.find_vpc_item_by_name(name)
+  end
+  def self.find_vpc_item_by_name(name)
+    res = Puppet::Type.type(:aws_vpc).provider(:api).instances.
+      find {|vpc| vpc.name == name || vpc.aws_item.id == name}
+    res.aws_item if res
   end
 
   def find_region_name_for_vpc_name(name)
     self.class.find_region_name_for_vpc_name(name)
   end
   def self.find_region_name_for_vpc_name(name)
-    regions.find do |region_name|
-      ec2.regions[region_name].vpcs.find do |vpc|
-        vpc_name = vpc.tags.to_h['Name'] || vpc.vpc_id
-        vpc_name == name
-      end
-    end
+    find_vpc_item_by_name(name).client.instance_variable_get('@region')
   end
 
   def self.find_hosted_zone_by_name(name)

From 86e26f1a561b318f36fd449aa513e3108cb87bd4 Mon Sep 17 00:00:00 2001
From: Maksym Melnychok <maksym.melnychok@gmail.com>
Date: Thu, 12 Mar 2015 15:34:53 +0100
Subject: [PATCH 4/5] fix specs

---
 .gitignore                                    |  3 ++
 Gemfile                                       |  2 +-
 lib/puppet/provider/aws_subnet/api.rb         |  1 -
 lib/puppet_x/bobtfish/ec2_api.rb              | 43 ++++++++++-------
 spec/unit/puppet/aws_cgw_provider_spec.rb     | 35 +++++---------
 spec/unit/puppet/aws_dopts_provider_spec.rb   | 36 +++++---------
 spec/unit/puppet/aws_igw_provider_spec.rb     | 35 +++++---------
 .../puppet/aws_routetable_provider_spec.rb    | 35 +++++---------
 spec/unit/puppet/aws_subnet_provider_spec.rb  | 47 ++++++-------------
 spec/unit/puppet/aws_vgw_provider_spec.rb     | 34 +++++---------
 spec/unit/puppet/aws_vpc_provider_spec.rb     | 41 ++++++----------
 spec/unit/puppet/aws_vpn_provider_spec.rb     |  9 ----
 12 files changed, 115 insertions(+), 206 deletions(-)

diff --git a/.gitignore b/.gitignore
index 80bf705..0556701 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,6 @@ pkg
 spec/fixtures
 .rspec_system
 Gemfile.lock
+.bundle
+.ruby-version
+.ruby-gemset
diff --git a/Gemfile b/Gemfile
index d757e3a..2081ef8 100644
--- a/Gemfile
+++ b/Gemfile
@@ -1,7 +1,7 @@
 source "http://rubygems.org"
 
 gem 'nokogiri', '~> 1.5.11'
-gem 'aws-sdk'
+gem 'aws-sdk', '1.55.0'
 
 group :test do
   gem "rake"
diff --git a/lib/puppet/provider/aws_subnet/api.rb b/lib/puppet/provider/aws_subnet/api.rb
index 2370d17..eedfff7 100644
--- a/lib/puppet/provider/aws_subnet/api.rb
+++ b/lib/puppet/provider/aws_subnet/api.rb
@@ -67,4 +67,3 @@ def destroy
     @property_hash[:ensure] = :absent
   end
 end
-
diff --git a/lib/puppet_x/bobtfish/ec2_api.rb b/lib/puppet_x/bobtfish/ec2_api.rb
index 2b8bbf9..77d21c0 100644
--- a/lib/puppet_x/bobtfish/ec2_api.rb
+++ b/lib/puppet_x/bobtfish/ec2_api.rb
@@ -26,31 +26,23 @@ def self.instances_class
   end
 
   def self.instances
+    describe_call = instances_class.send :describe_call_name
+    set_call      = :"#{instances_class.send :inflected_name}_set"
+    id_call       = :"#{instances_class.send :inflected_name}_id"
+
     @instance_names ||= []
-    @instances ||= regions.map do |region_name|
-      region        = ec2.regions[region_name]
-      describe_call = instances_class.send :describe_call_name
-      set_call      = :"#{instances_class.send :inflected_name}_set"
-      id_call       = :"#{instances_class.send :inflected_name}_id"
-      item_set      = region.client.send(describe_call).send(set_call)
+    @instances      ||= regions.map do |region_name|
+      region   = ec2.regions[region_name]
+      item_set = region.client.send(describe_call).send(set_call)
 
       item_set.map do |item_attrs|
-        item = instances_class.new_from(
-          describe_call, item_attrs, item_attrs.send(id_call), :config => region.config)
-
-        item_attrs.keys.each {|k| item.define_singleton_method(:"pre_#{k}") { item_attrs[k] } }
-
-        original_tags = item.tags
-        item.define_singleton_method(:set_tags) {|tags| original_tags.set(tags) }
-
-        item_tags = item.pre_tag_set.inject({}) {|tags, tag| tags.merge!(tag[:key] => tag[:value]) }
-        item.define_singleton_method(:tags) { item_tags }
-
+        item = preload(region, item_attrs, describe_call, id_call)
         name = [region_name, item.tags['Name']].join('/')
+
         raise "#{item} : #{name} is a duplicate" if @instance_names.include? name
         @instance_names << name
 
-        new_from_aws(region_name, item, item_tags.clone)
+        new_from_aws(region_name, item, item.tags.clone)
       end
     end.flatten.compact.sort_by{|i| i.aws_item.id}
   rescue Exception => e
@@ -61,6 +53,21 @@ def self.instances
     Kernel.exit 1
   end
 
+  def self.preload(region, item_attrs, describe_call, id_call)
+    item = instances_class.new_from(
+      describe_call, item_attrs, item_attrs.send(id_call), :config => region.config)
+
+    item_attrs.keys.each {|k| item.define_singleton_method(:"pre_#{k}") { item_attrs[k] } }
+
+    original_tags = item.tags
+    item.define_singleton_method(:set_tags) {|tags| original_tags.set(tags) }
+
+    item_tags = item.pre_tag_set.inject({}) {|tags, tag| tags.merge!(tag[:key] => tag[:value]) }
+    item.define_singleton_method(:tags) { item_tags }
+
+    item
+  end
+
   def self.prefetch(resources)
     instances.each do |provider|
       if resource = resources[provider.name]
diff --git a/spec/unit/puppet/aws_cgw_provider_spec.rb b/spec/unit/puppet/aws_cgw_provider_spec.rb
index 3600c71..3f74270 100644
--- a/spec/unit/puppet/aws_cgw_provider_spec.rb
+++ b/spec/unit/puppet/aws_cgw_provider_spec.rb
@@ -5,33 +5,20 @@
 
 describe provider_class do
   context "with 2 resources in each of 2 regions in 2 accounts" do
-    let(:two_gateways) {[:gateway1, :gateway2]}
-    let(:two_regions) { [:region1, :region2] }
-    let(:ec2_mock) {
-      ec2_mock = double 'object'
-      ec2_mock.stub_chain('regions.[].customer_gateways.reject').and_return(two_gateways)
-      ec2_mock
-    }
+    let(:type) { :customer_gateway }
 
-    before :each do
-      provider_class.should_receive(:regions).and_return(two_regions)
-      expect(provider_class).to receive(:new_from_aws) do |a1, a2|
-        [:region1, :region2].include?(a1).should be(true)
-        [:gateway1, :gateway2].include?(a2).should be(true)
-        :blah
-      end.at_least(:once)
-    end
+    def thing(name); double(:name => name, :tags => {'Name' => name}); end
+    let(:client) { double('client',
+      :"describe_#{type}s" => double(:"#{type}_set" => [:"#{type}1", :"#{type}2"])) }
+    let(:region) { double('region', :client => client) }
+    let(:ec2) { double('ec2', :regions => double(:[] => region)) }
+    let(:aws_thing) { double('aws_thing', :aws_item => double(:id => nil)) }
 
     it "should find 4 instances" do
-      provider_class.should_receive(:ec2).at_least(:once).and_return(ec2_mock)
-      provider_class.instances.count.should eq(4)
-    end
-    it "should send a key hash to the ec2 method" do
-      expect(provider_class).to receive(:ec2) do
-        ec2_mock
-      end.at_least(:once)
-      provider_class.instances.count.should eq(4)
+      provider_class.stub(:ec2 => ec2, :regions => [:region1, :region2])
+      expect(provider_class).to receive(:preload) {|r, n, _, _| thing(n)}.at_least(:once)
+      expect(provider_class).to receive(:new_from_aws).and_return(aws_thing).at_least(:once)
+      expect(provider_class.instances.count).to eq(4)
     end
   end
 end
-
diff --git a/spec/unit/puppet/aws_dopts_provider_spec.rb b/spec/unit/puppet/aws_dopts_provider_spec.rb
index dbe58c4..7eedac6 100644
--- a/spec/unit/puppet/aws_dopts_provider_spec.rb
+++ b/spec/unit/puppet/aws_dopts_provider_spec.rb
@@ -5,33 +5,21 @@
 
 describe provider_class do
   context "with 2 resources in each of 2 regions" do
-    let(:two_doptss) {[:dopts1, :dopts2]}
-    let(:two_regions) { [:region1, :region2] }
-    let(:ec2_mock) {
-      ec2_mock = double 'object'
-      ec2_mock.stub_chain('regions.[].dhcp_options').and_return(two_doptss)
-      ec2_mock
-    }
+    let(:type) { :dhcp_options }
 
-    before :each do
-      provider_class.should_receive(:regions).and_return(two_regions)
-      expect(provider_class).to receive(:new_from_aws) do |a1, a2|
-        [:region1, :region2].include?(a1).should be(true)
-        [:dopts1, :dopts2].include?(a2).should be(true)
-        :blah
-      end.at_least(:once)
-    end
+    def thing(name); double(:name => name, :tags => {'Name' => name}); end
+    let(:client) { double('client',
+      :"describe_#{type}" => double(:"#{type}_set" => [:"#{type}1", :"#{type}2"])) }
+    let(:region) { double('region', :client => client) }
+    let(:ec2) { double('ec2', :regions => double(:[] => region)) }
+    let(:aws_thing) { double('aws_thing', :aws_item => double(:id => nil)) }
 
     it "should find 4 instances" do
-      provider_class.should_receive(:ec2).at_least(:once).and_return(ec2_mock)
-      provider_class.instances.count.should eq(4)
-    end
-    it "should send a key hash to the ec2 method" do
-      expect(provider_class).to receive(:ec2) do
-        ec2_mock
-      end.at_least(:once)
-      provider_class.instances.count.should eq(4)
+      provider_class.stub(:ec2 => ec2, :regions => [:region1, :region2])
+      expect(client).to receive(:"describe_#{type}")
+      expect(provider_class).to receive(:preload) {|r, n, _, _| thing(n)}.at_least(:once)
+      expect(provider_class).to receive(:new_from_aws).and_return(aws_thing).at_least(:once)
+      expect(provider_class.instances.count).to eq(4)
     end
   end
 end
-
diff --git a/spec/unit/puppet/aws_igw_provider_spec.rb b/spec/unit/puppet/aws_igw_provider_spec.rb
index 62df1e8..f6797dd 100644
--- a/spec/unit/puppet/aws_igw_provider_spec.rb
+++ b/spec/unit/puppet/aws_igw_provider_spec.rb
@@ -5,32 +5,21 @@
 
 describe provider_class do
   context "with 2 resources in each of 2 regions" do
-    let(:two_igws) {[:igw1, :igw2]}
-    let(:two_regions) { [:region1, :region2] }
-    let(:ec2_mock) {
-      ec2_mock = double 'object'
-      ec2_mock.stub_chain('regions.[].internet_gateways').and_return(two_igws)
-      ec2_mock
-    }
+    let(:type) { :internet_gateway }
 
-    before :each do
-      provider_class.should_receive(:regions).and_return(two_regions)
-      expect(provider_class).to receive(:new_from_aws) do |a1|
-        [:igw1, :igw2].include?(a1).should be(true)
-        :blah
-      end.at_least(:once)
-    end
+    def thing(name); double(:name => name, :tags => {'Name' => name}); end
+    let(:client) { double('client',
+      :"describe_#{type}s" => double(:"#{type}_set" => [:"#{type}1", :"#{type}2"])) }
+    let(:region) { double('region', :client => client) }
+    let(:ec2) { double('ec2', :regions => double(:[] => region)) }
+    let(:aws_thing) { double('aws_thing', :aws_item => double(:id => nil)) }
 
     it "should find 4 instances" do
-      provider_class.should_receive(:ec2).at_least(:once).and_return(ec2_mock)
-      provider_class.instances.count.should eq(4)
-    end
-    it "should send a key hash to the ec2 method" do
-      expect(provider_class).to receive(:ec2) do
-        ec2_mock
-      end.at_least(:once)
-      provider_class.instances.count.should eq(4)
+      provider_class.stub(:ec2 => ec2, :regions => [:region1, :region2])
+      expect(client).to receive(:"describe_#{type}s")
+      expect(provider_class).to receive(:preload) {|r, n, _, _| thing(n)}.at_least(:once)
+      expect(provider_class).to receive(:new_from_aws).and_return(aws_thing).at_least(:once)
+      expect(provider_class.instances.count).to eq(4)
     end
   end
 end
-
diff --git a/spec/unit/puppet/aws_routetable_provider_spec.rb b/spec/unit/puppet/aws_routetable_provider_spec.rb
index 47bae66..422dd93 100644
--- a/spec/unit/puppet/aws_routetable_provider_spec.rb
+++ b/spec/unit/puppet/aws_routetable_provider_spec.rb
@@ -5,32 +5,21 @@
 
 describe provider_class do
   context "with 2 resources in each of 2 regions" do
-    let(:two_routetables) {[:routetable1, :routetable2]}
-    let(:two_regions) { [:region1, :region2] }
-    let(:ec2_mock) {
-      ec2_mock = double 'object'
-      ec2_mock.stub_chain('regions.[].route_tables').and_return(two_routetables)
-      ec2_mock
-    }
+    let(:type) { :route_table }
 
-    before :each do
-      provider_class.should_receive(:regions).and_return(two_regions)
-      expect(provider_class).to receive(:new_from_aws) do |a1, a2|
-        [:region1, :region2].include?(a1).should be(true)
-        [:routetable1, :routetable2].include?(a2).should be(true)
-        :blah
-      end.at_least(:once)
-    end
+    def thing(name); double(:name => name, :tags => {'Name' => name}); end
+    let(:client) { double('client',
+      :"describe_#{type}s" => double(:"#{type}_set" => [:"#{type}1", :"#{type}2"])) }
+    let(:region) { double('region', :client => client) }
+    let(:ec2) { double('ec2', :regions => double(:[] => region)) }
+    let(:aws_thing) { double('aws_thing', :aws_item => double(:id => nil)) }
 
     it "should find 4 instances" do
-      provider_class.should_receive(:ec2).at_least(:once).and_return(ec2_mock)
-      provider_class.instances.count.should eq(4)
-    end
-    it "should send a key hash to the ec2 method" do
-      expect(provider_class).to receive(:ec2) do
-        ec2_mock
-      end.at_least(:once)
-      provider_class.instances.count.should eq(4)
+      provider_class.stub(:ec2 => ec2, :regions => [:region1, :region2])
+      expect(client).to receive(:"describe_#{type}s")
+      expect(provider_class).to receive(:preload) {|r, n, _, _| thing(n)}.at_least(:once)
+      expect(provider_class).to receive(:new_from_aws).and_return(aws_thing).at_least(:once)
+      expect(provider_class.instances.count).to eq(4)
     end
   end
 end
diff --git a/spec/unit/puppet/aws_subnet_provider_spec.rb b/spec/unit/puppet/aws_subnet_provider_spec.rb
index cee577d..cd81eaf 100644
--- a/spec/unit/puppet/aws_subnet_provider_spec.rb
+++ b/spec/unit/puppet/aws_subnet_provider_spec.rb
@@ -4,41 +4,22 @@
 provider_class = Puppet::Type.type(:aws_subnet).provider(:api)
 
 describe provider_class do
-  context "with 2 resources in each of 2 vpcs in each of 2 regions" do
-    let(:two_vpcs) {
-      [:vpc1, :vpc2].collect do |v|
-        vpc = double 'object'
-        vpc.stub('name').and_return v
-        vpc.stub('subnets').and_return [:subnet1, :subnet2]
-        vpc
-      end
-    }
-    let(:two_regions) { [:region1, :region2] }
-    let(:ec2_mock) {
-      ec2_mock = double 'object'
-      ec2_mock.stub_chain('regions.[].vpcs').and_return(two_vpcs)
-      ec2_mock
-    }
+  context "with 2 resources in each of 2 regions" do
+    let(:type) { :subnet }
 
-    before :each do
-      provider_class.should_receive(:regions).and_return(two_regions)
-      expect(provider_class).to receive(:new_from_aws) do |a1, a2|
-        [:vpc1, :vpc2].include?(a1).should be(true)
-        [:subnet1, :subnet2].include?(a2).should be(true)
-        :blah
-      end.at_least(:once)
-      expect(provider_class).to receive(:name_or_id) {|arg| arg.name}.at_least(:once)
-    end
+    def thing(name); double(:name => name, :tags => {'Name' => name}); end
+    let(:client) { double('client',
+      :"describe_#{type}s" => double(:"#{type}_set" => [:"#{type}1", :"#{type}2"])) }
+    let(:region) { double('region', :client => client) }
+    let(:ec2) { double('ec2', :regions => double(:[] => region)) }
+    let(:aws_thing) { double('aws_thing', :aws_item => double(:id => nil)) }
 
-    it "should find 8 instances" do
-      expect(provider_class).to receive(:ec2).and_return(ec2_mock).at_least(:once)
-      provider_class.instances.count.should eq(8)
-    end
-    it "should send a key hash to the ec2 method" do
-      expect(provider_class).to receive(:ec2) do
-        ec2_mock
-      end.at_least(:once)
-      provider_class.instances.count.should eq(8)
+    it "should find 4 instances" do
+      provider_class.stub(:ec2 => ec2, :regions => [:region1, :region2])
+      expect(client).to receive(:"describe_#{type}s")
+      expect(provider_class).to receive(:preload) {|r, n, _, _| thing(n)}.at_least(:once)
+      expect(provider_class).to receive(:new_from_aws).and_return(aws_thing).at_least(:once)
+      expect(provider_class.instances.count).to eq(4)
     end
   end
 end
diff --git a/spec/unit/puppet/aws_vgw_provider_spec.rb b/spec/unit/puppet/aws_vgw_provider_spec.rb
index f9457f8..682676e 100644
--- a/spec/unit/puppet/aws_vgw_provider_spec.rb
+++ b/spec/unit/puppet/aws_vgw_provider_spec.rb
@@ -5,32 +5,20 @@
 
 describe provider_class do
   context "with 2 resources in each of 2 regions" do
-    let(:two_gateways) {[:gateway1, :gateway2]}
-    let(:two_regions) { [:region1, :region2] }
-    let(:ec2_mock) {
-      ec2_mock = double 'object'
-      ec2_mock.stub_chain('regions.[].vpn_gateways.reject').and_return(two_gateways)
-      ec2_mock
-    }
+    let(:type) { :vpn_gateway }
 
-    before :each do
-      provider_class.should_receive(:regions).and_return(two_regions)
-      expect(provider_class).to receive(:new_from_aws) do |a1, a2|
-        [:gateway1, :gateway2].include?(a1).should be(true)
-        [:region1, :region2].include?(a2).should be(true)
-        :blah
-      end.at_least(:once)
-    end
+    def thing(name); double(:name => name, :tags => {'Name' => name}); end
+    let(:client) { double('client',
+      :"describe_#{type}s" => double(:"#{type}_set" => [:"#{type}1", :"#{type}2"])) }
+    let(:region) { double('region', :client => client) }
+    let(:ec2) { double('ec2', :regions => double(:[] => region)) }
+    let(:aws_thing) { double('aws_thing', :aws_item => double(:id => nil)) }
 
     it "should find 4 instances" do
-      provider_class.should_receive(:ec2).at_least(:once).and_return(ec2_mock)
-      provider_class.instances.count.should eq(4)
-    end
-    it "should send a key hash to the ec2 method" do
-      expect(provider_class).to receive(:ec2) do
-        ec2_mock
-      end.at_least(:once)
-      provider_class.instances.count.should eq(4)
+      provider_class.stub(:ec2 => ec2, :regions => [:region1, :region2])
+      expect(provider_class).to receive(:preload) {|r, n, _, _| thing(n)}.at_least(:once)
+      expect(provider_class).to receive(:new_from_aws).and_return(aws_thing).at_least(:once)
+      expect(provider_class.instances.count).to eq(4)
     end
   end
 end
diff --git a/spec/unit/puppet/aws_vpc_provider_spec.rb b/spec/unit/puppet/aws_vpc_provider_spec.rb
index 8beac29..d7ace20 100644
--- a/spec/unit/puppet/aws_vpc_provider_spec.rb
+++ b/spec/unit/puppet/aws_vpc_provider_spec.rb
@@ -2,9 +2,7 @@
 require 'spec_helper'
 
 class Hash
-  def to_h
-    self
-  end
+  def to_h; self; end
 end
 
 provider_class = Puppet::Type.type(:aws_vpc).provider(:api)
@@ -35,34 +33,23 @@ class Puppet::Type::Aws_vpc::ProviderApi
         eql({:region=>"us-west-1", :cidr=>"10.10.0.0/16", :dhcp_options=> 'mydopt', :instance_tenancy=>"default", :ensure=>:present, :name=>"foo", :tags=>{}, :id=>"vpc-6666"}) }
     end
   end
+
   context "with 2 resources in each of 2 regions" do
-    let(:two_vpcs) {[:vpc1, :vpc2]}
-    let(:two_regions) { [:region1, :region2] }
-    let(:ec2_mock) {
-      ec2_mock = double 'object'
-      ec2_mock.stub_chain('regions.[].vpcs').and_return(two_vpcs)
-      ec2_mock
-    }
+    let(:type) { :vpc }
 
-    before :each do
-      provider_class.should_receive(:regions).and_return(two_regions)
-      expect(provider_class).to receive(:new_from_aws) do |a1, a2, a3, a4|
-        [:region1, :region2].include?(a1).should be(true)
-        [:vpc1, :vpc2].include?(a2).should be(true)
-        :blah
-      end.at_least(:once)
-    end
+    def thing(name); double(:name => name, :tags => {'Name' => name}); end
+    let(:client) { double('client',
+      :"describe_#{type}s" => double(:"#{type}_set" => [:"#{type}1", :"#{type}2"])) }
+    let(:region) { double('region', :client => client) }
+    let(:ec2) { double('ec2', :regions => double(:[] => region)) }
+    let(:aws_thing) { double('aws_thing', :aws_item => double(:id => nil)) }
 
     it "should find 4 instances" do
-      provider_class.should_receive(:ec2).at_least(:once).and_return(ec2_mock)
-      provider_class.instances.count.should eq(4)
-    end
-    it "should send a key hash to the ec2 method" do
-      expect(provider_class).to receive(:ec2) do
-        ec2_mock
-      end.at_least(:once)
-      provider_class.instances.count.should eq(4)
+      provider_class.stub(:ec2 => ec2, :regions => [:region1, :region2])
+      expect(client).to receive(:"describe_#{type}s")
+      expect(provider_class).to receive(:preload) {|r, n, _, _| thing(n)}.at_least(:once)
+      expect(provider_class).to receive(:new_from_aws).and_return(aws_thing).at_least(:once)
+      expect(provider_class.instances.count).to eq(4)
     end
   end
 end
-
diff --git a/spec/unit/puppet/aws_vpn_provider_spec.rb b/spec/unit/puppet/aws_vpn_provider_spec.rb
index ec55994..ac0d20f 100644
--- a/spec/unit/puppet/aws_vpn_provider_spec.rb
+++ b/spec/unit/puppet/aws_vpn_provider_spec.rb
@@ -26,14 +26,5 @@
       provider_class.should_receive(:ec2).at_least(:once).and_return(ec2_mock)
       provider_class.instances.count.should eq(4)
     end
-    it "should send a key hash to the ec2 method" do
-      expect(provider_class).to receive(:ec2) do
-        ec2_mock
-      end.at_least(:once)
-      provider_class.instances.count.should eq(4)
-    end
   end
-
-
 end
-

From b1ed77420a9ec2f03bf9524f3b0dd2b48125c84a Mon Sep 17 00:00:00 2001
From: Maksym Melnychok <maksym.melnychok@gmail.com>
Date: Thu, 12 Mar 2015 18:12:23 +0100
Subject: [PATCH 5/5] dont fail on things without name tag

---
 lib/puppet_x/bobtfish/ec2_api.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/puppet_x/bobtfish/ec2_api.rb b/lib/puppet_x/bobtfish/ec2_api.rb
index 77d21c0..8c3f516 100644
--- a/lib/puppet_x/bobtfish/ec2_api.rb
+++ b/lib/puppet_x/bobtfish/ec2_api.rb
@@ -37,7 +37,7 @@ def self.instances
 
       item_set.map do |item_attrs|
         item = preload(region, item_attrs, describe_call, id_call)
-        name = [region_name, item.tags['Name']].join('/')
+        name = [region_name, item.tags['Name'] || item.id].join('/')
 
         raise "#{item} : #{name} is a duplicate" if @instance_names.include? name
         @instance_names << name