From a873902eee827626da25ebca9a5941237da7bfec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcos=20Magalh=C3=A3es?= <mmagalha@Z1.mycloudbr.local>
Date: Mon, 23 Dec 2019 07:31:47 -0300
Subject: [PATCH 1/7] Add Extras

---
 install_k8s/extras.yml                        |  5 ++
 install_k8s/hosts                             | 17 ++++--
 install_k8s/main.yml                          | 27 +++++++---
 .../roles/create-cluster/files/docker.json    |  8 +++
 .../create-cluster/tasks/init-cluster.yml     | 11 ++--
 .../roles/create-cluster/vars/main.yml        |  3 +-
 install_k8s/roles/install-extra/.travis.yml   | 29 ++++++++++
 install_k8s/roles/install-extra/README.md     | 38 +++++++++++++
 .../roles/install-extra/defaults/main.yml     |  2 +
 .../files/cert_manager_test.yaml              | 23 ++++++++
 .../install-extra/files/metallb_config.yaml   | 12 +++++
 .../roles/install-extra/handlers/main.yml     |  2 +
 install_k8s/roles/install-extra/meta/main.yml | 53 +++++++++++++++++++
 install_k8s/roles/install-extra/tasks/cni.yml |  3 ++
 .../roles/install-extra/tasks/main.yml        |  4 ++
 .../roles/install-extra/tasks/metallb.yml     | 17 ++++++
 .../roles/install-extra/tests/inventory       |  2 +
 .../roles/install-extra/tests/test.yml        |  5 ++
 install_k8s/roles/install-extra/vars/main.yml |  3 ++
 .../roles/install-k8s/tasks/install.yml       | 43 +++++++++++++--
 install_k8s/roles/install-vms/.travis.yml     | 29 ++++++++++
 install_k8s/roles/install-vms/README.md       | 38 +++++++++++++
 .../roles/install-vms/defaults/main.yml       |  2 +
 .../roles/install-vms/handlers/main.yml       |  2 +
 install_k8s/roles/install-vms/meta/main.yml   | 53 +++++++++++++++++++
 install_k8s/roles/install-vms/tasks/main.yml  | 31 +++++++++++
 install_k8s/roles/install-vms/tests/inventory |  2 +
 install_k8s/roles/install-vms/tests/test.yml  |  5 ++
 install_k8s/roles/install-vms/vars/main.yml   | 12 +++++
 .../roles/join-workers/tasks/join-cluster.yml |  1 -
 30 files changed, 462 insertions(+), 20 deletions(-)
 create mode 100644 install_k8s/extras.yml
 create mode 100644 install_k8s/roles/create-cluster/files/docker.json
 create mode 100644 install_k8s/roles/install-extra/.travis.yml
 create mode 100644 install_k8s/roles/install-extra/README.md
 create mode 100644 install_k8s/roles/install-extra/defaults/main.yml
 create mode 100644 install_k8s/roles/install-extra/files/cert_manager_test.yaml
 create mode 100644 install_k8s/roles/install-extra/files/metallb_config.yaml
 create mode 100644 install_k8s/roles/install-extra/handlers/main.yml
 create mode 100644 install_k8s/roles/install-extra/meta/main.yml
 create mode 100644 install_k8s/roles/install-extra/tasks/cni.yml
 create mode 100644 install_k8s/roles/install-extra/tasks/main.yml
 create mode 100644 install_k8s/roles/install-extra/tasks/metallb.yml
 create mode 100644 install_k8s/roles/install-extra/tests/inventory
 create mode 100644 install_k8s/roles/install-extra/tests/test.yml
 create mode 100644 install_k8s/roles/install-extra/vars/main.yml
 create mode 100644 install_k8s/roles/install-vms/.travis.yml
 create mode 100644 install_k8s/roles/install-vms/README.md
 create mode 100644 install_k8s/roles/install-vms/defaults/main.yml
 create mode 100644 install_k8s/roles/install-vms/handlers/main.yml
 create mode 100644 install_k8s/roles/install-vms/meta/main.yml
 create mode 100644 install_k8s/roles/install-vms/tasks/main.yml
 create mode 100644 install_k8s/roles/install-vms/tests/inventory
 create mode 100644 install_k8s/roles/install-vms/tests/test.yml
 create mode 100644 install_k8s/roles/install-vms/vars/main.yml

diff --git a/install_k8s/extras.yml b/install_k8s/extras.yml
new file mode 100644
index 0000000..5bf00b6
--- /dev/null
+++ b/install_k8s/extras.yml
@@ -0,0 +1,5 @@
+- hosts: k8s-master
+  become: yes
+  user: mmagalha
+  roles:
+  - { role: install-extra, tags: ["intall_extra"]} 
\ No newline at end of file
diff --git a/install_k8s/hosts b/install_k8s/hosts
index f775a1a..40eaf66 100644
--- a/install_k8s/hosts
+++ b/install_k8s/hosts
@@ -1,8 +1,19 @@
-[k8s-master]
+[pve]
+pve01
+
+[k8s]
+172.32.0.221
+172.32.0.22[2:4]
 
+[k8s-master]
+172.32.0.221
 
 [k8s-workers]
+172.32.0.22[2:4]
 
 [k8s-workers:vars]
-K8S_MASTER_NODE_IP=
-K8S_API_SECURE_PORT=6443 
+K8S_MASTER_NODE_IP=172.32.0.221
+K8S_API_SECURE_PORT=6443
+
+[all:vars]
+ansible_python_interpreter=/usr/bin/python3
diff --git a/install_k8s/main.yml b/install_k8s/main.yml
index 6e3980f..a358b4d 100644
--- a/install_k8s/main.yml
+++ b/install_k8s/main.yml
@@ -1,23 +1,34 @@
-- hosts: all
+- hosts: pve
+  user: root
+  gather_facts: no
+  pre_tasks:
+  - name: 'atualizando repo'
+    raw: 'apt-get update'
+  - name: 'instalando o python'
+    raw: 'apt-get install -y python3 python3-proxmoxer'
+  roles:
+  - { role: install-vms, tags: ["install_pve_vms"]}
+
+- hosts: k8s
   become: yes
-  user: ubuntu
+  user: mmagalha
   gather_facts: no
   pre_tasks:
   - name: 'atualizando repo'
     raw: 'apt-get update'
   - name: 'instalando o python'
-    raw: 'apt-get install -y python'
+    raw: 'apt-get install -y python3'
   roles:
-  - { role: install-k8s,	tags: ["install_k8s_role"]}
+  - { role: install-k8s, tags: ["install_k8s_role"]}
 
 - hosts: k8s-master
   become: yes
-  user: ubuntu
+  user: mmagalha
   roles:
-  - { role: create-cluster,	tags: ["create_cluster_role"]}
+  - { role: create-cluster, tags: ["create_cluster_role"]}
 
 - hosts: k8s-workers
   become: yes
-  user: ubuntu
+  user: mmagalha
   roles:
-  - { role: join-workers,	tags: ["join_workers_role"]}
+  - { role: join-workers, tags: ["join_workers_role"]}  
diff --git a/install_k8s/roles/create-cluster/files/docker.json b/install_k8s/roles/create-cluster/files/docker.json
new file mode 100644
index 0000000..92750f6
--- /dev/null
+++ b/install_k8s/roles/create-cluster/files/docker.json
@@ -0,0 +1,8 @@
+{
+    "exec-opts": ["native.cgroupdriver=systemd"],
+    "log-driver": "json-file",
+    "log-opts": {
+      "max-size": "100m"
+    },
+    "storage-driver": "overlay2"
+  }
\ No newline at end of file
diff --git a/install_k8s/roles/create-cluster/tasks/init-cluster.yml b/install_k8s/roles/create-cluster/tasks/init-cluster.yml
index 44b274f..bb4ef11 100644
--- a/install_k8s/roles/create-cluster/tasks/init-cluster.yml
+++ b/install_k8s/roles/create-cluster/tasks/init-cluster.yml
@@ -2,36 +2,41 @@
   command:
     kubeadm reset --force
   register: kubeadmin_init
+ 
 - name: Initialize Kubernetes master with kubeadm init.
   command:
     kubeadm init
   register: kubeadmin_init
+
 - name: Ensure .kube directory exists.
   file:
     path: ~/.kube
     state: directory
+
 - name: Symlink the kubectl admin.conf to ~/.kube/conf.
   file:
     src: /etc/kubernetes/admin.conf
     dest: ~/.kube/config
     state: link
-- name: Configure weavenet networking.
-  shell: kubectl apply -f {{ default_kubernetes_cni_weavenet_manifestUrl }}
-  register: weavenet_result
+
 - name: "Cluster token"
   shell: kubeadm token list | cut -d ' ' -f1 | sed -n '2p'
   register: K8S_TOKEN
+
 - name: "CA Hash"
   shell: openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
   register: K8S_MASTER_CA_HASH
+
 - name: "Add K8S Token and Hash to dummy host"
   add_host:
     name:   "K8S_TOKEN_HOLDER"
     token:  "{{ K8S_TOKEN.stdout }}"
     hash:   "{{ K8S_MASTER_CA_HASH.stdout }}"
+
 - name:
   debug:
     msg: "[Master] K8S_TOKEN_HOLDER K8S token is {{ hostvars['K8S_TOKEN_HOLDER']['token'] }}"
+
 - name:
   debug:
     msg: "[Master] K8S_TOKEN_HOLDER K8S Hash is  {{ hostvars['K8S_TOKEN_HOLDER']['hash'] }}"
diff --git a/install_k8s/roles/create-cluster/vars/main.yml b/install_k8s/roles/create-cluster/vars/main.yml
index dd326c7..db7ad8d 100644
--- a/install_k8s/roles/create-cluster/vars/main.yml
+++ b/install_k8s/roles/create-cluster/vars/main.yml
@@ -1,3 +1,2 @@
 ---
-# vars file for create-cluster
-default_kubernetes_cni_weavenet_manifestUrl: "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"
+# vars file for create-cluster
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/.travis.yml b/install_k8s/roles/install-extra/.travis.yml
new file mode 100644
index 0000000..36bbf62
--- /dev/null
+++ b/install_k8s/roles/install-extra/.travis.yml
@@ -0,0 +1,29 @@
+---
+language: python
+python: "2.7"
+
+# Use the new container infrastructure
+sudo: false
+
+# Install ansible
+addons:
+  apt:
+    packages:
+    - python-pip
+
+install:
+  # Install ansible
+  - pip install ansible
+
+  # Check ansible version
+  - ansible --version
+
+  # Create ansible.cfg with correct roles_path
+  - printf '[defaults]\nroles_path=../' >ansible.cfg
+
+script:
+  # Basic role syntax check
+  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
+
+notifications:
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/README.md b/install_k8s/roles/install-extra/README.md
new file mode 100644
index 0000000..225dd44
--- /dev/null
+++ b/install_k8s/roles/install-extra/README.md
@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).
diff --git a/install_k8s/roles/install-extra/defaults/main.yml b/install_k8s/roles/install-extra/defaults/main.yml
new file mode 100644
index 0000000..0d146b2
--- /dev/null
+++ b/install_k8s/roles/install-extra/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+# defaults file for install-extra
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/files/cert_manager_test.yaml b/install_k8s/roles/install-extra/files/cert_manager_test.yaml
new file mode 100644
index 0000000..c6fadf3
--- /dev/null
+++ b/install_k8s/roles/install-extra/files/cert_manager_test.yaml
@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager-test
+---
+apiVersion: cert-manager.io/v1alpha2
+kind: Issuer
+metadata:
+  name: test-selfsigned
+  namespace: cert-manager-test
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1alpha2
+kind: Certificate
+metadata:
+  name: selfsigned-cert
+  namespace: cert-manager-test
+spec:
+  commonName: example.com
+  secretName: selfsigned-cert-tls
+  issuerRef:
+    name: test-selfsigned
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/files/metallb_config.yaml b/install_k8s/roles/install-extra/files/metallb_config.yaml
new file mode 100644
index 0000000..63caf55
--- /dev/null
+++ b/install_k8s/roles/install-extra/files/metallb_config.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: default
+      protocol: layer2
+      addresses:
+      - 172.32.0.230-172.32.0.235
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/handlers/main.yml b/install_k8s/roles/install-extra/handlers/main.yml
new file mode 100644
index 0000000..91caaa5
--- /dev/null
+++ b/install_k8s/roles/install-extra/handlers/main.yml
@@ -0,0 +1,2 @@
+---
+# handlers file for install-extra
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/meta/main.yml b/install_k8s/roles/install-extra/meta/main.yml
new file mode 100644
index 0000000..227ad9c
--- /dev/null
+++ b/install_k8s/roles/install-extra/meta/main.yml
@@ -0,0 +1,53 @@
+galaxy_info:
+  author: your name
+  description: your role description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.9
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.
+  
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/tasks/cni.yml b/install_k8s/roles/install-extra/tasks/cni.yml
new file mode 100644
index 0000000..085b954
--- /dev/null
+++ b/install_k8s/roles/install-extra/tasks/cni.yml
@@ -0,0 +1,3 @@
+- name: Configure weavenet networking.
+  shell: kubectl apply -f {{ default_kubernetes_cni_weavenet_manifestUrl }}
+  register: weavenet_result
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/tasks/main.yml b/install_k8s/roles/install-extra/tasks/main.yml
new file mode 100644
index 0000000..9c41d36
--- /dev/null
+++ b/install_k8s/roles/install-extra/tasks/main.yml
@@ -0,0 +1,4 @@
+---
+# tasks file for install-extra
+- include: cni.yml
+- include: metallb.yml
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/tasks/metallb.yml b/install_k8s/roles/install-extra/tasks/metallb.yml
new file mode 100644
index 0000000..d47e484
--- /dev/null
+++ b/install_k8s/roles/install-extra/tasks/metallb.yml
@@ -0,0 +1,17 @@
+- name: Copy configure MetalLB
+  copy:
+    src: roles/install-extra/files/metallb_config.yaml
+    dest: /opt/metallb_config.yaml
+    owner: root
+    group: root
+    backup: yes    
+  register: metallb_install
+
+- name: Install MetalLB config.
+  shell: kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml
+  register: metallb_install
+
+- name: Install MetalLB.
+  shell: kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml
+  register: metallb_install
+
diff --git a/install_k8s/roles/install-extra/tests/inventory b/install_k8s/roles/install-extra/tests/inventory
new file mode 100644
index 0000000..878877b
--- /dev/null
+++ b/install_k8s/roles/install-extra/tests/inventory
@@ -0,0 +1,2 @@
+localhost
+
diff --git a/install_k8s/roles/install-extra/tests/test.yml b/install_k8s/roles/install-extra/tests/test.yml
new file mode 100644
index 0000000..f978fae
--- /dev/null
+++ b/install_k8s/roles/install-extra/tests/test.yml
@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - install-extra
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/vars/main.yml b/install_k8s/roles/install-extra/vars/main.yml
new file mode 100644
index 0000000..d71758e
--- /dev/null
+++ b/install_k8s/roles/install-extra/vars/main.yml
@@ -0,0 +1,3 @@
+---
+# vars file for install-extra
+default_kubernetes_cni_weavenet_manifestUrl: "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"
diff --git a/install_k8s/roles/install-k8s/tasks/install.yml b/install_k8s/roles/install-k8s/tasks/install.yml
index 9df4e24..0cfe554 100644
--- a/install_k8s/roles/install-k8s/tasks/install.yml
+++ b/install_k8s/roles/install-k8s/tasks/install.yml
@@ -1,5 +1,11 @@
-- name: Instalando o Docker
-  shell: curl -fsSL https://get.docker.com | bash -
+- name: Adicionando as chaves do repo do Docker
+  shell: https://download.docker.com/linux/ubuntu/gpg
+  state: present
+
+- name: Adicionando repositório do Docker
+  apt_repository:
+    repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs) stable"
+    state: present
 
 - name: Adicionando as chaves repo k8s no apt
   apt_key:
@@ -11,11 +17,42 @@
     repo: deb http://apt.kubernetes.io/ kubernetes-xenial main
     state: present
 
-- name: Install k8s packages
+- name: Install docker and k8s packages
   apt:
     name: "{{ packages }}"
   vars:
     packages:
+    - docker-ce
+    - docker-ce-cli
+    - containerd.io
     - kubelet
     - kubeadm
     - kubectl
+
+- name: Copy docker.json
+  copy:
+    src: /data/DevOps/ansible/install_k8s/roles/create-cluster/files/docker.json
+    dest: /etc/docker/daemon.json
+    owner: root
+    group: root
+    backup: yes    
+  register: kubeadmin_init
+
+- name: restart systemd
+  systemd:
+    daemon_reload: yes 
+
+- name: reload service docker, in all cases
+  systemd:
+    name: docker
+    state: reloaded
+
+- name: Disable SWAP since kubernetes can't work with swap enabled (1/2)
+  shell: |
+    swapoff -a
+
+- name: Disable SWAP in fstab since kubernetes can't work with swap enabled (2/2)
+  replace:
+    path: /etc/fstab
+    regexp: '^([^#].*?\sswap\s+sw\s+.*)$'
+    replace: '# \1' 
diff --git a/install_k8s/roles/install-vms/.travis.yml b/install_k8s/roles/install-vms/.travis.yml
new file mode 100644
index 0000000..36bbf62
--- /dev/null
+++ b/install_k8s/roles/install-vms/.travis.yml
@@ -0,0 +1,29 @@
+---
+language: python
+python: "2.7"
+
+# Use the new container infrastructure
+sudo: false
+
+# Install ansible
+addons:
+  apt:
+    packages:
+    - python-pip
+
+install:
+  # Install ansible
+  - pip install ansible
+
+  # Check ansible version
+  - ansible --version
+
+  # Create ansible.cfg with correct roles_path
+  - printf '[defaults]\nroles_path=../' >ansible.cfg
+
+script:
+  # Basic role syntax check
+  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
+
+notifications:
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/
\ No newline at end of file
diff --git a/install_k8s/roles/install-vms/README.md b/install_k8s/roles/install-vms/README.md
new file mode 100644
index 0000000..225dd44
--- /dev/null
+++ b/install_k8s/roles/install-vms/README.md
@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).
diff --git a/install_k8s/roles/install-vms/defaults/main.yml b/install_k8s/roles/install-vms/defaults/main.yml
new file mode 100644
index 0000000..2a65dcc
--- /dev/null
+++ b/install_k8s/roles/install-vms/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+# defaults file for install-vms
\ No newline at end of file
diff --git a/install_k8s/roles/install-vms/handlers/main.yml b/install_k8s/roles/install-vms/handlers/main.yml
new file mode 100644
index 0000000..308c1db
--- /dev/null
+++ b/install_k8s/roles/install-vms/handlers/main.yml
@@ -0,0 +1,2 @@
+---
+# handlers file for install-vms
\ No newline at end of file
diff --git a/install_k8s/roles/install-vms/meta/main.yml b/install_k8s/roles/install-vms/meta/main.yml
new file mode 100644
index 0000000..227ad9c
--- /dev/null
+++ b/install_k8s/roles/install-vms/meta/main.yml
@@ -0,0 +1,53 @@
+galaxy_info:
+  author: your name
+  description: your role description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.9
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.
+  
\ No newline at end of file
diff --git a/install_k8s/roles/install-vms/tasks/main.yml b/install_k8s/roles/install-vms/tasks/main.yml
new file mode 100644
index 0000000..5ca00dd
--- /dev/null
+++ b/install_k8s/roles/install-vms/tasks/main.yml
@@ -0,0 +1,31 @@
+---
+# tasks file for install-vm
+- name: Clone VMs
+  proxmox_kvm:
+    api_user    : "{{ pve_user }}"
+    api_password: "{{ pve_pass }}"
+    api_host    : "{{ pve_host }}"
+    clone       : ubnttpl   # The VM source
+    name        : k8s0{{ item }}  # The target VM name
+    newid       : 100{{ item }}
+    node        : pve01
+    storage     : Local-01
+    format      : qcow2
+    timeout     : 500
+#    state       : started
+  loop: "{{ vm_id }}"
+
+- name: Set VM IP
+  command:
+    qm set 100{{ item }} --ipconfig0 ip=172.32.0.22{{ item }}/24,gw=172.32.0.254
+  loop: "{{ vm_id }}"
+  ignore_errors: False
+  
+- name: start VMs
+  proxmox_kvm:
+    api_user    : "{{ pve_user }}"
+    api_password: "{{ pve_pass }}"
+    api_host    : "{{ pve_host }}"
+    vmid        : 100{{ item }}
+    state       : started
+  loop: "{{ vm_id }}"
diff --git a/install_k8s/roles/install-vms/tests/inventory b/install_k8s/roles/install-vms/tests/inventory
new file mode 100644
index 0000000..878877b
--- /dev/null
+++ b/install_k8s/roles/install-vms/tests/inventory
@@ -0,0 +1,2 @@
+localhost
+
diff --git a/install_k8s/roles/install-vms/tests/test.yml b/install_k8s/roles/install-vms/tests/test.yml
new file mode 100644
index 0000000..33cb5d6
--- /dev/null
+++ b/install_k8s/roles/install-vms/tests/test.yml
@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - install-vms
\ No newline at end of file
diff --git a/install_k8s/roles/install-vms/vars/main.yml b/install_k8s/roles/install-vms/vars/main.yml
new file mode 100644
index 0000000..2001e44
--- /dev/null
+++ b/install_k8s/roles/install-vms/vars/main.yml
@@ -0,0 +1,12 @@
+---
+# vars file for install-vms
+pve_host: pve01
+pve_user: root@pam
+pve_pass: ncc1701
+vm_id:
+- 1
+- 2
+- 3
+- 4
+vm_name: "k8s0"
+
diff --git a/install_k8s/roles/join-workers/tasks/join-cluster.yml b/install_k8s/roles/join-workers/tasks/join-cluster.yml
index d8da4d8..e9fddb4 100644
--- a/install_k8s/roles/join-workers/tasks/join-cluster.yml
+++ b/install_k8s/roles/join-workers/tasks/join-cluster.yml
@@ -9,7 +9,6 @@
 - name: "Kubeadm reset node cluster config"
   command: 
     kubeadm reset --force
-  register: kubeadm-reset_node
 
 - name: "Kubeadm join"
   shell: 

From b0d23224a126df04d7126703746d7b03e8da224f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcos=20Magalh=C3=A3es?= <mmagalha@Z1.mycloudbr.local>
Date: Mon, 23 Dec 2019 07:35:54 -0300
Subject: [PATCH 2/7] pass

---
 install_k8s/roles/install-vms/vars/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/install_k8s/roles/install-vms/vars/main.yml b/install_k8s/roles/install-vms/vars/main.yml
index 2001e44..49db894 100644
--- a/install_k8s/roles/install-vms/vars/main.yml
+++ b/install_k8s/roles/install-vms/vars/main.yml
@@ -2,7 +2,7 @@
 # vars file for install-vms
 pve_host: pve01
 pve_user: root@pam
-pve_pass: ncc1701
+pve_pass: 
 vm_id:
 - 1
 - 2

From 5adb90efc4d4764bfc3df9394561df01b5ad979c Mon Sep 17 00:00:00 2001
From: Marcos Magalhaes <mmagalha@gmail.com>
Date: Mon, 23 Dec 2019 18:03:55 -0300
Subject: [PATCH 3/7] wait_for

---
 install_k8s/extras.yml                          | 2 +-
 install_k8s/hosts                               | 6 +++---
 install_k8s/main.yml                            | 4 ++--
 install_k8s/roles/install-k8s/tasks/install.yml | 5 +++--
 install_k8s/roles/install-vms/tasks/main.yml    | 3 +++
 install_k8s/roles/install-vms/vars/main.yml     | 2 +-
 install_k8s/valt.yml                            | 6 ++++++
 7 files changed, 19 insertions(+), 9 deletions(-)
 create mode 100644 install_k8s/valt.yml

diff --git a/install_k8s/extras.yml b/install_k8s/extras.yml
index 5bf00b6..204785c 100644
--- a/install_k8s/extras.yml
+++ b/install_k8s/extras.yml
@@ -1,4 +1,4 @@
-- hosts: k8s-master
+- hosts: k8s_master
   become: yes
   user: mmagalha
   roles:
diff --git a/install_k8s/hosts b/install_k8s/hosts
index 40eaf66..1ff6d81 100644
--- a/install_k8s/hosts
+++ b/install_k8s/hosts
@@ -5,13 +5,13 @@ pve01
 172.32.0.221
 172.32.0.22[2:4]
 
-[k8s-master]
+[k8s_master]
 172.32.0.221
 
-[k8s-workers]
+[k8s_workers]
 172.32.0.22[2:4]
 
-[k8s-workers:vars]
+[k8s_workers:vars]
 K8S_MASTER_NODE_IP=172.32.0.221
 K8S_API_SECURE_PORT=6443
 
diff --git a/install_k8s/main.yml b/install_k8s/main.yml
index f389f88..62bd270 100644
--- a/install_k8s/main.yml
+++ b/install_k8s/main.yml
@@ -21,13 +21,13 @@
   roles:
   - { role: install-k8s, tags: ["install_k8s_role"]}
 
-- hosts: k8s-master
+- hosts: k8s_master
   become: yes
   user: mmagalha
   roles:
   - { role: create-cluster, tags: ["create_cluster_role"]}
 
-- hosts: k8s-workers
+- hosts: k8s_workers
   become: yes
   user: mmagalha
   roles:
diff --git a/install_k8s/roles/install-k8s/tasks/install.yml b/install_k8s/roles/install-k8s/tasks/install.yml
index 0cfe554..bc516cb 100644
--- a/install_k8s/roles/install-k8s/tasks/install.yml
+++ b/install_k8s/roles/install-k8s/tasks/install.yml
@@ -1,6 +1,7 @@
 - name: Adicionando as chaves do repo do Docker
-  shell: https://download.docker.com/linux/ubuntu/gpg
-  state: present
+  apt_key:
+    url: https://download.docker.com/linux/ubuntu/gpg
+    state: present
 
 - name: Adicionando repositório do Docker
   apt_repository:
diff --git a/install_k8s/roles/install-vms/tasks/main.yml b/install_k8s/roles/install-vms/tasks/main.yml
index 5ca00dd..9fdeca0 100644
--- a/install_k8s/roles/install-vms/tasks/main.yml
+++ b/install_k8s/roles/install-vms/tasks/main.yml
@@ -29,3 +29,6 @@
     vmid        : 100{{ item }}
     state       : started
   loop: "{{ vm_id }}"
+
+- name: Wait 600 seconds for target connection to become reachable/usable
+  wait_for_connection:
\ No newline at end of file
diff --git a/install_k8s/roles/install-vms/vars/main.yml b/install_k8s/roles/install-vms/vars/main.yml
index 49db894..184cc2f 100644
--- a/install_k8s/roles/install-vms/vars/main.yml
+++ b/install_k8s/roles/install-vms/vars/main.yml
@@ -2,7 +2,7 @@
 # vars file for install-vms
 pve_host: pve01
 pve_user: root@pam
-pve_pass: 
+pve_pass: "{{ lookup('env','PVE_PASS') }}"
 vm_id:
 - 1
 - 2
diff --git a/install_k8s/valt.yml b/install_k8s/valt.yml
new file mode 100644
index 0000000..48d2c21
--- /dev/null
+++ b/install_k8s/valt.yml
@@ -0,0 +1,6 @@
+$ANSIBLE_VAULT;1.2;AES256;api_password
+33343366393464373066303033613732306631393861333865353430343139363735366363396138
+6466373538633537386164633464346536363733636234340a656162326537303433363632623737
+66313434653962386462653862613433633461316437306562356635363862666239306438393539
+6133636438643661660a306233343166616439313361643663316166306431396563663333373835
+3235

From 3c575b697c031705f601eeddfc358a454891c58e Mon Sep 17 00:00:00 2001
From: Marcos Magalhaes <mmagalha@gmail.com>
Date: Tue, 24 Dec 2019 08:46:08 -0300
Subject: [PATCH 4/7] PVE

---
 install_k8s/extras.yml                        |  5 ++
 install_k8s/hosts                             | 21 ++++++--
 install_k8s/main.yml                          | 27 +++++++---
 .../roles/create-cluster/files/docker.json    |  8 +++
 .../create-cluster/tasks/init-cluster.yml     | 11 ++--
 .../roles/create-cluster/vars/main.yml        |  3 +-
 install_k8s/roles/install-extra/.travis.yml   | 29 ++++++++++
 install_k8s/roles/install-extra/README.md     | 38 +++++++++++++
 .../roles/install-extra/defaults/main.yml     |  2 +
 .../files/cert_manager_test.yaml              | 23 ++++++++
 .../install-extra/files/metallb_config.yaml   | 12 +++++
 .../roles/install-extra/handlers/main.yml     |  2 +
 install_k8s/roles/install-extra/meta/main.yml | 53 +++++++++++++++++++
 install_k8s/roles/install-extra/tasks/cni.yml |  3 ++
 .../roles/install-extra/tasks/main.yml        |  4 ++
 .../roles/install-extra/tasks/metallb.yml     | 17 ++++++
 .../roles/install-extra/tests/inventory       |  2 +
 .../roles/install-extra/tests/test.yml        |  5 ++
 install_k8s/roles/install-extra/vars/main.yml |  3 ++
 .../roles/install-k8s/tasks/install.yml       | 44 +++++++++++++--
 install_k8s/roles/install-vms/.travis.yml     | 29 ++++++++++
 install_k8s/roles/install-vms/README.md       | 38 +++++++++++++
 .../roles/install-vms/defaults/main.yml       |  2 +
 .../roles/install-vms/handlers/main.yml       |  2 +
 install_k8s/roles/install-vms/meta/main.yml   | 53 +++++++++++++++++++
 install_k8s/roles/install-vms/tasks/main.yml  | 34 ++++++++++++
 install_k8s/roles/install-vms/tests/inventory |  2 +
 install_k8s/roles/install-vms/tests/test.yml  |  5 ++
 install_k8s/roles/install-vms/vars/main.yml   | 12 +++++
 .../roles/join-workers/tasks/join-cluster.yml |  1 -
 install_k8s/valt.yml                          |  6 +++
 31 files changed, 474 insertions(+), 22 deletions(-)
 create mode 100644 install_k8s/extras.yml
 create mode 100644 install_k8s/roles/create-cluster/files/docker.json
 create mode 100644 install_k8s/roles/install-extra/.travis.yml
 create mode 100644 install_k8s/roles/install-extra/README.md
 create mode 100644 install_k8s/roles/install-extra/defaults/main.yml
 create mode 100644 install_k8s/roles/install-extra/files/cert_manager_test.yaml
 create mode 100644 install_k8s/roles/install-extra/files/metallb_config.yaml
 create mode 100644 install_k8s/roles/install-extra/handlers/main.yml
 create mode 100644 install_k8s/roles/install-extra/meta/main.yml
 create mode 100644 install_k8s/roles/install-extra/tasks/cni.yml
 create mode 100644 install_k8s/roles/install-extra/tasks/main.yml
 create mode 100644 install_k8s/roles/install-extra/tasks/metallb.yml
 create mode 100644 install_k8s/roles/install-extra/tests/inventory
 create mode 100644 install_k8s/roles/install-extra/tests/test.yml
 create mode 100644 install_k8s/roles/install-extra/vars/main.yml
 create mode 100644 install_k8s/roles/install-vms/.travis.yml
 create mode 100644 install_k8s/roles/install-vms/README.md
 create mode 100644 install_k8s/roles/install-vms/defaults/main.yml
 create mode 100644 install_k8s/roles/install-vms/handlers/main.yml
 create mode 100644 install_k8s/roles/install-vms/meta/main.yml
 create mode 100644 install_k8s/roles/install-vms/tasks/main.yml
 create mode 100644 install_k8s/roles/install-vms/tests/inventory
 create mode 100644 install_k8s/roles/install-vms/tests/test.yml
 create mode 100644 install_k8s/roles/install-vms/vars/main.yml
 create mode 100644 install_k8s/valt.yml

diff --git a/install_k8s/extras.yml b/install_k8s/extras.yml
new file mode 100644
index 0000000..204785c
--- /dev/null
+++ b/install_k8s/extras.yml
@@ -0,0 +1,5 @@
+- hosts: k8s_master
+  become: yes
+  user: mmagalha
+  roles:
+  - { role: install-extra, tags: ["intall_extra"]} 
\ No newline at end of file
diff --git a/install_k8s/hosts b/install_k8s/hosts
index f775a1a..1ff6d81 100644
--- a/install_k8s/hosts
+++ b/install_k8s/hosts
@@ -1,8 +1,19 @@
-[k8s-master]
+[pve]
+pve01
 
+[k8s]
+172.32.0.221
+172.32.0.22[2:4]
 
-[k8s-workers]
+[k8s_master]
+172.32.0.221
 
-[k8s-workers:vars]
-K8S_MASTER_NODE_IP=
-K8S_API_SECURE_PORT=6443 
+[k8s_workers]
+172.32.0.22[2:4]
+
+[k8s_workers:vars]
+K8S_MASTER_NODE_IP=172.32.0.221
+K8S_API_SECURE_PORT=6443
+
+[all:vars]
+ansible_python_interpreter=/usr/bin/python3
diff --git a/install_k8s/main.yml b/install_k8s/main.yml
index 90c40b8..62bd270 100644
--- a/install_k8s/main.yml
+++ b/install_k8s/main.yml
@@ -1,23 +1,34 @@
-- hosts: all
+- hosts: pve
+  user: root
+  gather_facts: no
+  pre_tasks:
+  - name: 'atualizando repo'
+    raw: 'apt-get update'
+  - name: 'instalando o python'
+    raw: 'apt-get install -y python3 python3-proxmoxer'
+  roles:
+  - { role: install-vms, tags: ["install_pve_vms"]}
+
+- hosts: k8s
   become: yes
-  user: ubuntu
+  user: mmagalha
   gather_facts: no
   pre_tasks:
   - name: 'atualizando repo'
     raw: 'apt-get update'
   - name: 'instalando o python'
-    raw: 'apt-get install -y python'
+    raw: 'apt-get install -y python3'
   roles:
   - { role: install-k8s, tags: ["install_k8s_role"]}
 
-- hosts: k8s-master
+- hosts: k8s_master
   become: yes
-  user: ubuntu
+  user: mmagalha
   roles:
   - { role: create-cluster, tags: ["create_cluster_role"]}
 
-- hosts: k8s-workers
+- hosts: k8s_workers
   become: yes
-  user: ubuntu
+  user: mmagalha
   roles:
-  - { role: join-workers, tags: ["join_workers_role"]}
+  - { role: join-workers, tags: ["join_workers_role"]} 
diff --git a/install_k8s/roles/create-cluster/files/docker.json b/install_k8s/roles/create-cluster/files/docker.json
new file mode 100644
index 0000000..92750f6
--- /dev/null
+++ b/install_k8s/roles/create-cluster/files/docker.json
@@ -0,0 +1,8 @@
+{
+    "exec-opts": ["native.cgroupdriver=systemd"],
+    "log-driver": "json-file",
+    "log-opts": {
+      "max-size": "100m"
+    },
+    "storage-driver": "overlay2"
+  }
\ No newline at end of file
diff --git a/install_k8s/roles/create-cluster/tasks/init-cluster.yml b/install_k8s/roles/create-cluster/tasks/init-cluster.yml
index 44b274f..bb4ef11 100644
--- a/install_k8s/roles/create-cluster/tasks/init-cluster.yml
+++ b/install_k8s/roles/create-cluster/tasks/init-cluster.yml
@@ -2,36 +2,41 @@
   command:
     kubeadm reset --force
   register: kubeadmin_init
+ 
 - name: Initialize Kubernetes master with kubeadm init.
   command:
     kubeadm init
   register: kubeadmin_init
+
 - name: Ensure .kube directory exists.
   file:
     path: ~/.kube
     state: directory
+
 - name: Symlink the kubectl admin.conf to ~/.kube/conf.
   file:
     src: /etc/kubernetes/admin.conf
     dest: ~/.kube/config
     state: link
-- name: Configure weavenet networking.
-  shell: kubectl apply -f {{ default_kubernetes_cni_weavenet_manifestUrl }}
-  register: weavenet_result
+
 - name: "Cluster token"
   shell: kubeadm token list | cut -d ' ' -f1 | sed -n '2p'
   register: K8S_TOKEN
+
 - name: "CA Hash"
   shell: openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
   register: K8S_MASTER_CA_HASH
+
 - name: "Add K8S Token and Hash to dummy host"
   add_host:
     name:   "K8S_TOKEN_HOLDER"
     token:  "{{ K8S_TOKEN.stdout }}"
     hash:   "{{ K8S_MASTER_CA_HASH.stdout }}"
+
 - name:
   debug:
     msg: "[Master] K8S_TOKEN_HOLDER K8S token is {{ hostvars['K8S_TOKEN_HOLDER']['token'] }}"
+
 - name:
   debug:
     msg: "[Master] K8S_TOKEN_HOLDER K8S Hash is  {{ hostvars['K8S_TOKEN_HOLDER']['hash'] }}"
diff --git a/install_k8s/roles/create-cluster/vars/main.yml b/install_k8s/roles/create-cluster/vars/main.yml
index dd326c7..db7ad8d 100644
--- a/install_k8s/roles/create-cluster/vars/main.yml
+++ b/install_k8s/roles/create-cluster/vars/main.yml
@@ -1,3 +1,2 @@
 ---
-# vars file for create-cluster
-default_kubernetes_cni_weavenet_manifestUrl: "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"
+# vars file for create-cluster
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/.travis.yml b/install_k8s/roles/install-extra/.travis.yml
new file mode 100644
index 0000000..36bbf62
--- /dev/null
+++ b/install_k8s/roles/install-extra/.travis.yml
@@ -0,0 +1,29 @@
+---
+language: python
+python: "2.7"
+
+# Use the new container infrastructure
+sudo: false
+
+# Install ansible
+addons:
+  apt:
+    packages:
+    - python-pip
+
+install:
+  # Install ansible
+  - pip install ansible
+
+  # Check ansible version
+  - ansible --version
+
+  # Create ansible.cfg with correct roles_path
+  - printf '[defaults]\nroles_path=../' >ansible.cfg
+
+script:
+  # Basic role syntax check
+  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
+
+notifications:
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/README.md b/install_k8s/roles/install-extra/README.md
new file mode 100644
index 0000000..225dd44
--- /dev/null
+++ b/install_k8s/roles/install-extra/README.md
@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).
diff --git a/install_k8s/roles/install-extra/defaults/main.yml b/install_k8s/roles/install-extra/defaults/main.yml
new file mode 100644
index 0000000..0d146b2
--- /dev/null
+++ b/install_k8s/roles/install-extra/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+# defaults file for install-extra
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/files/cert_manager_test.yaml b/install_k8s/roles/install-extra/files/cert_manager_test.yaml
new file mode 100644
index 0000000..c6fadf3
--- /dev/null
+++ b/install_k8s/roles/install-extra/files/cert_manager_test.yaml
@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager-test
+---
+apiVersion: cert-manager.io/v1alpha2
+kind: Issuer
+metadata:
+  name: test-selfsigned
+  namespace: cert-manager-test
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1alpha2
+kind: Certificate
+metadata:
+  name: selfsigned-cert
+  namespace: cert-manager-test
+spec:
+  commonName: example.com
+  secretName: selfsigned-cert-tls
+  issuerRef:
+    name: test-selfsigned
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/files/metallb_config.yaml b/install_k8s/roles/install-extra/files/metallb_config.yaml
new file mode 100644
index 0000000..63caf55
--- /dev/null
+++ b/install_k8s/roles/install-extra/files/metallb_config.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: default
+      protocol: layer2
+      addresses:
+      - 172.32.0.230-172.32.0.235
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/handlers/main.yml b/install_k8s/roles/install-extra/handlers/main.yml
new file mode 100644
index 0000000..91caaa5
--- /dev/null
+++ b/install_k8s/roles/install-extra/handlers/main.yml
@@ -0,0 +1,2 @@
+---
+# handlers file for install-extra
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/meta/main.yml b/install_k8s/roles/install-extra/meta/main.yml
new file mode 100644
index 0000000..227ad9c
--- /dev/null
+++ b/install_k8s/roles/install-extra/meta/main.yml
@@ -0,0 +1,53 @@
+galaxy_info:
+  author: your name
+  description: your role description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.9
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.
+  
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/tasks/cni.yml b/install_k8s/roles/install-extra/tasks/cni.yml
new file mode 100644
index 0000000..085b954
--- /dev/null
+++ b/install_k8s/roles/install-extra/tasks/cni.yml
@@ -0,0 +1,3 @@
+- name: Configure weavenet networking.
+  shell: kubectl apply -f {{ default_kubernetes_cni_weavenet_manifestUrl }}
+  register: weavenet_result
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/tasks/main.yml b/install_k8s/roles/install-extra/tasks/main.yml
new file mode 100644
index 0000000..9c41d36
--- /dev/null
+++ b/install_k8s/roles/install-extra/tasks/main.yml
@@ -0,0 +1,4 @@
+---
+# tasks file for install-extra
+- include: cni.yml
+- include: metallb.yml
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/tasks/metallb.yml b/install_k8s/roles/install-extra/tasks/metallb.yml
new file mode 100644
index 0000000..d47e484
--- /dev/null
+++ b/install_k8s/roles/install-extra/tasks/metallb.yml
@@ -0,0 +1,17 @@
+- name: Copy configure MetalLB
+  copy:
+    src: roles/install-extra/files/metallb_config.yaml
+    dest: /opt/metallb_config.yaml
+    owner: root
+    group: root
+    backup: yes    
+  register: metallb_install
+
+- name: Install MetalLB config.
+  shell: kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml
+  register: metallb_install
+
+- name: Install MetalLB.
+  shell: kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml
+  register: metallb_install
+
diff --git a/install_k8s/roles/install-extra/tests/inventory b/install_k8s/roles/install-extra/tests/inventory
new file mode 100644
index 0000000..878877b
--- /dev/null
+++ b/install_k8s/roles/install-extra/tests/inventory
@@ -0,0 +1,2 @@
+localhost
+
diff --git a/install_k8s/roles/install-extra/tests/test.yml b/install_k8s/roles/install-extra/tests/test.yml
new file mode 100644
index 0000000..f978fae
--- /dev/null
+++ b/install_k8s/roles/install-extra/tests/test.yml
@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - install-extra
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/vars/main.yml b/install_k8s/roles/install-extra/vars/main.yml
new file mode 100644
index 0000000..d71758e
--- /dev/null
+++ b/install_k8s/roles/install-extra/vars/main.yml
@@ -0,0 +1,3 @@
+---
+# vars file for install-extra
+default_kubernetes_cni_weavenet_manifestUrl: "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"
diff --git a/install_k8s/roles/install-k8s/tasks/install.yml b/install_k8s/roles/install-k8s/tasks/install.yml
index 9df4e24..bc516cb 100644
--- a/install_k8s/roles/install-k8s/tasks/install.yml
+++ b/install_k8s/roles/install-k8s/tasks/install.yml
@@ -1,5 +1,12 @@
-- name: Instalando o Docker
-  shell: curl -fsSL https://get.docker.com | bash -
+- name: Adicionando as chaves do repo do Docker
+  apt_key:
+    url: https://download.docker.com/linux/ubuntu/gpg
+    state: present
+
+- name: Adicionando repositório do Docker
+  apt_repository:
+    repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs) stable"
+    state: present
 
 - name: Adicionando as chaves repo k8s no apt
   apt_key:
@@ -11,11 +18,42 @@
     repo: deb http://apt.kubernetes.io/ kubernetes-xenial main
     state: present
 
-- name: Install k8s packages
+- name: Install docker and k8s packages
   apt:
     name: "{{ packages }}"
   vars:
     packages:
+    - docker-ce
+    - docker-ce-cli
+    - containerd.io
     - kubelet
     - kubeadm
     - kubectl
+
+- name: Copy docker.json
+  copy:
+    src: /data/DevOps/ansible/install_k8s/roles/create-cluster/files/docker.json
+    dest: /etc/docker/daemon.json
+    owner: root
+    group: root
+    backup: yes    
+  register: kubeadmin_init
+
+- name: restart systemd
+  systemd:
+    daemon_reload: yes 
+
+- name: reload service docker, in all cases
+  systemd:
+    name: docker
+    state: reloaded
+
+- name: Disable SWAP since kubernetes can't work with swap enabled (1/2)
+  shell: |
+    swapoff -a
+
+- name: Disable SWAP in fstab since kubernetes can't work with swap enabled (2/2)
+  replace:
+    path: /etc/fstab
+    regexp: '^([^#].*?\sswap\s+sw\s+.*)$'
+    replace: '# \1' 
diff --git a/install_k8s/roles/install-vms/.travis.yml b/install_k8s/roles/install-vms/.travis.yml
new file mode 100644
index 0000000..36bbf62
--- /dev/null
+++ b/install_k8s/roles/install-vms/.travis.yml
@@ -0,0 +1,29 @@
+---
+language: python
+python: "2.7"
+
+# Use the new container infrastructure
+sudo: false
+
+# Install ansible
+addons:
+  apt:
+    packages:
+    - python-pip
+
+install:
+  # Install ansible
+  - pip install ansible
+
+  # Check ansible version
+  - ansible --version
+
+  # Create ansible.cfg with correct roles_path
+  - printf '[defaults]\nroles_path=../' >ansible.cfg
+
+script:
+  # Basic role syntax check
+  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
+
+notifications:
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/
\ No newline at end of file
diff --git a/install_k8s/roles/install-vms/README.md b/install_k8s/roles/install-vms/README.md
new file mode 100644
index 0000000..225dd44
--- /dev/null
+++ b/install_k8s/roles/install-vms/README.md
@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).
diff --git a/install_k8s/roles/install-vms/defaults/main.yml b/install_k8s/roles/install-vms/defaults/main.yml
new file mode 100644
index 0000000..2a65dcc
--- /dev/null
+++ b/install_k8s/roles/install-vms/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+# defaults file for install-vms
\ No newline at end of file
diff --git a/install_k8s/roles/install-vms/handlers/main.yml b/install_k8s/roles/install-vms/handlers/main.yml
new file mode 100644
index 0000000..308c1db
--- /dev/null
+++ b/install_k8s/roles/install-vms/handlers/main.yml
@@ -0,0 +1,2 @@
+---
+# handlers file for install-vms
\ No newline at end of file
diff --git a/install_k8s/roles/install-vms/meta/main.yml b/install_k8s/roles/install-vms/meta/main.yml
new file mode 100644
index 0000000..227ad9c
--- /dev/null
+++ b/install_k8s/roles/install-vms/meta/main.yml
@@ -0,0 +1,53 @@
+galaxy_info:
+  author: your name
+  description: your role description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.9
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.
+  
\ No newline at end of file
diff --git a/install_k8s/roles/install-vms/tasks/main.yml b/install_k8s/roles/install-vms/tasks/main.yml
new file mode 100644
index 0000000..9fdeca0
--- /dev/null
+++ b/install_k8s/roles/install-vms/tasks/main.yml
@@ -0,0 +1,34 @@
+---
+# tasks file for install-vm
+- name: Clone VMs
+  proxmox_kvm:
+    api_user    : "{{ pve_user }}"
+    api_password: "{{ pve_pass }}"
+    api_host    : "{{ pve_host }}"
+    clone       : ubnttpl   # The VM source
+    name        : k8s0{{ item }}  # The target VM name
+    newid       : 100{{ item }}
+    node        : pve01
+    storage     : Local-01
+    format      : qcow2
+    timeout     : 500
+#    state       : started
+  loop: "{{ vm_id }}"
+
+- name: Set VM IP
+  command:
+    qm set 100{{ item }} --ipconfig0 ip=172.32.0.22{{ item }}/24,gw=172.32.0.254
+  loop: "{{ vm_id }}"
+  ignore_errors: False
+  
+- name: start VMs
+  proxmox_kvm:
+    api_user    : "{{ pve_user }}"
+    api_password: "{{ pve_pass }}"
+    api_host    : "{{ pve_host }}"
+    vmid        : 100{{ item }}
+    state       : started
+  loop: "{{ vm_id }}"
+
+- name: Wait 600 seconds for target connection to become reachable/usable
+  wait_for_connection:
\ No newline at end of file
diff --git a/install_k8s/roles/install-vms/tests/inventory b/install_k8s/roles/install-vms/tests/inventory
new file mode 100644
index 0000000..878877b
--- /dev/null
+++ b/install_k8s/roles/install-vms/tests/inventory
@@ -0,0 +1,2 @@
+localhost
+
diff --git a/install_k8s/roles/install-vms/tests/test.yml b/install_k8s/roles/install-vms/tests/test.yml
new file mode 100644
index 0000000..33cb5d6
--- /dev/null
+++ b/install_k8s/roles/install-vms/tests/test.yml
@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - install-vms
\ No newline at end of file
diff --git a/install_k8s/roles/install-vms/vars/main.yml b/install_k8s/roles/install-vms/vars/main.yml
new file mode 100644
index 0000000..184cc2f
--- /dev/null
+++ b/install_k8s/roles/install-vms/vars/main.yml
@@ -0,0 +1,12 @@
+---
+# vars file for install-vms
+pve_host: pve01
+pve_user: root@pam
+pve_pass: "{{ lookup('env','PVE_PASS') }}"
+vm_id:
+- 1
+- 2
+- 3
+- 4
+vm_name: "k8s0"
+
diff --git a/install_k8s/roles/join-workers/tasks/join-cluster.yml b/install_k8s/roles/join-workers/tasks/join-cluster.yml
index 8fc172a..e9fddb4 100644
--- a/install_k8s/roles/join-workers/tasks/join-cluster.yml
+++ b/install_k8s/roles/join-workers/tasks/join-cluster.yml
@@ -9,7 +9,6 @@
 - name: "Kubeadm reset node cluster config"
   command: 
     kubeadm reset --force
-  register: kubeadm_reset_node
 
 - name: "Kubeadm join"
   shell: 
diff --git a/install_k8s/valt.yml b/install_k8s/valt.yml
new file mode 100644
index 0000000..48d2c21
--- /dev/null
+++ b/install_k8s/valt.yml
@@ -0,0 +1,6 @@
+$ANSIBLE_VAULT;1.2;AES256;api_password
+33343366393464373066303033613732306631393861333865353430343139363735366363396138
+6466373538633537386164633464346536363733636234340a656162326537303433363632623737
+66313434653962386462653862613433633461316437306562356635363862666239306438393539
+6133636438643661660a306233343166616439313361643663316166306431396563663333373835
+3235

From 3826725ead908d83aac4d42e65347be1897fe1e2 Mon Sep 17 00:00:00 2001
From: Marcos Magalhaes <mmagalha@gmail.com>
Date: Tue, 24 Dec 2019 09:56:11 -0300
Subject: [PATCH 5/7] docker JSON Path

---
 install_k8s/roles/install-k8s/tasks/install.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/install_k8s/roles/install-k8s/tasks/install.yml b/install_k8s/roles/install-k8s/tasks/install.yml
index bc516cb..6efafa1 100644
--- a/install_k8s/roles/install-k8s/tasks/install.yml
+++ b/install_k8s/roles/install-k8s/tasks/install.yml
@@ -5,7 +5,7 @@
 
 - name: Adicionando repositório do Docker
   apt_repository:
-    repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs) stable"
+    repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu  bionic stable"
     state: present
 
 - name: Adicionando as chaves repo k8s no apt
@@ -32,7 +32,7 @@
 
 - name: Copy docker.json
   copy:
-    src: /data/DevOps/ansible/install_k8s/roles/create-cluster/files/docker.json
+    src: install_k8s/roles/create-cluster/files/docker.json
     dest: /etc/docker/daemon.json
     owner: root
     group: root

From 56966218eed167faeedd06c99a810a781cf313d0 Mon Sep 17 00:00:00 2001
From: Marcos Magalhaes <mmagalha@gmail.com>
Date: Tue, 24 Dec 2019 10:05:14 -0300
Subject: [PATCH 6/7] Docker Json

---
 install_k8s/roles/install-k8s/tasks/install.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/install_k8s/roles/install-k8s/tasks/install.yml b/install_k8s/roles/install-k8s/tasks/install.yml
index 6efafa1..b7ffa6f 100644
--- a/install_k8s/roles/install-k8s/tasks/install.yml
+++ b/install_k8s/roles/install-k8s/tasks/install.yml
@@ -32,7 +32,7 @@
 
 - name: Copy docker.json
   copy:
-    src: install_k8s/roles/create-cluster/files/docker.json
+    src: /home/mmagalha/DevOps/ansible/install_k8s/roles/create-cluster/files/docker.json
     dest: /etc/docker/daemon.json
     owner: root
     group: root

From cd69218ba082243dfce34f1a0bd48c3302038868 Mon Sep 17 00:00:00 2001
From: Marcos Magalhaes <mmagalha@gmail.com>
Date: Mon, 30 Dec 2019 18:45:47 -0300
Subject: [PATCH 7/7] Metal

---
 README.md                                         | 6 +++++-
 install_k8s/roles/install-extra/tasks/metallb.yml | 8 ++++----
 install_k8s/roles/install-vms/tasks/main.yml      | 3 ++-
 3 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 66a4149..48b67fd 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,10 @@
-# IaaSWeek - Ansible
+# IaaSWeek - Ansible - K8S on Proxmox
 
 Ansible is a radically simple IT automation platform that makes your applications and systems easier to deploy. Avoid writing scripts or custom code to deploy and update your applications — automate in a language that approaches plain English, using SSH, with no agents to install on remote systems. ([Fonte](https://www.ansible.com/))
 
 - [Documentação](https://docs.ansible.com/ansible/latest/index.html)
 - [Github](https://github.com/ansible/ansible)
+
+- Clona as VMs no Proxmox
+- Configura o CloudInit
+- 
\ No newline at end of file
diff --git a/install_k8s/roles/install-extra/tasks/metallb.yml b/install_k8s/roles/install-extra/tasks/metallb.yml
index d47e484..17c111b 100644
--- a/install_k8s/roles/install-extra/tasks/metallb.yml
+++ b/install_k8s/roles/install-extra/tasks/metallb.yml
@@ -7,11 +7,11 @@
     backup: yes    
   register: metallb_install
 
-- name: Install MetalLB config.
-  shell: kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml
-  register: metallb_install
-
 - name: Install MetalLB.
   shell: kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml
   register: metallb_install
 
+- name: Install MetalLB config.
+  shell: kubectl apply -n metallb-system-f https://raw.githubusercontent.com/google/metallb/v0.8.3/manifests/metallb.yaml
+  register: metallb_install  
+
diff --git a/install_k8s/roles/install-vms/tasks/main.yml b/install_k8s/roles/install-vms/tasks/main.yml
index 9fdeca0..a5ea161 100644
--- a/install_k8s/roles/install-vms/tasks/main.yml
+++ b/install_k8s/roles/install-vms/tasks/main.yml
@@ -31,4 +31,5 @@
   loop: "{{ vm_id }}"
 
 - name: Wait 600 seconds for target connection to become reachable/usable
-  wait_for_connection:
\ No newline at end of file
+  wait_for_connection:
+        delay: 600
\ No newline at end of file