From 6b87932f236c05db844fb3ebb82bdfab2502e9fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sat, 18 Aug 2018 00:03:25 +0200 Subject: [PATCH 01/51] initial commit for project transition to ansible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- README.md | 31 +++++++++++-- pxe-server.yml | 92 +++++++++++++++++++++++++++++++++++++ roles/apt/tasks/main.yml | 37 +++++++++++++++ roles/docker/tasks/main.yml | 53 +++++++++++++++++++++ 4 files changed, 208 insertions(+), 5 deletions(-) create mode 100644 pxe-server.yml create mode 100644 roles/apt/tasks/main.yml create mode 100644 roles/docker/tasks/main.yml diff --git a/README.md b/README.md index d75735c..02f286c 100755 --- a/README.md +++ b/README.md @@ -1,21 +1,42 @@ pxe-server ========== -This repository contains PXE server that should help in installing, testing and -developing operating systems and firmware for PXE-capable platforms. +This repository contains PXE server (TFTP+NFS) that should help in installing, +testing and developing operating systems and firmware for PXE-capable +platforms. It was inspired by effort required to test PC Engines apu2 platform. +We use PXE server without DHCP, what may cause problems to BSD systems and is +subject of our further work on this project. Usage ----- +# pxe-server deployment + +## Ansible setup + +``` +virtualenv ansible-venv +source ansible-venv/bin/activate +pip install ansible +ansible-galaxy install angstwad.docker_ubuntu +ansible-galaxy install debops.apt_preferences +ssh-keygen -f ~/.ssh/ansible +ssh-add ~/.ssh/ansible +ssh-copy-id -i ~/.ssh/ansible @ +``` + +## Initial deployment + +Following procedure assume deployment on clean Debian system: + ``` -git clone https://github.com/3mdeb/pxe-server.git -cd pxe-server -NFS_SRV_IP= ./init.sh +ansible-playbook -i "," -b --ask-become-pass pxe-server.yml ``` + `init.sh` downloads all necessary files, OS images, PXE and extracts them in proper directories. diff --git a/pxe-server.yml b/pxe-server.yml new file mode 100644 index 0000000..87aff48 --- /dev/null +++ b/pxe-server.yml @@ -0,0 +1,92 @@ +--- +- hosts: all + user: debian + become: yes + become_user: root + become_method: su + roles: + - debops.apt_preferences + - apt + - docker + - netboot + + tasks: + + - name: Get kernels + get_url: + url: https://cloud.3mdeb.com/index.php/s/UQQVYrNIhg7ddwj/download + dest: /tmp/kernels.tar.gz + checksum: sha256:59f6355e210452b4ab4c7a68ee2730860dfd9f15bb5db2b3666f157d26ae919c + + - name: Get Debian rootfs + get_url: + url: https://cloud.3mdeb.com/index.php/s/9b8h6WmJcNsuB57/download + dest: /tmp/debian-stable.tar.gz + checksum: sha256:bb7fb5d9e23d6759458cf9e415fab7a5650112b7ede069633d9372e4e0443dcc + + - name: Get Voyage + get_url: + url: https://cloud.3mdeb.com/index.php/s/rUZPwRHOjxpSxN4/download + dest: /tmp/voyage.tar.gz + checksum: sha256:86934186fde2cbc749b2e33d027977f1b3a0cf02f69c2ffc9446e620b3d6e5c6 + + - name: Get Core-6.4 + get_url: + url: https://cloud.3mdeb.com/index.php/s/AQuUdsYkBzO9UJz/download + dest: /tmp/core.tar.gz + checksum: sha256:7fb624fe34b02b4df54c1e8c3b823b794441ef2b152480de7bb2c2b39f381be9 + + - name: Create /var/voyage + file: + path: /var/voyage + state: directory + + - name: Unarchive kernels + unarchive: + src: /tmp/kernels.tar.gz + dest: /var/netboot + remote_src: yes + keep_newer: yes + group: debian + owner: debian + + - name: Unarchive Debian rootfs + unarchive: + src: /tmp/debian-stable.tar.gz + dest: /var + remote_src: yes + keep_newer: yes + group: debian + owner: debian + + - name: Unarchive Voyage + unarchive: + src: /tmp/voyage.tar.gz + dest: /var/voyage + remote_src: yes + keep_newer: yes + group: debian + owner: debian + + - name: Unarchive Core + unarchive: + src: /tmp/core.tar.gz + dest: /var/netboot/ + remote_src: yes + keep_newer: yes + group: debian + owner: debian + + - name: Mount nfsd + mount: + path: /proc/fs/nfsd + src: nfsd + fstype: nfsd + state: present + + - name: Restart server + command: /sbin/shutdown -r +1 + async: 0 + poll: 0 + ignore_errors: true + diff --git a/roles/apt/tasks/main.yml b/roles/apt/tasks/main.yml new file mode 100644 index 0000000..7d94bdd --- /dev/null +++ b/roles/apt/tasks/main.yml @@ -0,0 +1,37 @@ +- name: Remove cdrom repo + apt_repository: + repo: deb cdrom:[Debian GNU/Linux 9.4.0 _Stretch_ - Official amd64 xfce-CD Binary-1 20180310-11:21]/ stretch main + state: absent + +- name: Add trffic manager stable deb repo + apt_repository: + repo: deb http://debian-archive.trafficmanager.net/debian/ stable main contrib non-free + state: present + +- name: Add trffic manager stable deb-src repo + apt_repository: + repo: deb-src http://debian-archive.trafficmanager.net/debian/ stable main contrib non-free + state: present + +- name: Install apt-transport-https + apt: + name: apt-transport-https + state: present +- name: Add Docker CE key to apt + apt_key: + url: https://download.docker.com/linux/debian/gpg + state: present + +- name: Add Docker repo + apt_repository: + repo: deb [arch=amd64] https://download.docker.com/linux/debian stretch stable + state: present + +- name: Install essential packages + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - docker-ce + - python-pip diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml new file mode 100644 index 0000000..9ebfa0c --- /dev/null +++ b/roles/docker/tasks/main.yml @@ -0,0 +1,53 @@ + +- name: Install docker-py + pip: + name: docker-py + state: absent + +- name: Install docker + pip: + name: docker + +- name: add default user to docker group + user: + name: debian + groups: docker + append: yes + +- name: Start 3mdeb/pxe-server Docker container + docker_container: + name: pxe-server + state: started + image: 3mdeb/pxe-server:latest + pull: yes + command: bash /usr/local/bin/run.sh + restart: yes + restart_policy: always + privileged: yes + published_ports: + - "111:111/tcp" + - "2049:2049/tcp" + - "8000:8000/tcp" + - "627:627/tcp" + - "627:627/udp" + - "875:875/tcp" + - "875:875/udp" + - "892:892/tcp" + - "892:892/udp" + - "111:111/udp" + - "2049:2049/udp" + - "10053:10053/udp" + - "10053:10053/tcp" + - "32769:32769/tcp" + - "32769:32769/udp" + - "32765:32765/tcp" + - "32765:32765/udp" + - "32766:32766/tcp" + - "32766:32766/udp" + - "32767:32767/tcp" + - "32767:32767/udp" + volumes: + - /var/netboot:/srv/http + - /var/debian-stable:/srv/nfs/debian + - /var/xen:/srv/nfs/xen + - /var/voyage:/srv/nfs/voyage From b3e261e01e51ef75f17ae7e9940f80cb5d15becc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sat, 18 Aug 2018 00:07:54 +0200 Subject: [PATCH 02/51] roles/netboot: replace 3mdeb/netboot repository with files and template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/netboot/files/preseed.cfg | 48 ++++++++++++++ roles/netboot/files/preseed_ubuntu.cfg | 54 ++++++++++++++++ roles/netboot/tasks/main.yml | 15 +++++ roles/netboot/templates/menu.ipxe.j2 | 90 ++++++++++++++++++++++++++ 4 files changed, 207 insertions(+) create mode 100644 roles/netboot/files/preseed.cfg create mode 100644 roles/netboot/files/preseed_ubuntu.cfg create mode 100644 roles/netboot/tasks/main.yml create mode 100644 roles/netboot/templates/menu.ipxe.j2 diff --git a/roles/netboot/files/preseed.cfg b/roles/netboot/files/preseed.cfg new file mode 100644 index 0000000..9f771c6 --- /dev/null +++ b/roles/netboot/files/preseed.cfg @@ -0,0 +1,48 @@ +d-i debian-installer/locale string en_US +d-i debian-installer/language string en +d-i debian-installer/country string US + +# Skip creation of a normal user account. +d-i passwd/make-user boolean false +d-i passwd/root-login boolean true +# printf "debian" | mkpasswd -s -m sha-512 +d-i passwd/root-password-crypted password $6$H/WJeEJc$0HnpUXUtjPR/RMpD3qxvb.OGJgTY425jnZn6a9X0YrhGXyEifkR5kTJ20zpv9etzI0k.a9j2G4jMUZjx1XCIH0 +d-i user-setup/allow-password-weak boolean true +d-i netcfg/choose_interface select auto +d-i netcfg/get_hostname string unassigned-hostname +d-i netcfg/get_domain string unassigned-domain +d-i mirror/country string manual +d-i mirror/http/hostname string ftp.pl.debian.org +d-i mirror/http/directory string /debian +d-i mirror/http/proxy string +d-i clock-setup/utc boolean true +d-i clock-setup/ntp boolean true +d-i time/zone string Europe/Warsaw +d-i partman/mount_style select uuid +d-i partman/confirm boolean true +d-i partman/choose_partition select finish +d-i partman/confirm_nooverwrite boolean true +d-i partman-auto/disk string /dev/disk/by-path/pci-0000:00:10.0-usb-0:1:1.0-scsi-0:0:0:0 +d-i partman-auto/method string regular +d-i partman-auto/choose_recipe select atomic +d-i partman-lvm/device_remove_lvm boolean true +d-i partman-lvm/confirm boolean true +d-i partman-lvm/confirm_nooverwrite boolean true +d-i partman-auto-lvm/guided_size string max +d-i partman-partitioning/confirm_write_new_label boolean true + +d-i grub-installer/grub2_instead_of_grub_legacy boolean true +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=666974 +d-i grub-installer/only_debian boolean false +d-i grub-installer/bootdev string /dev/disk/by-path/pci-0000:00:10.0-usb-0:1:1.0-scsi-0:0:0:0 +d-i pkgsel/update-policy select none +d-i pkgsel/include string openssh-server +d-i pkgsel/install-language-support boolean false +d-i finish-install/reboot_in_progress note + +d-i base-installer/install-recommends boolean false +popularity-contest popularity-contest/participate boolean false +tasksel tasksel/first multiselect minimal + +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=852323 + SSH +d-i preseed/late_command string update-dev; in-target update-grub; in-target sh -c 'sed -i "s/^#PermitRootLogin.*\$/PermitRootLogin yes/g" /etc/ssh/sshd_config'; diff --git a/roles/netboot/files/preseed_ubuntu.cfg b/roles/netboot/files/preseed_ubuntu.cfg new file mode 100644 index 0000000..43b3b8a --- /dev/null +++ b/roles/netboot/files/preseed_ubuntu.cfg @@ -0,0 +1,54 @@ +d-i debian-installer/locale string en_US +d-i debian-installer/language string en +d-i debian-installer/country string US + +# Skip creation of a normal user account. +d-i passwd/make-user boolean false +d-i passwd/root-login boolean true +# printf "ubuntu" | mkpasswd -s -m sha-512 +d-i passwd/root-password-crypted password $6$KC5aQVTfAEj$cfUlQi1422C09If79rNXZjavnrUWcXz5EVYBJ77sX9sb8EZoKBWpNNAerBo.Rix/4s/oryqKsws9dL3IKrwLt1 +d-i user-setup/allow-password-weak boolean true +d-i netcfg/choose_interface select auto +d-i netcfg/get_hostname string unassigned-hostname +d-i netcfg/get_domain string unassigned-domain +d-i mirror/country string manual +d-i mirror/http/hostname string http://pl.archive.ubuntu.com +d-i mirror/http/directory string /ubuntu +d-i mirror/http/proxy string +d-i clock-setup/utc boolean true +d-i clock-setup/ntp boolean true +d-i time/zone string Europe/Warsaw +d-i partman/mount_style select uuid +d-i partman/confirm boolean true +d-i partman/choose_partition select finish +d-i partman/confirm_nooverwrite boolean true +d-i partman-auto/disk string /dev/disk/by-path/pci-0000:00:10.0-usb-0:1:1.0-scsi-0:0:0:0 +d-i partman-auto/method string regular +d-i partman-auto/choose_recipe select atomic +d-i partman-lvm/device_remove_lvm boolean true +d-i partman-lvm/confirm boolean true +d-i partman-lvm/confirm_nooverwrite boolean true +d-i partman-auto-lvm/guided_size string max +d-i partman-partitioning/confirm_write_new_label boolean true + +d-i grub-installer/grub2_instead_of_grub_legacy boolean true +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=666974 +d-i grub-installer/only_debian boolean false +d-i grub-installer/bootdev string /dev/disk/by-path/pci-0000:00:10.0-usb-0:1:1.0-scsi-0:0:0:0 +d-i pkgsel/update-policy select none +d-i pkgsel/include string openssh-server +d-i pkgsel/install-language-support boolean false +d-i finish-install/reboot_in_progress note + +# disable /home encryption +d-i user-setup/encrypt-home boolean false + +# https://ubuntuforums.org/showthread.php?t=2215103 +d-i preseed/early_command string umount /media || true + +d-i base-installer/install-recommends boolean false +popularity-contest popularity-contest/participate boolean false +tasksel tasksel/first multiselect minimal + +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=852323 + SSH +d-i preseed/late_command string update-dev; in-target update-grub; in-target sh -c 'sed -i "s/^#*PermitRootLogin.*\$/PermitRootLogin yes/g" /etc/ssh/sshd_config'; diff --git a/roles/netboot/tasks/main.yml b/roles/netboot/tasks/main.yml new file mode 100644 index 0000000..7b6eb87 --- /dev/null +++ b/roles/netboot/tasks/main.yml @@ -0,0 +1,15 @@ +- name: copy preseed.cfg + copy: + src: files/preseed.cfg + dest: /var/netboot/preseed.cfg + +- name: copy preseed_ubuntu.cfg + copy: + src: files/preseed_ubuntu.cfg + dest: /var/netboot/preseed_ubuntu.cfg + +- name: deploy menu.ipxe + template: + src: templates/menu.ipxe.j2 + dest: /var/netboot/menu.ipxe + diff --git a/roles/netboot/templates/menu.ipxe.j2 b/roles/netboot/templates/menu.ipxe.j2 new file mode 100644 index 0000000..f0cf915 --- /dev/null +++ b/roles/netboot/templates/menu.ipxe.j2 @@ -0,0 +1,90 @@ +#!ipxe +# +:MENU +menu +item --gap -- ---------------- iPXE boot menu ---------------- +item shell ipxe shell +item xen Xen +item deb-netboot-4.14.y Debian stable netboot 4.14.y +item deb-netboot-4.15.y Debian stable netboot 4.15.y +item deb-netboot-4.16.y Debian stable netboot 4.16.y +item deb-stable-netinst Debian stable netinst +item deb-i386-stable-netinst Debian i386 stable netinst +item deb-testing-netinst TODO:Debian testing netinst +item deb-testing-netinst-uefi TODO:Debian testing netinst (UEFI-aware) +item voyage-netinst Voyage netinst 0.11.0 +item ubuntu-lts-netinst Ubuntu LTS netinst +item coreos-netinst Core OS netinst +item core-6.4 Core 6.4 +item --gap -- ------------ iPXE boot menu end ---------------- +choose --default boot --timeout 3000 target && goto ${target} + +:xen +kernel kernels/xen-4.8-amd64 dom0_mem=512M loglvl=all guest_loglvl=all com1=115200,8n1 console=com1 +module kernels/vmlinuz-4.14.y console=hvc0 earlyprintk=xen nomodeset root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug +boot +goto MENU + +:deb-netboot-4.14.y +kernel kernels/vmlinuz-4.14.y bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 +boot +goto MENU + +:deb-netboot-4.15.y +kernel kernels/vmlinuz-4.15.y bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 +boot +goto MENU + +:deb-netboot-4.16.y +kernel kernels/vmlinuz-4.16.y bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 +boot +goto MENU + +:deb-stable-netinst +kernel http://ftp.nl.debian.org/debian/dists/stable/main/installer-amd64/current/images/netboot/debian-installer/amd64/linux bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 initrd=http://ftp.nl.debian.org/debian/dists/stable/main/installer-amd64/current/images/netboot/debian-installer/amd64/initrd.gz +initrd http://ftp.nl.debian.org/debian/dists/stable/main/installer-amd64/current/images/netboot/debian-installer/amd64/initrd.gz +boot +goto MENU + +:deb-i386-stable-netinst +kernel http://ftp.nl.debian.org/debian/dists/stable/main/installer-i386/current/images/netboot/debian-installer/i386/linux bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 initrd=http://ftp.nl.debian.org/debian/dists/stable/main/installer-i386/current/images/netboot/debian-installer/i386/initrd.gz +initrd http://ftp.nl.debian.org/debian/dists/stable/main/installer-i386/current/images/netboot/debian-installer/i386/initrd.gz +boot +goto MENU + +:ubuntu-lts-netinst +kernel http://archive.ubuntu.com/ubuntu/dists/xenial-updates/main/installer-amd64/current/images/hwe-netboot/ubuntu-installer/amd64/linux bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 initrd=initrd.gz +initrd http://archive.ubuntu.com/ubuntu/dists/xenial-updates/main/installer-amd64/current/images/hwe-netboot/ubuntu-installer/amd64/initrd.gz +boot +goto MENU + +:coreos-netinst +set base-url http://stable.release.core-os.net/amd64-usr/current +kernel ${base-url}/coreos_production_pxe.vmlinuz initrd=coreos_production_pxe_image.cpio.gz coreos.first_boot=1 --- console=ttyS0,115200 coreos.autologin=ttyS0 earlyprint=serial,ttyS0,115200 +initrd ${base-url}/coreos_production_pxe_image.cpio.gz +boot +goto MENU + +:core-6.4 +kernel core-6.4/vmlinuz --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 +initrd core-6.4/core.gz +boot +goto MENU + +deb-testing-netinst +boot +goto MENU + +:deb-testing-netinst-uefi +boot +goto MENU + +:voyage-netinst +kernel kernels/voyage/vmlinuz bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe boot=live netboot=nfs root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/voyage --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 +initrd kernels/voyage/initrd.img +boot +goto MENU + +:shell +shell || +goto MENU From 24df55386ffcf14ae2d2f329981319bf18554ded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sat, 18 Aug 2018 00:08:36 +0200 Subject: [PATCH 03/51] gitignore: add ansible related files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- .gitignore | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index e540866..73f7127 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -debian -voyage -netboot +ansible-venv +pxe-server.retry From 92ea8bbeb886e668dd4a3ee8a5e9c6cdc37898e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Tue, 21 Aug 2018 17:50:31 +0200 Subject: [PATCH 04/51] README.md: add performance notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 02f286c..b7284f1 100755 --- a/README.md +++ b/README.md @@ -30,12 +30,57 @@ ssh-copy-id -i ~/.ssh/ansible @ ## Initial deployment -Following procedure assume deployment on clean Debian system: +Following procedure assume deployment on clean Debian as target system: ``` ansible-playbook -i "," -b --ask-become-pass pxe-server.yml ``` +### Performance + +``` +Tuesday 21 August 2018 17:47:35 +0200 (0:00:00.820) 0:05:09.644 ******** +=============================================================================== +apt ------------------------------------------------------------------- 136.75s +copy ------------------------------------------------------------------- 63.61s +docker ----------------------------------------------------------------- 51.06s +unarchive -------------------------------------------------------------- 36.18s +get_url ---------------------------------------------------------------- 10.50s +netboot ----------------------------------------------------------------- 4.56s +setup ------------------------------------------------------------------- 2.49s +file -------------------------------------------------------------------- 2.33s +mount ------------------------------------------------------------------- 0.91s +command ----------------------------------------------------------------- 0.82s +debops.apt_preferences -------------------------------------------------- 0.25s +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +total ----------------------------------------------------------------- 309.46s +Tuesday 21 August 2018 17:47:35 +0200 (0:00:00.820) 0:05:09.633 ******** +=============================================================================== +apt : Install essential packages --------------------------------------- 99.51s +Copy Debian rootfs ----------------------------------------------------- 59.49s +docker : Start 3mdeb/pxe-server Docker container ----------------------- 35.68s +Unarchive Debian rootfs ------------------------------------------------ 31.32s +apt : Remove cdrom repo ------------------------------------------------ 12.19s +docker : Install docker ------------------------------------------------ 10.67s +Get Voyage ------------------------------------------------------------- 10.50s +apt : Add trffic manager stable deb repo -------------------------------- 8.19s +apt : Add trffic manager stable deb-src repo ---------------------------- 6.65s +Unarchive Voyage -------------------------------------------------------- 4.86s +apt : Add Docker repo --------------------------------------------------- 4.47s +apt : Add Docker CE key to apt ------------------------------------------ 4.02s +docker : Install docker-py ---------------------------------------------- 3.88s +Gathering Facts --------------------------------------------------------- 2.49s +Copy Linux 4.14.y ------------------------------------------------------- 2.24s +Copy Linux 4.9.y -------------------------------------------------------- 1.88s +apt : Install apt-transport-https --------------------------------------- 1.73s +netboot : deploy menu.ipxe ---------------------------------------------- 1.43s +netboot : copy preseed.cfg ---------------------------------------------- 1.04s +Create /var/voyage ------------------------------------------------------ 1.01s +Playbook run took 0 days, 0 hours, 5 minutes, 9 seconds +``` + +==== + `init.sh` downloads all necessary files, OS images, PXE and extracts them in proper directories. @@ -126,4 +171,3 @@ Requesting configuration that many times makes a little mess, so as a temporary workaround add a static IP for the `net0/eth0` interface on Your DHCP server. The IP address requested will remain the same and so the problems will be gone too. - From 9bd92a4c4eeb1ddb825bbd451a0fc6529fb0d5f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Tue, 21 Aug 2018 17:50:47 +0200 Subject: [PATCH 05/51] pxe-server: use common variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- pxe-server.yml | 1 + roles/common/vars/main.yml | 1 + 2 files changed, 2 insertions(+) create mode 100644 roles/common/vars/main.yml diff --git a/pxe-server.yml b/pxe-server.yml index 87aff48..f60ad31 100644 --- a/pxe-server.yml +++ b/pxe-server.yml @@ -5,6 +5,7 @@ become_user: root become_method: su roles: + - common - debops.apt_preferences - apt - docker diff --git a/roles/common/vars/main.yml b/roles/common/vars/main.yml new file mode 100644 index 0000000..7b97e7a --- /dev/null +++ b/roles/common/vars/main.yml @@ -0,0 +1 @@ +release_version: "v1.0.0" From 5a1af4d4d64ef1b193b214507d44ef163c0aae28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Tue, 21 Aug 2018 17:51:17 +0200 Subject: [PATCH 06/51] pxe-server: switch to new rootfs deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- pxe-server.yml | 65 +++++++++++++++++-------------------- roles/docker/tasks/main.yml | 3 +- 2 files changed, 31 insertions(+), 37 deletions(-) diff --git a/pxe-server.yml b/pxe-server.yml index f60ad31..9c12bb6 100644 --- a/pxe-server.yml +++ b/pxe-server.yml @@ -13,47 +13,51 @@ tasks: - - name: Get kernels - get_url: - url: https://cloud.3mdeb.com/index.php/s/UQQVYrNIhg7ddwj/download - dest: /tmp/kernels.tar.gz - checksum: sha256:59f6355e210452b4ab4c7a68ee2730860dfd9f15bb5db2b3666f157d26ae919c + #TODO: we should rely on stable release available on cloud + - name: Copy Debian rootfs + copy: + src: /home/pietrushnic/storage/projects/2017/3mdeb/tmp/release-v1.0.0/rootfs-v1.0.0.tar.gz + dest: /tmp/rootfs-{{ release_version }}.tar.gz - - name: Get Debian rootfs - get_url: - url: https://cloud.3mdeb.com/index.php/s/9b8h6WmJcNsuB57/download - dest: /tmp/debian-stable.tar.gz - checksum: sha256:bb7fb5d9e23d6759458cf9e415fab7a5650112b7ede069633d9372e4e0443dcc + - name: Copy Linux 4.9.y + copy: + src: /home/pietrushnic/storage/projects/2017/3mdeb/tmp/release-v1.0.0/vmlinuz-4.9.122 + dest: /var/netboot/kernels/vmlinuz-4.9.122 + + - name: Copy Linux 4.14.y + copy: + src: /home/pietrushnic/storage/projects/2017/3mdeb/tmp/release-v1.0.0/vmlinuz-4.14.65 + dest: /var/netboot/kernels/vmlinuz-4.14.65 + - name: create Linux 4.14.y + file: + src: /var/netboot/kernels/vmlinuz-4.14.65 + dest: /var/netboot/kernels/vmlinuz-4.14.y + state: link + + - name: create Linux 4.9.y + file: + src: /var/netboot/kernels/vmlinuz-4.9.122 + dest: /var/netboot/kernels/vmlinuz-4.9.y + state: link + + # TODO: because of iso_extract lack of support for directories, + # implementation of ISO extraction for Voyage would be to convolutes, we + # leaving it as it is - name: Get Voyage get_url: url: https://cloud.3mdeb.com/index.php/s/rUZPwRHOjxpSxN4/download dest: /tmp/voyage.tar.gz checksum: sha256:86934186fde2cbc749b2e33d027977f1b3a0cf02f69c2ffc9446e620b3d6e5c6 - - name: Get Core-6.4 - get_url: - url: https://cloud.3mdeb.com/index.php/s/AQuUdsYkBzO9UJz/download - dest: /tmp/core.tar.gz - checksum: sha256:7fb624fe34b02b4df54c1e8c3b823b794441ef2b152480de7bb2c2b39f381be9 - - name: Create /var/voyage file: path: /var/voyage state: directory - - name: Unarchive kernels - unarchive: - src: /tmp/kernels.tar.gz - dest: /var/netboot - remote_src: yes - keep_newer: yes - group: debian - owner: debian - - name: Unarchive Debian rootfs unarchive: - src: /tmp/debian-stable.tar.gz + src: /tmp/rootfs-{{ release_version }}.tar.gz dest: /var remote_src: yes keep_newer: yes @@ -69,15 +73,6 @@ group: debian owner: debian - - name: Unarchive Core - unarchive: - src: /tmp/core.tar.gz - dest: /var/netboot/ - remote_src: yes - keep_newer: yes - group: debian - owner: debian - - name: Mount nfsd mount: path: /proc/fs/nfsd diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index 9ebfa0c..b862bad 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -48,6 +48,5 @@ - "32767:32767/udp" volumes: - /var/netboot:/srv/http - - /var/debian-stable:/srv/nfs/debian - - /var/xen:/srv/nfs/xen + - /var/rootfs:/srv/nfs/debian - /var/voyage:/srv/nfs/voyage From 17130b452bfe7728c797e26650381124f561824e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Tue, 21 Aug 2018 17:51:45 +0200 Subject: [PATCH 07/51] netboot: create symlinks for required kernels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/netboot/tasks/main.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/roles/netboot/tasks/main.yml b/roles/netboot/tasks/main.yml index 7b6eb87..2b2128d 100644 --- a/roles/netboot/tasks/main.yml +++ b/roles/netboot/tasks/main.yml @@ -13,3 +13,32 @@ src: templates/menu.ipxe.j2 dest: /var/netboot/menu.ipxe +- name: create kernels directory + file: + path: /var/netboot/kernels + state: directory + +- name: create Xen dev symlink + file: + src: /var/netboot/kernels/xen-4.8-amd64 + dest: /var/netboot/kernels/xen-dev + state: link + #TODO: remove force after adding xen kernel + force: yes + +- name: create Linux dev symlink + file: + src: /var/netboot/kernels/vmlinuz-4.14.y + dest: /var/netboot/kernels/vmlinuz-dev + state: link + #TODO: remove force after adding xen kernel + force: yes + +#Following kernels should be deployed to /var/netboot/kernels + +#TODO: obtain Xen kernel from debian - artifact of rootfs building +#TODO: obtain Debian with 4.9.y - artifact of rootfs building +#TODO: obtain Debian with 4.14.y - artifact of rootfs building + +#TODO: obtain Voyage Linux - have to be obtained from our cloud, since we do +#not support extraction from ISO From 5a5b9742789906fff4f74109d803672e16864c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Tue, 21 Aug 2018 17:52:28 +0200 Subject: [PATCH 08/51] netboot: modify menu.ipxe template with kernels requred for v1.0.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/netboot/templates/menu.ipxe.j2 | 50 +++++++++++++++------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/roles/netboot/templates/menu.ipxe.j2 b/roles/netboot/templates/menu.ipxe.j2 index f0cf915..24edbbb 100644 --- a/roles/netboot/templates/menu.ipxe.j2 +++ b/roles/netboot/templates/menu.ipxe.j2 @@ -3,19 +3,19 @@ :MENU menu item --gap -- ---------------- iPXE boot menu ---------------- -item shell ipxe shell -item xen Xen +item shell ipxe shell +item xen Xen +item xen-dev Xen dev +item xen-linux-dev Xen Linux dev +item deb-netboot-dev Debian stable netboot dev +item deb-netboot-4.9.y Debian stable netboot 4.9.y item deb-netboot-4.14.y Debian stable netboot 4.14.y -item deb-netboot-4.15.y Debian stable netboot 4.15.y -item deb-netboot-4.16.y Debian stable netboot 4.16.y item deb-stable-netinst Debian stable netinst item deb-i386-stable-netinst Debian i386 stable netinst -item deb-testing-netinst TODO:Debian testing netinst -item deb-testing-netinst-uefi TODO:Debian testing netinst (UEFI-aware) item voyage-netinst Voyage netinst 0.11.0 item ubuntu-lts-netinst Ubuntu LTS netinst item coreos-netinst Core OS netinst -item core-6.4 Core 6.4 +item core-6.4 Core 6.4 item --gap -- ------------ iPXE boot menu end ---------------- choose --default boot --timeout 3000 target && goto ${target} @@ -25,18 +25,30 @@ module kernels/vmlinuz-4.14.y console=hvc0 earlyprintk=xen nomodeset root=/dev/n boot goto MENU -:deb-netboot-4.14.y -kernel kernels/vmlinuz-4.14.y bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 +:xen-dev +kernel kernels/xen-dev dom0_mem=512M loglvl=all guest_loglvl=all com1=115200,8n1 console=com1 +module kernels/vmlinuz-4.14.y console=hvc0 earlyprintk=xen nomodeset root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug +boot +goto MENU + +:xen-linux-dev +kernel kernels/xen-4.8-amd64 dom0_mem=512M loglvl=all guest_loglvl=all com1=115200,8n1 console=com1 +module kernels/vmlinuz-dev console=hvc0 earlyprintk=xen nomodeset root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug +boot +goto MENU + +:deb-netboot-dev +kernel kernels/vmlinuz-dev bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 boot goto MENU -:deb-netboot-4.15.y -kernel kernels/vmlinuz-4.15.y bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 +:deb-netboot-4.9.y +kernel kernels/vmlinuz-4.9.y bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 boot goto MENU -:deb-netboot-4.16.y -kernel kernels/vmlinuz-4.16.y bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 +:deb-netboot-4.14.y +kernel kernels/vmlinuz-4.14.y bootfile=http://{{ ansible_default_ipv4.address }}:8000/menu.ipxe root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 boot goto MENU @@ -66,16 +78,8 @@ boot goto MENU :core-6.4 -kernel core-6.4/vmlinuz --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 -initrd core-6.4/core.gz -boot -goto MENU - -deb-testing-netinst -boot -goto MENU - -:deb-testing-netinst-uefi +kernel http://www.tinycorelinux.net/6.x/x86/archive/6.4/distribution_files/vmlinuz --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 +initrd http://www.tinycorelinux.net/6.x/x86/archive/6.4/distribution_files/core.gz boot goto MENU From 8281c4ed8dea7bab9ce0706c16ea032b199434a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Tue, 21 Aug 2018 17:52:42 +0200 Subject: [PATCH 09/51] ansible.cfg: initial commit with performance counters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- ansible.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 ansible.cfg diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..e209fcb --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,2 @@ +[defaults] +callback_whitelist = profile_tasks, profile_roles, timer From 20c8d5b87e551871988b5818703fd8d1be5de8ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Tue, 18 Sep 2018 00:15:31 +0200 Subject: [PATCH 10/51] README.md: add pre-release test results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index b7284f1..13103be 100755 --- a/README.md +++ b/README.md @@ -36,6 +36,24 @@ Following procedure assume deployment on clean Debian as target system: ansible-playbook -i "," -b --ask-become-pass pxe-server.yml ``` +### Tests + +`v1.0.0` tests results: + +| Description | Result | +| --- | --- | +| Boot Xen 4.8 and Verify if IOMMU is enabled | FAIL | +| Boot Xen 4.8 and Verify if IOMMU is enabled on Linux development kernel | FAIL | +| Boot Xen development kernel and Linux 4.14.y | PASS | +| Boot to Core 6.4 booted over iPXE | FAIL | +| Voyage installation | FAIL | +| Ubuntu installation | PASS | +| Debian i386 installation | FAIL | +| Debian installation | PASS | +| pfSense 2.4.x installation | FAIL | + +Test duration: ~2h15min + ### Performance ``` From 74409d662d9fe0f27a82e71e8677e1369ec9dcc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Tue, 18 Sep 2018 00:16:04 +0200 Subject: [PATCH 11/51] add temporary paths for kernels and rootfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- pxe-server.yml | 15 ++++++++++----- roles/netboot/tasks/main.yml | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pxe-server.yml b/pxe-server.yml index 9c12bb6..84ace62 100644 --- a/pxe-server.yml +++ b/pxe-server.yml @@ -16,28 +16,33 @@ #TODO: we should rely on stable release available on cloud - name: Copy Debian rootfs copy: - src: /home/pietrushnic/storage/projects/2017/3mdeb/tmp/release-v1.0.0/rootfs-v1.0.0.tar.gz + src: /home/pietrushnic/storage/projects/2017/3mdeb/debian-rootfs-builder/rootfs-v1.0.0.tar.gz dest: /tmp/rootfs-{{ release_version }}.tar.gz - name: Copy Linux 4.9.y copy: - src: /home/pietrushnic/storage/projects/2017/3mdeb/tmp/release-v1.0.0/vmlinuz-4.9.122 + src: /home/pietrushnic/storage/projects/2017/3mdeb/debian-rootfs-builder/vmlinuz-4.9.122 dest: /var/netboot/kernels/vmlinuz-4.9.122 - name: Copy Linux 4.14.y copy: - src: /home/pietrushnic/storage/projects/2017/3mdeb/tmp/release-v1.0.0/vmlinuz-4.14.65 + src: /home/pietrushnic/storage/projects/2017/3mdeb/debian-rootfs-builder/vmlinuz-4.14.65 dest: /var/netboot/kernels/vmlinuz-4.14.65 + - name: Copy Xen 4.8 + copy: + src: /home/pietrushnic/storage/projects/2017/3mdeb/debian-rootfs-builder/xen-4.8-amd64 + dest: /var/netboot/kernels/xen-4.8-amd64 + - name: create Linux 4.14.y file: - src: /var/netboot/kernels/vmlinuz-4.14.65 + src: vmlinuz-4.14.65 dest: /var/netboot/kernels/vmlinuz-4.14.y state: link - name: create Linux 4.9.y file: - src: /var/netboot/kernels/vmlinuz-4.9.122 + src: vmlinuz-4.9.122 dest: /var/netboot/kernels/vmlinuz-4.9.y state: link diff --git a/roles/netboot/tasks/main.yml b/roles/netboot/tasks/main.yml index 2b2128d..c7c81e4 100644 --- a/roles/netboot/tasks/main.yml +++ b/roles/netboot/tasks/main.yml @@ -20,7 +20,7 @@ - name: create Xen dev symlink file: - src: /var/netboot/kernels/xen-4.8-amd64 + src: xen-4.8-amd64 dest: /var/netboot/kernels/xen-dev state: link #TODO: remove force after adding xen kernel @@ -28,7 +28,7 @@ - name: create Linux dev symlink file: - src: /var/netboot/kernels/vmlinuz-4.14.y + src: vmlinuz-4.14.y dest: /var/netboot/kernels/vmlinuz-dev state: link #TODO: remove force after adding xen kernel From 7292fd544e5115b87bc023e449839f9b40acefed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Tue, 18 Sep 2018 00:58:06 +0200 Subject: [PATCH 12/51] roles: add tinycoreos role MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/tinycoreos/files/inittab | 20 ++++++++++++++++++ roles/tinycoreos/files/securetty | 16 +++++++++++++++ roles/tinycoreos/tasks/main.yml | 35 ++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 roles/tinycoreos/files/inittab create mode 100644 roles/tinycoreos/files/securetty create mode 100644 roles/tinycoreos/tasks/main.yml diff --git a/roles/tinycoreos/files/inittab b/roles/tinycoreos/files/inittab new file mode 100644 index 0000000..aecc022 --- /dev/null +++ b/roles/tinycoreos/files/inittab @@ -0,0 +1,20 @@ +# /etc/inittab: init configuration for busybox init. +# Boot-time system configuration/initialization script. +# +::sysinit:/etc/init.d/rcS + +# /sbin/getty respawn shell invocations for selected ttys. +/dev/ttyS0::respawn:/sbin/getty -nl /sbin/autologin 115200 ttyS0 +#tty2::respawn:/sbin/getty 38400 tty2 +#tty3::respawn:/sbin/getty 38400 tty3 +#tty4::askfirst:/sbin/getty 38400 tty4 +#tty5::askfirst:/sbin/getty 38400 tty5 +#tty6::askfirst:/sbin/getty 38400 tty6 + +# Stuff to do when restarting the init +# process, or before rebooting. +::restart:/etc/init.d/rc.shutdown +::restart:/sbin/init +::ctrlaltdel:/sbin/reboot +::shutdown:/etc/init.d/rc.shutdown + diff --git a/roles/tinycoreos/files/securetty b/roles/tinycoreos/files/securetty new file mode 100644 index 0000000..688fdfd --- /dev/null +++ b/roles/tinycoreos/files/securetty @@ -0,0 +1,16 @@ +# /etc/securetty: List of terminals on which root is allowed to login. +# +console + +# For people with serial port consoles +ttyS0 + +# Standard consoles +tty1 +tty2 +tty3 +tty4 +tty5 +tty6 +tty7 + diff --git a/roles/tinycoreos/tasks/main.yml b/roles/tinycoreos/tasks/main.yml new file mode 100644 index 0000000..7ca8bb2 --- /dev/null +++ b/roles/tinycoreos/tasks/main.yml @@ -0,0 +1,35 @@ +- name: get Tiny Core Linux 6.4 initrd + get_url: + url: http://www.tinycorelinux.net/6.x/x86/archive/6.4/distribution_files/core.gz + dest: /tmp/core.gz + checksum: sha256:a0824dc1a65d0b5f1969fe72e03c682a1716df8eb5bb179d9baf6b8f28dc8e74 + +- name: unarchive core.gz + command: gunzip /tmp/core.gz + args: + chdir: /tmp + +- name: Create /tmp/cpio + file: + path: /tmp/cpio + state: directory + +- name: unarchive core cpio + command: cpio -i --file /tmp/core + args: + chdir: /tmp/cpio + +- name: copy etc/securetty + copy: + src: files/securetty + dest: /tmp/cpio/etc/securetty + +- name: copy etc/inittab + copy: + src: files/inittab + dest: /tmp/cpio/etc/inittab + +- name: archive core cpio + shell: find | cpio -o -H newc --file /var/netboot/core.gz + args: + chdir: /tmp/cpio From 9f1ea5832f2f3ea7937c7c1660bff904b36c2c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Tue, 18 Sep 2018 00:58:20 +0200 Subject: [PATCH 13/51] pxe-server: add tags for each role MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- pxe-server.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pxe-server.yml b/pxe-server.yml index 84ace62..a165d4b 100644 --- a/pxe-server.yml +++ b/pxe-server.yml @@ -5,11 +5,11 @@ become_user: root become_method: su roles: - - common - - debops.apt_preferences - - apt - - docker - - netboot + - { role: 'debops.apt_preferences', tags: 'debops.apt_preferences' } + - { role: 'apt', tags: 'apt' } + - { role: 'docker', tags: 'docker' } + - { role: 'netboot', tags: 'netboot' } + - { role: 'tinycoreos', tags: 'tinycoreos' } tasks: From bc34ca1aca99eec4b0c3ab0bafd7ce30044e3a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Tue, 18 Sep 2018 00:58:38 +0200 Subject: [PATCH 14/51] roles/apt: add gzip package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/apt/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/apt/tasks/main.yml b/roles/apt/tasks/main.yml index 7d94bdd..242da5f 100644 --- a/roles/apt/tasks/main.yml +++ b/roles/apt/tasks/main.yml @@ -35,3 +35,4 @@ with_items: - docker-ce - python-pip + - gzip From 57d68371a292358d957ae463c13f03f69b81e033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:14:31 +0200 Subject: [PATCH 15/51] gitignore: ignore results and rootfs directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 73f7127..5ecf631 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ ansible-venv pxe-server.retry +results +rootfs From 711bd30e9c0dca8a64c7e6827322f45a2b70488f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:15:07 +0200 Subject: [PATCH 16/51] README.md: split whole initial deployment into 3 steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 13103be..b59afe9 100755 --- a/README.md +++ b/README.md @@ -30,6 +30,26 @@ ssh-copy-id -i ~/.ssh/ansible @ ## Initial deployment +### Rootfs components creation + +``` +docker run --privileged --rm -v $HOME/.ansible:/root/.ansible \ +-v $HOME/.ccache:/home/debian/.ccache \ -v $PWD:/home/debian/scripts \ +-t -i 3mdeb/debian-rootfs-builder ansible-playbook -vvv \ -i hosts \ +/home/debian/scripts/create-rootfs-components.yml +``` + +### Rootfs preparation + +``` +docker run --privileged --rm -v $HOME/.ansible:/root/.ansible \ +-v $HOME/.ccache:/home/debian/.ccache \ -v $PWD:/home/debian/scripts \ +-t -i 3mdeb/debian-rootfs-builder ansible-playbook -vvv \ -i hosts \ +/home/debian/scripts/prepare_rootfs.yml +``` + +### Deploy + Following procedure assume deployment on clean Debian as target system: ``` From 1f305ea7fda3b73bcc1c2ab5531dd80b1874b60f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:15:49 +0200 Subject: [PATCH 17/51] roles/chroot_cleanup: initial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/chroot_cleanup/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 roles/chroot_cleanup/tasks/main.yml diff --git a/roles/chroot_cleanup/tasks/main.yml b/roles/chroot_cleanup/tasks/main.yml new file mode 100644 index 0000000..912e15c --- /dev/null +++ b/roles/chroot_cleanup/tasks/main.yml @@ -0,0 +1,2 @@ +- name: clean apt cache + command: apt clean From 6c17bdc2a271175db1dc8ecf42cf03b199eede46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:16:01 +0200 Subject: [PATCH 18/51] roles/chroot_mount: initial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/chroot_mount/tasks/main.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 roles/chroot_mount/tasks/main.yml diff --git a/roles/chroot_mount/tasks/main.yml b/roles/chroot_mount/tasks/main.yml new file mode 100644 index 0000000..421da59 --- /dev/null +++ b/roles/chroot_mount/tasks/main.yml @@ -0,0 +1,21 @@ +- name: mount /proc + mount: + path: "{{ rootfs_dir }}/proc" + src: /proc + opts: bind + fstype: proc + state: present +- name: mount /dev + mount: + path: "{{ rootfs_dir }}/dev" + src: /dev + opts: bind + fstype: devtmpfs + state: present +- name: mount /dev/pts + mount: + path: "{{ rootfs_dir }}/dev/pts" + src: /dev/pts + opts: bind + fstype: devpts + state: present From a119ee1a2d9bb5754dbea067fb6f6bef5ac12bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:16:11 +0200 Subject: [PATCH 19/51] roles/chroot_umount: initial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/chroot_umount/tasks/main.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 roles/chroot_umount/tasks/main.yml diff --git a/roles/chroot_umount/tasks/main.yml b/roles/chroot_umount/tasks/main.yml new file mode 100644 index 0000000..d82c31f --- /dev/null +++ b/roles/chroot_umount/tasks/main.yml @@ -0,0 +1,18 @@ +- name: unmount /dev/pts + mount: + path: "{{ rootfs_dir }}/dev/pts" + src: /dev/pts + opts: bind + state: absent +- name: unmount /dev + mount: + path: "{{ rootfs_dir }}/dev/pts" + src: /dev/pts + opts: bind + state: absent +- name: unmount /proc + mount: + path: "{{ rootfs_dir }}/dev/pts" + src: /dev/pts + opts: bind + state: absent From b0c8a50b3f64e264516be2b7ec9283c4ab99c522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:16:27 +0200 Subject: [PATCH 20/51] roles/config: initial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/config/files/fstab | 2 ++ roles/config/files/issue | 2 ++ roles/config/tasks/main.yml | 16 ++++++++++++++++ 3 files changed, 20 insertions(+) create mode 100644 roles/config/files/fstab create mode 100644 roles/config/files/issue create mode 100644 roles/config/tasks/main.yml diff --git a/roles/config/files/fstab b/roles/config/files/fstab new file mode 100644 index 0000000..43061d2 --- /dev/null +++ b/roles/config/files/fstab @@ -0,0 +1,2 @@ +/proc /proc proc defaults 0 0 +/sys /sys sysfs defaults 0 0 diff --git a/roles/config/files/issue b/roles/config/files/issue new file mode 100644 index 0000000..319563e --- /dev/null +++ b/roles/config/files/issue @@ -0,0 +1,2 @@ +Debian GNU/Linux 9 \n \l [root:debian] + diff --git a/roles/config/tasks/main.yml b/roles/config/tasks/main.yml new file mode 100644 index 0000000..308e88c --- /dev/null +++ b/roles/config/tasks/main.yml @@ -0,0 +1,16 @@ +- name: replace fstab + copy: + src: files/fstab + dest: /etc/fstab + +- name: replace issue + copy: + src: files/issue + dest: /etc/issue + +- name: change root password + shell: echo root:debian|chpasswd + +- name: configure hostname + hostname: + name: rootfs-{{ release_version }} From b279de237dc139d02b7767ab60ed80a587902c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:16:38 +0200 Subject: [PATCH 21/51] roles/debootstrap: initial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/debootstrap/tasks/main.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 roles/debootstrap/tasks/main.yml diff --git a/roles/debootstrap/tasks/main.yml b/roles/debootstrap/tasks/main.yml new file mode 100644 index 0000000..7db58bf --- /dev/null +++ b/roles/debootstrap/tasks/main.yml @@ -0,0 +1,24 @@ +- name: check if "{{ rootfs_tar_gz }}" exist + stat: + path: "{{ rootfs_tar_gz }}" + register: rootfs_file + +- name: cleanup rootfs dir + file: + state: absent + path: "{{ rootfs_dir }}" + when: not rootfs_file.stat.exists + +- name: cleanup rootfs dir + file: + state: directory + path: "{{ rootfs_dir }}" + when: not rootfs_file.stat.exists + +- name: debootstrap first stage + command: debootstrap --foreign --include=python --arch amd64 stable {{ rootfs_dir }} http://deb.debian.org/debian + when: not rootfs_file.stat.exists + +- name: debootstrap second stage + command: chroot {{ rootfs_dir }} /debootstrap/debootstrap --second-stage + when: not rootfs_file.stat.exists From 5052596fddeab532ca6b89a6eeb86662bf1822bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:16:50 +0200 Subject: [PATCH 22/51] roles/deploy_artifacts: initial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/deploy_artifacts/tasks/main.yml | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 roles/deploy_artifacts/tasks/main.yml diff --git a/roles/deploy_artifacts/tasks/main.yml b/roles/deploy_artifacts/tasks/main.yml new file mode 100644 index 0000000..9399c23 --- /dev/null +++ b/roles/deploy_artifacts/tasks/main.yml @@ -0,0 +1,36 @@ +- name: Copy Debian rootfs + copy: + src: "results/rootfs-{{ release_version }}.tar.gz" + dest: /tmp/rootfs-{{ release_version }}.tar.gz +- name: Copy Linux 4.9.y + copy: + src: "results/vmlinuz-{{ linux_4_9 }}" + dest: /var/netboot/kernels/vmlinuz-{{ linux_4_9 }} +- name: Copy Linux 4.14.y + copy: + src: "results/vmlinuz-{{ linux_4_9 }}" + dest: /var/netboot/kernels/vmlinuz-{{ linux_4_9 }} +- name: Copy Xen 4.8 + copy: + src: "results/xen-4.8-amd64" + dest: /var/netboot/kernels/xen-4.8-amd64 +- name: create Linux 4.14.y symlink + file: + src: vmlinuz-{{ linux_4_14 }} + dest: /var/netboot/kernels/vmlinuz-{{ linux_4_14 }} + state: link + force: yes +- name: create Linux 4.9.y symlink + file: + src: vmlinuz-{{ linux_4_9 }} + dest: /var/netboot/kernels/vmlinuz-{{ linux_4_9 }} + state: link + force: yes +- name: Unarchive Debian rootfs + unarchive: + src: /tmp/rootfs-{{ release_version }}.tar.gz + dest: /var + remote_src: yes + keep_newer: yes + group: debian + owner: debian From 727bdbdd3378b9e74679d5a58903c5761a378dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:18:03 +0200 Subject: [PATCH 23/51] roles/linux-install: intial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/linux-install/tasks/main.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 roles/linux-install/tasks/main.yml diff --git a/roles/linux-install/tasks/main.yml b/roles/linux-install/tasks/main.yml new file mode 100644 index 0000000..7874cdc --- /dev/null +++ b/roles/linux-install/tasks/main.yml @@ -0,0 +1,13 @@ +- name: check if /lib/modules/{{ version }} exist + stat: + path: "/lib/modules/{{ version }}" + register: mod_dir +- name: install Linux "{{ version }}" + apt: + deb: "{{ item }}" + with_items: + - linux-headers-{{ version }}_{{ version }}-1_amd64.deb + - linux-image-{{ version }}_{{ version }}-1_amd64.deb + - linux-image-{{ version }}-dbg_{{ version }}-1_amd64.deb + - linux-libc-dev_{{ version }}-1_amd64.deb + when: not mod_dir.stat.exists From 4576977cc353a052f5513373c946b6e14c9b47f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:18:18 +0200 Subject: [PATCH 24/51] roles/linux-kernel: initial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/linux-kernel/tasks/main.yml | 154 ++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 roles/linux-kernel/tasks/main.yml diff --git a/roles/linux-kernel/tasks/main.yml b/roles/linux-kernel/tasks/main.yml new file mode 100644 index 0000000..d86d211 --- /dev/null +++ b/roles/linux-kernel/tasks/main.yml @@ -0,0 +1,154 @@ +- name: check linux_headers + stat: + path: "{{ rootfs_dir }}/linux-headers-{{ version }}_{{ version }}-1_amd64.deb" + register: linux_headers +- name: check linux_image_dbg + stat: + path: "{{ rootfs_dir }}/linux-image-{{ version }}-dbg_{{ version }}-1_amd64.deb" + register: linux_image_dbg +- name: check linux_image + stat: + path: "{{ rootfs_dir }}/linux-image-{{ version }}_{{ version }}-1_amd64.deb" + register: linux_image +- name: check linux_libc + stat: + path: "{{ rootfs_dir }}/linux-libc-dev_{{ version }}-1_amd64.deb" + register: linux_libc +- name: check vmlinuz + stat: + path: "{{ results_dir }}/vmlinuz-{{ version }}" + register: vmlinuz +- name: check config + stat: + path: "{{ results_dir }}/config-{{ version }}" + register: config_ver + +- name: get Linux "{{ version }}" + get_url: + url: https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-{{ version }}.tar.gz + # TODO: nice would be remote checksum that can verify if package is fine + dest: "{{ rootfs_dir }}/linux-{{ version }}.tar.gz" + register: result + until: result is succeeded + retries: 3 + delay: 3 + when: + - not config_ver.stat.exists + - not vmlinuz.stat.exists + - not linux_libc.stat.exists + - not linux_image.stat.exists + - not linux_image_dbg.stat.exists + - not linux_headers.stat.exists + +- name: decompress Linux "{{ version }}" + unarchive: + src: "{{ rootfs_dir }}/linux-{{ version }}.tar.gz" + dest: "{{ rootfs_dir }}" + creates: "{{ rootfs_dir }}/linux-{{ version }}" + when: + - not config_ver.stat.exists + - not vmlinuz.stat.exists + - not linux_libc.stat.exists + - not linux_image.stat.exists + - not linux_image_dbg.stat.exists + - not linux_headers.stat.exists + +- name: make mrproper + command: make mrproper + args: + chdir: "{{ rootfs_dir }}/linux-{{ version }}" + when: + - not config_ver.stat.exists + - not vmlinuz.stat.exists + - not linux_libc.stat.exists + - not linux_image.stat.exists + - not linux_image_dbg.stat.exists + - not linux_headers.stat.exists + +- name: get apu_config + get_url: + url: "{{ config }}" + dest: "{{ rootfs_dir }}/linux-{{ version }}/.config" + when: + - not config_ver.stat.exists + - not vmlinuz.stat.exists + - not linux_libc.stat.exists + - not linux_image.stat.exists + - not linux_image_dbg.stat.exists + - not linux_headers.stat.exists + +- name: make olddefconfig + command: make olddefconfig + args: + chdir: "{{ rootfs_dir }}/linux-{{ version }}" + when: + - not config_ver.stat.exists + - not vmlinuz.stat.exists + - not linux_libc.stat.exists + - not linux_image.stat.exists + - not linux_image_dbg.stat.exists + - not linux_headers.stat.exists + +- name: make deb-pkg + command: make -j{{ ansible_processor_vcpus }} deb-pkg bzImage + #command: make deb-pkg bzImage + args: + chdir: "{{ rootfs_dir }}/linux-{{ version }}" + creates: + - "{{ rootfs_dir }}/linux-headers-{{ version }}_{{ version }}-1_amd64.deb" + - "{{ rootfs_dir }}/linux-image-{{ version }}-dbg_{{ version }}-1_amd64.deb" + - "{{ rootfs_dir }}/linux-image-{{ version }}_{{ version }}-1_amd64.deb" + - "{{ rootfs_dir }}/linux-libc-dev_{{ version }}-1_amd64.deb" + ignore_errors: yes + when: + - not config_ver.stat.exists + - not vmlinuz.stat.exists + - not linux_libc.stat.exists + - not linux_image.stat.exists + - not linux_image_dbg.stat.exists + - not linux_headers.stat.exists + +#TODO: this is for netboot/kernels directory +- name: copy bzImage to known location + copy: + src: "{{ rootfs_dir }}/linux-{{ version }}/arch/x86/boot/bzImage" + dest: "{{ results_dir }}/vmlinuz-{{ version }}" + when: + - not config_ver.stat.exists + - not vmlinuz.stat.exists + - not linux_libc.stat.exists + - not linux_image.stat.exists + - not linux_image_dbg.stat.exists + - not linux_headers.stat.exists + +# TODO: save config for further commit to apu2-documentation +- name: copy .config to known location + copy: + src: "{{ rootfs_dir }}/linux-{{ version }}/.config" + dest: "{{ results_dir }}/config-{{ version }}" + when: + - not config_ver.stat.exists + - not vmlinuz.stat.exists + - not linux_libc.stat.exists + - not linux_image.stat.exists + - not linux_image_dbg.stat.exists + - not linux_headers.stat.exists + +- name: remove everything except artifacts + file: + path: "{{ rootfs_dir }}/{{ item }}" + state: absent + with_items: + - linux-{{ version }} + - linux-{{ version }}.tar.gz + - linux-{{ version }}_{{ version }}-1_amd64.changes + - linux-{{ version }}_{{ version }}-1.debian.tar.gz + - linux-{{ version }}_{{ version }}-1.dsc + - linux-{{ version }}_{{ version }}.orig.tar.gz + when: + - not config_ver.stat.exists + - not vmlinuz.stat.exists + - not linux_libc.stat.exists + - not linux_image.stat.exists + - not linux_image_dbg.stat.exists + - not linux_headers.stat.exists From 55eb1fa157f976022c48386cd073be707dc1c4b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:18:27 +0200 Subject: [PATCH 25/51] roles/packages: initial commmit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/packages/tasks/main.yml | 45 +++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 roles/packages/tasks/main.yml diff --git a/roles/packages/tasks/main.yml b/roles/packages/tasks/main.yml new file mode 100644 index 0000000..736c384 --- /dev/null +++ b/roles/packages/tasks/main.yml @@ -0,0 +1,45 @@ +- name: install packages + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - apt-utils + - autoconf + - bc + - binutils + - bison + - build-essential + - ca-certificates + - cmake + - dialog + - dmidecode + - doxygen + - flex + - g++ + - gcc-multilib + - gdb + - gettext + - git + - iasl + - liblzma-dev + - locales + - m4 + - make + - makedev + - ncurses-dev + - nfs-common + - ntpdate + - python + - python-dev + - ssh + - sudo + - tmux + - vim + - wget + - wpasupplicant + - zlib1g-dev + - xen-system-amd64 + - xen-tools + - xen-linux-system-amd64 + From 6c7a7c8369ede3a635c88d7e73c7e1b52875236b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:18:38 +0200 Subject: [PATCH 26/51] roles/prepare_artifacts: initial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/prepare_artifacts/tasks/main.yml | 33 ++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 roles/prepare_artifacts/tasks/main.yml diff --git a/roles/prepare_artifacts/tasks/main.yml b/roles/prepare_artifacts/tasks/main.yml new file mode 100644 index 0000000..df7ac1f --- /dev/null +++ b/roles/prepare_artifacts/tasks/main.yml @@ -0,0 +1,33 @@ +- name: check if "{{ rootfs_tar_gz }}" exist + stat: + path: "{{ rootfs_tar_gz }}" + register: rootfs_file +- name: check if "{{ results_dir }}/xen-4.8-amd64" exist + stat: + path: "{{ results_dir }}/xen-4.8-amd64" + register: xen_file +# for some reason archive return error, so we use tar directly An exception +# occurred during task execution. To see the full traceback, use -vvv. The +# error was: OSError: [Errno 2] No such file or directory: +# 'rootfs/sbin/runlevel' +# +# fatal: [localhost]: FAILED! => {"changed": false, "msg": "Error when +# writing tar.gz archive at rootfs-v1.0.0.tar.gz: [Errno 2] No such file or +# directory: 'rootfs/sbin/runlevel'"} +- name: compress rootfs + command: tar czvf "{{ rootfs_tar_gz }}" rootfs + args: + chdir: "{{ rootfs_dir }}/.." + when: not rootfs_file.stat.exists + +- name: preserver Xen kernel + copy: + src: "{{ rootfs_dir }}/boot/xen-4.8-amd64.gz" + dest: "{{ results_dir }}/xen-4.8-amd64.gz" + when: not xen_file.stat.exists + +- name: unarchive Xen kernel + command: gunzip -f "{{ results_dir }}/xen-4.8-amd64.gz" + args: + chdir: "{{ results_dir }}" + when: not xen_file.stat.exists From 57a06ed80bf7dae9f64fe805dd831ea8086a7a63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:18:48 +0200 Subject: [PATCH 27/51] roles/voyage: initial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/voyage/tasks/main.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 roles/voyage/tasks/main.yml diff --git a/roles/voyage/tasks/main.yml b/roles/voyage/tasks/main.yml new file mode 100644 index 0000000..91684a8 --- /dev/null +++ b/roles/voyage/tasks/main.yml @@ -0,0 +1,22 @@ +# TODO: because of iso_extract lack of support for directories, +# implementation of ISO extraction for Voyage would be to convolutes, we +# leaving it as it is +- name: Get Voyage + get_url: + url: https://cloud.3mdeb.com/index.php/s/rUZPwRHOjxpSxN4/download + dest: /tmp/voyage.tar.gz + checksum: sha256:86934186fde2cbc749b2e33d027977f1b3a0cf02f69c2ffc9446e620b3d6e5c6 + +- name: Create /var/voyage + file: + path: /var/voyage + state: directory + +- name: Unarchive Voyage + unarchive: + src: /tmp/voyage.tar.gz + dest: /var/voyage + remote_src: yes + keep_newer: yes + group: debian + owner: debian From 962e458781617ac1fc97279cbb1c0b5ea0d4a339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:19:08 +0200 Subject: [PATCH 28/51] pxe-server.yml: use predefined roles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- pxe-server.yml | 80 +++++--------------------------------------------- 1 file changed, 8 insertions(+), 72 deletions(-) diff --git a/pxe-server.yml b/pxe-server.yml index a165d4b..f5ed8d5 100644 --- a/pxe-server.yml +++ b/pxe-server.yml @@ -5,86 +5,22 @@ become_user: root become_method: su roles: - - { role: 'debops.apt_preferences', tags: 'debops.apt_preferences' } - - { role: 'apt', tags: 'apt' } - - { role: 'docker', tags: 'docker' } - - { role: 'netboot', tags: 'netboot' } - - { role: 'tinycoreos', tags: 'tinycoreos' } + - { role: 'common' } + - { role: 'debops.apt_preferences' } + - { role: 'apt' } + - { role: 'docker' } + - { role: 'netboot' } + - { role: 'tinycoreos' } + - { role: 'deploy_artifacts' } + - { role: 'voyage' } tasks: - - #TODO: we should rely on stable release available on cloud - - name: Copy Debian rootfs - copy: - src: /home/pietrushnic/storage/projects/2017/3mdeb/debian-rootfs-builder/rootfs-v1.0.0.tar.gz - dest: /tmp/rootfs-{{ release_version }}.tar.gz - - - name: Copy Linux 4.9.y - copy: - src: /home/pietrushnic/storage/projects/2017/3mdeb/debian-rootfs-builder/vmlinuz-4.9.122 - dest: /var/netboot/kernels/vmlinuz-4.9.122 - - - name: Copy Linux 4.14.y - copy: - src: /home/pietrushnic/storage/projects/2017/3mdeb/debian-rootfs-builder/vmlinuz-4.14.65 - dest: /var/netboot/kernels/vmlinuz-4.14.65 - - - name: Copy Xen 4.8 - copy: - src: /home/pietrushnic/storage/projects/2017/3mdeb/debian-rootfs-builder/xen-4.8-amd64 - dest: /var/netboot/kernels/xen-4.8-amd64 - - - name: create Linux 4.14.y - file: - src: vmlinuz-4.14.65 - dest: /var/netboot/kernels/vmlinuz-4.14.y - state: link - - - name: create Linux 4.9.y - file: - src: vmlinuz-4.9.122 - dest: /var/netboot/kernels/vmlinuz-4.9.y - state: link - - # TODO: because of iso_extract lack of support for directories, - # implementation of ISO extraction for Voyage would be to convolutes, we - # leaving it as it is - - name: Get Voyage - get_url: - url: https://cloud.3mdeb.com/index.php/s/rUZPwRHOjxpSxN4/download - dest: /tmp/voyage.tar.gz - checksum: sha256:86934186fde2cbc749b2e33d027977f1b3a0cf02f69c2ffc9446e620b3d6e5c6 - - - name: Create /var/voyage - file: - path: /var/voyage - state: directory - - - name: Unarchive Debian rootfs - unarchive: - src: /tmp/rootfs-{{ release_version }}.tar.gz - dest: /var - remote_src: yes - keep_newer: yes - group: debian - owner: debian - - - name: Unarchive Voyage - unarchive: - src: /tmp/voyage.tar.gz - dest: /var/voyage - remote_src: yes - keep_newer: yes - group: debian - owner: debian - - name: Mount nfsd mount: path: /proc/fs/nfsd src: nfsd fstype: nfsd state: present - - name: Restart server command: /sbin/shutdown -r +1 async: 0 From 5a86e39b1f4bea7c73b3f0e85222244a073bebdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:19:35 +0200 Subject: [PATCH 29/51] roles/common: add required config variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/common/vars/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/common/vars/main.yml b/roles/common/vars/main.yml index 7b97e7a..9c80d43 100644 --- a/roles/common/vars/main.yml +++ b/roles/common/vars/main.yml @@ -1 +1,7 @@ release_version: "v1.0.0" +linux_4_9: "4.9.128" +linux_4_14: "4.14.71" +apu_config: "https://raw.githubusercontent.com/pcengines/apu2-documentation/master/configs/config-4.14.59" +rootfs_tar_gz: "{{ results_dir }}/rootfs-{{ release_version }}.tar.gz" +rootfs_dir: "/home/debian/scripts/rootfs" +results_dir: "/home/debian/scripts/results" From 335e2523c5b4414e60306e47d65c82d5133efa95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:19:55 +0200 Subject: [PATCH 30/51] roles/docker: install setuptools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/docker/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index b862bad..c53e0fd 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -4,6 +4,10 @@ name: docker-py state: absent +- name: Install stuptools + pip: + name: setuptools + - name: Install docker pip: name: docker From b93267376cacc0f4dd10f2020cd81e3fa07e4aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:20:12 +0200 Subject: [PATCH 31/51] roles/tinycoreos: force gunzip of core.gz MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/tinycoreos/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/tinycoreos/tasks/main.yml b/roles/tinycoreos/tasks/main.yml index 7ca8bb2..425f3cb 100644 --- a/roles/tinycoreos/tasks/main.yml +++ b/roles/tinycoreos/tasks/main.yml @@ -5,7 +5,7 @@ checksum: sha256:a0824dc1a65d0b5f1969fe72e03c682a1716df8eb5bb179d9baf6b8f28dc8e74 - name: unarchive core.gz - command: gunzip /tmp/core.gz + command: gunzip -f /tmp/core.gz args: chdir: /tmp From 085545cbe0687f6d9987908dd3c56c203eb9b80c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 00:20:37 +0200 Subject: [PATCH 32/51] add remaining steps for initial pxe-server deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- create-rootfs-components.yml | 9 +++++++++ hosts | 2 ++ prepare-rootfs.yml | 25 +++++++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 create-rootfs-components.yml create mode 100644 hosts create mode 100644 prepare-rootfs.yml diff --git a/create-rootfs-components.yml b/create-rootfs-components.yml new file mode 100644 index 0000000..075b086 --- /dev/null +++ b/create-rootfs-components.yml @@ -0,0 +1,9 @@ +--- +- name: build rootfs, kernels and prepare chroot + hosts: localhost + connection: local + roles: + - { role: 'common', tags: 'rootfs_prepare' } + - { role: 'debootstrap', tags: 'rootfs_prepare' } + - { role: 'linux-kernel', version: "{{ linux_4_9 }}", config: "{{ apu_config }}", tags: 'rootfs_prepare' } + - { role: 'linux-kernel', version: "{{ linux_4_14 }}", config: "{{ apu_config }}", tags: 'rootfs_prepare' } diff --git a/hosts b/hosts new file mode 100644 index 0000000..86f2ad4 --- /dev/null +++ b/hosts @@ -0,0 +1,2 @@ +[rootfs_chroot] +/home/debian/scripts/rootfs diff --git a/prepare-rootfs.yml b/prepare-rootfs.yml new file mode 100644 index 0000000..d66f979 --- /dev/null +++ b/prepare-rootfs.yml @@ -0,0 +1,25 @@ +- name: chroot mount + hosts: localhost + connection: local + roles: + - { role: 'common', tags: 'rootfs_prepare' } + - { role: 'chroot_mount', tags: 'install_in_rootfs' } + +- name: intall kernels and all remaining packages + hosts: rootfs_chroot + connection: chroot + roles: + - { role: 'common', tags: 'install_in_rootfs' } + - { role: 'config', tags: 'install_in_rootfs' } + - { role: 'packages', tags: 'install_in_rootfs' } + - { role: 'linux-install', version: "{{ linux_4_9 }}", tags: 'install_in_rootfs' } + - { role: 'linux-install', version: "{{ linux_4_14 }}", tags: 'install_in_rootfs' } + - { role: 'chroot_cleanup', tags: 'install_in_rootfs' } + +- name: umount chroot and prepare artifacts + hosts: localhost + connection: local + roles: + - { role: 'chroot_umount', tags: 'install_in_rootfs' } + - { role: 'common', tags: 'install_in_rootfs' } + - { role: 'prepare_artifacts', tags: 'install_in_rootfs' } From 9b7a7e68485898474b6ecf98c09569e3cd7fb7ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 01:12:05 +0200 Subject: [PATCH 33/51] roles/deploy_artifacts: fix kernel deployment code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/deploy_artifacts/tasks/main.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/roles/deploy_artifacts/tasks/main.yml b/roles/deploy_artifacts/tasks/main.yml index 9399c23..842cfc8 100644 --- a/roles/deploy_artifacts/tasks/main.yml +++ b/roles/deploy_artifacts/tasks/main.yml @@ -2,30 +2,36 @@ copy: src: "results/rootfs-{{ release_version }}.tar.gz" dest: /tmp/rootfs-{{ release_version }}.tar.gz + - name: Copy Linux 4.9.y copy: src: "results/vmlinuz-{{ linux_4_9 }}" dest: /var/netboot/kernels/vmlinuz-{{ linux_4_9 }} + - name: Copy Linux 4.14.y copy: - src: "results/vmlinuz-{{ linux_4_9 }}" - dest: /var/netboot/kernels/vmlinuz-{{ linux_4_9 }} + src: "results/vmlinuz-{{ linux_4_14 }}" + dest: /var/netboot/kernels/vmlinuz-{{ linux_4_14 }} + - name: Copy Xen 4.8 copy: src: "results/xen-4.8-amd64" dest: /var/netboot/kernels/xen-4.8-amd64 + - name: create Linux 4.14.y symlink file: src: vmlinuz-{{ linux_4_14 }} - dest: /var/netboot/kernels/vmlinuz-{{ linux_4_14 }} + dest: /var/netboot/kernels/vmlinuz-4.14.y state: link force: yes + - name: create Linux 4.9.y symlink file: src: vmlinuz-{{ linux_4_9 }} - dest: /var/netboot/kernels/vmlinuz-{{ linux_4_9 }} + dest: /var/netboot/kernels/vmlinuz-4.9.y state: link force: yes + - name: Unarchive Debian rootfs unarchive: src: /tmp/rootfs-{{ release_version }}.tar.gz From 5303fec2c124e5406fc5f66599fa8d82a5d1b570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 12:36:28 +0200 Subject: [PATCH 34/51] README.md: add test results for v1.0.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- README.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b59afe9..6da15b9 100755 --- a/README.md +++ b/README.md @@ -62,17 +62,15 @@ ansible-playbook -i "," -b --ask-become-pass pxe-server.yml | Description | Result | | --- | --- | -| Boot Xen 4.8 and Verify if IOMMU is enabled | FAIL | -| Boot Xen 4.8 and Verify if IOMMU is enabled on Linux development kernel | FAIL | -| Boot Xen development kernel and Linux 4.14.y | PASS | -| Boot to Core 6.4 booted over iPXE | FAIL | -| Voyage installation | FAIL | -| Ubuntu installation | PASS | -| Debian i386 installation | FAIL | -| Debian installation | PASS | -| pfSense 2.4.x installation | FAIL | - -Test duration: ~2h15min +| XEN1.2 Verify if IOMMU is enabled | PASS | +| XEN1.4 Verify if IOMMU is enabled on Xen Linux dev | PASS | +| XEN1.5 Verify if IOMMU is enabled on Xen dev | PASS | +| DEB1.1 Debian from iPXE 4.14.y | PASS | +| DEB1.5 Debian from iPXE 4.9.y | PASS | +| TCL1.1 Boot to Core 6.4 booted over iPXE | PASS | +| VOY1.1 Boot into Voyage installer | PASS | +| PFS1.1 pfSense 2.4.x install test | PASS | + ### Performance From c037091148a10500ebaff32fc9c7e3a25dd7c918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 12:36:56 +0200 Subject: [PATCH 35/51] roles/deploy_artifact: correct artificats deployment for Core 6.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/deploy_artifacts/tasks/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/deploy_artifacts/tasks/main.yml b/roles/deploy_artifacts/tasks/main.yml index 842cfc8..e2b1a79 100644 --- a/roles/deploy_artifacts/tasks/main.yml +++ b/roles/deploy_artifacts/tasks/main.yml @@ -40,3 +40,9 @@ keep_newer: yes group: debian owner: debian + +- name: copy core.gz to /var/netboot + copy: + src: /tmp/core.gz + dest: /var/netboot/core.gz + remote_src: yes From e81964e257da604e2469ed8a69955c5229363f82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 12:37:33 +0200 Subject: [PATCH 36/51] roles/netboot: correct ipxe menu entry for Core 6.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/netboot/templates/menu.ipxe.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/netboot/templates/menu.ipxe.j2 b/roles/netboot/templates/menu.ipxe.j2 index 24edbbb..700d009 100644 --- a/roles/netboot/templates/menu.ipxe.j2 +++ b/roles/netboot/templates/menu.ipxe.j2 @@ -78,8 +78,8 @@ boot goto MENU :core-6.4 -kernel http://www.tinycorelinux.net/6.x/x86/archive/6.4/distribution_files/vmlinuz --- console=ttyS0,115200 earlyprint=serial,ttyS0,115200 -initrd http://www.tinycorelinux.net/6.x/x86/archive/6.4/distribution_files/core.gz +kernel http://www.tinycorelinux.net/6.x/x86/archive/6.4/distribution_files/vmlinuz --- root=/dev/ram0 console=ttyS0,115200 earlyprint=serial,ttyS0,115200 +initrd http://{{ ansible_default_ipv4.address }}:8000/core.gz boot goto MENU From 5707aec4740c821cf705e14d5a914189e67001ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Mon, 24 Sep 2018 12:37:54 +0200 Subject: [PATCH 37/51] roles/tinycoreos: fix initird preparation for Core 6.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/tinycoreos/tasks/main.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/roles/tinycoreos/tasks/main.yml b/roles/tinycoreos/tasks/main.yml index 425f3cb..f76dc90 100644 --- a/roles/tinycoreos/tasks/main.yml +++ b/roles/tinycoreos/tasks/main.yml @@ -29,7 +29,17 @@ src: files/inittab dest: /tmp/cpio/etc/inittab -- name: archive core cpio - shell: find | cpio -o -H newc --file /var/netboot/core.gz +- name: Remove /tmp/core + file: + path: /tmp/core + state: absent + +- name: create core cpio + shell: find | cpio -o -H newc --file /tmp/core args: chdir: /tmp/cpio + +- name: archive core + command: gzip core + args: + chdir: /tmp From e71c74d067a0d1ed4e82beb80bb453dbe392c8b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sat, 6 Oct 2018 23:28:54 +0200 Subject: [PATCH 38/51] gitignore: add retry files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5ecf631..cd059d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ ansible-venv -pxe-server.retry +*.retry results rootfs From 26f002729e0912c70cc95ccd10a11f6048b9c08c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sun, 7 Oct 2018 00:15:06 +0200 Subject: [PATCH 39/51] add support for xen-dev MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- pxe-server.yml | 6 ++++++ roles/deploy_artifacts/tasks/main.yml | 15 +++++++++++++++ roles/docker/tasks/main.yml | 1 + 3 files changed, 22 insertions(+) diff --git a/pxe-server.yml b/pxe-server.yml index f5ed8d5..d93641b 100644 --- a/pxe-server.yml +++ b/pxe-server.yml @@ -21,6 +21,12 @@ src: nfsd fstype: nfsd state: present + - name: Mount xen-image-minimal-genericx86-64.ext4 + mount: + path: /var/xen-dev + src: /var/xen-image-minimal-genericx86-64.ext4 + fstype: ext4 + state: present - name: Restart server command: /sbin/shutdown -r +1 async: 0 diff --git a/roles/deploy_artifacts/tasks/main.yml b/roles/deploy_artifacts/tasks/main.yml index e2b1a79..504d456 100644 --- a/roles/deploy_artifacts/tasks/main.yml +++ b/roles/deploy_artifacts/tasks/main.yml @@ -13,11 +13,26 @@ src: "results/vmlinuz-{{ linux_4_14 }}" dest: /var/netboot/kernels/vmlinuz-{{ linux_4_14 }} +- name: Copy Linux Xen dev + copy: + src: "results/vmlinuz-xen-dev" + dest: /var/netboot/kernels/vmlinuz-xen-dev + - name: Copy Xen 4.8 copy: src: "results/xen-4.8-amd64" dest: /var/netboot/kernels/xen-4.8-amd64 +- name: Copy Xen dev + copy: + src: "results/xen-dev" + dest: /var/netboot/kernels/xen-dev + +- name: Copy xen-image-minimal-genericx86-64.ext4 + copy: + src: "results/xen-image-minimal-genericx86-64.ext4" + dest: /var/xen-image-minimal-genericx86-64.ext4 + - name: create Linux 4.14.y symlink file: src: vmlinuz-{{ linux_4_14 }} diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index c53e0fd..41c0618 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -54,3 +54,4 @@ - /var/netboot:/srv/http - /var/rootfs:/srv/nfs/debian - /var/voyage:/srv/nfs/voyage + - /var/xen-dev:/srv/nfs/xen-dev From 79d7ceec2aec2e7beab9fc660eae56488fa2ab53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sun, 7 Oct 2018 00:15:59 +0200 Subject: [PATCH 40/51] roles/netboot: use vmlinuz-xen-dev instead of 4.14.y MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/netboot/templates/menu.ipxe.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/netboot/templates/menu.ipxe.j2 b/roles/netboot/templates/menu.ipxe.j2 index 700d009..61797a5 100644 --- a/roles/netboot/templates/menu.ipxe.j2 +++ b/roles/netboot/templates/menu.ipxe.j2 @@ -27,7 +27,7 @@ goto MENU :xen-dev kernel kernels/xen-dev dom0_mem=512M loglvl=all guest_loglvl=all com1=115200,8n1 console=com1 -module kernels/vmlinuz-4.14.y console=hvc0 earlyprintk=xen nomodeset root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug +module kernels/vmlinuz-xen-dev console=hvc0 earlyprintk=xen nomodeset root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/xen-dev,vers=3,udp nfsrootdebug boot goto MENU From 422d51542e0efd4f169059da4f96fbed101d2d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Thu, 11 Oct 2018 14:51:28 +0200 Subject: [PATCH 41/51] deploy-meta-virtualization.yml: initial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- deploy-meta-virtualization.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 deploy-meta-virtualization.yml diff --git a/deploy-meta-virtualization.yml b/deploy-meta-virtualization.yml new file mode 100644 index 0000000..c39cb01 --- /dev/null +++ b/deploy-meta-virtualization.yml @@ -0,0 +1,29 @@ +--- +- hosts: all + user: debian + become: yes + become_user: root + become_method: su + roles: + - { role: 'common' } + - { role: 'docker' } + - { role: 'deploy_artifacts' } + + tasks: + - name: Mount nfsd + mount: + path: /proc/fs/nfsd + src: nfsd + fstype: nfsd + state: present + - name: Mount xen-image-minimal-genericx86-64.ext4 + mount: + path: /var/xen-dev + src: /var/xen-image-minimal-genericx86-64.ext4 + fstype: ext4 + state: present + - name: Restart server + command: /sbin/shutdown -r +1 + async: 0 + poll: 0 + ignore_errors: true From 474508f80fbbf7d714880382eaea99a4a49e2cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Thu, 11 Oct 2018 14:51:59 +0200 Subject: [PATCH 42/51] move TinyCore deployment to pxe-server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- pxe-server.yml | 6 ++++++ roles/deploy_artifacts/tasks/main.yml | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pxe-server.yml b/pxe-server.yml index d93641b..5db59ac 100644 --- a/pxe-server.yml +++ b/pxe-server.yml @@ -15,6 +15,12 @@ - { role: 'voyage' } tasks: + + - name: copy core.gz to /var/netboot + copy: + src: /tmp/core.gz + dest: /var/netboot/core.gz + remote_src: yes - name: Mount nfsd mount: path: /proc/fs/nfsd diff --git a/roles/deploy_artifacts/tasks/main.yml b/roles/deploy_artifacts/tasks/main.yml index 504d456..8670e6f 100644 --- a/roles/deploy_artifacts/tasks/main.yml +++ b/roles/deploy_artifacts/tasks/main.yml @@ -55,9 +55,3 @@ keep_newer: yes group: debian owner: debian - -- name: copy core.gz to /var/netboot - copy: - src: /tmp/core.gz - dest: /var/netboot/core.gz - remote_src: yes From ccc06f6b29eb4540feb37a28c2a4bfc1da3de6db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Thu, 11 Oct 2018 14:52:23 +0200 Subject: [PATCH 43/51] roles/docker: change mounted directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/docker/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index 41c0618..a68dc2c 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -54,4 +54,4 @@ - /var/netboot:/srv/http - /var/rootfs:/srv/nfs/debian - /var/voyage:/srv/nfs/voyage - - /var/xen-dev:/srv/nfs/xen-dev + - /var/xen-dev:/srv/nfs/xen From 58628dab240018d0a10ccdff0e7e02e138954667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Fri, 12 Oct 2018 11:19:47 +0200 Subject: [PATCH 44/51] add roles/deploy_xen to speed up deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- deploy-meta-virtualization.yml | 2 +- roles/deploy_artifacts/tasks/main.yml | 15 --------------- roles/deploy_xen/tasks/main.yml | 15 +++++++++++++++ 3 files changed, 16 insertions(+), 16 deletions(-) create mode 100644 roles/deploy_xen/tasks/main.yml diff --git a/deploy-meta-virtualization.yml b/deploy-meta-virtualization.yml index c39cb01..d6d04d2 100644 --- a/deploy-meta-virtualization.yml +++ b/deploy-meta-virtualization.yml @@ -7,7 +7,7 @@ roles: - { role: 'common' } - { role: 'docker' } - - { role: 'deploy_artifacts' } + - { role: 'deploy_xen' } tasks: - name: Mount nfsd diff --git a/roles/deploy_artifacts/tasks/main.yml b/roles/deploy_artifacts/tasks/main.yml index 8670e6f..842cfc8 100644 --- a/roles/deploy_artifacts/tasks/main.yml +++ b/roles/deploy_artifacts/tasks/main.yml @@ -13,26 +13,11 @@ src: "results/vmlinuz-{{ linux_4_14 }}" dest: /var/netboot/kernels/vmlinuz-{{ linux_4_14 }} -- name: Copy Linux Xen dev - copy: - src: "results/vmlinuz-xen-dev" - dest: /var/netboot/kernels/vmlinuz-xen-dev - - name: Copy Xen 4.8 copy: src: "results/xen-4.8-amd64" dest: /var/netboot/kernels/xen-4.8-amd64 -- name: Copy Xen dev - copy: - src: "results/xen-dev" - dest: /var/netboot/kernels/xen-dev - -- name: Copy xen-image-minimal-genericx86-64.ext4 - copy: - src: "results/xen-image-minimal-genericx86-64.ext4" - dest: /var/xen-image-minimal-genericx86-64.ext4 - - name: create Linux 4.14.y symlink file: src: vmlinuz-{{ linux_4_14 }} diff --git a/roles/deploy_xen/tasks/main.yml b/roles/deploy_xen/tasks/main.yml new file mode 100644 index 0000000..765f865 --- /dev/null +++ b/roles/deploy_xen/tasks/main.yml @@ -0,0 +1,15 @@ +- name: Copy Linux Xen dev + copy: + src: "results/vmlinuz-xen-dev" + dest: /var/netboot/kernels/vmlinuz-xen-dev + +- name: Copy Xen dev + copy: + src: "results/xen-dev" + dest: /var/netboot/kernels/xen-dev + +- name: Copy xen-image-minimal-genericx86-64.ext4 + copy: + src: "results/xen-image-minimal-genericx86-64.ext4" + dest: /var/xen-image-minimal-genericx86-64.ext4 + From 0eda2d039d6326588235c02e7e746370332f1be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sat, 20 Oct 2018 00:00:25 +0200 Subject: [PATCH 45/51] get rid of xen from meta-virtualization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- pxe-server.yml | 8 ++------ roles/common/vars/main.yml | 5 +++-- roles/deploy_artifacts/tasks/main.yml | 6 +++--- roles/deploy_xen/tasks/main.yml | 18 ++++++++++++------ roles/docker/tasks/main.yml | 1 - roles/packages/tasks/main.yml | 2 -- roles/prepare_artifacts/tasks/main.yml | 10 +++++----- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/pxe-server.yml b/pxe-server.yml index 5db59ac..35cc811 100644 --- a/pxe-server.yml +++ b/pxe-server.yml @@ -21,18 +21,14 @@ src: /tmp/core.gz dest: /var/netboot/core.gz remote_src: yes + - name: Mount nfsd mount: path: /proc/fs/nfsd src: nfsd fstype: nfsd state: present - - name: Mount xen-image-minimal-genericx86-64.ext4 - mount: - path: /var/xen-dev - src: /var/xen-image-minimal-genericx86-64.ext4 - fstype: ext4 - state: present + - name: Restart server command: /sbin/shutdown -r +1 async: 0 diff --git a/roles/common/vars/main.yml b/roles/common/vars/main.yml index 9c80d43..ef826e2 100644 --- a/roles/common/vars/main.yml +++ b/roles/common/vars/main.yml @@ -1,7 +1,8 @@ -release_version: "v1.0.0" +release_version: "v1.1.1" linux_4_9: "4.9.128" linux_4_14: "4.14.71" -apu_config: "https://raw.githubusercontent.com/pcengines/apu2-documentation/master/configs/config-4.14.59" +apu_config: "" rootfs_tar_gz: "{{ results_dir }}/rootfs-{{ release_version }}.tar.gz" rootfs_dir: "/home/debian/scripts/rootfs" results_dir: "/home/debian/scripts/results" +xen_version: "4.11-amd64" diff --git a/roles/deploy_artifacts/tasks/main.yml b/roles/deploy_artifacts/tasks/main.yml index 842cfc8..495a4e6 100644 --- a/roles/deploy_artifacts/tasks/main.yml +++ b/roles/deploy_artifacts/tasks/main.yml @@ -13,10 +13,10 @@ src: "results/vmlinuz-{{ linux_4_14 }}" dest: /var/netboot/kernels/vmlinuz-{{ linux_4_14 }} -- name: Copy Xen 4.8 +- name: Copy Xen {{ xen_version }} copy: - src: "results/xen-4.8-amd64" - dest: /var/netboot/kernels/xen-4.8-amd64 + src: "results/xen-{{ xen_version }}" + dest: "/var/netboot/kernels/xen-{{ xen_version }}" - name: create Linux 4.14.y symlink file: diff --git a/roles/deploy_xen/tasks/main.yml b/roles/deploy_xen/tasks/main.yml index 765f865..0954d45 100644 --- a/roles/deploy_xen/tasks/main.yml +++ b/roles/deploy_xen/tasks/main.yml @@ -3,13 +3,19 @@ src: "results/vmlinuz-xen-dev" dest: /var/netboot/kernels/vmlinuz-xen-dev -- name: Copy Xen dev +- name: Copy Xen dev gz copy: - src: "results/xen-dev" - dest: /var/netboot/kernels/xen-dev + src: "results/xen-dev.gz" + dest: /var/netboot/kernels/xen-dev.gz -- name: Copy xen-image-minimal-genericx86-64.ext4 +- name: Unarchive Xen dev gz + command: gunzip -f /var/netboot/kernels/xen-dev.gz + +- name: Copy xen-image-minimal-genericx86-64.ext4.gz copy: - src: "results/xen-image-minimal-genericx86-64.ext4" - dest: /var/xen-image-minimal-genericx86-64.ext4 + src: "results/xen-image-minimal-genericx86-64.ext4.gz" + dest: /var/xen-image-minimal-genericx86-64.ext4.gz + +- name: Unarchive xen-image-minimal-genericx86-64.ext4.gz + command: gunzip -f /var/xen-image-minimal-genericx86-64.ext4.gz diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index a68dc2c..c53e0fd 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -54,4 +54,3 @@ - /var/netboot:/srv/http - /var/rootfs:/srv/nfs/debian - /var/voyage:/srv/nfs/voyage - - /var/xen-dev:/srv/nfs/xen diff --git a/roles/packages/tasks/main.yml b/roles/packages/tasks/main.yml index 736c384..25a1a92 100644 --- a/roles/packages/tasks/main.yml +++ b/roles/packages/tasks/main.yml @@ -41,5 +41,3 @@ - zlib1g-dev - xen-system-amd64 - xen-tools - - xen-linux-system-amd64 - diff --git a/roles/prepare_artifacts/tasks/main.yml b/roles/prepare_artifacts/tasks/main.yml index df7ac1f..c63a04c 100644 --- a/roles/prepare_artifacts/tasks/main.yml +++ b/roles/prepare_artifacts/tasks/main.yml @@ -2,9 +2,9 @@ stat: path: "{{ rootfs_tar_gz }}" register: rootfs_file -- name: check if "{{ results_dir }}/xen-4.8-amd64" exist +- name: check if "{{ results_dir }}/xen-{{ xen_version }}" exist stat: - path: "{{ results_dir }}/xen-4.8-amd64" + path: "{{ results_dir }}/xen-{{ xen_version }}" register: xen_file # for some reason archive return error, so we use tar directly An exception # occurred during task execution. To see the full traceback, use -vvv. The @@ -22,12 +22,12 @@ - name: preserver Xen kernel copy: - src: "{{ rootfs_dir }}/boot/xen-4.8-amd64.gz" - dest: "{{ results_dir }}/xen-4.8-amd64.gz" + src: "{{ rootfs_dir }}/boot/xen-{{ xen_version }}.gz" + dest: "{{ results_dir }}/xen-{{ xen_version }}.gz" when: not xen_file.stat.exists - name: unarchive Xen kernel - command: gunzip -f "{{ results_dir }}/xen-4.8-amd64.gz" + command: gunzip -f "{{ results_dir }}/xen-{{ xen_version }}.gz" args: chdir: "{{ results_dir }}" when: not xen_file.stat.exists From 6414fde9dc5d0aed0f33a768f54309cf1ca446b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sat, 20 Oct 2018 00:00:36 +0200 Subject: [PATCH 46/51] use sid for debootstrap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/debootstrap/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/debootstrap/tasks/main.yml b/roles/debootstrap/tasks/main.yml index 7db58bf..73ddabb 100644 --- a/roles/debootstrap/tasks/main.yml +++ b/roles/debootstrap/tasks/main.yml @@ -16,7 +16,7 @@ when: not rootfs_file.stat.exists - name: debootstrap first stage - command: debootstrap --foreign --include=python --arch amd64 stable {{ rootfs_dir }} http://deb.debian.org/debian + command: debootstrap --foreign --include=python --arch amd64 sid {{ rootfs_dir }} http://deb.debian.org/debian when: not rootfs_file.stat.exists - name: debootstrap second stage From 00e5e69f967004aeeceb4b1548f082bff77f6a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sat, 20 Oct 2018 00:01:00 +0200 Subject: [PATCH 47/51] correct building conditions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/linux-install/tasks/main.yml | 3 +- roles/linux-kernel/tasks/main.yml | 71 ++++++------------------------ 2 files changed, 15 insertions(+), 59 deletions(-) diff --git a/roles/linux-install/tasks/main.yml b/roles/linux-install/tasks/main.yml index 7874cdc..498c97d 100644 --- a/roles/linux-install/tasks/main.yml +++ b/roles/linux-install/tasks/main.yml @@ -10,4 +10,5 @@ - linux-image-{{ version }}_{{ version }}-1_amd64.deb - linux-image-{{ version }}-dbg_{{ version }}-1_amd64.deb - linux-libc-dev_{{ version }}-1_amd64.deb - when: not mod_dir.stat.exists + when: + - not mod_dir.stat.exists diff --git a/roles/linux-kernel/tasks/main.yml b/roles/linux-kernel/tasks/main.yml index d86d211..945bed7 100644 --- a/roles/linux-kernel/tasks/main.yml +++ b/roles/linux-kernel/tasks/main.yml @@ -31,14 +31,9 @@ register: result until: result is succeeded retries: 3 - delay: 3 + delay: 3 when: - - not config_ver.stat.exists - - not vmlinuz.stat.exists - - not linux_libc.stat.exists - - not linux_image.stat.exists - - not linux_image_dbg.stat.exists - - not linux_headers.stat.exists + - not config_ver.stat.exists or not vmlinuz.stat.exists or not linux_libc.stat.exists or not linux_image.stat.exists or not linux_image_dbg.stat.exists or not linux_headers.stat.exists - name: decompress Linux "{{ version }}" unarchive: @@ -46,48 +41,28 @@ dest: "{{ rootfs_dir }}" creates: "{{ rootfs_dir }}/linux-{{ version }}" when: - - not config_ver.stat.exists - - not vmlinuz.stat.exists - - not linux_libc.stat.exists - - not linux_image.stat.exists - - not linux_image_dbg.stat.exists - - not linux_headers.stat.exists + - not config_ver.stat.exists or not vmlinuz.stat.exists or not linux_libc.stat.exists or not linux_image.stat.exists or not linux_image_dbg.stat.exists or not linux_headers.stat.exists - name: make mrproper command: make mrproper args: chdir: "{{ rootfs_dir }}/linux-{{ version }}" when: - - not config_ver.stat.exists - - not vmlinuz.stat.exists - - not linux_libc.stat.exists - - not linux_image.stat.exists - - not linux_image_dbg.stat.exists - - not linux_headers.stat.exists + - not config_ver.stat.exists or not vmlinuz.stat.exists or not linux_libc.stat.exists or not linux_image.stat.exists or not linux_image_dbg.stat.exists or not linux_headers.stat.exists -- name: get apu_config - get_url: - url: "{{ config }}" +- name: copy kernel config + copy: + src: "{{ results_dir }}/config-{{ version }}" dest: "{{ rootfs_dir }}/linux-{{ version }}/.config" when: - - not config_ver.stat.exists - - not vmlinuz.stat.exists - - not linux_libc.stat.exists - - not linux_image.stat.exists - - not linux_image_dbg.stat.exists - - not linux_headers.stat.exists + - not config_ver.stat.exists or not vmlinuz.stat.exists or not linux_libc.stat.exists or not linux_image.stat.exists or not linux_image_dbg.stat.exists or not linux_headers.stat.exists - name: make olddefconfig command: make olddefconfig args: chdir: "{{ rootfs_dir }}/linux-{{ version }}" when: - - not config_ver.stat.exists - - not vmlinuz.stat.exists - - not linux_libc.stat.exists - - not linux_image.stat.exists - - not linux_image_dbg.stat.exists - - not linux_headers.stat.exists + - not config_ver.stat.exists or not vmlinuz.stat.exists or not linux_libc.stat.exists or not linux_image.stat.exists or not linux_image_dbg.stat.exists or not linux_headers.stat.exists - name: make deb-pkg command: make -j{{ ansible_processor_vcpus }} deb-pkg bzImage @@ -101,12 +76,7 @@ - "{{ rootfs_dir }}/linux-libc-dev_{{ version }}-1_amd64.deb" ignore_errors: yes when: - - not config_ver.stat.exists - - not vmlinuz.stat.exists - - not linux_libc.stat.exists - - not linux_image.stat.exists - - not linux_image_dbg.stat.exists - - not linux_headers.stat.exists + - not config_ver.stat.exists or not vmlinuz.stat.exists or not linux_libc.stat.exists or not linux_image.stat.exists or not linux_image_dbg.stat.exists or not linux_headers.stat.exists #TODO: this is for netboot/kernels directory - name: copy bzImage to known location @@ -114,12 +84,7 @@ src: "{{ rootfs_dir }}/linux-{{ version }}/arch/x86/boot/bzImage" dest: "{{ results_dir }}/vmlinuz-{{ version }}" when: - - not config_ver.stat.exists - - not vmlinuz.stat.exists - - not linux_libc.stat.exists - - not linux_image.stat.exists - - not linux_image_dbg.stat.exists - - not linux_headers.stat.exists + - not config_ver.stat.exists or not vmlinuz.stat.exists or not linux_libc.stat.exists or not linux_image.stat.exists or not linux_image_dbg.stat.exists or not linux_headers.stat.exists # TODO: save config for further commit to apu2-documentation - name: copy .config to known location @@ -127,12 +92,7 @@ src: "{{ rootfs_dir }}/linux-{{ version }}/.config" dest: "{{ results_dir }}/config-{{ version }}" when: - - not config_ver.stat.exists - - not vmlinuz.stat.exists - - not linux_libc.stat.exists - - not linux_image.stat.exists - - not linux_image_dbg.stat.exists - - not linux_headers.stat.exists + - not config_ver.stat.exists or not vmlinuz.stat.exists or not linux_libc.stat.exists or not linux_image.stat.exists or not linux_image_dbg.stat.exists or not linux_headers.stat.exists - name: remove everything except artifacts file: @@ -146,9 +106,4 @@ - linux-{{ version }}_{{ version }}-1.dsc - linux-{{ version }}_{{ version }}.orig.tar.gz when: - - not config_ver.stat.exists - - not vmlinuz.stat.exists - - not linux_libc.stat.exists - - not linux_image.stat.exists - - not linux_image_dbg.stat.exists - - not linux_headers.stat.exists + - not config_ver.stat.exists or not vmlinuz.stat.exists or not linux_libc.stat.exists or not linux_image.stat.exists or not linux_image_dbg.stat.exists or not linux_headers.stat.exists From 728aecd32b92a1253d177b440ae97e26a3409595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sat, 20 Oct 2018 00:01:31 +0200 Subject: [PATCH 48/51] fix menu.ipxe and disable cache updating for apt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- roles/netboot/templates/menu.ipxe.j2 | 6 +++--- roles/packages/tasks/main.yml | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/roles/netboot/templates/menu.ipxe.j2 b/roles/netboot/templates/menu.ipxe.j2 index 61797a5..b4f260c 100644 --- a/roles/netboot/templates/menu.ipxe.j2 +++ b/roles/netboot/templates/menu.ipxe.j2 @@ -20,19 +20,19 @@ item --gap -- ------------ iPXE boot menu end ---------------- choose --default boot --timeout 3000 target && goto ${target} :xen -kernel kernels/xen-4.8-amd64 dom0_mem=512M loglvl=all guest_loglvl=all com1=115200,8n1 console=com1 +kernel kernels/xen-{{ xen_version }} dom0_mem=512M loglvl=all guest_loglvl=all com1=115200,8n1 console=com1 module kernels/vmlinuz-4.14.y console=hvc0 earlyprintk=xen nomodeset root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug boot goto MENU :xen-dev kernel kernels/xen-dev dom0_mem=512M loglvl=all guest_loglvl=all com1=115200,8n1 console=com1 -module kernels/vmlinuz-xen-dev console=hvc0 earlyprintk=xen nomodeset root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/xen-dev,vers=3,udp nfsrootdebug +module kernels/vmlinuz-xen-dev console=hvc0 earlyprintk=xen nomodeset root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug boot goto MENU :xen-linux-dev -kernel kernels/xen-4.8-amd64 dom0_mem=512M loglvl=all guest_loglvl=all com1=115200,8n1 console=com1 +kernel kernels/xen-{{ xen_version }} dom0_mem=512M loglvl=all guest_loglvl=all com1=115200,8n1 console=com1 module kernels/vmlinuz-dev console=hvc0 earlyprintk=xen nomodeset root=/dev/nfs rw ip=dhcp nfsroot={{ ansible_default_ipv4.address }}:/srv/nfs/debian,vers=3,udp nfsrootdebug boot goto MENU diff --git a/roles/packages/tasks/main.yml b/roles/packages/tasks/main.yml index 25a1a92..ae22145 100644 --- a/roles/packages/tasks/main.yml +++ b/roles/packages/tasks/main.yml @@ -2,7 +2,6 @@ apt: name: "{{ item }}" state: present - update_cache: yes with_items: - apt-utils - autoconf From 772318b64ce28b73e34d79fde784d0765e1b646c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sat, 20 Oct 2018 00:02:28 +0200 Subject: [PATCH 49/51] add apt-cacher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- apt-cacher/Dockerfile | 17 +++++++++++++++++ docker-compose.yml | 7 +++++++ 2 files changed, 24 insertions(+) create mode 100644 apt-cacher/Dockerfile create mode 100644 docker-compose.yml diff --git a/apt-cacher/Dockerfile b/apt-cacher/Dockerfile new file mode 100644 index 0000000..a44a417 --- /dev/null +++ b/apt-cacher/Dockerfile @@ -0,0 +1,17 @@ +# +# Build: docker build -t apt-cacher . +# Run: docker run -d -p 3142:3142 --name apt-cacher-run apt-cacher +# +# and then you can run containers with: +# docker run -t -i --rm -e http_proxy http://dockerhost:3142/ debian bash +# +# Here, `dockerhost` is the IP address or FQDN of a host running the Docker daemon +# which acts as an APT proxy server. +FROM ubuntu + +VOLUME ["/var/cache/apt-cacher-ng"] +RUN apt-get update && apt-get install -y apt-cacher-ng + +RUN sed -i "s|\# PassThroughPattern: .* \# this would allow CONNECT to everything|PassThroughPattern: .* \# this would allow CONNECT to everything|g" /etc/apt-cacher-ng/acng.conf +EXPOSE 3142 +CMD chmod 777 /var/cache/apt-cacher-ng && /etc/init.d/apt-cacher-ng start && tail -f /var/log/apt-cacher-ng/* diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e7a7813 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,7 @@ +version: '3' +services: + apt-cacher: + image: apt-cacher + ports: + - "3142:3142" + container_name: apt-cacher-run From dbd28ae871d606d8d16343502a54598394320688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sat, 20 Oct 2018 23:50:56 +0200 Subject: [PATCH 50/51] README.md: fix documentation: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6da15b9..3b8b80d 100755 --- a/README.md +++ b/README.md @@ -34,8 +34,8 @@ ssh-copy-id -i ~/.ssh/ansible @ ``` docker run --privileged --rm -v $HOME/.ansible:/root/.ansible \ --v $HOME/.ccache:/home/debian/.ccache \ -v $PWD:/home/debian/scripts \ --t -i 3mdeb/debian-rootfs-builder ansible-playbook -vvv \ -i hosts \ +-v $HOME/.ccache:/home/debian/.ccache -v $PWD:/home/debian/scripts \ +-t -i 3mdeb/rootfs-builder ansible-playbook -i hosts \ /home/debian/scripts/create-rootfs-components.yml ``` @@ -43,9 +43,9 @@ docker run --privileged --rm -v $HOME/.ansible:/root/.ansible \ ``` docker run --privileged --rm -v $HOME/.ansible:/root/.ansible \ --v $HOME/.ccache:/home/debian/.ccache \ -v $PWD:/home/debian/scripts \ --t -i 3mdeb/debian-rootfs-builder ansible-playbook -vvv \ -i hosts \ -/home/debian/scripts/prepare_rootfs.yml +-v $HOME/.ccache:/home/debian/.ccache -v $PWD:/home/debian/scripts \ +-t -i 3mdeb/rootfs-builder ansible-playbook -i hosts \ +/home/debian/scripts/prepare-rootfs.yml ``` ### Deploy From edca42352db0d736f77596b333a11d16eac12b64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kr=C3=B3l?= Date: Sat, 20 Oct 2018 23:51:16 +0200 Subject: [PATCH 51/51] apt-cacher,rootfs-builder: initial commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Piotr Król --- apt-cacher/build.sh | 1 + rootfs-builder/Dockerfile | 34 ++++++++++++++++++++++++++++++++++ rootfs-builder/build.sh | 1 + 3 files changed, 36 insertions(+) create mode 100755 apt-cacher/build.sh create mode 100644 rootfs-builder/Dockerfile create mode 100755 rootfs-builder/build.sh diff --git a/apt-cacher/build.sh b/apt-cacher/build.sh new file mode 100755 index 0000000..40c193e --- /dev/null +++ b/apt-cacher/build.sh @@ -0,0 +1 @@ +docker build -t apt-cacher . diff --git a/rootfs-builder/Dockerfile b/rootfs-builder/Dockerfile new file mode 100644 index 0000000..957f30f --- /dev/null +++ b/rootfs-builder/Dockerfile @@ -0,0 +1,34 @@ +FROM debian:stretch-backports + +MAINTAINER Piotr Król + +ARG HTTP_PROXY + +ENV http_proxy ${HTTP_PROXY} + +RUN \ + useradd -p locked -m debian && \ + apt-get -qq update && \ + apt-get -qqy -t stretch-backports install \ + ansible \ + bc \ + build-essential \ + ccache \ + debootstrap \ + kmod \ + libelf-dev \ + libssl-dev \ + lsb-release \ + python \ + unzip \ + tar \ + && apt-get clean + +ENV PATH="/usr/lib/ccache:${PATH}" +ENV ANSIBLE_CONFIG="/home/debian/scripts/ansible.cfg" +ENV CCACHE_DIR="/home/debian/.ccache" +RUN mkdir /home/debian/.ccache && \ + chown debian:debian /home/debian/.ccache +WORKDIR /home/debian/scripts +RUN sed -i "s|#http_proxy = http://proxy.yoyodyne.com:18023/|http_proxy=${HTTP_PROXY}|g" /etc/wgetrc +RUN echo "Acquire::http::Proxy \"${HTTP_PROXY}\";" > /etc/apt/apt.conf.d/99HttpProxy diff --git a/rootfs-builder/build.sh b/rootfs-builder/build.sh new file mode 100755 index 0000000..0c00d07 --- /dev/null +++ b/rootfs-builder/build.sh @@ -0,0 +1 @@ +docker build --build-arg HTTP_PROXY=http://$1:3142 -t 3mdeb/rootfs-builder:latest .