init

Readme.md

@@ -0,0 +1,25 @@
+# Ganeti cluster deployment using Ansible
+Setup a Ganeti cluster on Debian VMs.
+
+## How to use
+- First, edit the inventory file.
+- Disable secure boot (if using UEFI).
+- Then:
+```sh
+# allow ansible to use the ssh key
+ssh-agent $SHELL
+ssh-add ~/.ssh/id_rsa
+
+# setup nodes
+ansible-playbook -i inventory -u root node.yml
+
+# setup master
+ansible-playbook -i inventory -u root master.yml
+
+# setup workers
+ansible-playbook -i inventory -u root worker.yml
+```
+
+## Features
+- static IP using systemd-networkd (using the last IP of the server before running the playbook)
+- zfs extstorage

group_vars/all.yml

@@ -0,0 +1,46 @@
+packages:
+  - git
+  - lvm2
+  - linux-headers-amd64
+  - zfs-dkms
+  - zfsutils-linux
+  - ganeti
+  - drbd-utils
+  - socat
+  - python3
+  - systemd-resolved # needs to be the last one
+                     # breaks dns resolution until
+                     # systemd-networkd is configured
+
+# network
+cluster_name: cluster.ganeti
+
+interface_name: lan0
+bridge_name: xenbr0
+mac_prefix: "02:42:ac"
+gateway: "192.168.50.254"
+dns_servers:
+  - "192.168.11.1"
+  - "1.1.1.1"
+
+# hostnames:
+# - ip: x.x.x.x
+#   name: example
+# ...
+hostnames:
+  - ip: "192.168.50.30"
+    name: "{{ cluster_name }}"
+  - ip: "192.168.50.31"
+    name: test-31.ganeti
+  - ip: "192.168.50.32"
+    name: test-32.ganeti
+  - ip: "192.168.50.33"
+    name: test-33.ganeti
+
+# storage
+zpool_name: ganeti-pool
+zpool_dev: /dev/vdc
+
+vg_name: xenvg
+pvs:
+  - /dev/vdb

inventory

@@ -0,0 +1,9 @@
+[nodes]
+192.168.50.20
+192.168.50.21
+
+[master]
+192.168.50.20
+
+[workers]
+192.168.50.21

master.yml

@@ -0,0 +1,6 @@
+- name: Cluster master setup
+  hosts: master
+  become: true
+  gather_facts: true
+  roles:
+    - master

node.yml

@@ -0,0 +1,6 @@
+- name: Cluster node setup
+  hosts: nodes
+  become: true
+  gather_facts: true
+  roles:
+    - node

roles/master/tasks/main.yml

@@ -0,0 +1,37 @@
+- name: Check if the cluster is intalized
+  ansible.builtin.command:
+    cmd: gnt-cluster info
+  ignore_errors: true
+  changed_when: false
+  register: ganeti_cluster_initalized
+
+- name: Initalize cluster
+  ansible.builtin.command:
+    cmd: |-
+      gnt-cluster init
+      --enabled-hypervisors kvm
+      --no-etc-hosts
+      --master-netdev {{ bridge_name }}
+      --nic-parameters link={{ bridge_name }},mode=bridged
+      --enabled-disk-templates drbd,plain
+      {{ cluster_name }}
+  register: ganeti_cluster_init_result
+  changed_when: ganeti_cluster_init_result.rc == 0
+  when: ganeti_cluster_initalized.rc != 0
+
+- name: Add worker {{ item }}
+  ansible.builtin.command:
+    cmd: |-
+      bash -c "
+      (gnt-node list | grep 'ganeti-{{ groups['nodes'].index(item) + 1 }}.ganeti') || \
+      gnt-node add \
+      --no-ssh-key-check \
+      --no-node-setup \
+      ganeti-{{ groups['nodes'].index(item) + 1 }}.ganeti"
+  register: node_add_result
+  changed_when: node_add_result.rc == 0
+  loop: "{{ groups['workers'] }}"
+
+- name: Install ZFS extstorage
+  ansible.builtin.include_role:
+    name: zfs-extstorage

roles/node/handlers/main.yml

@@ -0,0 +1,12 @@
+- name: Update initramfs
+  listen:
+    - update initramfs
+    - update initrd
+  ansible.builtin.command:
+    cmd: >
+      update-initramfs -k all -u
+
+- name: Reboot
+  listen:
+    - reboot
+  ansible.builtin.reboot:

roles/node/tasks/hostnames.yml

@@ -0,0 +1,11 @@
+- name: Set hostname
+  ansible.builtin.template:
+    src: etc-hostname.j2
+    dest: /etc/hostname
+    mode: "0644"
+
+- name: Set hosts
+  ansible.builtin.template:
+    src: etc-hosts.j2
+    dest: /etc/hosts
+    mode: "0644"

roles/node/tasks/keys.yml

@@ -0,0 +1,35 @@
+- name: Check if node has a key
+  ansible.builtin.stat:
+    path: /root/.ssh/id_rsa.pub
+  register: key_check
+
+- name: Generate an OpenSSH keypair
+  community.crypto.openssh_keypair:
+    path: /root/.ssh/id_rsa
+  when: not key_check.stat.exists
+
+- name: Fetch keys to local machine
+  ansible.builtin.fetch:
+    src: /root/.ssh/id_rsa.pub
+    dest: /tmp/fetched_keys/
+
+- name: Copy keys
+  ansible.builtin.copy:
+    src: /tmp/fetched_keys
+    dest: /tmp
+    mode: "0644"
+
+- name: Add key to authorized_keys
+  ansible.posix.authorized_key:
+    user: root
+    state: present
+    key: "{{ lookup('file', '/tmp/fetched_keys/' + item + '/root/.ssh/id_rsa.pub') }}"
+  loop: "{{ groups['nodes'] }}"
+
+- name: Add key to known_hosts
+  ansible.builtin.known_hosts:
+    path: /root/.ssh/known_hosts
+    name: "ganeti-{{ groups['nodes'].index(item) + 1 }}.ganeti"
+    key: "ganeti-{{ groups['nodes'].index(item) + 1 }}.ganeti {{ lookup('file', '/tmp/fetched_keys/' + item + '/root/.ssh/id_rsa.pub') }}"
+    state: present
+  loop: "{{ groups['nodes'] }}"

roles/node/tasks/main.yml

@@ -0,0 +1,27 @@
+- name: Update system
+  ansible.builtin.include_tasks:
+    file: update.yml
+
+- name: Install packages
+  ansible.builtin.include_tasks:
+    file: packages.yml
+
+- name: Set hostname
+  ansible.builtin.include_tasks:
+    file: hostnames.yml
+
+- name: Configure network
+  ansible.builtin.include_tasks:
+    file: network.yml
+
+- name: Enable modules
+  ansible.builtin.include_tasks:
+    file: modules.yml
+
+- name: Create storages
+  ansible.builtin.include_tasks:
+    file: storage.yml
+
+- name: Exchange keys
+  ansible.builtin.include_tasks:
+    file: keys.yml

roles/node/tasks/modules.yml

@@ -0,0 +1,18 @@
+- name: Enable ZFS
+  community.general.modprobe:
+    name: zfs
+    state: present
+    persistent: present
+
+- name: Enable KVM
+  community.general.modprobe:
+    name: kvm
+    state: present
+    persistent: present
+
+- name: Enable DRBD
+  community.general.modprobe:
+    name: drbd
+    params: usermode_helper=/bin/true
+    state: present
+    persistent: present

roles/node/tasks/network.yml

@@ -0,0 +1,69 @@
+- name: Check if default interface is configured
+  ansible.builtin.set_fact:
+    interface_configured: "{{ interface_name in ansible_interfaces }}"
+
+- name: Check if bridge is configured
+  ansible.builtin.set_fact:
+    bridge_configured: "{{ bridge_name in ansible_interfaces }}"
+
+- name: Configure default interface name
+  ansible.builtin.template:
+    src: etc-systemd-network-10-lan0.link.j2
+    dest: /etc/systemd/network/10-{{ interface_name }}.link
+    mode: "0644"
+  when: not interface_configured
+  notify:
+    - update initramfs
+    - reboot
+
+- name: Create bridge interface
+  ansible.builtin.template:
+    src: etc-systemd-network-20-xenbr0.netdev.j2
+    dest: /etc/systemd/network/20-{{ bridge_name }}.netdev
+    mode: "0644"
+  when: not bridge_configured
+  notify:
+    - update initramfs
+    - reboot
+
+- name: Configure bridge interface
+  ansible.builtin.template:
+    src: etc-systemd-network-30-xenbr0.link.j2
+    dest: /etc/systemd/network/30-{{ bridge_name }}.link
+    mode: "0644"
+  when: not bridge_configured
+  notify:
+    - update initramfs
+    - reboot
+
+- name: Create bridge network
+  ansible.builtin.template:
+    src: etc-systemd-network-40-xenbr0.network.j2
+    dest: /etc/systemd/network/40-{{ bridge_name }}.network
+    mode: "0644"
+  when: not bridge_configured
+  notify:
+    - update initramfs
+    - reboot
+
+- name: Configure network for default interface
+  ansible.builtin.template:
+    src: etc-systemd-network-50-lan0.network.j2
+    dest: /etc/systemd/network/50-{{ interface_name }}.network
+    mode: "0644"
+  when: not interface_configured
+  notify:
+    - update initramfs
+    - reboot
+
+- name: Enable systemd-networkd
+  ansible.builtin.systemd_service:
+    name: systemd-networkd
+    enabled: true
+    state: restarted
+
+- name: Enable systemd-resolved
+  ansible.builtin.systemd_service:
+    name: systemd-resolved
+    enabled: true
+    state: restarted

roles/node/tasks/packages.yml

@@ -0,0 +1,20 @@
+- name: Add backports
+  ansible.builtin.apt_repository:
+    repo: deb http://deb.debian.org/debian/ bookworm-backports main contrib non-free non-free-firmware
+    state: present
+
+- name: Add backports src
+  ansible.builtin.apt_repository:
+    repo: deb-src http://deb.debian.org/debian/ bookworm-backports main contrib non-free non-free-firmware
+    state: present
+
+- name: Install packages
+  ansible.builtin.apt:
+    name: "{{ item }}"
+    state: present
+    install_recommends: false
+  loop: "{{ packages }}"
+
+- name: Remove dependencies that are no longer required
+  ansible.builtin.apt:
+    autoremove: true

roles/node/tasks/storage.yml

@@ -0,0 +1,21 @@
+- name: Create zpool
+  ansible.builtin.command:
+    cmd: zpool create {{ zpool_name }} {{ zpool_dev }}
+    creates: /{{ zpool_name }}
+
+- name: Check if the folder exists
+  ansible.builtin.stat:
+    path: /usr/share/ganeti/extstorage/zfs
+  register: folder_check
+
+- name: Reinstall lvm2 if ZFS extstorage is installed
+  ansible.builtin.command:
+    cmd: apt reinstall lvm2
+  register: lvm2_reinstall_result
+  changed_when: lvm2_reinstall_result.rc == 0
+  when: folder_check.stat.exists
+
+- name: Create LVM vg
+  community.general.lvg:
+    vg: "{{ vg_name }}"
+    pvs: "{{ pvs }}"

roles/node/tasks/update.yml

@@ -0,0 +1,5 @@
+- name: Update and upgrade system
+  ansible.builtin.apt:
+    upgrade: true
+    update_cache: true
+    cache_valid_time: 86400

roles/node/templates/etc-hostname.j2

@@ -0,0 +1 @@
+ganeti-{{ groups['nodes'].index(inventory_hostname) + 1 }}.ganeti

roles/node/templates/etc-hosts.j2

@@ -0,0 +1,13 @@
+127.0.0.1  localhost
+
+::1      localhost ip6-localhost ip6-loopback
+ff02::1  ip6-allnodes
+ff02::2  ip6-allrouters
+
+{% for node in groups['nodes'] %}
+{{ node }}  ganeti-{{ groups['nodes'].index(node) + 1 }}.ganeti
+{% endfor %}
+
+{% for item in hostnames %}
+{{ item.ip }}  {{ item.name }}
+{% endfor %}

roles/node/templates/etc-systemd-network-10-lan0.link.j2

@@ -0,0 +1,5 @@
+[Match]
+MACAddress={{ ansible_default_ipv4.macaddress }}
+
+[Link]
+Name={{ interface_name }}

roles/node/templates/etc-systemd-network-20-xenbr0.netdev.j2

@@ -0,0 +1,4 @@
+[NetDev]
+Name={{ bridge_name }}
+Kind=bridge
+MACAddress=none

roles/node/templates/etc-systemd-network-30-xenbr0.link.j2

@@ -0,0 +1,5 @@
+[Match]
+OriginalName={{ bridge_name }}
+
+[Link]
+MACAddressPolicy=none

roles/node/templates/etc-systemd-network-40-xenbr0.network.j2

@@ -0,0 +1,9 @@
+[Match]
+Name={{ bridge_name }}
+
+[Network]
+Address={{ ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0]) }}/24
+Gateway={{ gateway }}
+{% for dns in dns_servers %}
+DNS={{ dns }}
+{% endfor %}

roles/node/templates/etc-systemd-network-50-lan0.network.j2

@@ -0,0 +1,5 @@
+[Match]
+Name={{ interface_name }}
+
+[Network]
+Bridge={{ bridge_name }}

roles/worker/tasks/main.yml

@@ -0,0 +1,3 @@
+- name: Install ZFS extstorage
+  ansible.builtin.include_role:
+    name: zfs-extstorage

roles/zfs-extstorage/tasks/main.yml

@@ -0,0 +1,41 @@
+- name: Clone ZFS extsotarge module
+  ansible.builtin.git:
+    repo: https://github.com/brigriffin/ganeti-extstorage-zfs.git
+    dest: /usr/share/ganeti/extstorage/zfs
+    single_branch: true
+    version: master
+    force: true
+
+- name: Set zpool for extstorage module
+  ansible.builtin.template:
+    src: exstorage.sh.j2
+    dest: /usr/share/ganeti/extstorage/zfs/etc/ganeti-{{ groups['nodes'].index(inventory_hostname) + 1 }}.sh
+    mode: "0644"
+
+- name: Make everything executable
+  ansible.builtin.file:
+    dest: /usr/share/ganeti/extstorage/zfs
+    recurse: true
+    mode: "0755"
+
+- name: Enable ext template
+  ansible.builtin.command:
+    cmd: /usr/share/ganeti/extstorage/zfs/install/1-enable-ext-template.sh
+    chdir: /usr/share/ganeti/extstorage/zfs/install/
+  register: enable_ext_template_result
+  changed_when: enable_ext_template_result.rc == 0
+  when: inventory_hostname in groups['master']
+
+- name: Create log directory
+  ansible.builtin.command:
+    cmd: /usr/share/ganeti/extstorage/zfs/install/2-create-log-directory.sh
+    chdir: /usr/share/ganeti/extstorage/zfs/install/
+    creates: /var/log/ganeti/extstorage
+  register: create_log_directory_result
+
+- name: Create lvm wrappers
+  ansible.builtin.command:
+    cmd: /usr/share/ganeti/extstorage/zfs/install/3-lvm-wrappers.sh
+    chdir: /usr/share/ganeti/extstorage/zfs/install/
+  register: lvm_wrappers_result
+  changed_when: lvm_wrappers_result.rc == 0

roles/zfs-extstorage/templates/exstorage.sh.j2

@@ -0,0 +1 @@
+EXTP_ZFS={{ zpool_name }}

worker.yml

@@ -0,0 +1,6 @@
+- name: Cluster worker setup
+  hosts: workers
+  become: true
+  gather_facts: true
+  roles:
+    - worker