From ca334ee30373aacc7802bf40ca66da067d471629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?BENEDEK=20L=C3=A1szl=C3=B3?= Date: Tue, 26 Nov 2024 15:56:58 +0100 Subject: [PATCH] init --- Readme.md | 25 +++++++ group_vars/all.yml | 46 +++++++++++++ inventory | 9 +++ master.yml | 6 ++ node.yml | 6 ++ roles/master/tasks/main.yml | 37 ++++++++++ roles/node/defaults/main.yml | 0 roles/node/handlers/main.yml | 12 ++++ roles/node/tasks/hostnames.yml | 11 +++ roles/node/tasks/keys.yml | 35 ++++++++++ roles/node/tasks/main.yml | 27 ++++++++ roles/node/tasks/modules.yml | 18 +++++ roles/node/tasks/network.yml | 69 +++++++++++++++++++ roles/node/tasks/packages.yml | 20 ++++++ roles/node/tasks/storage.yml | 21 ++++++ roles/node/tasks/update.yml | 5 ++ roles/node/templates/etc-hostname.j2 | 1 + roles/node/templates/etc-hosts.j2 | 13 ++++ .../etc-systemd-network-10-lan0.link.j2 | 5 ++ .../etc-systemd-network-20-xenbr0.netdev.j2 | 4 ++ .../etc-systemd-network-30-xenbr0.link.j2 | 5 ++ .../etc-systemd-network-40-xenbr0.network.j2 | 9 +++ .../etc-systemd-network-50-lan0.network.j2 | 5 ++ roles/worker/tasks/main.yml | 3 + roles/zfs-extstorage/tasks/main.yml | 41 +++++++++++ .../zfs-extstorage/templates/exstorage.sh.j2 | 1 + worker.yml | 6 ++ 27 files changed, 440 insertions(+) create mode 100644 Readme.md create mode 100644 group_vars/all.yml create mode 100644 inventory create mode 100644 master.yml create mode 100644 node.yml create mode 100644 roles/master/tasks/main.yml create mode 100644 roles/node/defaults/main.yml create mode 100644 roles/node/handlers/main.yml create mode 100644 roles/node/tasks/hostnames.yml create mode 100644 roles/node/tasks/keys.yml create mode 100644 roles/node/tasks/main.yml create mode 100644 roles/node/tasks/modules.yml create mode 100644 roles/node/tasks/network.yml create mode 100644 roles/node/tasks/packages.yml create mode 100644 roles/node/tasks/storage.yml create mode 100644 roles/node/tasks/update.yml create mode 100644 roles/node/templates/etc-hostname.j2 create mode 100644 roles/node/templates/etc-hosts.j2 create mode 100644 roles/node/templates/etc-systemd-network-10-lan0.link.j2 create mode 100644 roles/node/templates/etc-systemd-network-20-xenbr0.netdev.j2 create mode 100644 roles/node/templates/etc-systemd-network-30-xenbr0.link.j2 create mode 100644 roles/node/templates/etc-systemd-network-40-xenbr0.network.j2 create mode 100644 roles/node/templates/etc-systemd-network-50-lan0.network.j2 create mode 100644 roles/worker/tasks/main.yml create mode 100644 roles/zfs-extstorage/tasks/main.yml create mode 100644 roles/zfs-extstorage/templates/exstorage.sh.j2 create mode 100644 worker.yml diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..d29f325 --- /dev/null +++ b/Readme.md @@ -0,0 +1,25 @@ +# Ganeti cluster deployment using Ansible +Setup a Ganeti cluster on Debian VMs. + +## How to use +- First, edit the inventory file. +- Disable secure boot (if using UEFI). +- Then: +```sh +# allow ansible to use the ssh key +ssh-agent $SHELL +ssh-add ~/.ssh/id_rsa + +# setup nodes +ansible-playbook -i inventory -u root node.yml + +# setup master +ansible-playbook -i inventory -u root master.yml + +# setup workers +ansible-playbook -i inventory -u root worker.yml +``` + +## Features +- static IP using systemd-networkd (using the last IP of the server before running the playbook) +- zfs extstorage \ No newline at end of file diff --git a/group_vars/all.yml b/group_vars/all.yml new file mode 100644 index 0000000..856be6b --- /dev/null +++ b/group_vars/all.yml @@ -0,0 +1,46 @@ +packages: + - git + - lvm2 + - linux-headers-amd64 + - zfs-dkms + - zfsutils-linux + - ganeti + - drbd-utils + - socat + - python3 + - systemd-resolved # needs to be the last one + # breaks dns resolution until + # systemd-networkd is configured + +# network +cluster_name: cluster.ganeti + +interface_name: lan0 +bridge_name: xenbr0 +mac_prefix: "02:42:ac" +gateway: "192.168.50.254" +dns_servers: + - "192.168.11.1" + - "1.1.1.1" + +# hostnames: +# - ip: x.x.x.x +# name: example +# ... +hostnames: + - ip: "192.168.50.30" + name: "{{ cluster_name }}" + - ip: "192.168.50.31" + name: test-31.ganeti + - ip: "192.168.50.32" + name: test-32.ganeti + - ip: "192.168.50.33" + name: test-33.ganeti + +# storage +zpool_name: ganeti-pool +zpool_dev: /dev/vdc + +vg_name: xenvg +pvs: + - /dev/vdb diff --git a/inventory b/inventory new file mode 100644 index 0000000..47ba6f6 --- /dev/null +++ b/inventory @@ -0,0 +1,9 @@ +[nodes] +192.168.50.20 +192.168.50.21 + +[master] +192.168.50.20 + +[workers] +192.168.50.21 diff --git a/master.yml b/master.yml new file mode 100644 index 0000000..704b21e --- /dev/null +++ b/master.yml @@ -0,0 +1,6 @@ +- name: Cluster master setup + hosts: master + become: true + gather_facts: true + roles: + - master diff --git a/node.yml b/node.yml new file mode 100644 index 0000000..ba9d560 --- /dev/null +++ b/node.yml @@ -0,0 +1,6 @@ +- name: Cluster node setup + hosts: nodes + become: true + gather_facts: true + roles: + - node diff --git a/roles/master/tasks/main.yml b/roles/master/tasks/main.yml new file mode 100644 index 0000000..5f2af4a --- /dev/null +++ b/roles/master/tasks/main.yml @@ -0,0 +1,37 @@ +- name: Check if the cluster is intalized + ansible.builtin.command: + cmd: gnt-cluster info + ignore_errors: true + changed_when: false + register: ganeti_cluster_initalized + +- name: Initalize cluster + ansible.builtin.command: + cmd: |- + gnt-cluster init + --enabled-hypervisors kvm + --no-etc-hosts + --master-netdev {{ bridge_name }} + --nic-parameters link={{ bridge_name }},mode=bridged + --enabled-disk-templates drbd,plain + {{ cluster_name }} + register: ganeti_cluster_init_result + changed_when: ganeti_cluster_init_result.rc == 0 + when: ganeti_cluster_initalized.rc != 0 + +- name: Add worker {{ item }} + ansible.builtin.command: + cmd: |- + bash -c " + (gnt-node list | grep 'ganeti-{{ groups['nodes'].index(item) + 1 }}.ganeti') || \ + gnt-node add \ + --no-ssh-key-check \ + --no-node-setup \ + ganeti-{{ groups['nodes'].index(item) + 1 }}.ganeti" + register: node_add_result + changed_when: node_add_result.rc == 0 + loop: "{{ groups['workers'] }}" + +- name: Install ZFS extstorage + ansible.builtin.include_role: + name: zfs-extstorage diff --git a/roles/node/defaults/main.yml b/roles/node/defaults/main.yml new file mode 100644 index 0000000..e69de29 diff --git a/roles/node/handlers/main.yml b/roles/node/handlers/main.yml new file mode 100644 index 0000000..4d5e24f --- /dev/null +++ b/roles/node/handlers/main.yml @@ -0,0 +1,12 @@ +- name: Update initramfs + listen: + - update initramfs + - update initrd + ansible.builtin.command: + cmd: > + update-initramfs -k all -u + +- name: Reboot + listen: + - reboot + ansible.builtin.reboot: diff --git a/roles/node/tasks/hostnames.yml b/roles/node/tasks/hostnames.yml new file mode 100644 index 0000000..7b2f6ff --- /dev/null +++ b/roles/node/tasks/hostnames.yml @@ -0,0 +1,11 @@ +- name: Set hostname + ansible.builtin.template: + src: etc-hostname.j2 + dest: /etc/hostname + mode: "0644" + +- name: Set hosts + ansible.builtin.template: + src: etc-hosts.j2 + dest: /etc/hosts + mode: "0644" diff --git a/roles/node/tasks/keys.yml b/roles/node/tasks/keys.yml new file mode 100644 index 0000000..0362650 --- /dev/null +++ b/roles/node/tasks/keys.yml @@ -0,0 +1,35 @@ +- name: Check if node has a key + ansible.builtin.stat: + path: /root/.ssh/id_rsa.pub + register: key_check + +- name: Generate an OpenSSH keypair + community.crypto.openssh_keypair: + path: /root/.ssh/id_rsa + when: not key_check.stat.exists + +- name: Fetch keys to local machine + ansible.builtin.fetch: + src: /root/.ssh/id_rsa.pub + dest: /tmp/fetched_keys/ + +- name: Copy keys + ansible.builtin.copy: + src: /tmp/fetched_keys + dest: /tmp + mode: "0644" + +- name: Add key to authorized_keys + ansible.posix.authorized_key: + user: root + state: present + key: "{{ lookup('file', '/tmp/fetched_keys/' + item + '/root/.ssh/id_rsa.pub') }}" + loop: "{{ groups['nodes'] }}" + +- name: Add key to known_hosts + ansible.builtin.known_hosts: + path: /root/.ssh/known_hosts + name: "ganeti-{{ groups['nodes'].index(item) + 1 }}.ganeti" + key: "ganeti-{{ groups['nodes'].index(item) + 1 }}.ganeti {{ lookup('file', '/tmp/fetched_keys/' + item + '/root/.ssh/id_rsa.pub') }}" + state: present + loop: "{{ groups['nodes'] }}" diff --git a/roles/node/tasks/main.yml b/roles/node/tasks/main.yml new file mode 100644 index 0000000..d4e547c --- /dev/null +++ b/roles/node/tasks/main.yml @@ -0,0 +1,27 @@ +- name: Update system + ansible.builtin.include_tasks: + file: update.yml + +- name: Install packages + ansible.builtin.include_tasks: + file: packages.yml + +- name: Set hostname + ansible.builtin.include_tasks: + file: hostnames.yml + +- name: Configure network + ansible.builtin.include_tasks: + file: network.yml + +- name: Enable modules + ansible.builtin.include_tasks: + file: modules.yml + +- name: Create storages + ansible.builtin.include_tasks: + file: storage.yml + +- name: Exchange keys + ansible.builtin.include_tasks: + file: keys.yml diff --git a/roles/node/tasks/modules.yml b/roles/node/tasks/modules.yml new file mode 100644 index 0000000..0783cf0 --- /dev/null +++ b/roles/node/tasks/modules.yml @@ -0,0 +1,18 @@ +- name: Enable ZFS + community.general.modprobe: + name: zfs + state: present + persistent: present + +- name: Enable KVM + community.general.modprobe: + name: kvm + state: present + persistent: present + +- name: Enable DRBD + community.general.modprobe: + name: drbd + params: usermode_helper=/bin/true + state: present + persistent: present diff --git a/roles/node/tasks/network.yml b/roles/node/tasks/network.yml new file mode 100644 index 0000000..532619b --- /dev/null +++ b/roles/node/tasks/network.yml @@ -0,0 +1,69 @@ +- name: Check if default interface is configured + ansible.builtin.set_fact: + interface_configured: "{{ interface_name in ansible_interfaces }}" + +- name: Check if bridge is configured + ansible.builtin.set_fact: + bridge_configured: "{{ bridge_name in ansible_interfaces }}" + +- name: Configure default interface name + ansible.builtin.template: + src: etc-systemd-network-10-lan0.link.j2 + dest: /etc/systemd/network/10-{{ interface_name }}.link + mode: "0644" + when: not interface_configured + notify: + - update initramfs + - reboot + +- name: Create bridge interface + ansible.builtin.template: + src: etc-systemd-network-20-xenbr0.netdev.j2 + dest: /etc/systemd/network/20-{{ bridge_name }}.netdev + mode: "0644" + when: not bridge_configured + notify: + - update initramfs + - reboot + +- name: Configure bridge interface + ansible.builtin.template: + src: etc-systemd-network-30-xenbr0.link.j2 + dest: /etc/systemd/network/30-{{ bridge_name }}.link + mode: "0644" + when: not bridge_configured + notify: + - update initramfs + - reboot + +- name: Create bridge network + ansible.builtin.template: + src: etc-systemd-network-40-xenbr0.network.j2 + dest: /etc/systemd/network/40-{{ bridge_name }}.network + mode: "0644" + when: not bridge_configured + notify: + - update initramfs + - reboot + +- name: Configure network for default interface + ansible.builtin.template: + src: etc-systemd-network-50-lan0.network.j2 + dest: /etc/systemd/network/50-{{ interface_name }}.network + mode: "0644" + when: not interface_configured + notify: + - update initramfs + - reboot + +- name: Enable systemd-networkd + ansible.builtin.systemd_service: + name: systemd-networkd + enabled: true + state: restarted + +- name: Enable systemd-resolved + ansible.builtin.systemd_service: + name: systemd-resolved + enabled: true + state: restarted diff --git a/roles/node/tasks/packages.yml b/roles/node/tasks/packages.yml new file mode 100644 index 0000000..34eae17 --- /dev/null +++ b/roles/node/tasks/packages.yml @@ -0,0 +1,20 @@ +- name: Add backports + ansible.builtin.apt_repository: + repo: deb http://deb.debian.org/debian/ bookworm-backports main contrib non-free non-free-firmware + state: present + +- name: Add backports src + ansible.builtin.apt_repository: + repo: deb-src http://deb.debian.org/debian/ bookworm-backports main contrib non-free non-free-firmware + state: present + +- name: Install packages + ansible.builtin.apt: + name: "{{ item }}" + state: present + install_recommends: false + loop: "{{ packages }}" + +- name: Remove dependencies that are no longer required + ansible.builtin.apt: + autoremove: true diff --git a/roles/node/tasks/storage.yml b/roles/node/tasks/storage.yml new file mode 100644 index 0000000..86680d5 --- /dev/null +++ b/roles/node/tasks/storage.yml @@ -0,0 +1,21 @@ +- name: Create zpool + ansible.builtin.command: + cmd: zpool create {{ zpool_name }} {{ zpool_dev }} + creates: /{{ zpool_name }} + +- name: Check if the folder exists + ansible.builtin.stat: + path: /usr/share/ganeti/extstorage/zfs + register: folder_check + +- name: Reinstall lvm2 if ZFS extstorage is installed + ansible.builtin.command: + cmd: apt reinstall lvm2 + register: lvm2_reinstall_result + changed_when: lvm2_reinstall_result.rc == 0 + when: folder_check.stat.exists + +- name: Create LVM vg + community.general.lvg: + vg: "{{ vg_name }}" + pvs: "{{ pvs }}" diff --git a/roles/node/tasks/update.yml b/roles/node/tasks/update.yml new file mode 100644 index 0000000..e64c9e1 --- /dev/null +++ b/roles/node/tasks/update.yml @@ -0,0 +1,5 @@ +- name: Update and upgrade system + ansible.builtin.apt: + upgrade: true + update_cache: true + cache_valid_time: 86400 diff --git a/roles/node/templates/etc-hostname.j2 b/roles/node/templates/etc-hostname.j2 new file mode 100644 index 0000000..b0fcf4b --- /dev/null +++ b/roles/node/templates/etc-hostname.j2 @@ -0,0 +1 @@ +ganeti-{{ groups['nodes'].index(inventory_hostname) + 1 }}.ganeti diff --git a/roles/node/templates/etc-hosts.j2 b/roles/node/templates/etc-hosts.j2 new file mode 100644 index 0000000..61c4c09 --- /dev/null +++ b/roles/node/templates/etc-hosts.j2 @@ -0,0 +1,13 @@ +127.0.0.1 localhost + +::1 localhost ip6-localhost ip6-loopback +ff02::1 ip6-allnodes +ff02::2 ip6-allrouters + +{% for node in groups['nodes'] %} +{{ node }} ganeti-{{ groups['nodes'].index(node) + 1 }}.ganeti +{% endfor %} + +{% for item in hostnames %} +{{ item.ip }} {{ item.name }} +{% endfor %} diff --git a/roles/node/templates/etc-systemd-network-10-lan0.link.j2 b/roles/node/templates/etc-systemd-network-10-lan0.link.j2 new file mode 100644 index 0000000..07cf3e7 --- /dev/null +++ b/roles/node/templates/etc-systemd-network-10-lan0.link.j2 @@ -0,0 +1,5 @@ +[Match] +MACAddress={{ ansible_default_ipv4.macaddress }} + +[Link] +Name={{ interface_name }} diff --git a/roles/node/templates/etc-systemd-network-20-xenbr0.netdev.j2 b/roles/node/templates/etc-systemd-network-20-xenbr0.netdev.j2 new file mode 100644 index 0000000..b6a8a89 --- /dev/null +++ b/roles/node/templates/etc-systemd-network-20-xenbr0.netdev.j2 @@ -0,0 +1,4 @@ +[NetDev] +Name={{ bridge_name }} +Kind=bridge +MACAddress=none diff --git a/roles/node/templates/etc-systemd-network-30-xenbr0.link.j2 b/roles/node/templates/etc-systemd-network-30-xenbr0.link.j2 new file mode 100644 index 0000000..af3068d --- /dev/null +++ b/roles/node/templates/etc-systemd-network-30-xenbr0.link.j2 @@ -0,0 +1,5 @@ +[Match] +OriginalName={{ bridge_name }} + +[Link] +MACAddressPolicy=none \ No newline at end of file diff --git a/roles/node/templates/etc-systemd-network-40-xenbr0.network.j2 b/roles/node/templates/etc-systemd-network-40-xenbr0.network.j2 new file mode 100644 index 0000000..c0869b8 --- /dev/null +++ b/roles/node/templates/etc-systemd-network-40-xenbr0.network.j2 @@ -0,0 +1,9 @@ +[Match] +Name={{ bridge_name }} + +[Network] +Address={{ ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0]) }}/24 +Gateway={{ gateway }} +{% for dns in dns_servers %} +DNS={{ dns }} +{% endfor %} diff --git a/roles/node/templates/etc-systemd-network-50-lan0.network.j2 b/roles/node/templates/etc-systemd-network-50-lan0.network.j2 new file mode 100644 index 0000000..83e1a49 --- /dev/null +++ b/roles/node/templates/etc-systemd-network-50-lan0.network.j2 @@ -0,0 +1,5 @@ +[Match] +Name={{ interface_name }} + +[Network] +Bridge={{ bridge_name }} diff --git a/roles/worker/tasks/main.yml b/roles/worker/tasks/main.yml new file mode 100644 index 0000000..2ed85fa --- /dev/null +++ b/roles/worker/tasks/main.yml @@ -0,0 +1,3 @@ +- name: Install ZFS extstorage + ansible.builtin.include_role: + name: zfs-extstorage diff --git a/roles/zfs-extstorage/tasks/main.yml b/roles/zfs-extstorage/tasks/main.yml new file mode 100644 index 0000000..43f9239 --- /dev/null +++ b/roles/zfs-extstorage/tasks/main.yml @@ -0,0 +1,41 @@ +- name: Clone ZFS extsotarge module + ansible.builtin.git: + repo: https://github.com/brigriffin/ganeti-extstorage-zfs.git + dest: /usr/share/ganeti/extstorage/zfs + single_branch: true + version: master + force: true + +- name: Set zpool for extstorage module + ansible.builtin.template: + src: exstorage.sh.j2 + dest: /usr/share/ganeti/extstorage/zfs/etc/ganeti-{{ groups['nodes'].index(inventory_hostname) + 1 }}.sh + mode: "0644" + +- name: Make everything executable + ansible.builtin.file: + dest: /usr/share/ganeti/extstorage/zfs + recurse: true + mode: "0755" + +- name: Enable ext template + ansible.builtin.command: + cmd: /usr/share/ganeti/extstorage/zfs/install/1-enable-ext-template.sh + chdir: /usr/share/ganeti/extstorage/zfs/install/ + register: enable_ext_template_result + changed_when: enable_ext_template_result.rc == 0 + when: inventory_hostname in groups['master'] + +- name: Create log directory + ansible.builtin.command: + cmd: /usr/share/ganeti/extstorage/zfs/install/2-create-log-directory.sh + chdir: /usr/share/ganeti/extstorage/zfs/install/ + creates: /var/log/ganeti/extstorage + register: create_log_directory_result + +- name: Create lvm wrappers + ansible.builtin.command: + cmd: /usr/share/ganeti/extstorage/zfs/install/3-lvm-wrappers.sh + chdir: /usr/share/ganeti/extstorage/zfs/install/ + register: lvm_wrappers_result + changed_when: lvm_wrappers_result.rc == 0 diff --git a/roles/zfs-extstorage/templates/exstorage.sh.j2 b/roles/zfs-extstorage/templates/exstorage.sh.j2 new file mode 100644 index 0000000..fba036e --- /dev/null +++ b/roles/zfs-extstorage/templates/exstorage.sh.j2 @@ -0,0 +1 @@ +EXTP_ZFS={{ zpool_name }} \ No newline at end of file diff --git a/worker.yml b/worker.yml new file mode 100644 index 0000000..586b048 --- /dev/null +++ b/worker.yml @@ -0,0 +1,6 @@ +- name: Cluster worker setup + hosts: workers + become: true + gather_facts: true + roles: + - worker