Compare commits

...

7 Commits
main ... xen

26 changed files with 572 additions and 91 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
.vscode

View File

@ -3,7 +3,7 @@ Setup a Ganeti cluster on Debian VMs.
## How to use
- First, edit the inventory file.
- Disable secure boot (if using UEFI).
- DON'T use UEFI
- Then:
```sh
# allow ansible to use the ssh key
@ -16,10 +16,10 @@ ansible-playbook -i inventory -u root node.yml
# setup master
ansible-playbook -i inventory -u root master.yml
# setup workers
ansible-playbook -i inventory -u root worker.yml
# setup ganeti web manager
ansible-playbook -i inventory -u root web-manager.yml
```
## Features
- static IP using systemd-networkd (using the last IP of the server before running the playbook)
- zfs extstorage
- ganeti web manager

View File

@ -2,14 +2,19 @@ packages:
- git
- lvm2
- linux-headers-amd64
- zfs-dkms
- zfsutils-linux
- ganeti
- ganeti-instance-debootstrap
- drbd-utils
- socat
- python3
- xen-hypervisor
- xen-hypervisor-common
- xen-system-amd64
- xen-utils
- xen-tools
- qemu-system-x86
- qemu-system-xen
- qemu-utils
- systemd-resolved # needs to be the last one
# breaks dns resolution until
# systemd-networkd is configured
@ -22,7 +27,7 @@ bridge_name: xenbr0
mac_prefix: "02:42:ac"
gateway: "192.168.50.254"
dns_servers:
- "192.168.11.1"
- "192.168.1.1"
- "1.1.1.1"
# hostnames:
@ -40,16 +45,18 @@ hostnames:
name: test-33.ganeti
# storage
zpool_name: ganeti-pool
zpool_dev: /dev/vdc
vg_name: xenvg
pvs:
- /dev/vdb
- /dev/vda6
# ganeti
instance_debootstrap:
- name: default
arch: amd64
suite: bookworm
extra_pkgs: "acpi-support-base,udev,linux-image-amd64,sudo,vim,grub-pc,openssh-server"
extra_pkgs: "linux-image-amd64"
rapi:
- name: admin
hash: b0e8418ff15cda34f6942ead9ed96aae
write: true

View File

@ -1,11 +1,12 @@
[nodes]
192.168.50.20
192.168.50.21
192.168.50.22
; 192.168.50.21
[master]
192.168.50.20
[workers]
192.168.50.21
192.168.50.22
; 192.168.50.21
[web-manager]
192.168.50.20

View File

@ -9,7 +9,7 @@
ansible.builtin.command:
cmd: |-
gnt-cluster init
--enabled-hypervisors kvm
--enabled-hypervisors xen-hvm
--no-etc-hosts
--master-netdev {{ bridge_name }}
--nic-parameters link={{ bridge_name }},mode=bridged
@ -31,7 +31,3 @@
register: node_add_result
changed_when: node_add_result.rc == 0
loop: "{{ groups['workers'] }}"
- name: Install ZFS extstorage
ansible.builtin.include_role:
name: zfs-extstorage

View File

@ -0,0 +1,21 @@
#!/bin/sh
set -e
# Make sure we're not working on the root directory
if [ -z "$TARGET" -o "$TARGET" = "/" ]; then
echo "Invalid target directory '$TARGET', aborting." 1>&2
exit 1
fi
if [ "$(mountpoint -d /)" = "$(mountpoint -d "$TARGET")" ]; then
echo "The target directory seems to be the root dir, aborting." 1>&2
exit 1
fi
# Disable root's password, as the switch to enable shadow by default
# has left root with a disabled password, preventing the initial login
echo "Disabling root's password"
chroot "$TARGET" passwd -d root
exit 0

View File

@ -0,0 +1,57 @@
#!/bin/bash
#
# This is an example script that install and configure grub after installation.
# To use it put it in your CUSTOMIZE_DIR and make it executable.
#
# Do not include grub in EXTRA_PKGS of
# $sysconfdir/default/ganeti-instance-debootstrap because it will
# cause error of debootstrap.
set -e
. common.sh
CLEANUP=( )
trap cleanup EXIT
if [ -z "$TARGET" -o ! -d "$TARGET" ]; then
echo "Missing target directory"
exit 1
fi
# install grub
export LANG=C
if [ "$PROXY" ]; then
export http_proxy="$PROXY"
export https_proxy="$PROXY"
fi
umount_sys_fuse()
{
umount $TARGET/sys/fs/fuse/connections 2>/dev/null || true
}
mount --bind /dev $TARGET/dev
CLEANUP+=("umount $TARGET/dev")
mount --bind /proc $TARGET/proc
CLEANUP+=("umount $TARGET/proc")
mount --bind /sys $TARGET/sys
CLEANUP+=("umount $TARGET/sys")
CLEANUP+=("umount_sys_fuse")
DEBIAN_FRONTEND=noninteractive chroot "$TARGET" apt-get -y install grub-pc grub-common
echo "(hd0) $BLOCKDEV" > $TARGET/boot/grub/device.map
CLEANUP+=("rm $TARGET/boot/grub/device.map")
chroot "$TARGET" sed -Ei 's/^(GRUB_CMDLINE_LINUX=\".*)\"$/\1 net.ifnames=0"/' /etc/default/grub
chroot "$TARGET" grub-install "(hd0)"
GRUB_DISABLE_OS_PROBER=true chroot "$TARGET" update-grub
echo 'grub-pc grub-pc/install_devices multiselect /dev/xvda' | chroot "$TARGET" debconf-set-selections
# execute cleanups
cleanup
trap - EXIT
exit 0

View File

@ -6,6 +6,13 @@
cmd: >
update-initramfs -k all -u
- name: Update grub
listen:
- update grub
ansible.builtin.command:
cmd: >
update-grub
- name: Reboot
listen:
- reboot

View File

@ -4,3 +4,22 @@
dest: "/etc/ganeti/instance-debootstrap/variants/{{ item.name }}.conf"
mode: "0644"
loop: "{{ instance_debootstrap }}"
- name: List variants
ansible.builtin.template:
src: etc-ganeti-instance-debootstrap-variants.list.j2
dest: "/etc/ganeti/instance-debootstrap/variants.list"
mode: "0644"
- name: Install hooks
ansible.builtin.copy:
src: ganeti/
dest: /etc/ganeti
mode: "0755"
- name: Fix hook permissions
ansible.builtin.file:
path: "/etc/ganeti/instance-debootstrap/hooks/"
state: directory
mode: "0755"
recurse: true

View File

@ -0,0 +1,26 @@
- name: Make sure the directory exists
ansible.builtin.file:
state: directory
path: "/var/lib/ganeti/rapi"
mode: "0644"
- name: Add RAPI users
ansible.builtin.template:
src: var-lib-ganeti-rapi-users.j2
dest: /var/lib/ganeti/rapi/users
mode: "0644"
- name: Configure RAPI arguments
ansible.builtin.lineinfile:
regex: "RAPI_ARGS=.*"
line: RAPI_ARGS="-b 0.0.0.0 --require-authentication"
path: /etc/default/ganeti
- name: Clear VNC password
ansible.builtin.file:
path: /etc/ganeti/vnc-cluster-password
mode: "0644"
- name: Include debootstrap
ansible.builtin.include_tasks:
file: debootstrap.yml

View File

@ -18,6 +18,10 @@
ansible.builtin.include_tasks:
file: modules.yml
- name: Configure Xen
ansible.builtin.include_tasks:
file: xen.yml
- name: Create storages
ansible.builtin.include_tasks:
file: storage.yml
@ -26,6 +30,6 @@
ansible.builtin.include_tasks:
file: keys.yml
- name: Configure debootstrap
- name: Configure Ganeti
ansible.builtin.include_tasks:
file: debootstrap.yml
file: ganeti.yml

View File

@ -1,9 +1,3 @@
- name: Enable ZFS
community.general.modprobe:
name: zfs
state: present
persistent: present
- name: Enable KVM
community.general.modprobe:
name: kvm

View File

@ -1,20 +1,3 @@
- name: Create zpool
ansible.builtin.command:
cmd: zpool create {{ zpool_name }} {{ zpool_dev }}
creates: /{{ zpool_name }}
- name: Check if the folder exists
ansible.builtin.stat:
path: /usr/share/ganeti/extstorage/zfs
register: folder_check
- name: Reinstall lvm2 if ZFS extstorage is installed
ansible.builtin.command:
cmd: apt reinstall lvm2
register: lvm2_reinstall_result
changed_when: lvm2_reinstall_result.rc == 0
when: folder_check.stat.exists
- name: Create LVM vg
community.general.lvg:
vg: "{{ vg_name }}"

100
roles/node/tasks/xen.yml Normal file
View File

@ -0,0 +1,100 @@
- name: "Set Xen commandline"
ansible.builtin.lineinfile:
path: "/etc/default/grub.d/xen.cfg"
regexp: '^GRUB_CMDLINE_XEN_DEFAULT=.*$'
line: 'GRUB_CMDLINE_XEN_DEFAULT="dom0_mem=1024M,max:1024M dom0_max_vcpus=2 dom0_vcpus_pin loglvl=all guest_loglvl=all iommu=debug,verbose apic_verbosity=debug ivrs_ioapic[0]=00:14.0" console=com1 com1=115200'
notify:
- update grub
- name: "Set CPU pinning from dom0"
ansible.builtin.replace:
path: "/etc/xen/xl.conf"
regexp: '^#vm.cpumask=.*$'
replace: 'vm.cpumask="2-7"'
notify:
- update grub
- name: "Disable ballooning for dom0"
ansible.builtin.replace:
path: "/etc/xen/xl.conf"
regexp: '^#autoballoon=.*$'
replace: 'autoballoon="0"'
notify:
- update grub
- name: "Disable domain saving"
ansible.builtin.lineinfile:
path: "/etc/default/xendomains"
regexp: '^XENDOMAINS_SAVE=.*$'
line: 'XENDOMAINS_SAVE='
notify:
- update grub
- name: "Disable domain restore"
ansible.builtin.lineinfile:
path: "/etc/default/xendomains"
regexp: '^XENDOMAINS_RESTORE=.*$'
line: 'XENDOMAINS_RESTORE=false'
notify:
- update grub
- name: "Get latest kernel"
ansible.builtin.command:
cmd: bash -c 'find /boot -name "vmlinuz*" | sort -r | head -1'
register: latest_kernel
changed_when: false
failed_when: latest_kernel.rc != 0
- name: "Symlink to the latest kernel"
ansible.builtin.file:
state: link
src: "{{ latest_kernel.stdout }}"
dest: "/boot/vmlinuz-3-xenU"
- name: "Get latest initrd"
ansible.builtin.command:
cmd: bash -c 'find /boot -name "initrd.img*" | sort -r | head -1'
register: latest_initrd
changed_when: false
failed_when: latest_initrd.rc != 0
- name: "Symlink to initrd"
ansible.builtin.file:
state: link
src: "{{ latest_initrd.stdout }}"
dest: "/boot/initrd.img-3-xenU"
- name: "Get latest kernel config"
ansible.builtin.command:
cmd: bash -c 'find /boot -name "config*" | sort -r | head -1'
register: latest_config
changed_when: false
failed_when: latest_config.rc != 0
- name: "Symlink to kernel config"
ansible.builtin.file:
state: link
src: "{{ latest_config.stdout }}"
dest: "/boot/config-3-xenU"
- name: "Add Xen block drivers to modules"
ansible.builtin.lineinfile:
path: "/etc/initramfs-tools/modules"
regexp: "xen_blkfront"
line: "xen_blkfront"
notify:
- update initramfs
- reboot
- name: "Get latest Xen version"
ansible.builtin.command:
cmd: bash -c 'find /usr/lib -type d -name "xen-*" | sort -r | head -1'
register: latest_xen
changed_when: false
failed_when: latest_xen.rc != 0
- name: "Link Xen to latest version"
ansible.builtin.file:
state: link
src: "{{ latest_xen.stdout }}"
dest: /usr/lib/xen

View File

@ -0,0 +1,3 @@
{% for variant in instance_debootstrap %}
{{ variant.name }}
{% endfor %}

View File

@ -0,0 +1,3 @@
{% for user in rapi %}
{{ user.name }} {HA1}{{ user.hash }} {% if user.write %}write{% endif %}
{% endfor %}

View File

@ -0,0 +1,39 @@
web_manager:
dependencies:
- curl
- tar
- libldap2-dev
- libsasl2-dev
- libssl-dev
- libffi-dev
- build-essential
- make
version: 0.11.1
tz: Europe/Budapest
superuser:
username: admin
email: admin@cluster.ganeti
password: admin
python:
src: https://www.python.org/ftp/python/2.7.18/Python-2.7.18.tgz
pip_src: https://bootstrap.pypa.io/pip/2.7/get-pip.py
dependencies:
- build-essential
- gdb
- lcov
- pkg-config
- libbz2-dev
- libffi-dev
- libgdbm-dev
- libgdbm-compat-dev
- liblzma-dev
- libncurses5-dev
- libreadline6-dev
- libsqlite3-dev
- libssl-dev
- lzma
- lzma-dev
- tk-dev
- uuid-dev
- zlib1g-dev

View File

@ -0,0 +1,99 @@
- name: Install dependencies
ansible.builtin.apt:
name: "{{ item }}"
state: present
install_recommends: false
loop: "{{ web_manager.dependencies }}"
- name: Check for Python 2
ansible.builtin.command:
cmd: python2 -V
register: python2_version
changed_when: false
failed_when: false
- name: Install Python 2.7
ansible.builtin.include_tasks:
file: python.yml
when: python2_version.rc != 0
- name: Clone Ganeti Web Manager
ansible.builtin.git:
repo: "https://github.com/osuosl/ganeti_webmgr.git"
dest: "/tmp/ganeti_webmgr"
force: true
version: "{{ web_manager.version }}"
- name: Link virtualenv for the dumb Ganeti Web Manager setup script
ansible.builtin.file:
state: link
src: /usr/local/bin/virtualenv
dest: /usr/bin/virtualenv
- name: Create virtualenv for Ganeti Web Manager
ansible.builtin.command:
cmd: bash -c "python -m pip install virtualenv && python -m virtualenv /opt/ganeti_webmgr"
register: venv_result
changed_when: venv_result.rc == 0
- name: Fix Ganeti Web Manager setup script
ansible.builtin.replace:
path: "/tmp/ganeti_webmgr/scripts/setup.sh"
regexp: "(--use-wheel)|(--setuptools)|(--no-site-packages)"
replace: ""
- name: Install Ganeti Web Manager
ansible.builtin.command:
cmd: ./scripts/setup.sh -D sqlite -N -u
register: setup_result
changed_when: setup_result.rc == 0
args:
chdir: "/tmp/ganeti_webmgr"
- name: Configure Ganeti Web Manager
ansible.builtin.template:
src: "config.yml.j2"
dest: "/opt/ganeti_webmgr/config/config.yml"
mode: "0644"
vars:
secret_key: "{{ lookup('ansible.builtin.password', '/dev/null', chars=['ascii_letters', 'digits'], length=32) }}"
web_mgr_api_key: "{{ lookup('ansible.builtin.password', '/dev/null', chars=['ascii_letters', 'digits'], length=32) }}"
- name: Install VNCAuthProxy systemd service
ansible.builtin.copy:
src: "/tmp/ganeti_webmgr/scripts/vncauthproxy/init-systemd"
dest: "/lib/systemd/system/vncauthproxy.service"
mode: "0644"
remote_src: true
- name: Enable VNCAuthProxy systemd service
ansible.builtin.systemd_service:
daemon_reload: true
name: vncauthproxy
enabled: true
state: restarted
- name: Init Ganeti Web Manager
ansible.builtin.command:
cmd: bash -c "source /opt/ganeti_webmgr/bin/activate && export DJANGO_SETTINGS_MODULE=ganeti_webmgr.ganeti_web.settings && django-admin.py syncdb --migrate --noinput && django-admin.py refreshcache"
register: init_result
changed_when: init_result.rc == 0
- name: Add superuser
ansible.builtin.command:
cmd: bash -c "source /opt/ganeti_webmgr/bin/activate && export DJANGO_SETTINGS_MODULE=ganeti_webmgr.ganeti_web.settings && echo -e '{{ web_manager.superuser.password }}\n{{ web_manager.superuser.password }}\n' | django-admin.py createsuperuser --username {{ web_manager.superuser.username }} --email {{ web_manager.superuser.email }}"
register: superuser_result
changed_when: superuser_result.rc == 0
- name: Install Ganeti Web Manager systemd service
ansible.builtin.template:
src: "ganeti-web-manager.service.j2"
dest: "/etc/systemd/system/ganeti-web-manager.service"
mode: "0644"
- name: Enable Ganeti Web Manager
ansible.builtin.systemd_service:
daemon_reload: true
name: ganeti-web-manager
enabled: true
state: restarted

View File

@ -0,0 +1,38 @@
- name: Download sources
ansible.builtin.get_url:
url: "{{ python.src }}"
dest: /tmp/python.tgz
mode: "0644"
- name: Extract python sources
ansible.builtin.unarchive:
src: /tmp/python.tgz
dest: /tmp
remote_src: true
- name: Install dependencies
ansible.builtin.apt:
name: "{{ item }}"
state: present
install_recommends: false
loop: "{{ python.dependencies }}"
- name: Compile and install python
ansible.builtin.command:
cmd: bash -c "./configure && make install"
register: python_compile_result
changed_when: python_compile_result.rc == 0
args:
chdir: /tmp/Python-2.7.18
- name: Download get-pip.py
ansible.builtin.get_url:
url: "{{ python.pip_src }}"
dest: /tmp
mode: "0644"
- name: Install pip
ansible.builtin.command:
cmd: python /tmp/get-pip.py
register: pip_install_result
changed_when: pip_install_result.rc == 0

View File

@ -0,0 +1,111 @@
# config.yml
# Django settings for ganeti_webmgr project.
##### Database Configuration #####
DATABASES:
default:
ENGINE: django.db.backends.sqlite3
# django.db.backends.sqlite3
# django.db.backends.postgresql
# django.db.backends.mysql
# django.db.backends.oracle
# django.db.backends.postgresql_psycopg2
# Or path to database file if using sqlite3.
NAME: /opt/ganeti_webmgr/ganeti.db
USER: ""
PASSWORD: ""
HOST: ""
PORT: ""
##### End Database Configuration #####
# Site name and domain referenced by some modules to provide links back to
# the site.
SITE_NAME: Ganeti Web Manager
SITE_DOMAIN: "localhost:8000"
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
TIME_ZONE: "{{ web_manager.tz }}"
DATE_FORMAT: d/m/Y
DATETIME_FORMAT: "d/m/Y H:i"
# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE: "en-US"
##### End Locale Configuration #####
# Enable i18n (translations) and l10n (locales, currency, times).
# You really have no good reason to disable these unless you are only
# going to be using GWM in English.
USE_I18N: True
# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N: True
# prefix used for the site. ie. http://myhost.com/<SITE_ROOT>
# for the django standalone server this should be
# for apache this is the url the site is mapped to, probably /tracker
SITE_ROOT: ""
# Absolute path to the directory that holds media.
# Example: /home/media/media.lawrence.com/
STATIC_ROOT: /opt/ganeti_webmgr/collected_static
# URL that handles the media served from STATIC_ROOT.
# XXX contrary to django docs, do not use a trailing slash. It makes urls
# using this url easier to read. ie. <STATIC_URL>/images/foo.png
STATIC_URL: /static
##### Registration Settings #####
ACCOUNT_ACTIVATION_DAYS: 7
# Email settings for registration
EMAIL_HOST: localhost
EMAIL_PORT: 25
DEFAULT_FROM_EMAIL: noreply@example.org
# Whether users should be able to create their own accounts.
# False if accounts can only be created by admins.
ALLOW_OPEN_REGISTRATION: True
##### End Registration Settings #####
####### Haystack Search Index settings #######
HAYSTACK_WHOOSH_PATH: /opt/ganeti_webmgr/whoosh_index
####### End Haystack Search Index settings #######
# GWM Specifics
# The maximum number of items on a single list page
ITEMS_PER_PAGE: 15
# Ganeti Cached Cluster Objects Timeouts
# LAZY_CACHE_REFRESH (milliseconds) is the fallback cache timer that is
# checked when the object is instantiated. It defaults to 600000ms, or ten
# minutes.
LAZY_CACHE_REFRESH: 600000
# VNC Proxy. This will use a proxy to create local ports that are forwarded to
# the virtual machines. It allows you to control access to the VNC servers.
#
# Expected values:
# String syntax: HOST:CONTROL_PORT, for example: localhost:8888. If
# localhost is used then the proxy will only be accessible to clients and
# browsers on localhost. Production servers should use a publicly accessible
# hostname or IP
#
# Firewall Rules:
# Control Port: 8888, must be open between Ganeti Web Manager and Proxy
# Internal Ports: 12000+ must be open between the Proxy and Ganeti Nodes
# External Ports: default is 7000-8000, must be open between Proxy and Client
# Flash Policy Server: 843, must open between Proxy and Clients
VNC_PROXY: "localhost:8888"
# This is how long gwm will wait before timing out when requesting data from the
# ganeti cluster.
RAPI_CONNECT_TIMEOUT: 3
SECRET_KEY: "{{ secret_key }}"
WEB_MGR_API_KEY: "{{ web_mgr_api_key }}"

View File

@ -0,0 +1,11 @@
[Unit]
Description=Ganeti Web Manager
Wants=network-online.target
After=network-online.target
[Service]
ExecStart=bash -c "source /opt/ganeti_webmgr/bin/activate && django-admin.py runserver 0.0.0.0:8000 --insecure"
Environment="DJANGO_SETTINGS_MODULE=ganeti_webmgr.ganeti_web.settings"
[Install]
WantedBy=multi-user.target

View File

@ -1,3 +0,0 @@
- name: Install ZFS extstorage
ansible.builtin.include_role:
name: zfs-extstorage

View File

@ -1,41 +0,0 @@
- name: Clone ZFS extsotarge module
ansible.builtin.git:
repo: https://github.com/brigriffin/ganeti-extstorage-zfs.git
dest: /usr/share/ganeti/extstorage/zfs
single_branch: true
version: master
force: true
- name: Set zpool for extstorage module
ansible.builtin.template:
src: exstorage.sh.j2
dest: /usr/share/ganeti/extstorage/zfs/etc/ganeti-{{ groups['nodes'].index(inventory_hostname) + 1 }}.sh
mode: "0644"
- name: Make everything executable
ansible.builtin.file:
dest: /usr/share/ganeti/extstorage/zfs
recurse: true
mode: "0755"
- name: Enable ext template
ansible.builtin.command:
cmd: /usr/share/ganeti/extstorage/zfs/install/1-enable-ext-template.sh
chdir: /usr/share/ganeti/extstorage/zfs/install/
register: enable_ext_template_result
changed_when: enable_ext_template_result.rc == 0
when: inventory_hostname in groups['master']
- name: Create log directory
ansible.builtin.command:
cmd: /usr/share/ganeti/extstorage/zfs/install/2-create-log-directory.sh
chdir: /usr/share/ganeti/extstorage/zfs/install/
creates: /var/log/ganeti/extstorage
register: create_log_directory_result
# - name: Create lvm wrappers
# ansible.builtin.command:
# cmd: /usr/share/ganeti/extstorage/zfs/install/3-lvm-wrappers.sh
# chdir: /usr/share/ganeti/extstorage/zfs/install/
# register: lvm_wrappers_result
# changed_when: lvm_wrappers_result.rc == 0

View File

@ -1 +0,0 @@
EXTP_ZFS={{ zpool_name }}

6
web-manager.yml Normal file
View File

@ -0,0 +1,6 @@
- name: Ganeti Web Manager setup
hosts: web-manager
become: true
gather_facts: true
roles:
- web-manager