I was trying to stop myself from...well...anyway.
Problem #1: Windows. Enough said.
Problem #2: Closed source. Ahhh....not gonna work in this day and age.
It's trivial to setup PXE booting and subsequently into most OSes. Netboot XYZ is open source, runs easily within docker and all it needs is some DHCP support. The easiest way to do that is with Dnsmasq. I've even run dnsmasq in "supporting" mode when it was not the main DHCP server, and all it does is add to the DHCP response with PXE details.
Hell, my latest automation adventure led me to build a "bootstrap" server, i.e. the one thing you need before anything else and then Ansible gets to do its thing. What is it? A plain ol Ubuntu server. It's even automated!
Exhibit 1: The autoinstall script for the bootstrap (ubuntu) server itself. It:
- installs Ubuntu to a specific drive, sets up the networking and then installs;
- Docker, Hashicorp vault CLI (you do have secrets...right?) and Ansible (to act as a control node).
- Installs a few apt packages to support everything.
Then a simple shell script that installs and configures all needed containers:
This particular implementation is geared towards automating Proxmox installations, I could as esily (and I've done it in the past) add a Netboot XYZ container and it'll let you install anything it supports. Including Windows...
Problem #1: Windows. Enough said.
Problem #2: Closed source. Ahhh....not gonna work in this day and age.
It's trivial to setup PXE booting and subsequently into most OSes. Netboot XYZ is open source, runs easily within docker and all it needs is some DHCP support. The easiest way to do that is with Dnsmasq. I've even run dnsmasq in "supporting" mode when it was not the main DHCP server, and all it does is add to the DHCP response with PXE details.
Hell, my latest automation adventure led me to build a "bootstrap" server, i.e. the one thing you need before anything else and then Ansible gets to do its thing. What is it? A plain ol Ubuntu server. It's even automated!
Exhibit 1: The autoinstall script for the bootstrap (ubuntu) server itself. It:
- installs Ubuntu to a specific drive, sets up the networking and then installs;
- Docker, Hashicorp vault CLI (you do have secrets...right?) and Ansible (to act as a control node).
- Installs a few apt packages to support everything.
Code:
#cloud-config
# See the autoinstall documentation at:
# https://canonical-subiquity.readthedocs-hosted.com/en/latest/reference/autoinstall-reference.html
autoinstall:
active-directory:
admin-name: ''
domain-name: ''
apt:
disable_components: []
fallback: offline-install
geoip: false
mirror-selection:
primary:
- uri: http://us.archive.ubuntu.com/ubuntu/
preserve_sources_list: false
security:
- arches:
- amd64
- arm64
- i386
uri: http://security.ubuntu.com/ubuntu/
- arches:
- armhf
- powerpc
- ppc64el
- riscv64
- s390x
uri: http://ports.ubuntu.com/ubuntu-ports
codecs:
install: false
drivers:
install: false
identity:
hostname: vp-bootstrap
password: blah blah
realname: Administrator
username: vpadmin
kernel:
package: linux-generic
keyboard:
layout: us
toggle: null
variant: ''
locale: en_US.UTF-8
network:
ethernets:
enp2s0:
dhcp-identifier: mac
dhcp4: true
dhcp6: false
version: 2
oem:
install: auto
packages:
- network-manager
- cockpit
- cockpit-files
- pcp
- git
- wget
- unzip
- net-tools
- yq
- gpg
- curl
- lsb-release
- software-properties-common
source:
id: ubuntu-server
search_drivers: true
ssh:
allow-pw: true
authorized-keys: []
install-server: true
storage:
config:
- ptable: gpt
match:
serial: SH9MST6D128GHM*
path: /dev/sda
wipe: superblock-recursive
preserve: false
name: ''
grub_device: false
id: disk-sda
type: disk
- device: disk-sda
size: 1127219200
wipe: superblock
flag: boot
number: 1
preserve: false
grub_device: true
offset: 1048576
path: /dev/sda1
id: partition-0
type: partition
- fstype: fat32
volume: partition-0
preserve: false
id: format-0
type: format
- device: disk-sda
size: 126906007552
wipe: superblock
number: 2
preserve: false
offset: 1128267776
path: /dev/sda2
id: partition-1
type: partition
- fstype: ext4
volume: partition-1
preserve: false
id: format-1
type: format
- path: /
device: format-1
id: mount-1
type: mount
- path: /boot/efi
device: format-0
id: mount-0
type: mount
updates: security
late-commands:
- curtin in-target --target=/target -- mkdir -p /opt/config
- curtin in-target --target=/target -- sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config
- curtin in-target --target=/target -- sed -i 's/^[# ]*PasswordAuthentication[[:space:]]\+no/PasswordAuthentication yes/' /etc/ssh/sshd_config
user-data:
disable_root: false
runcmd:
#--------------------------------Configure network
- rm /etc/netplan/00*.yaml
- netplan apply
- systemctl stop systemd-networkd.service systemd-networkd.socket systemd-networkd-varlink.socket systemd-networkd-resolve-hook.socket systemd-networkd-wait-online.service networkd-dispatcher.service
- systemctl disable systemd-networkd.service systemd-networkd.socket systemd-networkd-varlink.socket systemd-networkd-resolve-hook.socket systemd-networkd-wait-online.service networkd-dispatcher.service
- systemctl mask systemd-networkd.service systemd-networkd.socket systemd-networkd-varlink.socket systemd-networkd-resolve-hook.socket systemd-networkd-wait-online.service networkd-dispatcher.service
- systemctl restart NetworkManager
#--------------------------------This is the isp network
# - nmcli connection add type ethernet con-name isp ifname enp2s0 ipv4.method auto ipv6.method disabled
#--------------------------------These are other networks that are/may be needed.
# - nmcli connection add type ethernet con-name wol ifname eno1 ipv4.method manual ipv4.addresses 10.10.2.253/24 ipv4.gateway 10.10.2.254 ipv6.method disabled ipv4.never-default yes
- nmcli connection add type ethernet con-name homelan ifname eno1 ipv4.method manual ipv4.addresses 10.10.20.253/24 ipv4.gateway 10.10.20.254 ipv6.method disabled ipv4.never-default yes
- nmcli connection add type vlan con-name oob ifname eno1.13 dev eno1 id 13 ipv4.method manual ipv4.addresses 10.10.13.253/24 ipv4.gateway 10.10.13.254 ipv6.method disabled ipv4.never-default yes
- nmcli connection add type vlan con-name infra ifname eno1.14 dev eno1 id 14 ipv4.method manual ipv4.addresses 10.10.14.253/24 ipv4.gateway 10.10.14.254 ipv6.method disabled # default route
- nmcli connection add type vlan con-name services ifname eno1.15 dev eno1 id 15 ipv4.method manual ipv4.addresses 10.10.15.253/24 ipv4.gateway 10.10.15.254 ipv6.method disabled ipv4.never-default yes
- nmcli connection add type vlan con-name provision ifname eno1.999 dev eno1 id 999 ipv4.method manual ipv4.addresses 172.16.100.253/24 ipv4.gateway 172.16.100.254 ipv6.method disabled ipv4.never-default yes # provisioning network
- nmcli connection up eno1
#--------------------------------Configure system parameters
- timedatectl set-timezone America/New_York
- systemctl enable pmcd pmlogger #Not always needed.
- echo 'blacklist spi_intel_pci' | sudo tee /etc/modprobe.d/blacklist-spi_intel_pci.conf #Not always needed.
#--------------------------------Configure linux command line
- sed -i '/GRUB_CMDLINE_LINUX_DEFAULT=/c\GRUB_CMDLINE_LINUX_DEFAULT="quiet mitigations=off"' /etc/default/grub #Remove mitigations=off if that concerns you, it doesn't to e. This is an internal network/server.
- sed -i '/GRUB_TIMEOUT=/c\GRUB_TIMEOUT=0' /etc/default/grub
- update-grub
#--------------------------------Install Docker
- install -m 0755 -d /etc/apt/keyrings
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
- chmod a+r /etc/apt/keyrings/docker.asc
- echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
- apt update
- apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
#--------------------------------Install Hashicorp vault cli
# Add the HashiCorp GPG key
- wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
# Add the official HashiCorp repository
- echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
# Update packages and install Vault
- apt update && apt install vault
#--------------------------------Update and Upgrade
- apt update
- apt upgrade -y
#--------------------------------
- reboot
users:
- lock_passwd: false
name: root
passwd: blah blah
ssh_authorized_keys:
- blah blah
write_files:
- content: "network:\n version: 2\n renderer: NetworkManager\n"
path: /etc/netplan/01-netcfg.yaml
version: 1
Code:
#--------------------------------Variables
PROVISION_NET_IP=$(nmcli -g IP4.ADDRESS connection show "provision" | head -n1 | cut -d/ -f1)
echo "The Provisioning IP address is: ${PROVISION_NET_IP}"
#--------------------------------Destroy previous containers, if any
docker compose -p pve-qdevice -f /opt/qdevice/docker-compose.yaml down -v
docker compose -p dnsmasq -f /opt/dnsmasq/docker-compose.yaml down -v
docker compose -p technitium_vp -f /opt/technitium_vp/docker-compose.yaml --env-file /opt/technitium_vp/compose.env down -v
docker compose -p autopve -f /opt/autopve/docker-compose.yaml down -v
docker compose -p komodo -f /opt/komodo/docker-compose.yaml --env-file /opt/komodo/compose.env down -v
docker compose -p hashi-vault1 -f /opt/hashi-vault1/docker-compose.yaml --env-file /opt/hashi-vault1/compose.env down -v
#--------------------------------Cleanup directories
rm -r /opt/autopve
rm -r /opt/dnsmasq
rm -r /opt/komodo
rm -r /opt/technitium_vp
rm -r /opt/qdevice
# rm -r /opt/semaphore
rm -r /opt/hashi-vault1
rm unseal.creds
#--------------------------------Create directories
mkdir -p /opt/autopve/data/files
mkdir -p /opt/autopve/data/playbooks/bare_metal/{api_secrets,roles,group_vars,host_vars}
mkdir -p /opt/dnsmasq/tftproot
mkdir -p /opt/komodo
mkdir -p /opt/qdevice
mkdir -p /opt/technitium_vp
mkdir -p /opt/hashi-vault1/{data,config,file,logs,certs}
chown -R 100:1000 /opt/hashi-vault1/data
chown -R 100:1000 /opt/hashi-vault1/logs
# chown -R 100:1000 /opt/hashi-vault1/certs
# chown -R 100:1000 /opt/hashi-vault1/file
#--------------------------------Install Hashicorp Vault
cp /opt/config/bootstrap2/14c/docker/hashi_vault/docker-compose.yaml /opt/hashi-vault1/
cp /opt/config/bootstrap2/14c/docker/hashi_vault/compose.env /opt/hashi-vault1/
cp /opt/config/bootstrap2/14c/docker/hashi_vault/config.hcl /opt/hashi-vault1/config/
# Set IP address to the provisioning IP in config files
# sed -i "/api_addr =/c\api_addr = \"http://${PROVISION_NET_IP}:8200\"" /opt/hashi-vault1/config/config.hcl
# sed -i "/cluster_addr =/c\cluster_addr = \"http://${PROVISION_NET_IP}:8201\"" /opt/hashi-vault1/config/config.hcl
# sed -i "/VAULT_ADDR =/c\VAULT_ADDR: \"http://${PROVISION_NET_IP}:8200\"" /opt/hashi-vault1/compose.env
# sed -i "/VAULT_API_ADDR =/c\VAULT_API_ADDR: \"http://${PROVISION_NET_IP}:8200\"" /opt/hashi-vault1/compose.env
# sed -i "/VAULT_ADDRESS =/c\VAULT_ADDRESS: \"http://${PROVISION_NET_IP}:8200\"" /opt/hashi-vault1/compose.env
# sed -i "/ - 8200:/c\ - ${PROVISION_NET_IP}:8200:8200" /opt/hashi-vault1/docker-compose.yaml
# sed -i "/ - 8201:/c\ - ${PROVISION_NET_IP}:8201:8201" /opt/hashi-vault1/docker-compose.yaml
cp /opt/config/secrets.json /opt/hashi-vault1/secrets.json
chown -R 100:1000 /opt/hashi-vault1//secrets.json
docker compose -p hashi-vault1 -f /opt/hashi-vault1/docker-compose.yaml --env-file /opt/hashi-vault1/compose.env up -d
echo 'Sleeping for 10 seconds...'
sleep 10
#---------------Setup Vault-------------------------
export VAULT_ADDR="http://10.10.14.253:8200"
## unseal
vault operator init | tee -a unseal.creds
for NUM in {1..3}; do
vault operator unseal \
$(grep -oP "(?<=Unseal Key $NUM: ).*" unseal.creds)
done
# export the results for use in other steps
export VAULT_ROOT_TOKEN="$(
grep -oP "(?<=Initial Root Token: ).*" unseal.creds
)"
vault login $VAULT_ROOT_TOKEN
#---------For TRANSIT secrets------------------------
vault audit enable file file_path=/vault/logs/audit.log
vault secrets enable transit
vault write -f transit/keys/autounseal
vault policy write autounseal -<<EOF
path "transit/encrypt/autounseal" {
capabilities = [ "update" ]
}
path "transit/decrypt/autounseal" {
capabilities = [ "update" ]
}
EOF
vault token create -orphan -policy="autounseal" \
-wrap-ttl=120 -period=24h \
-field=wrapping_token > wrapping-token.txt
# --------------------------------------------------------
vault auth enable approle # Enable approle based auth.
vault secrets enable -path=secret -version=2 kv #Enable KV v2
# Ansible Policy
vault policy write ansible -<<EOF
path "secret/data/ansible/*" {
capabilities = [ "read" ]
}
EOF
# Admin Policy
vault policy write admin -<<EOF
path "secret/data/*" {
capabilities = [ "create", "read", "update", "delete", "patch", "list" ]
}
path "secret/metadata/ansible/*" {
capabilities = [ "create", "read", "update", "delete", "patch", "list" ]
}
# Mount the AppRole auth method
path "sys/auth/approle" {
capabilities = [ "create", "read", "update", "delete", "sudo" ]
}
# Configure the AppRole auth method
path "sys/auth/approle/*" {
capabilities = [ "create", "read", "update", "delete" ]
}
# Create and manage roles
path "auth/approle/*" {
capabilities = [ "create", "read", "update", "delete", "list" ]
}
# Write ACL policies
path "sys/policies/acl/*" {
capabilities = [ "create", "read", "update", "delete", "list" ]
}
EOF
# ##############
# # create admin role
# ############################
vault write auth/approle/role/admin \
policies="admin" \
token_ttl="1h" \
token_max_ttl="4h"
# ##############
# # create ansible role
# ############################
vault write auth/approle/role/ansible \
policies="ansible" \
token_ttl="1h" \
token_max_ttl="4h"
##############
# login using admin role
############################
ROLE_ID=$(vault read auth/approle/role/admin/role-id -format=json \
| jq -r .data.role_id
)
SECRET_ID=$(vault write -f auth/approle/role/admin/secret-id -format=json \
| jq -r .data.secret_id)
# generate token for admin role
VAULT_ADMIN_TOKEN=$(vault write auth/approle/login \
role_id="$ROLE_ID" \
secret_id="$SECRET_ID" \
--format=json \
| jq -r .auth.client_token
)
# login using admin token
vault login $VAULT_ADMIN_TOKEN
#Upload secrets.
vault kv put secret/ansible/bare_metal/passwords @/opt/hashi-vault1/secrets.json
#cleanup
rm /opt/hashi-vault1/secrets.json
# #rm /opt/config/secrets.json
# -------------------------------DEBUGGING---------------------------
# login using ansible role
############################
export ANSIBLE_HASHI_VAULT_ROLE_ID=$(vault read auth/approle/role/ansible/role-id -format=json \
| jq -r .data.role_id
)
export ANSIBLE_HASHI_VAULT_SECRET_ID=$(vault write -f auth/approle/role/ansible/secret-id -format=json \
| jq -r .data.secret_id)
VAULT_ANSIBLE_TOKEN=$(vault write auth/approle/login \
role_id="$ANSIBLE_HASHI_VAULT_ROLE_ID" \
secret_id="$ANSIBLE_HASHI_VAULT_SECRET_ID" \
--format=json \
| jq -r .auth.client_token
)
vault login $VAULT_ANSIBLE_TOKEN
# Set environment secrets for other containers.
export KOMODO_ADMIN_PASSWORD=$(vault kv get -mount=secret -field=komodo_admin_password ansible/bare_metal/passwords)
export DNS_SERVER_PASSWORD=$(vault kv get -mount=secret -field=technitium_admin_password ansible/bare_metal/passwords)
export QDEVICE_ROOT_PASSWORD=$(vault kv get -mount=secret -field=proxmox_root_password ansible/bare_metal/passwords)
#-------------------------------------------------------------------
#--------------------------------Configure Komodo
cp /opt/config/bootstrap2/14c/docker/komodo/docker-compose.yaml /opt/komodo/
cp /opt/config/bootstrap2/14c/docker/komodo/compose.env /opt/komodo/
# sed -i "/KOMODO_HOST=/c\KOMODO_HOST=http://${PROVISION_NET_IP}:9120" /opt/komodo/compose.env
sed -i '/KOMODO_DATABASE_PASSWORD=/c\KOMODO_DATABASE_PASSWORD=6351ce0d611a7c85ffa1b4f0e4cf0af7' /opt/komodo/compose.env
sed -i '/KOMODO_WEBHOOK_SECRET=/c\KOMODO_WEBHOOK_SECRET=051efb10e44574f6803c01cbeeda3ddb' /opt/komodo/compose.env
sed -i '/KOMODO_JWT_SECRET=/c\KOMODO_JWT_SECRET=84b5d0354b97bb7afb9dc90e59269b1fdb9967dced9238c2' /opt/komodo/compose.env
sed -i '/KOMODO_DISABLE_CONFIRM_DIALOG=/c\KOMODO_DISABLE_CONFIRM_DIALOG=true' /opt/komodo/compose.env
sed -i '/TZ=/c\TZ=America/\New_York' /opt/komodo/compose.env
# sed -i "/ - 9120:/c\ - ${PROVISION_NET_IP}:9120:9120" /opt/komodo/docker-compose.yaml
docker compose -p komodo -f /opt/komodo/docker-compose.yaml --env-file /opt/komodo/compose.env up -d
echo 'Sleeping for 10 seconds...'
sleep 10
#--------------------------------Configure Technitium
cp /opt/config/bootstrap2/14c/docker/technitium_vp/docker-compose.yaml /opt/technitium_vp/
cp /opt/config/bootstrap2/14c/docker/technitium_vp/compose.env /opt/technitium_vp/
docker compose -p technitium_vp -f /opt/technitium_vp/docker-compose.yaml --env-file /opt/technitium_vp/compose.env up -d
echo 'Sleeping for 10 seconds...'
sleep 10
#--------------------------------Configure dnsmasq
cp /opt/config/bootstrap2/14c/docker/dnsmasq/docker-compose.yaml /opt/dnsmasq/
cp /opt/config/bootstrap2/14c/docker/dnsmasq/dnsmasq.conf /opt/dnsmasq/
cp -r /opt/config/bootstrap2/files/pxe/. /opt/dnsmasq/tftproot/
sed -i "/dhcp-boot=tag:ipxe-x86_64/c\dhcp-boot=tag:ipxe-x86_64,http://${PROVISION_NET_IP}:8080/files/proxmox_network_boot/current/boot.ipxe" /opt/dnsmasq/dnsmasq.conf
sed -i "/dhcp-option=250/c\dhcp-option=250,http://${PROVISION_NET_IP}:8080/answer" /opt/dnsmasq/dnsmasq.conf
docker compose -p dnsmasq -f /opt/dnsmasq/docker-compose.yaml up -d
echo 'Sleeping for 10 seconds...'
sleep 10
#--------------------------------Install autopve
#ssh-keygen -t rsa -b 4096 -f /opt/autopve/data/id_rsa -C "autopve@vppp.com" -N "" -q
cp -r /opt/config/bootstrap2/keys/ssh/ansible/. /opt/autopve/data/ #Ansible RSA keys
#change permissions on the private key file
chmod 600 /opt/autopve/data/id_rsa
cp /opt/config/bootstrap2/14c/docker/autopve/docker-compose.yaml /opt/autopve/
cp -r /opt/config/bootstrap2/files/proxmox_network_boot /opt/autopve/data/files #These are the proxmox boot files.
cp -r /opt/config/ansible/roles/. /opt/autopve/data/playbooks/bare_metal/roles/
cp -r /opt/config/ansible/inventories/14c/host_vars/. /opt/autopve/data/playbooks/bare_metal/host_vars/
cp -r /opt/config/ansible/inventories/14c/group_vars/. /opt/autopve/data/playbooks/bare_metal/group_vars/
cp /opt/config/ansible/playbooks/bare_metal_14c.yaml /opt/autopve/data/playbooks/bare_metal/playbook.yaml
cp -r /opt/config/bootstrap2/14c/autopve/. /opt/autopve/data/playbooks/bare_metal/
cp /opt/config/bootstrap2/files/autopve/storage-general.json /opt/autopve/data/
sed -i "s|xxx|http://${PROVISION_NET_IP}:8080/playbook/bare_metal|g" /opt/autopve/data/storage-general.json
cp -r /opt/config/ansible/. /opt/autopve/data/ansible/
# #--------------------------------create storage-general.json file.
# unset AUTOPVE_CUSTOM_ANSWER
# unset AUTOPVE_ANSWER
# AUTOPVE_CUSTOM_ANSWER=$(yq -r '.proxmox_default_answer' /opt/autopve/data/playbooks/bare_metal/group_vars/all.yaml | tr -d '\n' | jq)
# dir_path="/opt/autopve/data/playbooks/bare_metal/host_vars"
# for file in "$dir_path"/*.yaml; do
# # Check if it is a regular file (skips folders)
# if [ -f "$file" ]; then
# AUTOPVE_ANSWER=$(yq -r '.proxmox.answer' "$file" | tr -d '\n' | jq)
# AUTOPVE_CUSTOM_ANSWER=$(echo "$AUTOPVE_CUSTOM_ANSWER" | jq ".config_100 += $AUTOPVE_ANSWER")
# fi
# done
# echo "$AUTOPVE_CUSTOM_ANSWER" > /opt/autopve/data/storage-general.json #This is the answer file
# #--------------------------------
docker compose -p autopve -f /opt/autopve/docker-compose.yaml up -d
echo 'Sleeping for 10 seconds...'
sleep 10
# docker exec -it autopve ansible-galaxy collection install mellanox.onyx #Install Mellanox ansible collection
# docker exec -it autopve ansible-galaxy collection install commscope.icx #Install Brocade/CommScope ansible collection
# docker exec -it autopve pip install jmespath "proxmoxer>=2.0" requests #For JSON queries
docker exec -it autopve apt-get install bzip2 #For extracting ISO files
docker exec -it autopve pip install hvac #For Hashicorp vault support
docker exec -it autopve pip3 install netaddr #For IP address manipulation support
# docker exec -it autopve ansible-galaxy collection install mipsou.technitium #Install Technitium management ansible collection
docker exec -it autopve ansible-galaxy collection install effectivelywild.technitium_dns #Install Technitium management ansible collection
docker exec -it autopve ansible-galaxy role install bpbradley.komodo #Install Komodo periphery management ansible collection
#--------------------------------Configure Qdevice
cp /opt/config/bootstrap2/14c/docker/qdevice/docker-compose.yaml /opt/qdevice/
docker compose -p pve-qdevice -f /opt/qdevice/docker-compose.yaml up -d
Last edited:
