diff options
48 files changed, 3131 insertions, 7 deletions
@@ -13,7 +13,7 @@ frustration with Ansible. To execute boxconf on a target host, just run the following: - ./boxconf $TARGET_HOSTNAME + ./boxconf $HOSTNAME A deployment tarball will be generated and SCP'd to the remote box, where `boxconf` will re-exec itself. After gathering some information about the target system (such @@ -48,9 +48,8 @@ The `site/` directory does not exist in this repo. Its purpose is to hold person site-specific variables and scripts that you would rather not share in a public git repo. Ideally, you would use git submodules for this. -The `hostname` value is taken from the short hostname of the remote system. -If the remote hostname is incorrect (or unset), you can override the hostname -detection by passing the `-o $HOSTNAME` flag to boxconf. +If the hostname does not exist in DNS, you can manually specify the SSH +target by passing the `-s $IP_ADDRESS` option to `boxconf`. The `hostclass` value is matched based on the regular expressions listed in the [hostclasses](./hostclasses) file. diff --git a/files/boot.config.freebsd b/files/boot.config.freebsd new file mode 100644 index 0000000..73d9cb4 --- /dev/null +++ b/files/boot.config.freebsd @@ -0,0 +1 @@ +-D -S115200 diff --git a/files/etc/aliases.freebsd b/files/etc/aliases.freebsd new file mode 100644 index 0000000..b0aeb2d --- /dev/null +++ b/files/etc/aliases.freebsd @@ -0,0 +1,38 @@ +# All local mail should end up forwarded to this address: +root: ${root_mail_alias} + +# Basic system aliases -- these MUST be present +MAILER-DAEMON: postmaster +postmaster: root + +_dhcp: root +_pflogd: root +auditdistd: root +bin: root +bind: root +daemon: root +games: root +hast: root +kmem: root +mailnull: postmaster +man: root +news: root +nobody: root +operator: root +pop: root +proxy: root +smmsp: postmaster +sshd: root +system: root +toor: root +tty: root +usenet: news +uucp: root +manager: root +dumper:root + +# NETWORK OPERATIONS MAILBOX NAMES +abuse: root +security: root +ftp: root +ftp-bugs: ftp diff --git a/files/etc/cron.d/zfs-trim.freebsd b/files/etc/cron.d/zfs-trim.freebsd new file mode 100644 index 0000000..64b07b9 --- /dev/null +++ b/files/etc/cron.d/zfs-trim.freebsd @@ -0,0 +1,3 @@ +SHELL=/bin/sh +PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin +@weekly root zfs list -Ho name | xargs -r -n1 zpool trim diff --git a/files/etc/devfs.rules.freebsd_hypervisor b/files/etc/devfs.rules.freebsd_hypervisor new file mode 100644 index 0000000..fe40b9c --- /dev/null +++ b/files/etc/devfs.rules.freebsd_hypervisor @@ -0,0 +1,4 @@ +# Allow jails to access bpf device for DHCP. +[devfsrules_jail_vnet_bpf=${hypervisor_jail_bpf_ruleset}] +add include \$devfsrules_jail_vnet +add path 'bpf*' unhide diff --git a/files/etc/dma/dma.conf.freebsd b/files/etc/dma/dma.conf.freebsd new file mode 100644 index 0000000..ff8aae0 --- /dev/null +++ b/files/etc/dma/dma.conf.freebsd @@ -0,0 +1,5 @@ +SMARTHOST ${smtp_host} +SECURETRANSFER +STARTTLS +OPPORTUNISTIC_TLS +MAILNAME ${email_domain} diff --git a/files/etc/hosts.freebsd b/files/etc/hosts.freebsd new file mode 100644 index 0000000..5551ff0 --- /dev/null +++ b/files/etc/hosts.freebsd @@ -0,0 +1,4 @@ +::1 localhost localhost.${domain} +127.0.0.1 localhost localhost.${domain} + +${BOXCONF_DEFAULT_IPV4} ${BOXCONF_HOSTNAME}.${domain} ${BOXCONF_HOSTNAME} diff --git a/files/etc/login.conf.freebsd b/files/etc/login.conf.freebsd new file mode 100644 index 0000000..b7def42 --- /dev/null +++ b/files/etc/login.conf.freebsd @@ -0,0 +1,64 @@ +default:\\ + :passwd_format=sha512:\\ + :copyright=/etc/COPYRIGHT:\\ + :welcome=/var/run/motd:\\ + :setenv=BLOCKSIZE=K:\\ + :mail=/var/mail/$:\\ + :path=/sbin /bin /usr/sbin /usr/bin /usr/local/sbin /usr/local/bin ~/bin:\\ + :nologin=/var/run/nologin:\\ + :cputime=unlimited:\\ + :datasize=unlimited:\\ + :stacksize=unlimited:\\ + :memorylocked=64M:\\ + :memoryuse=unlimited:\\ + :filesize=unlimited:\\ + :coredumpsize=unlimited:\\ + :openfiles=unlimited:\\ + :maxproc=unlimited:\\ + :sbsize=unlimited:\\ + :vmemoryuse=unlimited:\\ + :swapuse=unlimited:\\ + :pseudoterminals=unlimited:\\ + :kqueues=unlimited:\\ + :umtxp=unlimited:\\ + :priority=0:\\ + :ignoretime@:\\ + :umask=022:\\ + :charset=UTF-8:\\ + :lang=${locale}: + +# +# A collection of common class names - forward them all to 'default' +# (login would normally do this anyway, but having a class name +# here suppresses the diagnostic) +# +standard:\\ + :tc=default: +xuser:\\ + :tc=default: +staff:\\ + :tc=default: + +# This PATH may be clobbered by individual applications. Notably, by default, +# rc(8), service(8), and cron(8) will all override it with a default PATH that +# may not include /usr/local/sbin and /usr/local/bin when starting services or +# jobs. +daemon:\\ + :path=/sbin /bin /usr/sbin /usr/bin /usr/local/sbin /usr/local/bin:\\ + :mail@:\\ + :memorylocked=128M:\\ + :tc=default: +news:\\ + :tc=default: +dialer:\\ + :tc=default: + +# +# Root can always login +# +# N.B. login_getpwclass(3) will use this entry for the root account, +# in preference to 'default'. +root:\\ + :ignorenologin:\\ + :memorylocked=unlimited:\\ + :tc=default: diff --git a/files/etc/ntp.conf.freebsd b/files/etc/ntp.conf.freebsd new file mode 100644 index 0000000..d4b1a03 --- /dev/null +++ b/files/etc/ntp.conf.freebsd @@ -0,0 +1,18 @@ +interface ignore wildcard +interface listen ${BOXCONF_DEFAULT_IPV4} + +tos minclock 3 maxclock 6 + +$(if [ -n "${ntp_servers:-}" ]; then + printf 'server %s iburst\n' $ntp_servers + elif [ -n "${ntp_pools:-}" ]; then + printf 'pool %s iburst\n' $ntp_pools + fi) + +restrict default limited kod nomodify notrap noquery nopeer +restrict source limited kod nomodify notrap noquery + +restrict 127.0.0.1 +restrict ::1 + +leapfile "/var/db/ntpd.leap-seconds.list" diff --git a/files/etc/pf.conf.freebsd b/files/etc/pf.conf.freebsd new file mode 100644 index 0000000..633f3ef --- /dev/null +++ b/files/etc/pf.conf.freebsd @@ -0,0 +1,37 @@ +egress = "${BOXCONF_DEFAULT_INTERFACE}" +allowed_tcp_ports = "{ $(join ', ' ${allowed_tcp_ports:-}) }" +allowed_udp_ports = "{ $(join ', ' ${allowed_udp_ports:-}) }" +acme_standalone_port = ${acme_standalone_port} +acme_standalone_user = ${acme_uid} +nfscbd_port = ${nfscbd_port} + +set block-policy return +set skip on lo +scrub in on \$egress all fragment reassemble no-df + +$([ "${acme_standalone:-}" = true ] && echo \ + 'rdr on $egress inet proto tcp to port http -> ($egress) port $acme_standalone_port' + +[ -n "${redirect_tcp_ports:-}" ] && printf \ + 'rdr on $egress inet proto tcp to port %s -> ($egress) port %s\n' $redirect_tcp_ports + +[ -n "${redirect_udp_ports:-}" ] && printf \ + 'rdr on $egress inet proto udp to port %s -> ($egress) port %s\n' $redirect_udp_ports) + +antispoof quick for \$egress + +block all +pass out quick on \$egress inet +pass in quick on \$egress inet proto icmp all icmp-type { echoreq, unreach } + +$([ "${acme_standalone:-}" = true ] && echo \ + 'pass in quick on $egress inet proto tcp to port $acme_standalone_port user $acme_standalone_user' + +[ -n "${allowed_tcp_ports:-}" ] && echo \ + 'pass in quick on $egress inet proto tcp to port $allowed_tcp_ports' + +[ -n "${allowed_udp_ports:-}" ] && echo \ + 'pass in quick on $egress inet proto udp to port $allowed_udp_ports' + +[ "$BOXCONF_VIRTUALIZATION_TYPE" == jail ] || echo \ + 'pass in quick on $egress inet proto { tcp, udp } to port $nfscbd_port') diff --git a/files/etc/profile.d/locale.sh.freebsd b/files/etc/profile.d/locale.sh.freebsd new file mode 100644 index 0000000..093e6d1 --- /dev/null +++ b/files/etc/profile.d/locale.sh.freebsd @@ -0,0 +1,2 @@ +export LANG=${locale} +export CHARSET=UTF-8 diff --git a/files/etc/resolv.conf.common b/files/etc/resolv.conf.common new file mode 100644 index 0000000..24c2044 --- /dev/null +++ b/files/etc/resolv.conf.common @@ -0,0 +1,3 @@ +$(printf 'nameserver %s\n' $resolvers) +domain ${domain} +options timeout:1 diff --git a/files/etc/ssh/ssh_config.freebsd_hypervisor b/files/etc/ssh/ssh_config.freebsd_hypervisor new file mode 120000 index 0000000..338cdba --- /dev/null +++ b/files/etc/ssh/ssh_config.freebsd_hypervisor @@ -0,0 +1 @@ +ssh_config.no_idm
\ No newline at end of file diff --git a/files/etc/ssh/ssh_config.no_idm b/files/etc/ssh/ssh_config.no_idm new file mode 100644 index 0000000..97f3ba8 --- /dev/null +++ b/files/etc/ssh/ssh_config.no_idm @@ -0,0 +1 @@ +# Intentionally empty. diff --git a/files/etc/ssh/sshd_config.freebsd_hypervisor b/files/etc/ssh/sshd_config.freebsd_hypervisor new file mode 120000 index 0000000..355377d --- /dev/null +++ b/files/etc/ssh/sshd_config.freebsd_hypervisor @@ -0,0 +1 @@ +sshd_config.no_idm
\ No newline at end of file diff --git a/files/etc/ssh/sshd_config.no_idm b/files/etc/ssh/sshd_config.no_idm new file mode 100644 index 0000000..f38720c --- /dev/null +++ b/files/etc/ssh/sshd_config.no_idm @@ -0,0 +1,10 @@ +PermitRootLogin prohibit-password +AuthorizedKeysFile .ssh/authorized_keys + +KbdInteractiveAuthentication no +PasswordAuthentication yes + +UsePAM yes +UseDNS no + +Subsystem sftp /usr/libexec/sftp-server diff --git a/files/etc/syslog.conf.freebsd b/files/etc/syslog.conf.freebsd new file mode 100644 index 0000000..dda6710 --- /dev/null +++ b/files/etc/syslog.conf.freebsd @@ -0,0 +1,12 @@ +*.err;kern.warning;auth.notice;mail.crit /dev/console +*.info;authpriv.none;auth.none;cron.none;kern.debug;mail.crit;news.err /var/log/messages +security.* /var/log/security +auth.info;authpriv.info /var/log/auth.log +mail.info /var/log/maillog +cron.* /var/log/cron +!-devd +*.=debug /var/log/debug.log +*.emerg * +!* +include /etc/syslog.d +include /usr/local/etc/syslog.d diff --git a/files/etc/ttys.freebsd b/files/etc/ttys.freebsd new file mode 100644 index 0000000..3ebdfe3 --- /dev/null +++ b/files/etc/ttys.freebsd @@ -0,0 +1,24 @@ +console none unknown off insecure +# +ttyv0 "/usr/libexec/getty Pc" xterm onifexists secure +# Virtual terminals +ttyv1 "/usr/libexec/getty Pc" xterm onifexists secure +ttyv2 "/usr/libexec/getty Pc" xterm onifexists secure +ttyv3 "/usr/libexec/getty Pc" xterm onifexists secure +ttyv4 "/usr/libexec/getty Pc" xterm onifexists secure +ttyv5 "/usr/libexec/getty Pc" xterm onifexists secure +ttyv6 "/usr/libexec/getty Pc" xterm onifexists secure +ttyv7 "/usr/libexec/getty Pc" xterm onifexists secure +ttyv8 "/usr/local/bin/xdm -nodaemon" xterm off secure +# Serial terminals +# The 'dialup' keyword identifies dialin lines to login, fingerd etc. +ttyu0 "/usr/libexec/getty 3wire.115200" vt100 onifexists secure +ttyu1 "/usr/libexec/getty 3wire" vt100 onifconsole secure +ttyu2 "/usr/libexec/getty 3wire" vt100 onifconsole secure +ttyu3 "/usr/libexec/getty 3wire" vt100 onifconsole secure +# Dumb console +dcons "/usr/libexec/getty std.9600" vt100 off secure +# Xen Virtual console +xc0 "/usr/libexec/getty Pc" xterm onifconsole secure +# RISC-V HTIF console +rcons "/usr/libexec/getty std.9600" vt100 onifconsole secure diff --git a/files/usr/local/etc/jailctl.conf.freebsd_hypervisor b/files/usr/local/etc/jailctl.conf.freebsd_hypervisor new file mode 100644 index 0000000..02b6065 --- /dev/null +++ b/files/usr/local/etc/jailctl.conf.freebsd_hypervisor @@ -0,0 +1,29 @@ +#!/bin/sh + +JAIL_HOME='${hypervisor_jail_home}' +JAIL_DATASET='${hypervisor_jail_dataset}' +TRUNK_INTERFACE='${hypervisor_trunk_interface}' + +DEFAULT_DOMAIN='${domain}' +DEFAULT_VLAN='${hypervisor_default_vlan}' +DEFAULT_NETMASK='$(prefix2netmask "$hypervisor_default_prefix")' +DEFAULT_OS_QUOTA='${hypervisor_default_os_quota}' +DEFAULT_DATA_QUOTA='${hypervisor_default_data_quota}' + +ZFS_OPTS='${hypervisor_jail_default_zfs_opts}' + +DEFAULT_DEVFS_RULESET='5' +BPF_ENABLED_DEVFS_RULESET='${hypervisor_jail_bpf_ruleset}' + +DEFAULT_PF_CONF='egress = "jail0" + +set block-policy return +set skip on lo +scrub in on \$egress all fragment reassemble no-df + +antispoof quick for \$egress + +block all +pass out quick on \$egress inet +pass in quick on \$egress inet proto icmp all icmp-type { echoreq, unreach } +pass in quick on \$egress inet proto tcp to port ssh' diff --git a/files/usr/local/etc/rc.d/vmctl.freebsd_hypervisor b/files/usr/local/etc/rc.d/vmctl.freebsd_hypervisor new file mode 100644 index 0000000..5f1a84b --- /dev/null +++ b/files/usr/local/etc/rc.d/vmctl.freebsd_hypervisor @@ -0,0 +1,24 @@ +#!/bin/sh +# +# $FreeBSD$ + +# PROVIDE: vmctl +# REQUIRE: NETWORKING SERVERS dmesg +# BEFORE: ipfw pf +# KEYWORD: shutdown nojail + +. /etc/rc.subr + +name="vmctl" +desc="Start and stop bhyve virtual machines" +rcvar="vmctl_enable" + +: ${vmctl_enable:="NO"} + +command="/usr/local/sbin/${name}" +start_cmd="${command} _start-all" +stop_cmd="${command} _stop-all" +status_cmd="${command} list" + +load_rc_config $name +run_rc_command "$1" diff --git a/files/usr/local/etc/vmctl.conf.freebsd_hypervisor b/files/usr/local/etc/vmctl.conf.freebsd_hypervisor new file mode 100644 index 0000000..7bef759 --- /dev/null +++ b/files/usr/local/etc/vmctl.conf.freebsd_hypervisor @@ -0,0 +1,18 @@ +#!/bin/sh + +VM_HOME='${hypervisor_vm_home}' +VM_DATASET='${hypervisor_vm_dataset}' +TRUNK_INTERFACE='${hypervisor_trunk_interface}' + +DEFAULT_DOMAIN='${domain}' +DEFAULT_CPUS='${hypervisor_vm_default_cpus}' +DEFAULT_MEMORY='${hypervisor_vm_default_mem}' +DEFAULT_OS_SIZE='${hypervisor_default_os_quota}' +DEFAULT_DATA_SIZE='${hypervisor_default_data_quota}' +DEFAULT_PREFIXLEN='${hypervisor_default_prefix}' +DEFAULT_VLAN='${hypervisor_default_vlan}' +TEMPLATE_ZVOL_SIZE='${hypervisor_vm_template_size}' +DEFAULT_AUTOSTART_DELAY='${hypervisor_vm_default_autostart_delay}' + +ZFS_OPTS='${hypervisor_vm_default_zfs_opts}' +ZFS_VOLBLOCKSIZE='${hypervisor_vm_zfs_volblocksize}' diff --git a/files/usr/local/sbin/jailctl.freebsd_hypervisor b/files/usr/local/sbin/jailctl.freebsd_hypervisor new file mode 100644 index 0000000..05c0158 --- /dev/null +++ b/files/usr/local/sbin/jailctl.freebsd_hypervisor @@ -0,0 +1,1098 @@ +#!/bin/sh +# +# Jail management utility. + +set -eu -o pipefail + +. /usr/local/etc/jailctl.conf + +cmd::main(){ + local usage="COMMAND [ARGS]... +FreeBSD jail management utility. +Commands: + create Create a new jail + create-snapshot Take a snapshot of a jail + create-template Create a template from a jail + destroy-snapshot Delete a jail snapshot + destroy-template Delete a template + destroy Delete a jail and its dataset + download-release Download and create a FreeBSD release template + edit Edit a jail's configuration + exec Run a command within the jail + list-snapshots List jail snapshots + list-templates List available templates + list List configured jails + reprovision Wipe and reprovision an OS dataset from template + restart Restart a jail + rollback Rollback a jail to a given snapshot + shell Run a shell within the jail + show Show jail configuration + start Start a jail + status Show running jail status + stop Stop a jail + update-release Update a FreeBSD release template" + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -ge 1 ] || cmd::usage 'no comand specified' + local cmd=$1; shift + + case $cmd in + create) cmd::create "$@" ;; + create-snapshot|snapshot|snap) cmd::create_snapshot "$@" ;; + create-template) cmd::create_template "$@" ;; + destroy-snapshot|rms) cmd::destroy_snapshot "$@" ;; + destroy-template|rmt) cmd::destroy_template "$@" ;; + destroy|rm) cmd::destroy "$@" ;; + download-release) cmd::download_release "$@" ;; + edit) cmd::edit "$@" ;; + exec) cmd::exec "$@" ;; + list-snapshots|lss) cmd::list_snapshots "$@" ;; + list-templates|lst) cmd::list_templates "$@" ;; + list|ls) cmd::list "$@" ;; + reprovision) cmd::reprovision "$@" ;; + restart) cmd::restart "$@" ;; + rollback) cmd::rollback "$@" ;; + shell|sh) cmd::shell "$@" ;; + show) cmd::show "$@" ;; + start) cmd::start "$@" ;; + status) cmd::status "$@" ;; + stop) cmd::stop "$@" ;; + update-release) cmd::update_release "$@" ;; + # The following commands are internal to jailctl. Don't run them manually. + _create-epair) cmd::_create_epair "$@" ;; + _destroy-epair) cmd::_destroy_epair "$@" ;; + *) cmd::usage "unknown command: ${cmd}" ;; + esac +} + +################################################################################ +# Standard helper functions. +################################################################################ +die(){ + printf '%s: %s\n' jailctl "$*" 1>&2 + exit 1 +} + +warn(){ + printf '%s\n' "$*" 1>&2 +} + + +################################################################################ +# CLI-related functions. +################################################################################ +cmd::help(){ + printf 'Usage: %s %s\n' jailctl "$usage" + [ -n "${help:-}" ] && printf '%s\n' "$help" + exit 0 +} + +cmd::usage(){ + [ $# -gt 0 ] && printf '%s: %s\n' "jailctl" "$1" 1>&2 + printf 'Usage: %s %s\n' jailctl "${usage}" 1>&2 + exit 2 +} + +cmd::getopt_help(){ + local opt + while getopts :h opt; do + case $opt in + h) cmd::help ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done +} + +cmd::create(){ + local usage='create [-a IP] [-b] [-c CPUSET] [-d DOMAIN] [-g GATEWAY] [-k SSHKEY] + [-m MEMLIMIT] [-n NETMASK] [-q QUOTA] [-Q OS_QUOTA] [-r NAMESERVER] + [-s SEARCHDOMAIN] [-v VLANID] NAME TEMPLATE' + local help="Create a new jail. +Options: + -a IP IPv4 address + -b Enable BPF device (allows DHCP, tcpdump, etc) + -c CPUSET CPU list for cpuset(1) + -d DOMAIN Host domain name + -g GATEWAY Default IPv4 gateway + -k SSHKEY Path to SSH pubkey for root's authorized_keys + -m MEMLIMIT Virtual memory limit + -n NETMASK IPv4 netmask + -q QUOTA Quota for delegated dataset + -Q OS_QUOTA Quota for root filesystem + -r NAMESERVER DNS resolver + -s SEARCHDOMAIN DNS search domain + -v VLANID VLAN ID number" + + local \ + bpf_enabled=false \ + cpuset \ + data_quota \ + data_quota=$DEFAULT_DATA_QUOTA \ + devfs_ruleset=$DEFAULT_DEVFS_RULESET \ + domain=$DEFAULT_DOMAIN \ + gateway \ + ip \ + memlimit \ + nameservers \ + netmask=$DEFAULT_NETMASK \ + os_quota=$DEFAULT_OS_QUOTA \ + searchdomains \ + snapshot \ + sshkey \ + vlan=$DEFAULT_VLAN \ + opt + + while getopts :a:bc:d:g:hk:m:n:q:Q:r:s:v: opt; do + case $opt in + a) ip=$OPTARG ;; + b) bpf_enabled=true ;; + c) cpuset=$OPTARG ;; + d) domain=$OPTARG ;; + g) gateway=$OPTARG ;; + h) cmd::help ;; + k) sshkey=$OPTARG ;; + m) memlimit=$OPTARG ;; + n) netmask=$OPTARG ;; + q) data_quota=$OPTARG ;; + Q) os_quota=$OPTARG ;; + r) nameservers="${nameservers:-} ${OPTARG}" ;; + s) searchdomains="${searchdomains:-} ${OPTARG}" ;; + v) vlan=$OPTARG ;; + :) cmd::usage "missing option value: -${OPTARG}" ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'NAME not specified' + [ $# -lt 2 ] && cmd::usage 'TEMPLATE not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local name=$1 template=$2 + + jail::exists "$name" && die "jail name already in use: ${name}" + template::exists "$template" || die "no such template: ${template}" + + interface::exists "bridge${vlan}" || interface::add_vlan "$vlan" + + if [ -n "${sshkey:-}" ]; then + [ -f "$sshkey" ] || die "ssh key ${sshkey}: file not found" + fi + + zfs::ensure_snapshot snapshot "${JAIL_DATASET}/templates/${template}" + + # Clone template into new 'os' dataset. + zfs create -v "${JAIL_DATASET}/${name}" + zfs clone \ + $ZFS_OPTS \ + -o quota="$os_quota" \ + "$snapshot" "${JAIL_DATASET}/${name}/os" + + # Create delegated 'data' dataset. + zfs create -v \ + $ZFS_OPTS \ + -o mountpoint=none \ + -o quota="$data_quota" \ + "${JAIL_DATASET}/${name}/data" + + # Copy timezone configuration from host. + cp -v /etc/localtime "${JAIL_HOME}/${name}/os/etc/localtime" + + # Generate /etc/resolv.conf in the jail (word-splitting intentional). + [ -n "${nameservers:-}" ] && printf 'nameserver %s\n' $nameservers >> "${JAIL_HOME}/${name}/os/etc/resolv.conf" + [ -n "${searchdomains:-}" ] && printf 'search %s\n' "$searchdomains" >> "${JAIL_HOME}/${name}/os/etc/resolv.conf" + + # Get the jail's virtual interface name. + local epair_name + epair_name=$(interface::epair::derive_name "$name") + + # Set /etc/rc.conf values within the jail. + sysrc -v -f "${JAIL_HOME}/${name}/os/etc/rc.conf" \ + "hostname=${name}.${domain}" \ + "ifconfig_ej_${epair_name}_name=jail0" \ + 'ipv6_activate_all_interfaces=NO' \ + 'syslogd_flags=-ss' \ + 'sendmail_enable=NONE' \ + 'dumpdev=NO' \ + 'pf_enable=YES' + + echo "$DEFAULT_PF_CONF" > "${JAIL_HOME}/${name}/os/etc/pf.conf" + + if [ -n "${ip:-}" ]; then + # If $ip set, configure /etc/rc.conf for static IP. + sysrc -v -f "${JAIL_HOME}/${name}/os/etc/rc.conf" \ + "ifconfig_jail0=inet ${ip} netmask ${netmask}" \ + "defaultrouter=${gateway}" + else + # Othersie, configure /etc/rc.conf for DHCP. + sysrc -v -f "${JAIL_HOME}/${name}/os/etc/rc.conf" "ifconfig_jail0=SYNCDHCP" + devfs_ruleset=$BPF_ENABLED_DEVFS_RULESET + fi + + if [ -n "${sshkey:-}" ]; then + # If $sshkey set, enable sshd and root login. + echo "PermitRootLogin prohibit-password" | tee -a "${JAIL_HOME}/${name}/os/etc/ssh/sshd_config" + sysrc -v -f "${JAIL_HOME}/${name}/os/etc/rc.conf" 'sshd_enable=YES' + # Copy the ssh key into root's authorized_keys within the jail. + install -v -d -m 0700 "${JAIL_HOME}/${name}/os/root/.ssh" + install -v -m 0600 "$sshkey" "${JAIL_HOME}/${name}/os/root/.ssh/authorized_keys" + fi + + # If BPF requested, use our custom bpf ruleset. + [ "${bpf_enabled}" = true ] && devfs_ruleset=$BPF_ENABLED_DEVFS_RULESET + + # Generate jail config file. + local jailcfg="${JAIL_HOME}/${name}/jail.conf" + cat <<EOF > "$jailcfg" +${name} { + path = "${JAIL_HOME}/\$name/os"; + host.hostname = "\$name.${domain}"; + + exec.prestart = "jailctl _create-epair \$name vlan${vlan} bridge${vlan}"; + exec.created = "zfs set jailed=on ${JAIL_DATASET}/\$name/data"; + exec.created += "zfs jail \$name ${JAIL_DATASET}/\$name/data"; + exec.start = "zfs mount -a"; + exec.start += "/bin/sh /etc/rc"; + exec.stop = "/bin/sh /etc/rc.shutdown"; + exec.stop += "zfs list -Ho name,jailed,mounted | awk '\$2 == \\"on\\" && \$3 == \\"yes\\" {print \$1}' | xargs -rtn1 zfs unmount"; + exec.poststop = "jailctl _destroy-epair \$name"; + exec.poststop += "rctl -r jail:\$name:"; + exec.clean; + + exec.system_user = "root"; + exec.jail_user = "root"; + + mount.devfs; + devfs_ruleset = "${devfs_ruleset}"; + + mount = "tmpfs \$path/tmp tmpfs rw,size=1G 0 0"; + allow.mount = true; + allow.mount.zfs = true; + enforce_statfs = 1; + + vnet; + vnet.interface = "ej_${epair_name}"; +EOF + + ln -sv "$jailcfg" "/etc/jail.conf.d/${name}.conf" + + # Configure resource limits. + [ -n "${cpuset:-}" ] && \ + echo " exec.created += \"cpuset -j \$name -cl ${cpuset}\";" >> "$jailcfg" + [ -n "${memlimit:-}" ] && \ + echo " exec.prestart += \"rctl -a jail:\$name:memoryuse:deny=${memlimit}\";" >> "$jailcfg" + + # End jail config file. + echo '}' >> "$jailcfg" + + # Update host's /etc/rc.conf to start the jail on boot. + sysrc -v jail_list+="$name" + + # Start the jail. + jail::start "$name" +} + +cmd::create_snapshot(){ + local usage='create-snapshot [-d] [-o] JAIL [SNAPNAME]' + local help='Create a jail snapshot. +Options: + -d Only snapshot the data dataset + -o Only snapshot the OS dataset' + + local opt both=true os_only=false data_only=false snapname + while getopts :dho opt; do + case $opt in + d) data_only=true; both=false ;; + h) cmd::help ;; + o) os_only=true; both=false ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + jail=$1 snapname=${2:-} + + jail::exists "$jail" || die "no such jail: ${jail}" + + [ -n "$snapname" ] || snapname=$(date +%Y-%m-%dT%H:%M:%S) + + # Snapshot the OS dataset. + if [ "$both" = true ] || [ "$os_only" = true ]; then + zfs snapshot "${JAIL_DATASET}/${jail}/os@${snapname}" + fi + + # Snapshot the data dataset. + if [ "$both" = true ] || [ "$data_only" = true ]; then + zfs snapshot "${JAIL_DATASET}/${jail}/data@${snapname}" + fi +} + +cmd::create_template(){ + local usage='create-template NAME JAIL' + local help='Create a template from a jail.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'NAME not specified' + [ $# -lt 2 ] && cmd::usage 'JAIL not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local name=$1 jail=$2 + + jail::exists "$jail" || die "no such jail: ${jail}" + template::exists "$name" && die "template already exists: $name" + jail::running "$jail" && die "refusing to create template while jail is running: ${jail}" + + local snapname + snapname=$(date +%Y-%m-%dT%H:%M:%S) + + zfs snapshot "${JAIL_DATASET}/${jail}/os@${snapname}" + zfs send "${JAIL_DATASET}/${jail}/os@${snapname}" | zfs receive -v "${JAIL_DATASET}/templates/${name}" + zfs destroy "${JAIL_DATASET}/${jail}/os@${snapname}" +} + +cmd::destroy_snapshot(){ + local usage='destroy-snapshot [-y] JAIL SNAPSHOT' + local help="Delete a jail snapshot +Options: + -y Don't prompt for confirmation" + + local noconfirm=false answer opt + + while getopts :hy opt; do + case $opt in + h) cmd::help ;; + y) noconfirm=true ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -lt 2 ] && cmd::usage 'SNAPSHOT not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local jail=$1 snapshot=$2 + + jail::exists "$jail" || die "no such jail: ${jail}" + + local datasets='' + + if [ "${snapshot#*@}" != "$snapshot" ]; then + # If the snapshot name contains '@', then we have something like 'os@snapname'. + zfs::dataset_exists "${JAIL_DATASET}/${jail}/${snapshot}" && \ + datasets="${JAIL_DATASET}/${jail}/${snapshot}" + else + # Otherwise, check if either os or data dataset contains a matching snapshot. + zfs::dataset_exists "${JAIL_DATASET}/${jail}/os@${snapshot}" && \ + datasets="${JAIL_DATASET}/${jail}/os@${snapshot}" + + zfs::dataset_exists "${JAIL_DATASET}/${jail}/data@${snapshot}" && \ + datasets="${datasets} ${JAIL_DATASET}/${jail}/data@${snapshot}" + fi + + [ -n "$datasets" ] || die "no such snapshot for jail ${jail}: ${snapshot}" + + if [ "$noconfirm" != true ]; then + read -rp "Really destroy ${jail} snapshot ${snapshot}? (y/N) " answer + case $answer in + [yY]|[yY][eE][sS]) : ;; + *) die 'operation cancelled' ;; + esac + fi + + local dataset + for dataset in $datasets; do + zfs destroy -v "$dataset" + done +} + +cmd::destroy_template(){ + local usage='destroy-template [-y] TEMPLATE' + local help="Delete a jail template. +Options: + -y Don't prompt for confirmation +Notes: + A template cannot be deleted while its clones still exist." + + local noconfirm=false answer opt + + while getopts :hy opt; do + case $opt in + h) cmd::help ;; + y) noconfirm=true ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'TEMPLATE not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local jail=$1 + + template::exists "$jail" || die "no such template: ${jail}" + + if [ "$noconfirm" != true ]; then + read -rp "Really destroy template ${jail}? (y/N) " answer + case $answer in + [yY]|[yY][eE][sS]) : ;; + *) die 'operation cancelled' ;; + esac + fi + + zfs destroy -v -r "${JAIL_DATASET}/templates/${jail}" +} + +cmd::destroy(){ + local usage='destroy [-y] JAIL' + local help="Delete a jail and its dataset. +Options: + -y Don't prompt for confirmation" + + local noconfirm=false answer opt + + while getopts :hy opt; do + case $opt in + h) cmd::help ;; + y) noconfirm=true ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local jail=$1 + + jail::exists "$jail" || die "no such jail: ${jail}" + + if [ "$noconfirm" != true ]; then + read -rp "Really destroy jail ${jail}? (y/N) " answer + case $answer in + [yY]|[yY][eE][sS]) : ;; + *) die 'operation cancelled' ;; + esac + fi + + jail::running "$jail" && jail::stop "$jail" + + # Delete jail config file. + rm "/etc/jail.conf.d/${jail}.conf" "${JAIL_HOME}/${jail}/jail.conf" + + # Remove the jail from the autostart list. + sysrc -v jail_list-="$jail" ||: + + # Destroy the jail's dataset. + zfs destroy -v -f -r "${JAIL_DATASET}/${jail}" +} + +cmd::download_release() { + local usage='download-release RELEASE' + local help='Download and create a FreeBSD release template.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'RELEASE not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local release=$1 + + local template + template::release2name template "$release" + + template::exists "$template" && die "template already exists: ${template}" + template::download_release "$release" +} + +cmd::edit(){ + local usage='edit JAIL' + local help='Edit jail configuration.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local jail=$1 + + jail::exists "$jail" || die "no such jail: ${jail}" + "$EDITOR" "${JAIL_HOME}/${jail}/jail.conf" +} + +cmd::exec(){ + local usage='exec JAIL COMMAND...' + local help='Run a command within the jail.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -lt 2 ] && cmd::usage 'COMMAND not specified' + jail=$1; shift + + jail::exists "$jail" || die "no such jail: ${jail}" + + jail::exec "$jail" "$@" +} + +cmd::list(){ + local usage='list [-t]' + local help='List configured jails. +Options: + -t Use terse output (jail names only)' + + local file name status opt terse=false + + while getopts :th opt; do + case $opt in + t) terse=true ;; + h) cmd::help ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -eq 0 ] || cmd::usage 'too many arguments' + + { [ $terse = true ] || echo 'JAIL STATUS' + for file in "$JAIL_HOME"/*/jail.conf; do + [ -e "$file" ] || continue + + name=$(basename "$(dirname "$file")") + + if [ $terse = true ]; then + printf '%s\n' "$name" + else + if jail::running "$name"; then + status=running + else + status=stopped + fi + printf '%s %s\n' "$name" "$status" + fi + done + } | column -t +} + +cmd::list_templates(){ + local usage='list-templates' + local help='list template datasets.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -eq 0 ] || cmd::usage 'too many arguments' + + ls -1 "${JAIL_HOME}/templates" +} + +cmd::list_snapshots(){ + local usage='list-snapshots JAIL' + local help='List jail snapshots +Options: + -d List snapshots from data dataset only + -o List snapshots from OS dataset only + -t Use terse output' + + local opt terse=false dataset='' + while getopts :dhot opt; do + case $opt in + d) dataset=/data ;; + h) cmd::help ;; + o) dataset=/os ;; + t) terse=true ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local jail=$1 + + jail::exists "$jail" || die "no such jail: ${jail}" + + if [ "$terse" = true ]; then + zfs list -r -t snapshot -H -o name -s creation "${JAIL_DATASET}/${jail}${dataset}" 2>/dev/null \ + | sed 's/^.*\///' \ + | sort -k1,1 -t@ -s + else + { echo 'DATASET SNAPSHOT USED REFER' + zfs list -r -t snapshot -H -o name,used,refer -s creation "${JAIL_DATASET}/${jail}${dataset}" 2>/dev/null + } | sed 's/^.*\///' \ + | sort -k1,1 -t@ -s \ + | tr '@' ' ' \ + | column -t + fi +} + +cmd::rollback(){ + local usage='rollback JAIL SNAPSHOT' + local help='Rollback a jail to a given snapshot. +Options: + -d Rollback the data dataset only + -f Attempt rollback even if the jail is running + -o Rollback the OS dataset only' + + local opt force=false both=true os_only=false data_only=false + while getopts :dfho opt; do + case $opt in + d) data_only=true; both=false ;; + f) force=true ;; + h) cmd::help ;; + o) os_only=true; both=false ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -lt 2 ] && cmd::usage 'SNAPSHOT not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local jail=$1 snapshot=$2 + + jail::exists "$jail" || die "no such jail: ${jail}" + + # Ensure OS snapshot exists, if requested. + if [ "$both" = true ] || [ "$os_only" = true ]; then + zfs::dataset_exists "${JAIL_DATASET}/${jail}/os@${snapshot}" \ + || die "no such snapshot for ${jail}/os: ${snapshot}" + fi + + # Ensure data snapshot exists, if requested. + if [ "$both" = true ] || [ "$data_only" = true ]; then + zfs::dataset_exists "${JAIL_DATASET}/${jail}/data@${snapshot}" \ + || die "no such snapshot for ${jail}/data: ${snapshot}" + fi + + # Don't rollback while jail is running. + if jail::running "$jail" && [ "$force" != true ]; then + die "jail ${jail} is running, refusing to rollback (-f to override)" + fi + + # Rollback the OS snapshot, if requested. + if [ "$both" = true ] || [ "$os_only" = true ]; then + zfs rollback -r "${JAIL_DATASET}/${jail}/os@${snapshot}" + fi + + # Rollback the data snapshot, if requested. + if [ "$both" = true ] || [ "$data_only" = true ]; then + zfs rollback -r "${JAIL_DATASET}/${jail}/data@${snapshot}" + fi +} + +cmd::start(){ + local usage='start JAIL' + local help='Start a jail.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local jail=$1 + + jail::exists "$jail" || die "no such jail: ${jail}" + jail::running "$jail" && die "jail already running: ${jail}" + + jail::start "$jail" +} + +cmd::stop(){ + local usage='stop JAIL' + local help='Stop a jail.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local jail=$1 + + jail::exists "$jail" || die "no such jail: ${jail}" + jail::running "$jail" || die "jail not running: ${jail}" + + jail::stop "$jail" +} + +cmd::reprovision(){ + local usage='reprovision JAIL TEMPLATE' + local help="Wipe and reprovision a jail's OS dataset from a template." + + local opt noconfirm=false running=false + while getopts :hy opt; do + case $opt in + h) cmd::help ;; + y) noconfirm=true ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -lt 2 ] && cmd::usage 'TEMPLATE not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local jail=$1 template=$2 + + jail::exists "$jail" || die "no such jail: ${jail}" + template::exists "$template" || die "no such template: ${template}" + + if [ "$noconfirm" != true ]; then + read -rp "Really reprovision ${jail}? (y/N) " answer + case $answer in + [yY]|[yY][eE][sS]) : ;; + *) die 'operation cancelled' ;; + esac + fi + + # If the jail is running, stop it. + if jail::running "$jail"; then + running=true + jail::stop "$jail" + fi + + local snapshot old_quota old_ifconfig old_defaultrouter old_hostname old_resolvconf + + # Get the latest snapshot for the template (if not specified). + zfs::ensure_snapshot snapshot "${JAIL_DATASET}/templates/${template}" + + # Stash old configuration data. + old_quota=$(zfs get -Hp -o value quota "${JAIL_DATASET}/${jail}/os") + old_hostname=$(sysrc -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" -qn hostname) + old_ifconfig=$(sysrc -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" -qn ifconfig_jail0) + old_defaultrouter=$(sysrc -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" -qn defaultrouter) ||: + old_pfenable=$(sysrc -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" -qn pf_enable) ||: + old_sshkey=$(cat "${JAIL_HOME}/${jail}/os/root/.ssh/authorized_keys") ||: + old_resolvconf=$(cat "${JAIL_HOME}/${jail}/os/etc/resolv.conf") ||: + + # Reprovision OS dataset from template. + zfs destroy -v -f -r "${JAIL_DATASET}/${jail}/os" + zfs clone \ + $ZFS_OPTS \ + -o quota="$old_quota" \ + "$snapshot" "${JAIL_DATASET}/${jail}/os" + + # Copy timezone configuration from host. + cp -v /etc/localtime "${JAIL_HOME}/${jail}/os/etc/localtime" + + # Restore stashed configuration data. + sysrc -v -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" \ + "hostname=${old_hostname}" \ + "ifconfig_ej_$(interface::epair::derive_name "$jail")_name=jail0" \ + "ifconfig_jail0=${old_ifconfig}" \ + 'ipv6_activate_all_interfaces=NO' \ + 'syslogd_flags=-ss' \ + 'sendmail_enable=NONE' \ + 'dumpdev=NO' \ + "pf_enable=YES" + + [ -n "$old_defaultrouter" ] && sysrc -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" "defaultrouter=${old_defaultrouter}" + + [ -n "$old_resolvconf" ] && printf '%s\n' "$old_resolvconf" | tee "${JAIL_HOME}/${jail}/os/etc/resolv.conf" + + echo "$DEFAULT_PF_CONF" > "${JAIL_HOME}/${jail}/os/etc/pf.conf" + + if [ -n "$old_sshkey" ]; then + echo "PermitRootLogin prohibit-password" | tee -a "${JAIL_HOME}/${jail}/os/etc/ssh/sshd_config" + sysrc -v -f "${JAIL_HOME}/${jail}/os/etc/rc.conf" 'sshd_enable=YES' + install -v -d -m 0700 "${JAIL_HOME}/${jail}/os/root/.ssh" + install -v -m 0600 /dev/null "${JAIL_HOME}/${jail}/os/root/.ssh/authorized_keys" + printf '%s\n' "$old_sshkey" | tee "${JAIL_HOME}/${jail}/os/root/.ssh/authorized_keys" + fi + + # If the jail was running, restart it. + if [ "$running" = true ]; then + jail::start "$jail" + fi +} + +cmd::restart(){ + local usage='restart JAIL' + local help='Restart a jail.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'no jail specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local jail=$1 + + jail::exists "$jail" || die "no such jail: ${jail}" + jail::running "$jail" || die "jail not running: ${jail}" + + jail::restart "$jail" +} + +cmd::shell(){ + local usage='shell JAIL' + local help='Run a shell within the jail.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local jail=$1 + + jail::exists "$jail" || die "no such jail: ${jail}" + jail::running "$jail" || die "jail not running: ${jail}" + + jail::exec "$jail" /bin/csh +} + +cmd::show(){ + local usage='show JAIL' + local help='Show jail configuration.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local jail=$1 + + jail::exists "$jail" || die "no such jail: ${jail}" + + printf -- '------------------------- JAIL CONFIGURATION -------------------------\n' + cat "${JAIL_HOME}/${jail}/jail.conf" + printf -- '\n---------------------------- ZFS DATASET -----------------------------\n' + zfs list -o name,quota,used,avail,mountpoint -S name \ + "${JAIL_DATASET}/${jail}/os" \ + "${JAIL_DATASET}/${jail}/data" +} + +cmd::status(){ + local usage='status JAIL' + local help='Show running jail status.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local jail=$1 + + jail::exists "$jail" || die "no such jail: ${jail}" + jail::running "$jail" || die "jail not running: ${jail}" + + printf -- '---------------------------- JAIL STATUS -----------------------------\n' + jls -j "$jail" -h jid name path osrelease host.hostname 2>/dev/null | column -t + printf -- '\n---------------------------- ZFS DATASET -----------------------------\n' + zfs list -o name,quota,used,avail,mountpoint -S name \ + "${JAIL_DATASET}/${jail}/os" \ + "${JAIL_DATASET}/${jail}/data" \ + | sed "s|^${JAIL_DATASET}/${jail}/||" \ + | column -t + printf -- '\n--------------------------- RESOURCE USAGE ---------------------------\n' + rctl -h -u "jail:${jail}:" \ + | grep -E '^(maxproc|memoryuse|openfiles|pcpu|memoryuse|swapuse|readbps|writebps|readiops|writeiops)=' \ + | rs -c= -C' ' -T \ + | column -t + printf -- '\n----------------------------- PROCESSES ------------------------------\n' + ps -auxdr -J "$jail" +} + +cmd::update_release(){ + local usage='update-release TEMPLATE' + local help='Update a FreeBSD release template.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'TEMPLATE not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local jail=$1 + + template::exists "$jail" || die "no such template: ${jail}" + template::update_release "$jail" +} + +cmd::_create_epair(){ + local usage='_create-epair JAIL INTERFACE BRIDGE' + local help="Create an epair for a VNET jail. +DO NOT RUN THIS COMMAND MANUALLY. It is invoked by each jail's exec.prestart." + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -lt 2 ] && cmd::usage 'INTERFACE not specified' + [ $# -lt 3 ] && cmd::usage 'BRIDGE not specified' + [ $# -gt 3 ] && cmd::usage 'too many arguments' + local jail=$1 interface=$2 bridge=$3 \ + new_epair epair_name epair_bridge_mac epair_jail_mac + + interface::exists "$interface" || die "no such interface: ${interface}" + interface::exists "$bridge" || die "no such bridge: ${bridge}" + + epair_name=$(interface::epair::derive_name "$jail") + interface::epair::derive_mac "$interface" "$jail" epair_bridge_mac epair_jail_mac + + new_epair=$(ifconfig epair create) + ifconfig "$bridge" addm "$new_epair" + + ifconfig "$new_epair" name "eb_${epair_name}" descr "jail/${jail}" + ifconfig "eb_${epair_name}" up + ifconfig "eb_${epair_name}" ether "$epair_bridge_mac" + + ifconfig "${new_epair%a}b" name "ej_${epair_name}" + ifconfig "ej_${epair_name}" up + ifconfig "ej_${epair_name}" ether "$epair_jail_mac" +} + +cmd::_destroy_epair(){ + local usage='_destroy-epair JAIL' + local help="Destroy an epair for a VNET jail. +DO NOT RUN THIS COMMAND MANUALLY. It is invoked by each jail's exec.poststop." + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'JAIL not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local jail=$1 epair_name + + epair_name=$(interface::epair::derive_name "$jail") + ifconfig "eb_${epair_name}" destroy +} + + +################################################################################ +# "Library" functions. +################################################################################ +interface::add_vlan(){ + local id=$1 + ifconfig "vlan${id}" create + ifconfig "vlan${id}" vlan "$id" vlandev "$TRUNK_INTERFACE" up + ifconfig "bridge${id}" create + ifconfig "bridge${id}" addm "vlan${id}" up + sysrc -v \ + "cloned_interfaces+=vlan${id} bridge${id}" \ + "ifconfig_vlan${id}=vlan ${id} vlandev ${TRUNK_INTERFACE} up" \ + "ifconfig_bridge${id}=addm vlan${id} up" +} + +interface::exists(){ + ifconfig "$1" > /dev/null 2>&1 +} + +interface::epair::derive_mac(){ + # Derive unique mac addresses for an epair interface, based on the physical + # interface address + jail name. + local iface=$1 name=$2 upvar_a=$3 upvar_b=$4 + local hwaddr checksum epair_mac_a epair_mac_b + + # Similar to /usr/share/examples/jails/jib + hwaddr=$(ifconfig "$iface" ether | awk '/ether/,$0=$2') + # ??:??:??:II:II:II + epair_mac_a=${hwaddr#??:??:??} # => :II:II:II + # => :SS:SS:II:II:II + epair_mac_a=":$(printf '%s' "$name" | md5 | sed 's/^\(..\)\(..\).*/\1:\2/')${epair_mac_a}" + # => NP:SS:SS:II:II:II + case $hwaddr in + ?2:*) epair_mac_a="0a${epair_mac_a}" epair_mac_b="0e${epair_mac_a}" ;; + ?[Ee]:*) epair_mac_a="02${epair_mac_a}" epair_mac_b="06${epair_mac_a}" ;; + *) epair_mac_a="02${epair_mac_a}" epair_mac_b="0e${epair_mac_a}" ;; + esac + + # Sanity check for duplicate MAC + for mac in $epair_mac_a $epair_mac_b; do + ifconfig | grep -qE "(ether|hwaddr) ${mac}" && die "MAC collision when configuring epair interface!" + done + + setvar "$upvar_a" "$epair_mac_a" + setvar "$upvar_b" "$epair_mac_b" +} + +interface::epair::derive_name(){ + # Generate an epair(4) interface suffix, based on the jail name. + # + # The maximum length of a network interface name on FreeBSD is 15 characters. + # We use a prefix of 'eb_' for the bridge side of the interface, and 'ej_' + # for the jail side. This leaves 12 characters for a unique suffix for each + # jail. + # + # If the sanitized jail name is less than 12 characters, we'll simply use it + # for the suffix. Otherwise, we'll use the last 12 characters of the jail + # name's SHA-1 hash. + local name=$1 sanitized + sanitized=$(printf '%s' "$name" | tr -dC '[:alnum:]_') + + if [ "${#sanitized}" -le 12 ]; then + printf '%s' "$sanitized" + else + printf '%s' "$name" | sha1 | tail -c 12 + fi +} + +jail::exec() { + local jail=$1; shift + jexec -l "$jail" "$@" +} + +jail::exists(){ + test -f "${JAIL_HOME}/${1}/jail.conf" +} + +jail::restart(){ + jail -v -f "${JAIL_HOME}/${1}/jail.conf" -rc "$1" +} + +jail::running(){ + jls -j "$1" > /dev/null 2>&1 +} + +jail::start(){ + jail -v -f "${JAIL_HOME}/${1}/jail.conf" -c "$1" +} + +jail::stop(){ + jail -v -f "${JAIL_HOME}/${1}/jail.conf" -r "$1" +} + +template::exists(){ + zfs list -H "${JAIL_DATASET}/templates/${1}" > /dev/null 2>&1 +} + +template::download_release(){ + # Download a given FreeBSD release and create a template jail. + local release=$1 arch base_tarball template + + arch=$(uname -p) + base_tarball="https://download.freebsd.org/releases/${arch}/${release}/base.txz" + + template::release2name template "$release" + + zfs create -v -p $ZFS_OPTS "${JAIL_DATASET}/templates/${template}" + + if ! fetch "$base_tarball" -o - | tar xzf - -C "${JAIL_HOME}/templates/${template}"; then + zfs destroy -v "${JAIL_DATASET}/templates/${template}" + die "failed to extract base tarball for ${release}" + fi + + template::update_release "$template" +} + +template::release2name(){ + # Convert a FreeBSD release version to a template name. + # e.g. for "13.2-RELEASE", return "freebsd13". + setvar "$1" "freebsd${2%.*}" +} + +template::update_release(){ + # Run freebsd-update within a the given template and take a fresh snapshot. + local template=$1 snapshot + + PAGER=/bin/cat freebsd-update -b "${JAIL_HOME}/templates/${template}" --not-running-from-cron fetch install + + snapshot=$("${JAIL_HOME}/templates/${template}/bin/freebsd-version") + template::exists "${template}@${snapshot}" || zfs snapshot "${JAIL_DATASET}/templates/${template}@${snapshot}" +} + +zfs::ensure_snapshot(){ + # If the given zfs dataset is a snapshot, return as-is. + # Otherwise, get the latest snapshot from the given dataset and return that. + local upvar=$1 dataset=$2 latest + + if [ "${dataset#*@}" = "$dataset" ]; then + latest=$(zfs list -t snapshot -o name -s creation -H "$dataset" | tail -1) + setvar "$upvar" "$latest" + else + zfs list -t snapshot -H "$dataset" > /dev/null 2>&1 || die "no such snapshot: ${dataset}" + setvar "$upvar" "$dataset" + fi +} + +zfs::dataset_exists(){ + zfs list "$1" > /dev/null 2>&1 +} + + +cmd::main "$@" diff --git a/files/usr/local/sbin/vmctl.freebsd_hypervisor b/files/usr/local/sbin/vmctl.freebsd_hypervisor new file mode 100644 index 0000000..b5bc9cf --- /dev/null +++ b/files/usr/local/sbin/vmctl.freebsd_hypervisor @@ -0,0 +1,1198 @@ +#!/bin/sh +# +# Bhyve management utility. + +set -eu -o pipefail + +. /usr/local/etc/vmctl.conf + +cmd::main(){ + local usage="COMMAND [ARGS]... +FreeBSD bhyve management utility. +Commands: + create Create a new VM + create-snapshot Take a snapshot of a VM + create-template Create a template from a VM disk image + console Connect to a VM's serial console + destroy Destroy a VM and its dataset + destroy-iso Delete an ISO + destroy-snapshot Delete a VM snapshot + destroy-template Delete a VM template + download-iso Download an ISO + download-template Download a disk image + edit Edit a VM's configuration + list List VMs + list-isos List available ISOs + list-snapshots List VM snapshots + list-templates List VM templates + reprovision Wipe and reprovision an OS disk from a template + restart Restart a VM + rollback Rollback a VM to a given snapshot + show Show VM configuration + start Start a VM + status Show VM status + stop Stop a VM" + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -ge 1 ] || cmd::usage 'no comand specified' + local cmd=$1; shift + + case $cmd in + console) cmd::console "$@" ;; + create) cmd::create "$@" ;; + create-snapshot|snapshot|snap) cmd::create_snapshot "$@" ;; + create-template) cmd::create_template "$@" ;; + destroy|rm) cmd::destroy "$@" ;; + destroy-iso|rmi) cmd::destroy_iso "$@" ;; + destroy-snapshot|rms) cmd::destroy_snapshot "$@" ;; + destroy-template|rmt) cmd::destroy_template "$@" ;; + download-iso|dli) cmd::download_iso "$@" ;; + download-template|dlt) cmd::download_template "$@" ;; + edit) cmd::edit "$@" ;; + list|ls) cmd::list "$@" ;; + list-isos|lsi) cmd::list_isos "$@" ;; + list-snapshots|lss) cmd::list_snapshots "$@" ;; + list-templates|lst) cmd::list_templates "$@" ;; + reprovision) cmd::reprovision "$@" ;; + restart|reboot) cmd::restart "$@" ;; + rollback) cmd::rollback "$@" ;; + show) cmd::show "$@" ;; + start|boot) cmd::start "$@" ;; + status) cmd::status "$@" ;; + stop|shutdown) cmd::stop "$@" ;; + # The following commands are internal and should not be invoked manually. + _start-all) cmd::_start_all "$@" ;; + _stop-all) cmd::_stop_all "$@" ;; + *) cmd::usage "unknown command: ${cmd}" ;; + esac +} + + +################################################################################ +# Standard helper functions. +################################################################################ +die(){ + printf '%s: %s\n' vmctl "$*" 1>&2 + exit 1 +} + + +################################################################################ +# CLI-related functions. +################################################################################ +cmd::help(){ + printf 'Usage: %s %s\n' vmctl "$usage" + [ -n "${help:-}" ] && printf '%s\n' "$help" + exit 0 +} + +cmd::usage(){ + [ $# -gt 0 ] && printf '%s: %s\n' "vmctl" "$1" 1>&2 + printf 'Usage: %s %s\n' vmctl "${usage}" 1>&2 + exit 2 +} + +cmd::getopt_help(){ + local opt + while getopts :h opt; do + case $opt in + h) cmd::help ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done +} + +cmd::console(){ + local usage='console VM' + local help='Connect to serial console. +Notes: + Type ~. to return to your shell. + See `man 1 cu` for additional escape codes.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local vm=$1 + + vm::exists "$vm" || die "no such vm: ${vm}" + vm::running "$vm" || die "vm not running: ${vm}" + + exec cu -l "/dev/nmdm_${vm}B" -s 9600 +} + +cmd::create(){ + local usage='create [-A] [-a IP] [-c CORES] [-d DOMAIN] [-g GATEWAY] [-i ISO] [-k SSHKEY] + [-m MEM] [-p PREFIXLEN] [-q DATA_SIZE] [-Q OS_SIZE] [-r NAMESERVER] + [-s SEARCHDOMAIN] [-v VLANID] NAME [TEMPLATE]' + local help="Create a new VM. +Options: + -A Don't autostart on boot + -a IP IPv4 address (cloud-init) + -c CORES Number of CPU cores + -d DOMAIN Host domain name (cloud-init) + -g GATEWAY Default IPv4 gateway (cloud-init) + -i ISO Boot from an ISO + -k SSHKEY Path to SSH pubkey for root's authorized_keys (cloud-init) + -m MEM Size of memory (RAM) + -p PREFIXLEN IPv4 network prefix length (cloud-init) + -q DATA_SIZE Size of data disk + -Q OS_SIZE Size of OS disk + -r NAMESERVER DNS resolver (cloud-init) + -s SEARCHDOMAIN DNS search domain (cloud-init) + -v VLANID VLAN ID number" + + local \ + autostart=true \ + cpus=$DEFAULT_CPUS \ + data_size=$DEFAULT_DATA_SIZE \ + domain=$DEFAULT_DOMAIN \ + gateway \ + ip \ + memory=$DEFAULT_MEMORY \ + nameservers \ + prefixlen=$DEFAULT_PREFIXLEN \ + os_size=$DEFAULT_OS_SIZE \ + searchdomains \ + vlan=$DEFAULT_VLAN \ + ssh_key_files \ + ssh_keys \ + iso \ + opt + + while getopts :Aa:c:d:g:hi:k:m:p:q:Q:r:s:v: opt; do + case $opt in + A) autostart=false ;; + a) ip=${OPTARG} ;; + c) cpus=$OPTARG ;; + d) domain=$OPTARG ;; + g) gateway=$OPTARG ;; + h) cmd::help ;; + i) iso=$OPTARG ;; + k) ssh_key_files="${ssh_key_files:-} ${OPTARG}" ;; + m) memory=$OPTARG ;; + p) prefixlen=$OPTARG ;; + q) data_size=$OPTARG ;; + Q) os_size=$OPTARG ;; + r) nameservers="${nameservers:-}${OPTARG}," ;; + s) searchdomains="${searchdomains:-}${OPTARG}," ;; + v) vlan=$OPTARG ;; + :) cmd::usage "missing option value: -${OPTARG}" ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'NAME not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local name=$1 template=${2:-} + + vm::exists "$name" && die "vm name already in use: ${name}" + + if [ -n "$template" ]; then + template::exists "$template" || die "no such template: ${template}" + fi + + if [ -n "${iso:-}" ]; then + iso::exists "$iso" || die "no such iso: ${iso}" + fi + + local ssh_keys key + for key in ${ssh_key_files:-}; do + [ -f "${key}" ] || die "no such ssh key: ${key}" + ssh_keys="${ssh_keys:-} - $(cat "$key") +" + done + + interface::exists "bridge${vlan}" || interface::add_vlan "$vlan" + + # Create the parent dataset. + zfs create -v "${VM_DATASET}/${name}" + + if [ -n "$template" ]; then + # Clone the OS zvol from template. + zfs::ensure_snapshot snapshot "${VM_DATASET}/templates/${template}" + zfs clone $ZFS_OPTS -o volmode=dev "$snapshot" "${VM_DATASET}/${name}/os" + else + # Create a new OS zvol. + zfs create -v -o volblocksize="$ZFS_VOLBLOCKSIZE" $ZFS_OPTS -o volmode=dev -V "$os_size" "${VM_DATASET}/${name}/os" + fi + + # Create a data zvol. + zfs create -v -o volblocksize="$ZFS_VOLBLOCKSIZE" $ZFS_OPTS -o volmode=dev -V "$data_size" "${VM_DATASET}/${name}/data" + + local macaddr uuid + macaddr=$(interface::derive_mac "$TRUNK_INTERFACE" "$name") + uuid=$(uuidgen) + + # Generate bhyve config file. + cat <<EOF > "${VM_HOME}/${name}/bhyve.conf" +name=${name} +uuid=${uuid} +cpus=${cpus} +memory.size=${memory} +acpi_tables=true +destroy_on_poweroff=true +keyboard.layout=us_unix +rtc.use_localtime=false +x86.strictmsr=true +x86.vmexit_on_hlt=true +x86.vmexit_on_pause=true +pci.0.0.0.device=hostbridge +pci.0.1.0.device=lpc +pci.0.2.0.device=virtio-net +pci.0.2.0.backend=$(interface::tap::derive_name "$name") +pci.0.2.0.mac=${macaddr} +pci.0.3.0.device=nvme +pci.0.3.0.path=/dev/zvol/${VM_DATASET}/%(name)/os +pci.0.4.0.device=nvme +pci.0.4.0.path=/dev/zvol/${VM_DATASET}/%(name)/data +pci.0.5.0.device=ahci +pci.0.5.0.port.0.type=cd +pci.0.5.0.port.0.path=${VM_HOME}/%(name)/seed.iso +pci.0.5.0.port.0.ro=true +lpc.com1.path=/dev/nmdm_%(name)A +lpc.bootrom=/usr/local/share/uefi-firmware/BHYVE_UEFI.fd +lpc.bootvars=${VM_HOME}/${name}/uefi-vars.fd +vmctl.vlan=${vlan} +EOF + + # Generate cloud-init network configuration. + if [ -z "${ip:-}" ]; then + cat <<EOF > "${VM_HOME}/${name}/network-config" +version: 2 +ethernets: + id0: + match: + macaddress: '${macaddr}' + dhcp4: True + dhcp6: False +EOF + else + cat <<EOF > "${VM_HOME}/${name}/network-config" +version: 2 +ethernets: + id0: + match: + macaddress: '${macaddr}' + addresses: [${ip}/${prefixlen}] + routes: + - to: 0.0.0.0/0 + via: ${gateway:-} + dhcp4: False + dhcp6: False + nameservers: + search: [${searchdomains:-}] + addresses: [${nameservers:-}] +EOF + fi + + # Generate cloud-init metadata. + cat <<EOF > "${VM_HOME}/${name}/meta-data" +instance-id: ${uuid} +local-hostname: ${name} +EOF + + # Generate cloud-init userdata. + cat <<EOF > "${VM_HOME}/${name}/user-data" +#cloud-config +disable_root: False +ssh_pwauth: False +resize_rootfs: True +ssh_authorized_keys: +${ssh_keys:-} +users: [] +fqdn: ${name}.${domain} +prefer_fqdn_over_hostname: True +runcmd: + - sed -i.bak -E '/^#?PermitRootLogin/s/^.*$/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config + - rm -f /etc/ssh/sshd_config.bak + - service ssh restart + - service sshd restart +EOF + + # Generate cloud-init ISO file. + genisoimage -output "${VM_HOME}/${name}/seed.iso" -volid cidata -joliet -input-charset utf-8 -rock \ + "${VM_HOME}/${name}/network-config" \ + "${VM_HOME}/${name}/meta-data" \ + "${VM_HOME}/${name}/user-data" + + # Copy initial UEFI vars. + cp /usr/local/share/uefi-firmware/BHYVE_UEFI_VARS.fd "${VM_HOME}/${name}/uefi-vars.fd" + + # Enable autostart (if requested). + [ "$autostart" = true ] && sysrc "vmctl_list+=${name}" + + # Start the VM. + vm::start "$name" "${iso:-}" +} + +cmd::create_snapshot(){ + local usage='create-snapshot [-d] [-o] VM [SNAPNAME]' + local help='Create a VM snapshot. +Options: + -d Only snapshot the data dataset + -o Only snapshot the OS dataset' + + local opt both=true os_only=false data_only=false snapname + while getopts :dho opt; do + case $opt in + d) data_only=true; both=false ;; + h) cmd::help ;; + o) os_only=true; both=false ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + vm=$1 snapname=${2:-} + + vm::exists "$vm" || die "no such vm: ${vm}" + + [ -n "$snapname" ] || snapname=$(date +%Y-%m-%dT%H:%M:%S) + + # Snapshot the OS disk. + if [ "$both" = true ] || [ "$os_only" = true ]; then + zfs snapshot "${VM_DATASET}/${vm}/os@${snapname}" + fi + + # Snapshot the data disk. + if [ "$both" = true ] || [ "$data_only" = true ]; then + zfs snapshot "${VM_DATASET}/${vm}/data@${snapname}" + fi +} + +cmd::create_template(){ + local usage='create-template NAME VM' + local help='Create a template from a VM.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'NAME not specified' + [ $# -lt 2 ] && cmd::usage 'VM not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local name=$1 vm=$2 + + vm::exists "$vm" || die "no such vm: ${vm}" + template::exists "$name" && die "template already exists: $name" + vm::running "$vm" && die "refusing to create template while vm is running: ${vm}" + + local snapname + snapname=$(date +%Y-%m-%dT%H:%M:%S) + zfs snapshot "${VM_DATASET}/${vm}/os@${snapname}" + zfs send "${VM_DATASET}/${vm}/os@${snapname}" | zfs receive -v $ZFS_OPTS -o volmode=dev "${VM_DATASET}/templates/${name}" + zfs destroy "${VM_DATASET}/${vm}/os@${snapname}" +} + +cmd::destroy_iso(){ + local usage='destroy-iso ISO' + local help='Delete an ISO.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'ISO not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + iso=$1 + + iso::exists "$iso" || die "no such iso: ${iso}" + rm "${VM_HOME}/isos/${iso}.iso" +} + +cmd::destroy(){ + local usage='destroy [-y] VM' + local help="Delete a VM and its dataset. +Options: + -y Don't prompt for confirmation" + + local noconfirm=false answer opt + + while getopts :hy opt; do + case $opt in + h) cmd::help ;; + y) noconfirm=true ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local vm=$1 + + vm::exists "$vm" || die "no such vm: ${vm}" + + if [ "$noconfirm" != true ]; then + read -rp "Really destroy vm ${vm}? (y/N) " answer + case $answer in + [yY]|[yY][eE][sS]) : ;; + *) die 'operation cancelled' ;; + esac + fi + + # If the VM is running, stop it. + vm::running "$1" && vm::stop "$1" KILL + + # Destroy the VM's dataset. + zfs destroy -v -r "${VM_DATASET}/${vm}" + + # Remove VM from autostart list. + sysrc -v "vmctl_list-=${vm}" +} + +cmd::destroy_snapshot(){ + local usage='destroy-snapshot [-y] VM SNAPSHOT' + local help="Delete a VM snapshot. +Options: + -y Don't prompt for confirmation" + + local noconfirm=false answer opt + + while getopts :hy opt; do + case $opt in + h) cmd::help ;; + y) noconfirm=true ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -lt 2 ] && cmd::usage 'SNAPSHOT not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local vm=$1 snapshot=$2 + + vm::exists "$vm" || die "no such vm: ${vm}" + + local datasets='' + + if [ "${snapshot#*@}" != "$snapshot" ]; then + # If the snapshot name contains '@', then we have something like 'os@snapname'. + zfs::dataset_exists "${VM_DATASET}/${vm}/${snapshot}" && \ + datasets="${VM_DATASET}/${vm}/${snapshot}" + else + # Otherwise, check if either os or data dataset contains a matching snapshot. + zfs::dataset_exists "${VM_DATASET}/${vm}/os@${snapshot}" && \ + datasets="${VM_DATASET}/${vm}/os@${snapshot}" + + zfs::dataset_exists "${VM_DATASET}/${vm}/data@${snapshot}" && \ + datasets="${datasets} ${VM_DATASET}/${vm}/data@${snapshot}" + fi + + [ -n "$datasets" ] || die "no such snapshot for vm ${vm}: ${snapshot}" + + if [ "$noconfirm" != true ]; then + read -rp "Really destroy ${vm} snapshot ${snapshot}? (y/N) " answer + case $answer in + [yY]|[yY][eE][sS]) : ;; + *) die 'operation cancelled' ;; + esac + fi + + local dataset + for dataset in $datasets; do + zfs destroy -v "$dataset" + done +} + +cmd::destroy_template(){ + local usage='destroy-template [-y] TEMPLATE' + local help="Delete a VM template image. +Options: + -y Don't prompt for confirmation +Notes: + A template cannot be deleted while its clones still exist." + + local noconfirm=false answer opt + + while getopts :hy opt; do + case $opt in + h) cmd::help ;; + y) noconfirm=true ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'TEMPLATE not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local template=$1 + + template::exists "$template" || die "no such template: ${template}" + + if [ "$noconfirm" != true ]; then + read -rp "Really destroy template ${template}? (y/N) " answer + case $answer in + [yY]|[yY][eE][sS]) : ;; + *) die 'operation cancelled' ;; + esac + fi + + zfs destroy -v -r "${VM_DATASET}/templates/${template}" +} + +cmd::download_iso(){ + local usage='download-iso URL [NAME]' + local help='Download an ISO.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'URL not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local url=$1 name=${2:-} + + [ -n "$name" ] || name=$(basename "$url" .iso) + + iso::exists "$name" && die "iso already exists: ${name}" + + fetch -o "${VM_HOME}/isos/${name}.iso" "$url" +} + +cmd::download_template(){ + local usage='download-template URL [NAME]' + local help='Download a disk image.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'URL not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local url=$1 name=${2:-} + + local ext image_ok=false + for ext in .raw .raw.xz .qcow2; do + if [ -z "${url%%*"$ext"}" ]; then + image_ok=true + [ -n "$name" ] || name=$(basename "$url" "$ext") + break + fi + done + + [ "$image_ok" = true ] || die "unknown image type: $(basename "$url")" + template::exists "$name" && die "template already exists: ${name}" + + local dataset="${VM_DATASET}/templates/${name}" \ + zvol="/dev/zvol/${VM_DATASET}/templates/${name}" + + # Create a ZFS dataset for the template. + zfs create -v -o volblocksize="$ZFS_VOLBLOCKSIZE" $ZFS_OPTS -o volmode=dev -V "$TEMPLATE_ZVOL_SIZE" "$dataset" + + # Extract the template into the zvol, based on its file type. + case $url in + http://*.raw|https://*.raw) + fetch "$url" -o "$zvol" || zfs::cleanup_and_die "$dataset" + ;; + *.raw) + dd if="$url" of="$zvol" bs=1M || zfs::cleanup_and_die "$dataset" + ;; + http://*.raw.xz|https://*.raw.xz) + fetch "$url" -o - | xz -d | dd of="$zvol" bs=1M || zfs::cleanup_and_die "$dataset" + ;; + *.raw.xz) + xz -cd "$url" | dd of="$zvol" bs=1M || zfs::cleanup_and_die "$dataset" + ;; + http://*.qcow2|https://*.qcow2) + fetch "$url" -o "/root/${name}.qcow2" || zfs::cleanup_and_die "$dataset" + qemu-img dd -O raw if="/root/${name}.qcow2" of="$zvol" bs=1M || zfs::cleanup_and_die "$dataset" + rm "/root/${name}.qcow2" + ;; + *.qcow2) + qemu-img dd -O raw if="$url" of="$zvol" bs=1M || zfs::cleanup_and_die "$dataset" + ;; + *) # NOTREACHED + ;; + esac + + # Snapshot the template so it can be cloned. + zfs snapshot "${VM_DATASET}/templates/${name}@$(date +%Y-%m-%dT%H:%M:%S)" +} + +cmd::edit(){ + local usage='edit VM' + local help='Edit VM configuration.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local vm=$1 + + vm::exists "$vm" || die "no such vm: ${vm}" + "$EDITOR" "${VM_HOME}/${vm}/bhyve.conf" +} + +cmd::list(){ + local usage='list [-t]' + local help='List configured VMs.' + + local opt terse=false + while getopts :th opt; do + case $opt in + t) terse=true ;; + h) cmd::help ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -eq 0 ] || cmd::usage 'too many arguments' + + local vm status + { [ $terse = true ] || echo 'VM STATUS' + for vm in $(vm::list); do + if [ $terse = true ]; then + printf '%s\n' "$vm" + else + if vm::running "${vm}"; then + status=running + else + status=stopped + fi + printf '%s %s\n' "$vm" "$status" + fi + done + } | column -t +} + +cmd::list_isos(){ + local usage='list-isos' + local help='List available ISOs.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + [ $# -gt 0 ] && cmd::usage 'too many arguments' + + local file + for file in "${VM_HOME}"/isos/*.iso; do + [ -e "$file" ] || continue + name=${file##*/} + name=${name%.iso} + printf '%s\n' "$name" + done +} + +cmd::list_snapshots(){ + local usage='list-snapshots VM' + local help='List VM snapshots. +Options: + -d List snapshots from data dataset only + -o List snapshots from OS dataset only + -t Use terse output' + + local opt terse=false dataset='' + while getopts :dhot opt; do + case $opt in + d) dataset=/data ;; + h) cmd::help ;; + o) dataset=/os ;; + t) terse=true ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local vm=$1 + + vm::exists "$vm" || die "no such vm: ${vm}" + + if [ "$terse" = true ]; then + zfs list -r -t snapshot -H -o name -s creation "${VM_DATASET}/${vm}${dataset}" 2>/dev/null \ + | sed 's/^.*\///' \ + | sort -k1,1 -t@ -s + else + { echo 'DATASET SNAPSHOT USED REFER' + zfs list -r -t snapshot -H -o name,used,refer -s creation "${VM_DATASET}/${vm}${dataset}" 2>/dev/null + } | sed 's/^.*\///' \ + | sort -k1,1 -t@ -s \ + | tr '@' ' ' \ + | column -t + fi +} + +cmd::list_templates(){ + local usage='list-templates' + local help='list template images.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -eq 0 ] || cmd::usage 'too many arguments' + + ls -1 /dev/zvol/${VM_DATASET}/templates 2>/dev/null +} + +cmd::reprovision(){ + local usage='reprovision VM TEMPLATE' + local help="Wipe and reprovision a VM's OS dataset from a template." + + local opt noconfirm=false running=false + while getopts :hy opt; do + case $opt in + h) cmd::help ;; + y) noconfirm=true ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -lt 2 ] && cmd::usage 'TEMPLATE not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local vm=$1 template=$2 + + vm::exists "$vm" || die "no such vm: ${vm}" + template::exists "$template" || die "no such template: ${template}" + + if [ "$noconfirm" != true ]; then + read -rp "Really reprovision ${vm}? (y/N) " answer + case $answer in + [yY]|[yY][eE][sS]) : ;; + *) die 'operation cancelled' ;; + esac + fi + + # If the vm is running, stop it. + if vm::running "$vm"; then + running=true + vm::stop "$vm" + fi + + local snapshot old_size + + # Get the latest snapshot for the template (if not specified). + zfs::ensure_snapshot snapshot "${VM_DATASET}/templates/${template}" + + # Stash old disk size. + old_size=$(zfs get -Hp -o value volsize "${VM_DATASET}/${vm}/os") + + # Reprovision OS dataset from template. + zfs destroy -v -r "${VM_DATASET}/${vm}/os" + zfs clone $ZFS_OPTS -o volmode=dev -o volsize="$old_size" "$snapshot" "${VM_DATASET}/${vm}/os" + + # If the jail was running, restart it. + if [ "$running" = true ]; then + vm::start "$vm" + fi +} + +cmd::restart(){ + local usage='restart VM' + local help='Restart a VM.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + + vm::exists "$1" || die "no such vm: ${1}" + vm::running "$1" || die "vm not running: ${1}" + + vm::stop "$1" + vm::start "$1" +} + +cmd::show(){ + local usage='show VM' + local help='Show VM configuration.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local vm=$1 + + vm::exists "$vm" || die "no such vm: ${vm}" + + printf -- '------------------------- BHYVE CONFIGURATION -------------------------\n' + cat "${VM_HOME}/${vm}/bhyve.conf" + printf -- '\n---------------------------- ZFS DATASET -----------------------------\n' + zfs list -o name,volsize,used,refer -S name \ + "${VM_DATASET}/${vm}/os" \ + "${VM_DATASET}/${vm}/data" +} + +cmd::rollback(){ + local usage='rollback VM SNAPSHOT' + local help='Rollback a VM to a given snapshot. +Options: + -d Rollback the data zvol only + -f Attempt rollback even if the jail is running + -o Rollback the OS zvol only' + + local opt force=false both=true os_only=false data_only=false + while getopts :dfho opt; do + case $opt in + d) data_only=true; both=false ;; + f) force=true ;; + h) cmd::help ;; + o) os_only=true; both=false ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -lt 2 ] && cmd::usage 'SNAPSHOT not specified' + [ $# -gt 2 ] && cmd::usage 'too many arguments' + local vm=$1 snapshot=$2 + + vm::exists "$vm" || die "no such vm: ${vm}" + + if [ "$both" = true ] || [ "$os_only" = true ]; then + zfs::dataset_exists "${VM_DATASET}/${vm}/os@${snapshot}" \ + || die "no such snapshot for ${vm}/os: ${snapshot}" + fi + if [ "$both" = true ] || [ "$data_only" = true ]; then + zfs::dataset_exists "${VM_DATASET}/${vm}/data@${snapshot}" \ + || die "no such snapshot for ${vm}/data: ${snapshot}" + fi + + if vm::running "$vm" && [ "$force" != true ]; then + die "vm ${vm} is running, refusing to rollback (-f to override)" + fi + + # Rollback the OS disk. + if [ "$both" = true ] || [ "$os_only" = true ]; then + zfs rollback -r "${VM_DATASET}/${vm}/os@${snapshot}" + fi + + # Rollback the data disk. + if [ "$both" = true ] || [ "$data_only" = true ]; then + zfs rollback -r "${VM_DATASET}/${vm}/data@${snapshot}" + fi +} + +cmd::start(){ + local usage='start [-i ISO] VM' + local help='Start a VM. +Options: + -i ISO Boot from an ISO' + + local iso opt + while getopts :hi: opt; do + case $opt in + h) cmd::help ;; + i) iso=$OPTARG ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local vm=$1 + + vm::exists "$vm" || die "no such vm: ${vm}" + vm::running "$vm" && die "vm already running: ${vm}" + + if [ -n "${iso:-}" ]; then + iso::exists "$iso" || die "no such iso: ${iso}" + fi + + vm::start "$vm" "${iso:-}" +} + +cmd::_start_all(){ + local usage='_start-all' + local help='Start all VMs.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + [ $# -eq 0 ] || cmd::usage 'too many arguments' + + rc::load_config + + local vm + for vm in ${vmctl_list:-}; do + echo "vmctl: starting ${vm}" + vm::start "$vm" + sleep "${vmctl_delay:-$DEFAULT_AUTOSTART_DELAY}" + done +} + +cmd::status(){ + local usage='status VM' + local help='Show running VM status.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local vm=$1 + + vm::exists "$vm" || die "no such vm: ${vm}" + vm::running "$vm" || die "vm not running: ${vm}" + + local pid + pid=$(vm::pid "$vm") + + printf -- '----------------------------- BHYVE PROCESS ------------------------------\n' + ps -auxdr -p "$pid" + printf -- '\n---------------------------- ZFS DATASET -----------------------------\n' + zfs list -o name,volsize,used,refer -S name \ + "${VM_DATASET}/${vm}/os" \ + "${VM_DATASET}/${vm}/data" \ + | sed "s|^${VM_DATASET}/${vm}/||" \ + | column -t + printf -- '\n--------------------------- RESOURCE USAGE ---------------------------\n' + rctl -h -u "process:${pid}" \ + | grep -E '^(pcpu|memoryuse|vmemoryuse|swapuse|readbps|writebps|readiops|writeiops)=' \ + | rs -c= -C' ' -T \ + | column -t +} + +cmd::stop(){ + local usage='start VM' + local help='Shutdown a VM. +Options: + -f Send SIGKILL' + + local opt signal=TERM + while getopts :fh opt; do + case $opt in + f) signal=KILL ;; + h) cmd::help ;; + ?) cmd::usage "unknown option: -${OPTARG}" ;; + esac + done + shift $((OPTIND - 1)) + + [ $# -lt 1 ] && cmd::usage 'VM not specified' + [ $# -gt 1 ] && cmd::usage 'too many arguments' + local vm=$1 + + vm::exists "$vm" || die "no such vm: ${vm}" + vm::running "$vm" || die "vm not running: ${vm}" + + vm::stop "$vm" "$signal" +} + +cmd::_stop_all(){ + local usage='_stop-all' + local help='Stop all VMs.' + + cmd::getopt_help "$@"; shift $((OPTIND - 1)) + [ $# -eq 0 ] || cmd::usage 'too many arguments' + + rc::load_config + + # Reverse the autostart list. + local vm vmctl_list_rev='' + for vm in ${vmctl_list:-}; do + vmctl_list_rev="${vm} ${vmctl_list_rev}" + done + + # Stop the VMs in reverse order. + for vm in $vmctl_list_rev; do + if vm::running "$vm"; then + echo "vmctl: stopping ${vm}" + vm::stop "$vm" + sleep "${vmctl_delay:-$DEFAULT_AUTOSTART_DELAY}" + fi + done + + # Stop any other VMs that might be running. + for vm in $(vm::list); do + if vm::running "$vm"; then + echo "vmctl: stopping ${vm}" + vm::stop "$vm" + fi + done +} + + +################################################################################ +# "Library" functions. +################################################################################ +interface::add_vlan(){ + local id=$1 + ifconfig "vlan${id}" create + ifconfig "vlan${id}" vlan "$id" vlandev "$TRUNK_INTERFACE" up + ifconfig "bridge${id}" create + ifconfig "bridge${id}" addm "vlan${id}" up + sysrc -v \ + "cloned_interfaces+=vlan${id} bridge${id}" \ + "ifconfig_vlan${id}=vlan ${id} vlandev ${TRUNK_INTERFACE} up" \ + "ifconfig_bridge${id}=addm vlan${id} up" +} + +interface::derive_mac(){ + # Derive unique mac addresses for a virtual NIC, based on the physical + # interface address + vm name. + local iface=$1 name=$2 + local hwaddr checksum virtual_mac + + # Similar to /usr/share/examples/jails/jib + hwaddr=$(ifconfig "$iface" ether | awk '/ether/,$0=$2') + # ??:??:??:II:II:II + virtual_mac=${hwaddr#??:??:??} # => :II:II:II + # => :SS:SS:II:II:II + virtual_mac=":$(printf '%s' "$name" | md5 | sed 's/^\(..\)\(..\).*/\1:\2/')${virtual_mac}" + # => NP:SS:SS:II:II:II + case $hwaddr in + ?2:*) virtual_mac="1e${virtual_mac}" ;; + ?[Ee]:*) virtual_mac="16${virtual_mac}" ;; + *) virtual_mac="1e${virtual_mac}" ;; + esac + + # Sanity check for duplicate MAC + ifconfig | grep -qE "(ether|hwaddr) ${virtual_mac}" && die "MAC collision when configuring virutal interface!" + + printf '%s' "${virtual_mac}" +} + +interface::exists(){ + ifconfig "$1" > /dev/null 2>&1 +} + +interface::tap::derive_name(){ + # Generate an tap(4) interface name, based on the VM name. + # + # The maximum length of a network interface name on FreeBSD is 15 characters. + # We use a prefix of 'tap_', leaving 11 characters for a unique suffix for each + # VM. + # + # If the sanitized VM name is less than 11 characters, we'll simply use it + # for the suffix. Otherwise, we'll use the last 11 characters of the VM + # name's SHA-1 hash. + local name=$1 sanitized hash + sanitized=$(printf '%s' "$name" | tr -dC '[:alnum:]_') + + if [ "${#sanitized}" -le 11 ]; then + printf 'tap_%s' "$sanitized" + else + hash=$(printf '%s' "$name" | sha1 | tail -c 11) + printf 'tap_%s' "$hash" + fi +} + +iso::exists(){ + test -f "${VM_HOME}/isos/${1}.iso" +} + +rc::load_config(){ + set +eu +o pipefail + . /etc/rc.subr + load_rc_config vmctl + set -eu -o pipefail +} + +template::exists(){ + zfs list -H "${VM_DATASET}/templates/${1}" > /dev/null 2>&1 +} + +vm::exists(){ + test -f "${VM_HOME}/${1}/bhyve.conf" +} + +vm::list(){ + for file in "${VM_HOME}"/*/bhyve.conf; do + [ -e "$file" ] || continue + printf '%s\n' "$(basename "$(dirname "$file")")" + done +} + +vm::pid(){ + pgrep -fx "bhyve: ${1}" +} + +vm::run(){ + local vm=$1 iso=${2:-} bootdisk zvol iso_args bootcount=0 rc=0 + + zvol="/dev/zvol/${VM_DATASET}/${vm}/os" + bootdisk=$zvol + iso_args='' + + # bhyve exit codes: + # 0: reboot + # 1: shutdown + # >1: error + # As long as rc=0, keep rebooting. + + while [ "$rc" -eq 0 ]; do + if [ -n "$iso" ]; then + # If this is the first boot, or no bootsector exists, boot from ISO. + if [ "$bootcount" -eq 0 ] || ! file -bs "$zvol" | grep -q 'boot sector'; then + bootdisk="${VM_HOME}/isos/${iso}.iso" + iso_args="-s 31:0,ahci-cd,${bootdisk}" + else + # Otherwise, boot from disk. + bootdisk=$zvol + iso_args='' + fi + fi + + bootcount=$(( bootcount + 1 )) + + bhyve -k "${VM_HOME}/${vm}/bhyve.conf" $iso_args "$vm" && rc=$? || rc=$? + done + + # If we reach this point, the bhyve process has terminated. Clean up the + # /dev/vmm device and tap interface. + bhyvectl --vm="${vm}" --destroy + ifconfig "$(interface::tap::derive_name "$vm")" destroy +} + +vm::running(){ + vm::pid "$1" > /dev/null +} + +vm::stop(){ + local vm=$1 signal=${2:-TERM} pid + pid=$(vm::pid "$vm") + + kill -s "$signal" "$pid" + + # Loop until the bhyve process terminates. + while kill -0 "$pid" 2>/dev/null; do + sleep 1 + done +} + +vm::start(){ + local vm=$1 iso=${2:-} vlan newif + + vlan=$(vm::config::get "$vm" vmctl.vlan) + + # Create the tap interface. + newif=$(ifconfig tap create) + ifconfig "bridge${vlan}" addm "$newif" + ifconfig "$newif" name "$(interface::tap::derive_name "$vm")" descr "vm/${vm}" + + # Start the bhyve process in the background. + vm::run "$vm" "$iso" > /dev/null 2>&1 & +} + +vm::config::get(){ + local vm=$1 key=$2 + awk -F= -v "key=${key}" \ + '$1 == key {rc=1; print $2} END {exit !rc}' \ + "${VM_HOME}/${vm}/bhyve.conf" \ + || die "config value not set for vm ${vm}: ${key}" +} + +zfs::ensure_snapshot(){ + # If the given zfs dataset is a snapshot, return as-is. + # Otherwise, get the latest snapshot from the given dataset and return that. + local upvar=$1 dataset=$2 latest + + if [ "${dataset#*@}" = "$dataset" ]; then + latest=$(zfs list -t snapshot -o name -s creation -H "$dataset" | tail -1) + setvar "$upvar" "$latest" + else + zfs list -t snapshot -H "$dataset" > /dev/null 2>&1 || die "no such snapshot: ${dataset}" + setvar "$upvar" "$dataset" + fi +} + +zfs::cleanup_and_die(){ + zfs destroy -v -r "$1" + exit 1 +} + +zfs::dataset_exists(){ + zfs list "$1" > /dev/null 2>&1 +} + + +cmd::main "$@" diff --git a/lib/10-core b/lib/10-core index d7280d9..5fa2a16 100644 --- a/lib/10-core +++ b/lib/10-core @@ -98,7 +98,7 @@ _boxconf_include(){ elif [ -d "$1" ]; then for _bci_file in "$1"/*; do if [ -f "$_bci_file" ]; then - log "sourcing ${1#${BOXCONF_ROOT}/}" + log "sourcing ${_bci_file#${BOXCONF_ROOT}/}" BOXCONF_SOURCE=$_bci_file . "$BOXCONF_SOURCE" fi @@ -141,8 +141,10 @@ _boxconf_stage(){ # Compex find expression to only copy files necessary for the target host. # This avoids leaking site-wide secrets to hosts that don't require them. - _bcs_relevant_files=$(find "${BOXCONF_ROOT}" -type f -and \( \ - -path "${BOXCONF_CA_DIR}/${_bcs_hostname}" \ + set -f + _bcs_relevant_files=$(find -L "$BOXCONF_ROOT" -type f -and \( \ + -path "${BOXCONF_CA_DIR}/ca.crt" \ + -or -path "${BOXCONF_CA_DIR}/${_bcs_hostname}" \ -or -path "${BOXCONF_VAR_DIR}/common" \ -or -path "${BOXCONF_VAR_DIR}/common/*" \ -or -path "${BOXCONF_VAR_DIR}/os/*" \ @@ -194,6 +196,7 @@ _boxconf_stage(){ -or -path "${BOXCONF_SITE_FILE_DIR}/*.${BOXCONF_HOSTCLASS}" \ -or -path "${BOXCONF_SITE_FILE_DIR}/*.${_bcs_hostname}" \ \) ) + set +f OIFS=$IFS; IFS=$'\n' set -- $_bcs_relevant_files diff --git a/lib/40-user b/lib/40-user new file mode 100644 index 0000000..42bbb82 --- /dev/null +++ b/lib/40-user @@ -0,0 +1,26 @@ +#!/bin/sh + +set_authorized_keys(){ + # Add authorized_keys for a user. + # $1 = username + # $2 = newline-separated string of authorized keys + _sak_homedir=$(eval echo "~${1}") + _sak_group=$(getent passwd "$1" | awk -F: '{ print $4}') + + # Create authorized keys file and set permissions. + install_directory -o "$1" -g "$_sak_group" -m 0700 "${_sak_homedir}/.ssh" + [ -f "${_sak_homedir}/.ssh/authorized_keys" ] || touch "${_sak_homedir}/.ssh/authorized_keys" + chown "$1" "${_sak_homedir}/.ssh/authorized_keys" + chgrp "$_sak_group" "${_sak_homedir}/.ssh/authorized_keys" + chmod 600 "${_sak_homedir}/.ssh/authorized_keys" + + printf '%s\n' "${2}" > "${_sak_homedir}/.ssh/authorized_keys" + log "added authorized_keys for ${1}:"$'\n'"$2" +} + +set_password(){ + # Set password for a local user. + # $1 = username + # $2 = password + printf '%s\n%s\n' "$2" "$2" | passwd "$1" > /dev/null +} diff --git a/scripts/common/10-root-user b/scripts/common/10-root-user new file mode 100644 index 0000000..9a9f5e6 --- /dev/null +++ b/scripts/common/10-root-user @@ -0,0 +1,7 @@ +#!/bin/sh + +# Add root SSH pubkeys. +set_authorized_keys root "$root_authorized_keys" + +# Set root password. +set_password root "$root_password" diff --git a/scripts/common/20-dns b/scripts/common/20-dns new file mode 100644 index 0000000..e2d5ad6 --- /dev/null +++ b/scripts/common/20-dns @@ -0,0 +1,9 @@ +#!/bin/sh + +# For IDM servers, the resolver is localhost. In that case, we delay copying +# this file until the IDM stack is fully up and running. +if [ "$BOXCONF_HOSTCLASS" = idm_server ]; then + return +fi + +install_template -m 0644 /etc/resolv.conf diff --git a/scripts/hostclass/freebsd_hypervisor b/scripts/hostclass/freebsd_hypervisor new file mode 100644 index 0000000..bdaa3c0 --- /dev/null +++ b/scripts/hostclass/freebsd_hypervisor @@ -0,0 +1,80 @@ +#!/bin/sh + +: ${hypervisor_trunk_interface:='lagg0'} +: ${hypervisor_default_vlan:='1'} +: ${hypervisor_default_prefix:='24'} +: ${hypervisor_default_os_quota:='24G'} +: ${hypervisor_default_data_quota:='8G'} + +: ${hypervisor_vm_home:='/usr/local/bhyve'} +: ${hypervisor_vm_dataset:='tank/bhyve'} +: ${hypervisor_vm_default_cpus:='2'} +: ${hypervisor_vm_default_mem:='4G'} +: ${hypervisor_vm_template_size:='10G'} +: ${hypervisor_vm_default_autostart_delay:='2'} +: ${hypervisor_vm_default_zfs_opts:='-o primarycache=metadata -o compress=off'} +: ${hypervisor_vm_zfs_volblocksize:='64k'} + +: ${hypervisor_jail_home:='/usr/local/jails'} +: ${hypervisor_jail_dataset:='tank/jails'} +: ${hypervisor_jail_default_zfs_opts:='-o compress=lz4'} + +hypervisor_jail_bpf_ruleset=1000 + +# Required for vnet jails. +set_sysctl net.link.tap.up_on_open=1 + +# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=262189 +set_sysctl vfs.zfs.vol.mode=2 + +# Load required kernel modules. +load_kernel_module vmm nmdm linux linux64 +set_loader_conf \ + vmm_load=YES \ + nmdm_load=YES \ + linux_load=YES \ + linux64_load=YES \ + kern.racct.enable=1 + +# Install vm/jail management dependencies. +pkg install -y \ + bhyve-firmware \ + cdrkit-genisoimage \ + qemu-tools + +# Create bhyve VM dataset. +create_dataset -o "mountpoint=${hypervisor_vm_home}" "$hypervisor_vm_dataset" + +# Create dataset for bhyve templates. +create_dataset -o volmode=none -o mountpoint=none "${hypervisor_vm_dataset}/templates" + +# Create jails dataset. +create_dataset -o mountpoint="${hypervisor_jail_home}" "$hypervisor_jail_dataset" + +# Create dataset for jail templates. +create_dataset -o mountpoint="${hypervisor_jail_home}/templates" "${hypervisor_jail_dataset}/templates" + +# Lock down permissions on the VM and jail directories. +chmod 700 "$hypervisor_vm_home" "$hypervisor_jail_home" + +# Create directory for VM ISO files. +install_directory -m 0755 "${hypervisor_vm_home}/isos" + +# Copy jail/bhyve management scripts. +install_directory -m 0755 /usr/local/etc/rc.d + +install_file -m 0555 \ + /usr/local/sbin/jailctl \ + /usr/local/sbin/vmctl \ + /usr/local/etc/rc.d/vmctl + +install_template -m 0644 \ + /usr/local/etc/jailctl.conf \ + /usr/local/etc/vmctl.conf + +install_template -m 0644 /etc/devfs.rules + +# Enable jails/bhyve to start on boot. +sysrc -v \ + vmctl_enable=YES \ + jail_enable=YES diff --git a/scripts/os/freebsd/10-bootloader b/scripts/os/freebsd/10-bootloader new file mode 100644 index 0000000..0506606 --- /dev/null +++ b/scripts/os/freebsd/10-bootloader @@ -0,0 +1,29 @@ +#!/bin/sh + +# Skip this file if running in a jail - jails don't have a bootloader. +if [ "$BOXCONF_VIRTUALIZATION_TYPE" = jail ]; then + return +fi + +# Configure serial console. +install_file -m 0644 /boot.config +install_file -m 0644 /etc/ttys +kill -HUP 1 + +set_loader_conf \ + autoboot_delay=1 \ + beastie_disable=YES \ + boot_multicons=YES \ + boot_serial=YES \ + cc_htcp_load=YES \ + console=comconsole,efi \ + comconsole_speed=115200 \ + kern.geom.label.disk_ident.enable=0 \ + kern.geom.label.gptid.enable=0 \ + net.inet.tcp.soreceive_stream=1 \ + net.inet6.ip6.auto_linklocal=0 \ + net.isr.defaultqlimit=2048 \ + net.link.ifqmaxlen=2048 \ + pf_load=YES \ + pflog_load=YES \ + security.bsd.allow_destructive_dtrace=0 diff --git a/scripts/os/freebsd/10-cpu b/scripts/os/freebsd/10-cpu new file mode 100644 index 0000000..adc27d4 --- /dev/null +++ b/scripts/os/freebsd/10-cpu @@ -0,0 +1,28 @@ +#!/bin/sh + +# Only run this file on baremetal hosts. +if [ "$BOXCONF_VIRTUALIZATION_TYPE" != none ]; then + return +fi + +# Allow lower C-states. As of FreeBSD 13, the default is to only allow C1. +# My Xeon processor supports C2, and enabling that resulted in 15 watts of +# power savings. +# +# Note that if your CPU supports *very* low C-states (likely for commodity +# desktop and laptop hardware), you may not want them enabled, as transitioning +# from a very low C-state can cause rather severe latency spikes. +# +# Experiment with your hardware and set $cx_lowest accordingly. +sysrc -v \ + microcode_update_enable=YES \ + performance_cx_lowest="$cx_lowest" \ + economy_cx_lowest="$cx_lowest" + +# Set energy/performance preference for Intel P-states. +# 0 = most performance, 100 = most power savings +if sysctl -n dev.hwpstate_intel.0.epp >/dev/null 2>&1; then + for n in $(seq 0 $(($(sysctl -n hw.ncpu)-1))); do + set_sysctl "dev.hwpstate_intel.${n}.epp=${intel_epp}" + done +fi diff --git a/scripts/os/freebsd/10-periodic b/scripts/os/freebsd/10-periodic new file mode 100644 index 0000000..36ddd95 --- /dev/null +++ b/scripts/os/freebsd/10-periodic @@ -0,0 +1,53 @@ +#!/bin/sh + +# Disable periodic(8) reports, as well as tasks that generate lots of I/O. +sysrc -v -f /etc/periodic.conf \ + daily_show_success=NO \ + daily_show_info=NO \ + daily_clean_disks_verbose=NO \ + daily_clean_tmps_verbose=NO \ + daily_clean_preserve_verbose=NO \ + daily_clean_rwho_verbose=NO \ + daily_backup_passwd_enable=NO \ + daily_backup_aliases_enable=NO \ + daily_backup_gpart_enable=NO \ + daily_status_disks_enable=NO \ + daily_status_zfs_zpool_list_enable=NO \ + daily_status_network_enable=NO \ + daily_status_uptime_enable=NO \ + daily_status_mailq_enable=NO \ + daily_status_security_enable=NO \ + daily_status_mail_rejects_enable=NO \ + daily_status_world_kernel=NO \ + weekly_show_success=NO \ + weekly_show_info=NO \ + weekly_locate_enable=NO \ + weekly_whatis_enable=NO \ + weekly_status_security_enable=NO \ + monthly_show_success=NO \ + monthly_show_info=NO \ + monthly_accounting_enable=NO \ + monthly_status_security_enable=NO \ + security_show_success=NO \ + security_show_info=NO \ + security_status_chksetuid_enable=NO \ + security_status_neggrpperm_enable=NO \ + security_status_chkmounts_enable=NO \ + security_status_chkuid0_enable=NO \ + security_status_passwdless_enable=NO \ + security_status_logincheck_enable=NO \ + security_status_ipfwdenied_enable=NO \ + security_status_ipfdenied_enable=NO \ + security_status_pfdenied_enable=NO \ + security_status_ipfwlimit_enable=NO \ + security_status_ipf6denied_enable=NO \ + security_status_kernelmsg_enable=NO \ + security_status_loginfail_enable=NO \ + security_status_tcpwrap_enable=NO + +# Sendmail-specific stuff +sysrc -v -f /etc/periodic.conf \ + daily_clean_hoststat_enable=NO \ + daily_status_mail_rejects_enable=NO \ + daily_status_include_submit_mailq=NO \ + daily_submit_queuerun=NO diff --git a/scripts/os/freebsd/10-rc-conf b/scripts/os/freebsd/10-rc-conf new file mode 100644 index 0000000..a8a3d22 --- /dev/null +++ b/scripts/os/freebsd/10-rc-conf @@ -0,0 +1,7 @@ +#!/bin/sh + +sysrc -v \ + clear_tmp_enable=YES \ + dumpdev=NO \ + ipv6_activate_all_interfaces=NO \ + syslogd_flags=-ss diff --git a/scripts/os/freebsd/10-sysctls b/scripts/os/freebsd/10-sysctls new file mode 100644 index 0000000..a59d54f --- /dev/null +++ b/scripts/os/freebsd/10-sysctls @@ -0,0 +1,80 @@ +#!/bin/sh + +case $BOXCONF_OS_VERSION in + 13.*) + set_sysctl \ + net.inet.ip.check_interface=1 \ + net.inet.tcp.rfc6675_pipe=1 + ;; + *) + set_sysctl \ + net.inet.ip.rfc1122_strong_es=1 + ;; +esac + +load_kernel_module cc_htcp + +set_sysctl \ + net.inet.icmp.drop_redirect=1 \ + net.inet.ip.process_options=0 \ + net.inet.ip.random_id=1 \ + net.inet.ip.redirect=0 \ + net.inet.tcp.abc_l_var=44 \ + net.inet.tcp.always_keepalive=0 \ + net.inet.tcp.cc.abe=1 \ + net.inet.tcp.cc.algorithm=htcp \ + net.inet.tcp.cc.htcp.adaptive_backoff=1 \ + net.inet.tcp.cc.htcp.rtt_scaling=1 \ + net.inet.tcp.drop_synfin=1 \ + net.inet.tcp.ecn.enable=1 \ + net.inet.tcp.fastopen.server_enable=1 \ + net.inet.tcp.icmp_may_rst=0 \ + net.inet.tcp.initcwnd_segments=44 \ + net.inet.tcp.minmss=536 \ + net.inet.tcp.msl=2500 \ + net.inet.tcp.mssdflt=1448 \ + net.inet.tcp.nolocaltimewait=1 \ + net.inet.tcp.path_mtu_discovery=0 \ + net.inet.tcp.recvbuf_max="$tcp_buffer_size" \ + net.inet.tcp.recvspace=65536 \ + net.inet.tcp.sendbuf_inc=65536 \ + net.inet.tcp.sendbuf_max="$tcp_buffer_size" \ + net.inet.tcp.sendspace=65536 \ + net.inet.tcp.syncookies=0 \ + net.inet6.ip6.redirect=0 \ + security.bsd.unprivileged_proc_debug="$allow_proc_debug" + +# Some sysctls cannot be set within jails. +if [ "$BOXCONF_VIRTUALIZATION_TYPE" != jail ]; then + set_sysctl \ + hw.kbd.keymap_restrict_change=4 \ + kern.coredump=0 \ + kern.elf32.allow_wx="$allow_wx" \ + kern.elf32.aslr.pie_enable=1 \ + kern.elf64.allow_wx="$allow_wx" \ + kern.ipc.maxsockbuf="$tcp_buffer_size" \ + kern.ipc.shm_use_phys=1 \ + kern.ipc.soacceptqueue=1024 \ + kern.ipc.somaxconn=1024 \ + kern.random.fortuna.minpoolsize=128 \ + kern.randompid=1 \ + net.inet.tcp.fast_finwait2_recycle=1 \ + net.inet.tcp.finwait2_timeout=5000 \ + net.inet.tcp.keepcnt=2 \ + net.inet.tcp.keepidle=62000 \ + net.inet.tcp.keepinit=5000 \ + net.inet.tcp.minmss=536 \ + net.inet.tcp.minmss=536 \ + security.bsd.hardlink_check_gid=0 \ + security.bsd.hardlink_check_uid=0 \ + security.bsd.see_other_gids=0 \ + security.bsd.see_other_uids=0 \ + security.bsd.unprivileged_read_msgbuf=0 \ + vfs.zfs.min_auto_ashift=12 + + # FreeBSD automatically scales kern.maxfilesperproc with the amount of memory. + # On systems with large amounts of RAM, this can cause strange lags with some + # applications that attempt to close every possible file descriptor. + # Therefore, we arbitrarily cap this value at 65535. + [ "$(sysctl -n kern.maxfilesperproc)" -le 65535 ] || set_sysctl kern.maxfilesperproc=65535 +fi diff --git a/scripts/os/freebsd/20-hostname b/scripts/os/freebsd/20-hostname new file mode 100644 index 0000000..bf7eeeb --- /dev/null +++ b/scripts/os/freebsd/20-hostname @@ -0,0 +1,6 @@ +#!/bin/sh + +# Set the fully qualified hostname. +sysrc -v hostname="${BOXCONF_HOSTNAME}.${domain}" +hostname "${BOXCONF_HOSTNAME}.${domain}" +install_template -m 0644 /etc/hosts diff --git a/scripts/os/freebsd/20-locale b/scripts/os/freebsd/20-locale new file mode 100644 index 0000000..cf72d8d --- /dev/null +++ b/scripts/os/freebsd/20-locale @@ -0,0 +1,8 @@ +#!/bin/sh + +# Set the default system locale. +install_template -m 0644 \ + /etc/profile.d/locale.sh \ + /etc/login.conf + +cap_mkdb /etc/login.conf diff --git a/scripts/os/freebsd/20-motd b/scripts/os/freebsd/20-motd new file mode 100644 index 0000000..9b1eadb --- /dev/null +++ b/scripts/os/freebsd/20-motd @@ -0,0 +1,5 @@ +#!/bin/sh + +# Disable motd. +sysrc -v update_motd=NO +rm -f /var/run/motd diff --git a/scripts/os/freebsd/20-ntp b/scripts/os/freebsd/20-ntp new file mode 100644 index 0000000..888bab4 --- /dev/null +++ b/scripts/os/freebsd/20-ntp @@ -0,0 +1,14 @@ +#!/bin/sh + +# Jails don't need NTP. +if [ "$BOXCONF_VIRTUALIZATION_TYPE" = jail ]; then + return +fi + +install_template -m 0644 /etc/ntp.conf + +sysrc -v \ + ntpd_enable=YES \ + ntpd_sync_on_start=YES + +service ntpd restart diff --git a/scripts/os/freebsd/20-root-ca b/scripts/os/freebsd/20-root-ca new file mode 100644 index 0000000..1f88c69 --- /dev/null +++ b/scripts/os/freebsd/20-root-ca @@ -0,0 +1,12 @@ +#!/bin/sh + +# Create local CA certificates directory. +install_directory -m 0755 \ + /usr/local/etc \ + /usr/local/etc/ssl \ + /usr/local/etc/ssl/certs + +# Install our root CA. +install_ca_certificate "$site_cacert_path" + +certctl rehash diff --git a/scripts/os/freebsd/20-timezone b/scripts/os/freebsd/20-timezone new file mode 100644 index 0000000..22a3729 --- /dev/null +++ b/scripts/os/freebsd/20-timezone @@ -0,0 +1,4 @@ +#!/bin/sh + +# Set the system timezone. +cp -v "/usr/share/zoneinfo/${timezone}" /etc/localtime diff --git a/scripts/os/freebsd/20-zfs b/scripts/os/freebsd/20-zfs new file mode 100644 index 0000000..aa37c0a --- /dev/null +++ b/scripts/os/freebsd/20-zfs @@ -0,0 +1,11 @@ +#!/bin/sh + +# Every host should have a "state" dataset, which is a ZFS dataset which +# persists across OS rebuilds. +[ -n "${state_dataset:-}" ] || die 'state_dataset not defined!' +create_dataset "$state_dataset" + +# If this is baremetal host or a VM, trim the zpools periodically. +if [ "$BOXCONF_VIRTUALIZATION_TYPE" != jail ]; then + install_file -m 0644 /etc/cron.d/zfs-trim +fi diff --git a/scripts/os/freebsd/30-mail b/scripts/os/freebsd/30-mail new file mode 100644 index 0000000..511ce69 --- /dev/null +++ b/scripts/os/freebsd/30-mail @@ -0,0 +1,9 @@ +#!/bin/sh + +if [ "$BOXCONF_HOSTCLASS" = smtp_server ]; then + return +fi + +# Configure local mail agent. +install_template -m 0644 /etc/dma/dma.conf +install_template -m 0644 /etc/aliases diff --git a/scripts/os/freebsd/30-ssh b/scripts/os/freebsd/30-ssh new file mode 100644 index 0000000..91b1991 --- /dev/null +++ b/scripts/os/freebsd/30-ssh @@ -0,0 +1,31 @@ +#!/bin/sh + +# Create state dataset to persist SSH host keys across OS rebuilds. +create_dataset -o "mountpoint=${ssh_host_key_dir}" "${state_dataset}/ssh" + +# If the state dataset contains existing host keys, symlink them into +# /etc/ssh. +# +# If not, this is the first time we are building this box, so copy the +# autogenerated host keys to the state partition. +for key in \ + ssh_host_ecdsa_key \ + ssh_host_ed25519_key \ + ssh_host_rsa_key +do + [ -f "${ssh_host_key_dir}/${key}" ] || \ + mv -v "/etc/ssh/${key}" "/etc/ssh/${key}.pub" "$ssh_host_key_dir" + + ln -snvf "${ssh_host_key_dir}/${key}" "/etc/ssh/${key}" + ln -snvf "${ssh_host_key_dir}/${key}.pub" "/etc/ssh/${key}.pub" +done + +# Copy SSH configs. +install_directory -m 0755 /etc/ssh/sshd_config.d + +install_template -m 0644 \ + /etc/ssh/sshd_config \ + /etc/ssh/ssh_config + +# Restart sshd. +service sshd restart diff --git a/scripts/os/freebsd/30-syslog b/scripts/os/freebsd/30-syslog new file mode 100644 index 0000000..6f3dc8c --- /dev/null +++ b/scripts/os/freebsd/30-syslog @@ -0,0 +1,7 @@ +#!/bin/sh + +# Copy syslog configuration. +install_file -m 0644 /etc/syslog.conf + +# Restart syslogd. +service syslogd restart diff --git a/scripts/os/freebsd/40-pkg b/scripts/os/freebsd/40-pkg new file mode 100644 index 0000000..7c1c828 --- /dev/null +++ b/scripts/os/freebsd/40-pkg @@ -0,0 +1,29 @@ +#!/bin/sh + +case $BOXCONF_HOSTCLASS in + pkg_repository) + return # Do nothing. + ;; + freebsd_hypervisor) + ;; # Keep default FreeBSD pkg repository. + *) + # Configure on-prem pkg repository. + install_directory -m 0755 \ + /usr/local/etc/pkg \ + /usr/local/etc/pkg/repos + + install_file -m 0644 \ + /usr/local/etc/ssl/repo.crt \ + /usr/local/etc/pkg/repos/FreeBSD.conf + + install_template -m 0644 /usr/local/etc/pkg/repos/onprem.conf + ;; +esac + +# Update packages. +pkg update -f + +# Install default packages. +if [ -n "${install_packages:-}" ]; then + pkg install -y $install_packages +fi diff --git a/scripts/os/freebsd/70-pf b/scripts/os/freebsd/70-pf new file mode 100644 index 0000000..9ec9961 --- /dev/null +++ b/scripts/os/freebsd/70-pf @@ -0,0 +1,15 @@ +#!/bin/sh + +if [ "$enable_pf" != true ]; then + return +fi + +# Enable pf. +sysrc -v pf_enable=YES + +# Copy pf configuration. +install_template -m 0600 /etc/pf.conf + +# Start (or reload) pf. +service pf status > /dev/null || service pf start +service pf reload diff --git a/vars/common b/vars/common new file mode 100644 index 0000000..bb7c4db --- /dev/null +++ b/vars/common @@ -0,0 +1,18 @@ +#!/bin/sh + +domain=idm.example.com +email_domain=example.com +locale=en_US.UTF-8 +ntp_pools='pool.ntp.org' +root_password=changeme +root_authorized_keys='ssh-ed25519 changeme +ssh-ed25519 changeme' +root_mail_alias="you@${email_domain}" +smtp_host_ip=1.2.3.4 +timezone=America/New_York + + +allowed_tcp_ports=ssh +bootstrap_resolvers='8.8.8.8 8.8.4.4' +smtp_host="smtp.${domain}" +tcp_buffer_size=2097152 # suitable for 1 GigE diff --git a/vars/hostclass/freebsd_hypervisor b/vars/hostclass/freebsd_hypervisor new file mode 100644 index 0000000..c38452f --- /dev/null +++ b/vars/hostclass/freebsd_hypervisor @@ -0,0 +1,5 @@ +#!/bin/sh + +enable_pf=false +smtp_host=${smtp_host_ip} +resolvers=$bootstrap_resolvers diff --git a/vars/os/freebsd b/vars/os/freebsd new file mode 100644 index 0000000..0d4a6fb --- /dev/null +++ b/vars/os/freebsd @@ -0,0 +1,34 @@ +#!/bin/sh + +allow_wx=1 +allow_proc_debug=0 +cx_lowest=Cmax +enable_pf=true +install_packages='sudo tmux vim' +intel_epp=50 + +export ASSUME_ALWAYS_YES=yes +acme_standalone_port=9080 +acme_uid=169 +nfscbd_port=7745 +site_cacert_path=/usr/local/etc/ssl/certs/ca.crt +ssh_host_key_dir=/var/db/ssh + +# For 10 Gbit ethernet, bump up the TCP buffers. +if ifconfig | grep -q '10Gbase-T'; then + tcp_buffer_size=16777216 +fi + +case $BOXCONF_VIRTUALIZATION_TYPE in + jail) + # For jails, the state dataset is delegated to the jail and named "data". + state_dataset=$(zfs list -Ho jailed,name | awk '$1 == "on" && $2 ~ /\/data$/ {print $2;exit}') + ;; + *) + # Otherwise, assume the state dataset is named "data" in the root zpool. + root_zpool=$(zfs list -Ho name,mountpoint | awk '$2 == "/" {print $1;exit}' | cut -d/ -f1) + if [ -n "$root_zpool" ]; then + state_dataset="${root_zpool}/data" + fi + ;; +esac |