Add systemupdate script

apps/noveriablcgen/init.sls

@@ -35,7 +35,7 @@ apps_noveriablcgen_config:
         ROOT_UUID: {{ salt['cmd.shell']('lsblk -o LABEL,UUID | grep ROOT | awk \'{print $2}\'') }}
         GRUB_CONFIG: "/boot/grub/grub.cfg"
     - mode: '0644'
-    - makedirs: true
+    - makedirs: True
     - user: root
     - group: root
     - require:

apps/poddoc/init.sls

@@ -32,7 +32,7 @@ apps_poddoc_config:
         GIT_REPO: "https://git.noveria.org/Podman"
         PODMAN_DIR: "/opt/podman"
     - mode: '0644'
-    - makedirs: true
+    - makedirs: True
     - user: root
     - group: root
     - require:

network/firewall.sls

@@ -14,15 +14,15 @@ network_firewall_rules:
     - user: root
     - group: root
     - mode: '0600'
-    - makedirs: true
+    - makedirs: True
     - require:
       - network_firewall_pkgs
 
 network_firewall_service_reload:
   service.running:
     - name: nftables
-    - enable: true
-    - reload: true
+    - enable: True
+    - reload: True
     - watch:
       - network_firewall_rules
     - require:

network/init.sls

@@ -15,7 +15,7 @@ network_interfaces:
 network_service_reload:
   service.running:
     - name: networking
-    - enable: true
-    - reload: true
+    - enable: True
+    - reload: True
     - watch:
-      - network_interfaces
+      - network_interfaces

podman/rootless.sls

@@ -11,7 +11,7 @@ podman_rootless_cgroup_mode:
     - uncomment: '#'
     - key_ignore_case: false
     - value_ignore_case: false
-    - append_if_not_found: true
+    - append_if_not_found: True
     - require:
       - podman_pkg
 

system/base.sls

@@ -1,3 +1,5 @@
+{% from "map.jinja" import noveria %}
+
 system_base_pkgs:
   pkg.installed:
     - pkgs:
@@ -14,8 +16,8 @@ system_base_pkgs:
 
 system_base_bin_dir:
   file.directory:
-    - name: /usr/local/noveria/bin
-    - makedirs: true
+    - name: {{ noveria.bin_dir }}
+    - makedirs: True
     - user: root
     - group: root
     - dir_mode: '0755'
@@ -23,13 +25,43 @@ system_base_bin_dir:
 
 system_base_apps_dir:
   file.directory:
-    - name: /usr/local/noveria/apps
-    - makedirs: true
+    - name: {{ noveria.app_dir }}
+    - makedirs: True
     - user: root
     - group: root
     - dir_mode: '0755'
     - file_mode: '0644'
 
+system_base_etc_dir:
+  file.directory:
+    - name: {{ noveria.etc_dir }}
+    - makedirs: True
+    - user: root
+    - group: root
+    - dir_mode: '0755'
+    - file_mode: '0644'
+
+system_base_systemupdate:
+  file.managed:
+    - name: {{ noveria.app_dir }}/systemupdate
+    - source: salt://{{ tpldir }}/files/base_systemupdate
+    - mode: '0755'
+    - user: root
+    - group: root
+    - require:
+      - system_base_apps_dir
+
+system_base_systemupdate_link:
+  file.symlink:
+    - name: {{ noveria.bin_dir }}/systemupdate
+    - target: {{ noveria.app_dir }}/systemupdate
+    - user: root
+    - group: root
+    - mode: '0755'
+    - require:
+      - system_base_bin_dir
+      - system_base_systemupdate
+
 system_base_modules_service:
   service.running:
     - name: modules

system/bootloader.sls

@@ -23,7 +23,7 @@ system_bootloader_mkinitfs_conf:
     - uncomment: '# '
     - key_ignore_case: false
     - value_ignore_case: false
-    - append_if_not_found: true
+    - append_if_not_found: True
     - require:
       - system_base_pkgs
       - system_bootloader_pkgs

system/files/base_systemupdate

@@ -0,0 +1,205 @@
+#!/usr/bin/env bash
+
+#######################################################################################################################
+##
+## Script to full update the system
+##
+##   0. Pre checks
+##   1. Generate new boot environment (BE)
+##   2. Update and Salt highstate
+##   3. Clean up
+##
+#######################################################################################################################
+
+
+#######################################################################################################################
+## Definitions
+#######################################################################################################################
+
+readonly TEMPDIR=$(mktemp -d /tmp/systemupdate.XXXXXX)
+readonly LOCK_FILE="${TEMPDIR}/systemupdate.lock"
+
+readonly TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S")
+
+readonly BTRFS_ROOT="/btrfs"
+readonly CURRENT_SUBVOLUME=$(LC_ALL=C btrfs sub show / | LC_ALL=C grep 'Name' | cut -d: -f2 | awk '{$1=$1};1')
+readonly NEW_SUBVOLUME="@root_${TIMESTAMP}"
+readonly MOUNTPOINT='/mnt'
+
+readonly EFI_DISK=$(findmnt -T /efi -o SOURCE | tail -n 1)
+readonly ROOT_DISK=$(findmnt / -o SOURCE | cut -d"[" -f1 | tail -n 1)
+
+readonly BE_HISTORY_COUNT=5
+
+
+#######################################################################################################################
+## Errorhandling
+#######################################################################################################################
+
+#----------------------------------------------------------------------------------------------------------------------
+#   systemupdate failed
+#----------------------------------------------------------------------------------------------------------------------
+systemupdateFailed() {
+  echo ""
+  echo "┌──────────────────────────────────────────┐"
+  echo "│  FAILED => clean up                      │"
+  echo "└──────────────────────────────────────────┘"
+
+  subtaskTitle "Unmount BE if mounted"
+  unmountMountpoint
+
+  subtaskTitle "Remove BE"
+  removeBEFromTimestamp ${TIMESTAMP}
+
+  rm -f ${LOCK_FILE}
+
+  subtaskTitle "Finished with exit code 1"
+  exit 1
+}
+
+# catch ^C and other signals and clean up
+trap "echo -e '\n=> Interrupted with CTRL+C' >&2; systemupdateFailed" SIGINT SIGHUP SIGTERM SIGABRT
+
+
+#######################################################################################################################
+## Helper Functions
+#######################################################################################################################
+
+#----------------------------------------------------------------------------------------------------------------------
+#   Subtask title output
+#----------------------------------------------------------------------------------------------------------------------
+subtaskTitle() {
+  echo -e "\n=> $1"
+}
+
+#----------------------------------------------------------------------------------------------------------------------
+#   Unmount ${MOUNTPOINT}
+#----------------------------------------------------------------------------------------------------------------------
+unmountMountpoint() {
+  # if mountpoint exists -> umount
+  [[ $(findmnt -M "${MOUNTPOINT}") ]] && umount -R "${MOUNTPOINT}"
+}
+
+#----------------------------------------------------------------------------------------------------------------------
+#   Recursive subvolume delete
+#----------------------------------------------------------------------------------------------------------------------
+btrfsSubDelRecursive() {
+  btrfs sub list -o "${BTRFS_ROOT}/${1}" | cut -d " " -f 9 | while read i; do
+    btrfsSubDelRecursive "$i"
+  done
+  btrfs sub del "${BTRFS_ROOT}/${1}"
+}
+
+#----------------------------------------------------------------------------------------------------------------------
+#   Remove BE from timestamp
+#----------------------------------------------------------------------------------------------------------------------
+removeBEFromTimestamp() {
+  # remove all subvolume with this timestamp
+  for f in $(btrfs sub list -o /btrfs | cut -d " " -f 9 | grep "@root"); do
+    if [[ "$f" =~ "$1" ]]; then
+      btrfsSubDelRecursive "$f"
+    fi
+  done
+}
+
+
+#######################################################################################################################
+## Main
+#######################################################################################################################
+
+echo "┌──────────────────────────────────────────┐"
+echo "│  0. Pre checks                           │"
+echo "└──────────────────────────────────────────┘"
+subtaskTitle "Check if another systemupgrade is in progress"
+if [ -f ${LOCK_FILE} ]; then
+  echo "[ERROR] Another systemupgrade is in progress (lockfile: ${LOCK_FILE}) => exit" >&2
+  exit 1
+fi
+
+subtaskTitle "Check if ${MOUNTPOINT} exists"
+if [ ! -d ${MOUNTPOINT} ]; then
+  mkdir -p "${MOUNTPOINT}"
+fi
+
+subtaskTitle "Check if ${MOUNTPOINT} is already a mountpoint"
+if [[ $(findmnt -M "${MOUNTPOINT}") ]]; then
+  echo "[ERROR] ${MOUNTPOINT} is already a mountpoint => exit" >&2
+  exit 1
+fi
+
+subtaskTitle "Checks finished and update can start"
+
+# Create lock file
+touch ${LOCK_FILE} || systemupdateFailed
+
+
+echo ""
+echo "┌──────────────────────────────────────────┐"
+echo "│  1. Generate new boot environment (BE)   │"
+echo "└──────────────────────────────────────────┘"
+
+subtaskTitle "Create snapshot of current running system"
+btrfs subvolume snapshot / ${BTRFS_ROOT}/${NEW_SUBVOLUME} || systemupdateFailed
+
+subtaskTitle "Mount new BE to ${MOUNTPOINT}"
+mount -o noatime,nodiratime,discard=async,space_cache=v2,subvol="${NEW_SUBVOLUME}" "${ROOT_DISK}" "${MOUNTPOINT}" || systemupdateFailed
+mount -o noatime,nodiratime,discard=async,space_cache=v2,subvol=@home "${ROOT_DISK}" "${MOUNTPOINT}/home" || systemupdateFailed
+mount -o noatime,nodiratime,discard=async,space_cache=v2,subvol=@podman "${ROOT_DISK}" "${MOUNTPOINT}/opt/podman" || systemupdateFailed
+mount -o noatime,nodiratime,discard=async,space_cache=v2,subvol=@mysql "${ROOT_DISK}" "${MOUNTPOINT}/var/lib/mysql" || systemupdateFailed
+mount -o noatime,nodiratime,discard=async,space_cache=v2,subvol=/ "${ROOT_DISK}" "${MOUNTPOINT}/btrfs" || systemupdateFailed
+
+mount -o nodev,nosuid,noexec "${EFI_DISK}" "${MOUNTPOINT}/efi" || systemupdateFailed
+mount -t proc /proc "${MOUNTPOINT}/proc/" || systemupdateFailed
+mount -t sysfs /sys "${MOUNTPOINT}/sys/" || systemupdateFailed
+mount -o bind /sys/firmware/efi/efivars "${MOUNTPOINT}/sys/firmware/efi/efivars/" || systemupdateFailed
+mount -o bind /dev "${MOUNTPOINT}/dev/" || systemupdateFailed
+mount -o bind /run "${MOUNTPOINT}/run/" || systemupdateFailed
+
+subtaskTitle "New BE mounted"
+
+echo ""
+echo "┌──────────────────────────────────────────┐"
+echo "│  2. Update and Salt highstate            │"
+echo "└──────────────────────────────────────────┘"
+
+subtaskTitle "Update Saltstack"
+
+chroot "${MOUNTPOINT}" /bin/bash -c "git -C /srv/salt pull" || systemupdateFailed
+chroot "${MOUNTPOINT}" /bin/bash -c "git -C /srv/pillar pull" || systemupdateFailed
+
+subtaskTitle "Update bootloader configs"
+chroot "${MOUNTPOINT}" /bin/bash -c "salt-call state.sls system.bootloader" >/dev/null || systemupdateFailed
+
+subtaskTitle "Alpine repositories & keyring update"
+chroot "${MOUNTPOINT}" /bin/bash -c "apk update" || systemupdateFailed
+
+subtaskTitle "Alpine packages update"
+chroot "${MOUNTPOINT}" /bin/bash -c "apk upgrade" || systemupdateFailed
+
+subtaskTitle "Salt highstate"
+chroot "${MOUNTPOINT}" /bin/bash -c "salt-call state.highstate" >/dev/null || systemupdateFailed
+
+subtaskTitle "Generate new initial ramdisk"
+chroot "${MOUNTPOINT}" /bin/bash -c "mkinitfs $(uname -r)" || systemupdateFailed
+
+subtaskTitle "Update motd"
+chroot "${MOUNTPOINT}" /bin/bash -c "/usr/local/noveria/bin/generate_motd" || systemupdateFailed
+
+subtaskTitle "Update GRUB"
+chroot "${MOUNTPOINT}" /bin/bash -c "grub-install --target=x86_64-efi --efi-directory=/efi --bootloader-id=alpine" || systemupdateFailed
+chroot "${MOUNTPOINT}" /bin/bash -c "/usr/local/noveria/bin/noveriablcgen --noconfirm" || systemupdateFailed
+
+subtaskTitle "Update finished"
+
+subtaskTitle "Unmount BE"
+unmountMountpoint
+
+echo ""
+echo "┌──────────────────────────────────────────┐"
+echo "│  3. Clean Up                             │"
+echo "└──────────────────────────────────────────┘"
+
+subtaskTitle "Clean up finished"
+
+# Remove lock file
+rm -f ${LOCK_FILE}