[Saltstack Cheat Sheet] #saltstack

saltstack.sh

# ============
# COMMON TASKS
# ============

# Run state.sls file on minion(s)
salt -G 'os:Ubuntu' state.apply ubuntu1401                                                                    # Formula file name without .sls at the end
salt -G 'os:Ubuntu' state.apply ubuntu1401 test=True                                                          # Test only (dryrun)
salt -G "minion_roles:ONLINE" state.apply execute_script_example pillar='{"single_instance_update": "aiat"}'  

# Webhook state.sls example
salt mssql1 state.apply webhook.mssql_webhook_deploy_on_push pillar='{"git_push_branch": "aiat"}'

# Hihgstate for single minions (all sls formulas assinged through top.sls)
salt online1 state.highstate test                                           # Test only (dry run)
salt online1 state.highstate                                                # Regular run
salt online1 state.highstate pillar='{"single_instance_update": "aiat"}'    # Single instance only (must be supported by Formula)
salt -G "minion_roles:INTDNS" state.highstate
salt -G "minion_roles:INTDNS" state.highstate pillar='{"single_instance_update": "arno"}'

# Full orchestration run
salt-run state.orchestrate orch.instance-setup                                                  # Update everything on all minions
salt-run state.orchestrate orch.instance-setup pillar='{"single_instance_update": "aiat"}'      # Single instance only (must be supported by Formula)

# Check free disk space
salt -G "minion_roles:ONLINE" cmd.run 'df -h'
salt -G "os:Ubuntu" cmd.run 'df -h'

# Check folder size
salt -G "minion_roles:ONLINE" cmd.run 'du -sh /opt/online/*'
salt -G "minion_roles:ONLINE" cmd.run 'du -shx /opt/online/*'				# Without mount points (e.g.: data_dir and update)
salt -G "minion_roles:ONLINE" cmd.run 'du -sh /opt/online/*/data_dir'		# All data_dir folders

# ===============
# FS-ONLINE TASKS
# ===============

# Update Online Tools
salt -G "minion_roles:ONLINE" cmd.run 'cd /opt/online/online_tools;git pull'
salt -G "minion_roles:BACKUP" cmd.run 'cd /opt/online/online_tools;git pull'

# Update Online Tools in the cores folder ONLY ON ONE MACHINE since it is shard by smb anyway!
salt 'online1' cmd.run 'cd /opt/online/cores/online_tools;git pull'

# Check Release Tag of Online Core
salt -G "minion_roles:ONLINE" cmd.run 'git -C /opt/online/online_o8r166 describe --tags --exact-match --match o8r*'

# Check instance.ini
salt -G "minion_roles:ONLINE" cmd.run 'grep -H "core" /opt/online/*/instance.ini'

# Check for failed Updates
salt -G "minion_roles:ONLINE" cmd.run 'grep -H "update_failed" /opt/online/*/status.ini'

# Manually start an update
salt 'online1' state.sls webhooks.online_webhook_update pillar='{"instance_to_update": "dadi"}'
salt 'online1' state.sls webhooks.online_webhook_update pillar='{"instance_list_to_update": "dadi,demo,aiat"}'

# DANGER: Remove status.ini of all instances
salt -G "minion_roles:ONLINE" cmd.run 'rm /opt/online/*/status.ini'

# Force update language(s) for addon(s)
salt -G "minion_roles:ONLINE" state.apply odoo_update_translation pillar='{"addons_to_update": "fso_con_zmr", "single_instance_update": "aiat"}'
salt -G "minion_roles:ONLINE" state.apply odoo_update_translation pillar='{"addons_to_update": "fso_con_zmr,fso_sosync"}'

# Update odoo addon(s):
salt online1 state.apply odoo_update_addons test=True pillar='{"addons_to_update": "website,website_crm,webstite_crm_extended", "single_instance_update": "demo"}'


# ========================
# TESTSING / DEBUG / Admin
# ========================

# Targetting minions
https://docs.saltstack.com/en/latest/topics/targeting/

# Basics
salt '*' test.ping
salt -G 'os:Ubuntu' test.ping
salt -G 'kernel:Linux' test.ping
salt -G 'minion_roles:ONLINE' test.ping

# Get Info
salt '*' state.show_top
salt '*' state.show_sls LIST,OF,STATES,WITHOUT,.sls   # salt filesrv1 state.show_sls FILESRV
salt '*' pillar.items
salt '*' grains.items

# Update all minion cahches on the master
salt '*' saltutil.sync_all

# Clear Pillar Cache
salt '*' saltutil.refresh_pillar

# Fix corrupted Minion Pillar Cache
# ATTENTION: A corrupted or non updated pillar cache on the minion will prevent job execution for matchers using -I!
#            https://github.com/saltstack/salt/issues/32144
salt 'online4' saltutil.refresh_pillar
salt -l debug -C "I@instances:cona" pillar.items
# Optional Steps if still not working:
salt cmd.run 'online4' 'rm -rf /var/cache/salt/minion/'
salt cmd.run 'online4' 'systemctl restart salt-minion'
salt 'online4' saltutil.refresh_pillar
salt -l debug -C "I@instances:cona" pillar.items

# Salt master fileserver
salt-run fileserver.update -l debug 2>&1        # Show status
salt-run fileserver.clear_cache backend=git     # Clear git-fileserver(s) cache(s)
salt-run cache.clear_git_lock gitfs             # Remove Gitfs locks

# Salt Jobs Queue
salt-run state.event pretty=True
salt-run state.event pretty=True | Kino
salt sosync1 saltutil.running
salt-run jobs.active                        # enspricht: salt '*' saltutil.running
salt-run jobs.list_jobs start_time='2018, Feb 22 15:12' display_progress=True
salt-run jobs.list_job 20180222153645037464

# Example: Call Salt API Webhook from internal network:
# HINT: "-k" means https insecure = accept self signed certificates
curl -ksS https://salt.datadialog.net:8000/hook/sosync/sync -d instance='care'

# Run shell commands on minions
salt 'online1' cmd.run 'ls -l | grep foo'
salt -G 'minion_roles:ONLINE' cmd.run 'reboot'
salt -G 'os:Ubuntu' cmd.run 'ntpq -p'

# List minion modules, functions and the docstring of functions
salt 'online4' sys.list_functions 'cmd'
salt 'online4' sys.doc 'cmd.run_bg'
salt 'online*' sys.list_modules
salt 'online*' sys.list_modules 's*'

# ======================
# UPDATE SALT ON MINIONS
# ======================

# List currently installed versions
salt '*' cmd.run 'salt-minion --version'
salt '*' test.version
salt-run manage.versions

# Target Minions with a specific salt version
salt -G 'saltversion:2019.2.4' test.ping
salt -C 'G@saltversion:2019.2.4' test.ping

# Target minions without a specific salt version
salt -C 'not G@saltversion:2019.2.4' test.ping

# Download the salt install script on all minions
salt -C 'G@os:Ubuntu and not G@saltversion:2019.2.4' cmd.run 'curl -o install_salt.sh -L https://bootstrap.saltstack.com'

# Update the salt minions 
# HINT: This will only install the salte minionon (it would update the salt master executables only if they are already installed)
salt -C 'G@os:Ubuntu and not G@saltversion:2019.2.4' cmd.run 'sudo sh install_salt.sh git v2019.2.4'

# OPTIONAL: Update the salt master
# ATTENTION: This makes only sense on the salt master server(s)
sudo sh install_salt.sh -M git v2019.2.4

# The minion update on the salt master if any is installed
sudo sh install_salt.sh git v2019.2.4



# =========================================
# JINJA TEMPLATING LANGUAGE TIPS AND TRICKS
# =========================================

# CREATE A SET IN JINJA (UNIQUE ITEM LIST)
# For loop test
# https://docs.saltstack.com/en/latest/topics/pillar/
# https://docs.saltstack.com/en/latest/topics/tutorials/pillar.html
# http://jinja.pocoo.org/docs/dev/templates/#list-of-control-structures
# https://docs.python.org/2/library/functions.html#func-set
# http://docs.ansible.com/ansible/playbooks_filters.html
{% set mikeslist = ['a1','b1','c1','a1'] %}
{% set mikesset = [] %}

{% for item in mikeslist %}
  {% if item not in mikesset %}
    {% do mikesset.append(item) %}
  {% endif %}
{% endfor %}

{% for item in mikesset %}
{{item}}:
  pgk.removed
{% endfor %}

# Run salt modules in jinja files:
{{ salt['file.group_to_gid']('some_group_that_exists') }}
{%- set online_admin_pw = salt['cmd.exec_code']('python2','from passlib.context import CryptContext; print CryptContext(["pbkdf2_sha512"]).encrypt("' + settings.online_admin_pw +'")') %}

# jinja debug for pillar files:
Context is: {{ show_full_context() }}


# Get the minion id (= hostname)
salt.grains.get('id')
# Example:
{% do online_releases.append(salt.pillar.get('hosts:' + salt.grains.get('id') + ':online_target_release')) %}


# get the network ip
salt.network.interfaces()['eth0']['inet'][0]['address']


# get the netmask as  e.g.: 255.255.255.0
salt.network.interfaces()['eth0']['inet'][0]['netmask']

# =======================
# ssh_known_hosts.present
# =======================
# https://github.com/saltstack/salt/issues/41653
# https://github.com/saltstack/salt/issues/46152
# echo -n 'nThbg6kXUpJWGl7E1IGOCspRomTxdCARLviKw6E5SY8' | base64 -d 2>/dev/null | od -A n -t x1 | xargs echo | sed 's/\ *//g;s/\(..\)/\1:/g;s/:$//'