additional-machineset.yaml

apiVersion: machine.openshift.io/v1beta1
kind: MachineSet
metadata:
  labels:
    machine.openshift.io/cluster-api-cluster: mandre-upshift-kvt6k
    machine.openshift.io/cluster-api-machine-role: custom
    machine.openshift.io/cluster-api-machine-type: custom
  name: mandre-upshift-kvt6k-custom
  namespace: openshift-machine-api
spec:
  replicas: 1
  selector:
    matchLabels:
      machine.openshift.io/cluster-api-cluster: mandre-upshift-kvt6k
      machine.openshift.io/cluster-api-machineset: mandre-upshift-kvt6k-custom
  template:
    metadata:
      labels:
        machine.openshift.io/cluster-api-cluster: mandre-upshift-kvt6k
        machine.openshift.io/cluster-api-machine-role: custom
        machine.openshift.io/cluster-api-machine-type: custom
        machine.openshift.io/cluster-api-machineset: mandre-upshift-kvt6k-custom
    spec:
      providerSpec:
        value:
          apiVersion: openstackproviderconfig.openshift.io/v1alpha1
          cloudName: openstack
          cloudsSecret:
            name: openstack-cloud-credentials
            namespace: openshift-machine-api
          flavor: ci.m1.large
          image: rhcos-4.6
          kind: OpenstackProviderSpec
          networks:
          - filter: {}
            subnets:
            - filter:
                name: mandre-upshift-kvt6k-additional
                tags: openshiftClusterID=mandre-upshift-kvt6k
          securityGroups:
          - filter: {}
            name: mandre-upshift-kvt6k-worker-additional
          serverMetadata:
            Name: mandre-upshift-kvt6k-custom
            openshiftClusterID: mandre-upshift-kvt6k
          tags:
          - openshiftClusterID=mandre-upshift-kvt6k
          trunk: true
          userDataSecret:
            name: custom-user-data
          availabilityZone: ''

custom-machineconfig.yaml

apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
  labels:
    machineconfiguration.openshift.io/role: custom
  name: 51-custom
spec:
  config:
    ignition:
      version: 3.1.0
    storage:
      files:
      - contents:
          source: data:,
        mode: 420
        path: /etc/kubernetes/manifests/keepalived.yaml
        overwrite: true
      - contents:
          source: data:,%23%21%2Fbin%2Fbash%0Aset%20-eo%20pipefail%0AIFACE%3D%241%0ASTATUS%3D%242%0A%0Acase%20%22%24STATUS%22%20in%0A%20%20%20%20up%7Cdown%7Cdhcp4-change%7Cdhcp6-change%29%0A%20%20%20%20logger%20-s%20%22NM%20resolv-prepender%20triggered%20by%20%24%7B1%7D%20%24%7B2%7D.%22%0A%0A%20%20%20%20%23%20Ensure%20resolv.conf%20exists%20before%20we%20try%20to%20run%20podman%0A%20%20%20%20if%20%5B%5B%20%21%20-e%20%2Fetc%2Fresolv.conf%20%5D%5D%20%7C%7C%20%21%20grep%20-q%20nameserver%20%2Fetc%2Fresolv.conf%3B%20then%0A%20%20%20%20%20%20%20%20cp%20%2Fvar%2Frun%2FNetworkManager%2Fresolv.conf%20%2Fetc%2Fresolv.conf%0A%20%20%20%20fi%0A%0A%20%20%20%20NAMESERVER_IP%3D%24%28%2Fusr%2Fbin%2Fpodman%20run%20--rm%20--authfile%20%2Fvar%2Flib%2Fkubelet%2Fconfig.json%20--net%3Dhost%20quay.io%2Fmandre%2Fbaremetal-runtimecfg%3Avipless%20node-ip%20show%29%0A%20%20%20%20DOMAIN%3D%22mandre-upshift.shiftstack.com%22%0A%20%20%20%20if%20%5B%5B%20-n%20%22%24NAMESERVER_IP%22%20%5D%5D%3B%20then%0A%20%20%20%20%20%20%20%20logger%20-s%20%22NM%20resolv-prepender%3A%20Prepending%20%27nameserver%20%24NAMESERVER_IP%27%20to%20%2Fetc%2Fresolv.conf%20%28other%20nameservers%20from%20%2Fvar%2Frun%2FNetworkManager%2Fresolv.conf%29%22%0A%20%20%20%20%20%20%20%20sed%20-e%20%22%2F%5Esearch%2Fd%22%20%5C%0A%20%20%20%20%20%20%20%20%20%20%20%20-e%20%22%2FGenerated%20by%2Fc%23%20Generated%20by%20OpenStack%20resolv%20prepender%20NM%20dispatcher%20script%5Cnsearch%20%24DOMAIN%5Cnnameserver%20%24NAMESERVER_IP%22%20%5C%0A%20%20%20%20%20%20%20%20%20%20%20%20%2Fvar%2Frun%2FNetworkManager%2Fresolv.conf%20%3E%20%2Fetc%2Fresolv.tmp%0A%20%20%20%20fi%0A%20%20%20%20%23%20Only%20leave%20the%20first%203%20nameservers%20in%20%2Fetc%2Fresolv.conf%0A%20%20%20%20sed%20-i%20%27%3Aa%20%24%21%7BN%3B%20ba%7D%3B%20s%2F%5C%28%5E%5C%7C%5Cn%5C%29nameserver%2F%5Cn%23%20nameserver%2F4g%27%20%2Fetc%2Fresolv.tmp%0A%20%20%20%20mv%20-f%20%2Fetc%2Fresolv.tmp%20%2Fetc%2Fresolv.conf%0A%20%20%20%20%3B%3B%0A%20%20%20%20%2A%29%0A%20%20%20%20%3B%3B%0Aesac
        mode: 493
        overwrite: true
        path: /etc/NetworkManager/dispatcher.d/30-resolv-prepender
    systemd:
      units:
      - name: nodeip-configuration.service
        enabled: true
        contents: |
          [Unit]
          Description=Writes IP address configuration so that kubelet and crio services select a valid node IP
          # This only applies to VIP managing environments where the kubelet and crio IP
          # address picking logic is flawed and may end up selecting an address from a
          # different subnet or a deprecated address
          Wants=network-online.target
          After=network-online.target ignition-firstboot-complete.service
          Before=kubelet.service crio.service

          [Service]
          # Need oneshot to delay kubelet
          Type=oneshot
          # Would prefer to do Restart=on-failure instead of this bash retry loop, but
          # the version of systemd we have right now doesn't support it. It should be
          # available in systemd v244 and higher.
          ExecStart=/bin/bash -c " \
            until \
            /usr/bin/podman run --rm \
            --authfile /var/lib/kubelet/config.json \
            --volume /etc/systemd/system:/etc/systemd/system:z \
            --net=host \
            quay.io/mandre/baremetal-runtimecfg:vipless \
            node-ip \
            set --retry-on-failure; \
            do \
            sleep 5; \
            done"

          [Install]
          WantedBy=multi-user.target

custom-machineconfigpool.yaml

apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfigPool
metadata:
  name: custom
spec:
  machineConfigSelector:
    matchExpressions:
      - {key: machineconfiguration.openshift.io/role, operator: In, values: [worker,custom]}
  nodeSelector:
    matchLabels:
      node-role.kubernetes.io/custom: ""

secret-custom-user-data.yaml

apiVersion: v1
data:
  disableTemplating: dHJ1ZQo=
  userData: eyJpZ25pdGlvbiI6eyJjb25maWciOnsibWVyZ2UiOlt7InNvdXJjZSI6Imh0dHBzOi8vMTAuMC4xMjguNToyMjYyMy9jb25maWcvY3VzdG9tIn1dfSwic2VjdXJpdHkiOnsidGxzIjp7ImNlcnRpZmljYXRlQXV0aG9yaXRpZXMiOlt7InNvdXJjZSI6ImRhdGE6dGV4dC9wbGFpbjtjaGFyc2V0PXV0Zi04O2Jhc2U2NCxMUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VSRlJFTkRRV1pwWjBGM1NVSkJaMGxKU2tWQmMybFpUMnREYmxsM1JGRlpTa3R2V2tsb2RtTk9RVkZGVEVKUlFYZEtha1ZUVFVKQlIwRXhWVVVLUTNoTlNtSXpRbXhpYms1dllWZGFNRTFTUVhkRVoxbEVWbEZSUkVWM1pIbGlNamt3VEZkT2FFMUNORmhFVkVsM1RWUkJlVTFxUVRWT1JHZDVUMFp2V0FwRVZFMTNUVlJCZVUxRVFUVk9SR2Q1VDBadmQwcHFSVk5OUWtGSFFURlZSVU40VFVwaU0wSnNZbTVPYjJGWFdqQk5Va0YzUkdkWlJGWlJVVVJGZDJSNUNtSXlPVEJNVjA1b1RVbEpRa2xxUVU1Q1oydHhhR3RwUnpsM01FSkJVVVZHUVVGUFEwRlJPRUZOU1VsQ1EyZExRMEZSUlVGMGVUVnhNVkZPTWt0V0x6SUtiakptYmpSNkswUjRZamxCWjJjemJFdFljMjk2VlUxWlJtVnBZbFZwTUVKMmJXdENaR2RNZDBWUEwzbFZkbUprYkZOUGVIZHVhbHBZVmpBNVZtNDRTZ28wWlV4WWVEUnZjQzlwUjAwMmJHWmljekJ0V1V0MU5qUXpjMlJTTXpjMWFUUkVTelV3UW5oblYyWndTaTlyTW1KRWRFTTNTRlpZTDBWa1MwTm9lREZwQ25aU2FERnplR3d2ZEhNNFIyVkZXU3RJUjFWc1dIVkdZVkZwUXlzeVJHczRNMnh3VW1oU2FWY3JVRkJHTlVSWVNuTmFWVXgyUW00cmN5OXlUVk42U0RBS01DOTRRVzlrT1hKaFpVRlpZMGR3ZFhOQ2RUUjNSakFyV0VGdk1tcFNRblEwZVd0Q1dsZFJVVXNyVkROcGMwdExVbk14WjJsM01FZ3hkekJCVEdoWmJ3cEhlRk4zYjNwbmVHNTJjbmxvZFhGcE5tMDNlaXRUSzJNMVpDdDJPU3R1ZFdkUFRuVlJNbXBpVTJKSk16aEJhVzFSTUZaRWJIZFlVVk4zUkRsaE1UbFlDbTFDV0VsQ1Zua3dhMUZKUkVGUlFVSnZNRWwzVVVSQlQwSm5UbFpJVVRoQ1FXWTRSVUpCVFVOQmNWRjNSSGRaUkZaU01GUkJVVWd2UWtGVmQwRjNSVUlLTDNwQlpFSm5UbFpJVVRSRlJtZFJWV1l5TlRoUmNsTmljRVJwWTJoQlZtODVUM0l6UWxvdmFWTXhOSGRFVVZsS1MyOWFTV2gyWTA1QlVVVk1RbEZCUkFwblowVkNRVVZ6TTFWNFVqRndVV05MVTJvdmJVSTBibXd6ZFd4UFYzVnRSRUZsTDJJNE56QXlhV3RzZDJOVmRUSTNSMms1T0RJNVMzVndTM0J6YWxseUNuUkdRbTR6ZEVkRk9UTXhjbnBTZFhkMVJUY3ZTR0pGV2xCbVNYQlNRVkpYVTNSeWFFUk9Xa1I0YVdodFoyUnhiRzkwUkdoTVdtaFFMemRDTHk5VmFIRUtNalZQYm5SM1ExUlBWM1pzZDFJNFMwTkVSV3R0V0RodFpIcEJibmxOTlhoemRFNTRSV05tUlc5MU5qQnpLM3BoTm1oU2JUQlRUVGRRUVVkVWRWWm1hQXBhYTFGcVZ5dFVSbTV2U1VnclZVRXdOSGs1U2poUFRXOUJLelZ2WlZKNVVVNWtaSFJwVUZrM1FuWXhhWGt2T1VsSVVDOXVPRXRNWjB4Wk9GcDRZVmxRQ25odmJVTm9UR1p0YVhOUWNUaG1kVU16V0V4amJGUlNTRlZJTDBGRWIzTkJaemg2TTBkaFNXZFpUbmQzVTBKWFUyeEZUMVpJVjBkQ1JUQk1ZelZVYlRjS1lVTlNNbVppVTBOa2FqaGFWMWh5Yld0T2FFRnBaWGhYU2xaclBRb3RMUzB0TFVWT1JDQkRSVkpVU1VaSlEwRlVSUzB0TFMwdENnPT0ifV19fSwidmVyc2lvbiI6IjMuMS4wIn19
kind: Secret
metadata:
  managedFields:
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:data:
        .: {}
        f:disableTemplating: {}
        f:userData: {}
      f:type: {}
    manager: cluster-bootstrap
    operation: Update
  name: custom-user-data
  namespace: openshift-machine-api
  selfLink: /api/v1/namespaces/openshift-machine-api/secrets/custom-user-data
type: Opaque