apiVersion: dataplane.openstack.org/v1beta1
kind: OpenStackDataPlaneNodeSet
metadata:
  name: std-nodeset
  namespace: openstack
spec:
  env:
    - name: ANSIBLE_FORCE_COLOR
      value: "True"
  networkAttachments:
    - ctlplane
  preProvisioned: false
  baremetalSetTemplate:
    deploymentSSHSecret: dataplane-ansible-ssh-private-key-secret
    bmhNamespace: openstack
    cloudUserName: cloud-admin
    #bmhLabelSelector:
    #  compute: std 
    ctlplaneInterface: eno1
    dnsSearchDomains:
      - n46-rhoso.local 
  nodeTemplate:
    ansibleSSHPrivateKeySecret: dataplane-ansible-ssh-private-key-secret
    extraMounts:
      - extraVolType: Logs
        volumes:
        - name: ansible-logs
          persistentVolumeClaim:
            claimName: ansible-logs-pvc
        mounts:
        - name: ansible-logs
          mountPath: "/runner/artifacts"
    managementNetwork: ctlplane
    networks:
      - name: ctlplane
        subnetName: ctlplane-subnet
        defaultRoute: true
      - name: internalapi
        subnetName: internalapi-subnet
      #- name: external
      #  subnetName: subnet1
      - name: storage
        subnetName: storage-subnet
      - name: tenant
        subnetName: tenant-subnet
    ansible:
      ansibleUser: cloud-admin
      ansiblePort: 22
      ansibleVarsFrom:
        - prefix: subscription_manager_
          secretRef:
            name: subscription-manager
        - secretRef:
            name: redhat-registry
      ansibleVars:
        timesync_ntp_servers:
        - hostname: 10.221.16.34
        - hostname: 10.221.16.50
        edpm_timezone: "Europe/Amsterdam"
        edpm_kernel_args: "default_hugepagesz=1GB hugepagesz=1G hugepages=232 hugepagesz=2M hugepages=4096 isolcpus=2-39,42-79 iommu=pt intel_iommu=on"
#DEPRECATED DO NOT USE        edpm_ovs_dpdk_lcore_list: 0,1,40,41
        edpm_ovs_dpdk_memory_channels: "4"
        edpm_ovs_dpdk_pmd_auto_lb: "true"
        edpm_ovs_dpdk_pmd_core_list: 2,3,4,5,42,44
        edpm_ovs_dpdk_pmd_improvement_threshold: "25"
        edpm_ovs_dpdk_pmd_load_threshold: "70"
        edpm_ovs_dpdk_pmd_rebal_interval: "2"
        edpm_ovs_dpdk_socket_memory: 8192,8192
        edpm_ovs_dpdk_vhost_postcopy_support: "true"
        edpm_tuned_isolated_cores: 2-39,42-79
        edpm_tuned_profile: cpu-partitioning
        edpm_bootstrap_command: |
          echo "{\"dmi.system.uuid\":\"$(uuidgen)\"}" > /etc/rhsm/facts/redhat.facts
          subscription-manager register --org="15990867" --activationkey="4d5d2157-5d6e-46fb-b85a-3de940bd1ec5"
          subscription-manager release --set=9.4
          subscription-manager repos --disable=*
          subscription-manager repos --enable=rhel-9-for-x86_64-baseos-eus-rpms --enable=rhel-9-for-x86_64-appstream-eus-rpms --enable=rhel-9-for-x86_64-highavailability-eus-rpms --enable=fast-datapath-for-rhel-9-x86_64-rpms --enable=rhoso-18.0-for-rhel-9-x86_64-rpms --enable=rhceph-7-tools-for-rhel-9-x86_64-rpms
        edpm_bootstrap_release_version_package: []
        neutron_physical_bridge_name: br-ex
        neutron_public_interface_name: eth0
        service_net_map:
          nova_api_network: internalapi
          nova_libvirt_network: internalapi
        edpm_ovn_bridge_mappings:
        - datacentre:br-ex
#       - datacentre2:br-datacentre2
        edpm_nova_libvirt_qemu_group: hugetlbfs
        edpm_network_config_nmstate: true
        edpm_network_config_update: false
        edpm_network_config_template: |
          ---
          {% set mtu_list = [ctlplane_mtu] %}
          {% for network in nodeset_networks %}
          {{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }}
          {%- endfor %}
          {% set min_viable_mtu = mtu_list | max %}
          network_config:
          - type: interface
            name: eno1
            mtu: {{ ctlplane_mtu }}
            dns_servers: {{ ctlplane_dns_nameservers }}
            domain: {{ dns_search_domains }}
            routes:
            - default: true
              next_hop: {{ ctlplane_gateway_ip }}
            use_dhcp: false
            addresses:
            - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }}
          - type: linux_bond
            name: bond0
            mtu: 9100
            use_dhcp: false
            bonding_options: "mode=active-backup"
            members:
            - type: interface
              name: ens1f1
              primary: true
            - type: interface
              name: ens2f0
          {% for network in nodeset_networks if network not in ['tenant'] %}
          - type: vlan
            device: bond0
            mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }}
            vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }}
            addresses:
            - ip_netmask:
                {{ lookup('vars', networks_lower[network] ~ '_ip') }}/{{ lookup('vars', networks_lower[network] ~ '_cidr') }}
            routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }}
          {% endfor %}
          - type: interface
            name: ens3f0
            mtu: 1500
            use_dhcp: false
          - type: interface
            name: ens3f1
            mtu: 1500
            use_dhcp: false
          - type: ovs_user_bridge
            name: {{ neutron_physical_bridge_name }}
            mtu: 9100
            ovs_extra: "set port {{ neutron_physical_bridge_name }} tag={{ lookup('vars', networks_lower['tenant'] ~ '_vlan_id') }}"
            addresses:
            - ip_netmask: 
                {{ lookup('vars', networks_lower['tenant'] ~ '_ip') }}/{{ lookup('vars', networks_lower['tenant'] ~ '_cidr')}}
            routes: {{ lookup('vars', networks_lower['tenant'] ~ '_host_routes') }}
            members:
            - type: ovs_dpdk_bond
              name: dpdkbond1
              mtu: 9100
              rx_queue: 3
              ovs_options: "bond_mode=balance-slb other_config:bond-miimon-interval=100"
              ovs_extra:
                - "set interface dpdk0 other_config:pmd-rxq-affinity=0:42"
                - "set interface dpdk1 other_config:pmd-rxq-affinity=0:44"
              members:
              - type: ovs_dpdk_port
                name: dpdk0
                mtu: 9100
                members:
                - type: interface
                  name: ens1f0
              - type: ovs_dpdk_port
                name: dpdk1
                mtu: 9100
                members:
                - type: interface
                  name: ens2f1 
#          - type: ovs_user_bridge
#            name: br-datacentre2
#            mtu: 9100
#            members:
#            - type: ovs_dpdk_bond
#              name: dpdkbond2
#              mtu: 9100
#              rx_queue: 3
#              ovs_options: "bond_mode=balance-slb other_config:bond-miimon-interval=100"
#              ovs_extra:
#                - "set interface dpdk3 other_config:pmd-rxq-affinity=0:43"
#                - "set interface dpdk4 other_config:pmd-rxq-affinity=0:45"
#              members:
#              - type: ovs_dpdk_port
#                name: dpdk3
#                mtu: 9100
#                members:
#                - type: interface
#                  name: ens3f0
#              - type: ovs_dpdk_port
#                name: dpdk4
#                mtu: 9100
#                members:
#                - type: interface
#                  name: ens3f1
  services:
    - bootstrap
    - download-cache
    - reboot-os
    - configure-ovs-dpdk
    - configure-network
    - validate-network
    - install-os
    - configure-os
    - ssh-known-hosts
    - run-os
    - install-certs
    - ovn
    - neutron-ovn-igmp
    - neutron-metadata
    #- neutron-sriov
    - libvirt
    - std-nova-custom
    - telemetry
  nodes:
    seroics03354:
      bmhLabelSelector:
        nodeName: seroics03354
      hostName: seroics03354
      networks:
      - name: ctlplane
        subnetName: ctlplane-subnet
        fixedIP: 192.168.18.151
      - name: internalapi
        subnetName: internalapi-subnet
      - name: storage
        subnetName: storage-subnet
      - name: tenant
        subnetName: tenant-subnet
      ansible:
        ansibleHost: 192.168.18.151
        ansibleUser: cloud-admin
        ansibleVars:
          fqdn_internal_api: seroics03354.n46-rhoso.local
#    seroics03356:
#      bmhLabelSelector:
#        nodeName: seroics03356
#      hostName: seroics03356
#      networks:
#      - name: ctlplane
#        subnetName: ctlplane-subnet
#        fixedIP: 192.168.18.152
#      - name: internalapi
#        subnetName: internalapi-subnet
#      - name: storage
#        subnetName: storage-subnet
#      - name: tenant
#        subnetName: tenant-subnet
#      ansible:
#        ansibleHost: 192.168.18.152
#        ansibleUser: cloud-admin
#        ansibleVars:
#          fqdn_internal_api: seroics03356.n46-rhoso.local
    seroics03357:
      bmhLabelSelector:
        nodeName: seroics03357
      hostName: seroics03357
      networks:
      - name: ctlplane
        subnetName: ctlplane-subnet
        fixedIP: 192.168.18.153
      - name: internalapi
        subnetName: internalapi-subnet
      - name: storage
        subnetName: storage-subnet
      - name: tenant
        subnetName: tenant-subnet
      ansible:
        ansibleHost: 192.168.18.153
        ansibleUser: cloud-admin
        ansibleVars:
          fqdn_internal_api: seroics03357.n46-rhoso.local
#    seroics03364:
#      bmhLabelSelector:
#        nodeName: seroics03364
#      hostName: seroics03364
#      networks:
#      - name: ctlplane
#        subnetName: ctlplane-subnet
#        fixedIP: 192.168.18.154
#      - name: internalapi
#        subnetName: internalapi-subnet
#      - name: storage
#        subnetName: storage-subnet
#      - name: tenant
#        subnetName: tenant-subnet
#      ansible:
#        ansibleHost: 192.168.18.154
#        ansibleUser: cloud-admin
#        ansibleVars:
#          fqdn_internal_api: seroics03364.n46-rhoso.local
#    seroics03377:
#      bmhLabelSelector:
#        nodeName: seroics03377
#      hostName: seroics03377
#      networks:
#      - name: ctlplane
#        subnetName: ctlplane-subnet
#        fixedIP: 192.168.18.155
#      - name: internalapi
#        subnetName: internalapi-subnet
#      - name: storage
#        subnetName: storage-subnet
#      - name: tenant
#        subnetName: tenant-subnet
#      ansible:
#        ansibleHost: 192.168.18.155
#        ansibleUser: cloud-admin
#        ansibleVars:
#          fqdn_internal_api: seroics03377.n46-rhoso.local
