apiVersion: kubeflow.org/v2beta1
kind: MPIJob
metadata:
  name: latency-2procss
spec:
  sshAuthMountPath: /home/mpi/.ssh
  cleanPodPolicy: Running
  slotsPerWorker: 1
  mpiReplicaSpecs:
    Launcher:
      replicas: 1
      template:
        spec:
          initContainers:
          - name: wait-hostfilename
            image: image-registry.openshift-image-registry.svc:5000/mpi-benchmarking/mpi-bench:osu-bench
            command:
            - bash
            - -cx
            - "true"
            - "[[ $(cat /etc/mpi/hostfile | wc -l) != 0 ]] && (date; echo 'Hostfile is ready'; cat /etc/mpi/hostfile) || (date; echo 'Hostfile not ready ...'; sleep 10; exit 1) && while read host; do while ! ssh -p 20022 $host echo $host ; do date; echo \"Pod $host is not up ...\"; sleep 10; done; date; echo \"Pod $host is ready\"; done <<< \"$(cat /etc/mpi/hostfile)\""
            volumeMounts:
            - mountPath: /etc/mpi
              name: mpi-job-config
            - mountPath: /home/mpi/.ssh
              name: ssh-auth
          containers:
          - name: mpi-launcher
            command:
            - mpirun
            - -np
            - "2"
            - -bind-to
            - nonem
            - -map-by
            - slot
            - -mca
            - pml
            - ob1
            - -mca
            - btl
            - ^openib
            - bash
            - -c
            - /opt/osu-micro-benchmarks/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency
            image: image-registry.openshift-image-registry.svc:5000/mpi-benchmarking/mpi-bench:osu-bench
            imagePullPolicy: Always
            env:
            - name: OMPI_MCA_plm_rsh_agent
              value: "bash /mnt/sshd-script/connect.sh"
            volumeMounts:
            - mountPath: /mnt/sshd-script
              name: sshd-script
            - mountPath: /mnt/sshd-secret
              name: sshd-secret
          volumes:
          - name: sshd-script
            configMap:
              defaultMode: 0777
              name: sshd-script
          - name: sshd-secret
            secret:
              secretName: sshd-secret
    Worker:
      replicas: 2
      template:
        metadata:
          labels:
            app: latency-2procs-worker
        spec:
          topologySpreadConstraints:
          - maxSkew: 1
            topologyKey: kubernetes.io/hostname
            whenUnsatisfiable: DoNotSchedule
            labelSelector:
              matchLabels:
                app: latency-2procs-worker
          containers:
          - name: mpi-worker
            image: image-registry.openshift-image-registry.svc:5000/mpi-benchmarking/mpi-bench:osu-bench
            imagePullPolicy: Always
            env:
            - name: OMPI_MCA_plm_rsh_agent
              value: "bash /mnt/sshd-script/connect.sh"
            command:
            - python3
            - /mnt/sshd-script/server.py
            volumeMounts:
            - mountPath: /mnt/sshd-script
              name: sshd-script
            - mountPath: /mnt/sshd-secret
              name: sshd-secret
          volumes:
          - name: sshd-script
            configMap:
              defaultMode: 0777
              name: sshd-script
          - name: sshd-secret
            secret:
              secretName: sshd-secret
