dask-mnp-eks.json

{
  "jobDefinitionName": "dask-mnp-hello-world",
  "type": "multinode",
  "platformCapabilities": [
    "EC2"
  ],
  "nodeProperties": {
    "numNodes": 3,
    "mainNode": 0,
    "nodeRangeProperties": [
      {
        "targetNodes": "0:0",
        "instanceTypes": [
          "c5.xlarge"
        ],
        "eksProperties": {
          "podProperties": {
            "hostNetwork": true,
            "shareProcessNamespace": true,
            "volumes": [
              {
                "name": "dask",
                "hostPath": {
                  "path": "/tmp/dask"
                }
              }
            ],
            "containers": [
              {
                "name": "dask-scheduler",
                "image": "$CONTAINER_URI",
                "command": [
                  "dask",
                  "scheduler"
                ],
                "args": ["--no-show", "--port", "8786", "--pid-file", "/dask/dask-scheduler.pid", "--scheduler-file", "/dask/dask-scheduler-info.json"],
                "resources": {
                  "requests": {
                    "cpu": "1",
                    "memory": "1000Mi"
                  },
                  "limits": {
                    "cpu": "2"
                  }
                },
                "volumeMounts": [
                  {
                    "name": "dask",
                    "mountPath": "/dask"
                  }
                ]
              },
              {
                "name": "application",
                "image": "$CONTAINER_URI",
                "imagePullPolicy": "Always",
                "command":[
                  "/usr/local/bin/run-dask-mnp.sh"
                ],
                "args": [
                  "/dask/dask-scheduler-info.json"
                ],
                "resources": {
                  "requests": {
                    "cpu": "1",
                    "memory": "1000Mi"
                  },
                  "limits": {
                    "cpu": "2"
                  }
                },
                "volumeMounts": [
                  {
                    "name": "dask",
                    "mountPath": "/dask"
                  }
                ]
              }
            ]
          }
        }
      },
      {
        "targetNodes": "1:",
        "instanceTypes": [
          "c5.xlarge"
        ],
        "eksProperties": {
          "podProperties": {
            "hostNetwork": true,
            "containers": [
              {
                "name": "dask-workers",
                "image": "$CONTAINER_URI",
                "command": [
                  "/bin/sh",
                  "-c"
                ],
                "args": [
                  "dask worker \"${AWS_BATCH_JOB_MAIN_NODE_PRIVATE_IPV4_ADDRESS}:8786\""
                ],
                "resources": {
                  "requests": {
                    "cpu": "3",
                    "memory": "3000Mi"
                  },
                  "limits": {
                    "cpu": "4"
                  }
                }
              }
            ]
          }
        }
      }
    ]
  },
}