forked from coiled/benchmarks
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcluster_kwargs.yaml
89 lines (72 loc) · 2.58 KB
/
cluster_kwargs.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
# Static kwargs passed to coiled.Cluster
# In A/B tests, these can be overridden by AB_environments/AB_<name>.cluster.yaml
# The override priority is as follows (bottom wins):
# 1. default parameters of coiled.Cluster
# 2. default section of this file
# 3. default section of AB_environments/AB_<name>.cluster.yaml
# 4. specific sections of this file
# 5. specific sections of AB_environments/AB_<name>.cluster.yaml
# The keys 'name', 'environ', and 'tags' must not be used.
# Settings for all clusters, unless overriden below
default:
package_sync: true
wait_for_workers: true
scheduler_vm_types: [m6i.large]
spot_policy: spot_with_fallback
# For all tests using the small_client fixture
small_cluster:
n_workers: 10
worker_vm_types: [m6i.large] # 2CPU, 8GiB
# For tests/benchmarks/test_parquet.py
parquet_cluster:
n_workers: 15
worker_vm_types: [m5.xlarge] # 4 CPU, 16 GiB
# For tests/benchmarks/test_spill.py
spill_cluster:
n_workers: 5
worker_disk_size: 64
worker_vm_types: [m6i.large] # 2CPU, 8GiB
# For tests/workflows/test_embarrassingly_parallel.py
embarrassingly_parallel:
n_workers: 100
worker_vm_types: [m6i.xlarge] # 4 CPU, 16 GiB (preferred default instance)
region: "us-east-1" # Same region as dataset
# For tests/workflows/test_xgboost_optuna.py
xgboost_optuna:
n_workers: 50
worker_vm_types: [m6i.xlarge] # 4 CPU, 16 GiB (preferred default instance)
# For tests/workflows/test_uber_lyft.py
uber_lyft:
n_workers: 20
worker_vm_types: [m6i.xlarge] # 4 CPU, 16 GiB (preferred default instance)
uber_lyft_large:
n_workers: 50
worker_vm_types: [m6i.xlarge] # 4 CPU, 16 GiB (preferred default instance)
# For tests/workflows/test_pytorch_optuna.py
pytorch_optuna:
n_workers: 10
worker_vm_types: [g4dn.xlarge] # 1 GPU, 4 CPU, 16 GiB
worker_options:
# Making workers single-threaded to avoid GPU contention. See discussion in
# https://github.com/coiled/benchmarks/pull/787#discussion_r1177004248 for
# more details.
nthreads: 1
# For tests/workflows/test_snowflake.py
snowflake:
n_workers: 20
worker_vm_types: [m6i.xlarge] # 4 CPU, 16 GiB (preferred default instance)
# Specific tests
test_work_stealing_on_scaling_up:
n_workers: 1
worker_vm_types: [t3.medium]
test_work_stealing_on_straggling_worker:
n_workers: 10
worker_vm_types: [t3.medium]
test_repeated_merge_spill:
n_workers: 20
worker_vm_types: [m6i.large]
# For tests/workflows/test_from_csv_to_parquet.py
from_csv_to_parquet:
n_workers: 10
worker_vm_types: [m6i.xlarge] # 4 CPU, 16 GiB (preferred default instance)
region: "us-east-1" # Same region as dataset