forked from replicate/cog-sdxl
/
create_hyperparam_sweep.py
executable file
·178 lines (146 loc) · 7.88 KB
/
create_hyperparam_sweep.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
import random, os, ast
from itertools import product
import time
random.seed(int(1000*time.time()))
def hamming_distance(dict1, dict2):
distance = 0
for key in dict1.keys():
if dict1[key] != dict2.get(key, None):
distance += 1
return distance
#######################################################################################
#######################################################################################
"""
Faces:
https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/xander_2.zip
https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/xander_5.zip
https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/xander_best.zip
https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/steel.zip
Objects:
https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/banny_all.zip
https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/banny_best.zip
https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/koji_color.zip
https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/plantoid_imgs.zip
Styles:
https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/does.zip
https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/clipx_200.zip
"""
# Setup the base experiment config:
#lora_training_urls = "https://minio.aws.abraham.fun/creations-stg/d6f8446d13a82bc159f4b26aadca90a888493e92cf0bab1e510cb5354fb065a7.zip|https://minio.aws.abraham.fun/creations-stg/991d70ba870022aef6c893b8335fee53ed9a32e8f998e23ec9dcf2adc0ee3f76.zip|https://minio.aws.abraham.fun/creations-stg/6b25015c2f655915c90c41fc35cc5f42f8a877307c2a8affc2d47ed812cf23c3.zip|https://minio.aws.abraham.fun/creations-stg/fbdc59246ee841bb8303787155a6a0c5cae56d7545a9bd0d5d077a9d8193baff.zip"
run_name = "n_tokens"
caption_prefix = "" # "" to activate chatgpt
mask_target_prompts = "" # "" to activate chatgpt
n_exp = 300 # how many random experiment settings to generate
min_hamming_distance = 1 # min_n_params that have to be different from any previous experiment to be scheduled
# Define training hyperparameters and their possible values
# The params are sampled stochastically, so if you want to use a specific value more often, just put it in multiple times
if 0:
hyperparameters = {
'lora_training_urls': [
"https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/banny_best.zip",
"https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/plantoid_imgs_9.zip",
],
'concept_mode': ['object'],
'left_right_flip_augmentation': ['True'],
'augment_imgs_up_to_n': ['20'],
'resolution': [896, 1024],
'lora_rank': ['12', '24'],
'prodigy_d_coef': ['0.75'],
'max_train_steps': ['600'],
'lora_param_scaler': ['1.0', '0.3', '0.1'],
'prodigy_d_coef': ['0.5', '1.0'],
'train_batch_size': ['4'],
'seed': ['0'],
'debug': ['True'] # avoid sending the entire .rar file back after each training run (takes a long time)
}
elif 0:
hyperparameters = {
'lora_training_urls': [
"https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/does.zip",
"https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/leavingthisplace.zip",
],
'concept_mode': ['style'],
'left_right_flip_augmentation': ['True'],
'augment_imgs_up_to_n': ['20'],
'resolution': [896, 1024],
'lora_rank': ['12', '24'],
'prodigy_d_coef': ['0.5', '1.0'],
'lora_param_scaler': ['1.0', '0.3', '0.1'],
'max_train_steps': ['600'],
'train_batch_size': ['4'],
'seed': ['0'],
'debug': ['True'] # avoid sending the entire .rar file back after each training run (takes a long time)
}
else:
hyperparameters = {
'lora_training_urls': [
"https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/steel.zip",
"https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/xander_5.zip",
"https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/mira.zip"
],
'concept_mode': ['face'],
'left_right_flip_augmentation': ['False'],
'caption_prefix': ["a photo of TOK"],
'augment_imgs_up_to_n': ['20'],
'resolution': [960],
'hard_pivot': ['False'],
'lora_weight_decay': ['0.002'],
'lora_param_scaler': ['0.25'],
'l1_penalty': ['0.1'],
'lora_rank': ['12'],
'prodigy_d_coef': ['0.8'],
'max_train_steps': ['600'],
'checkpointing_steps': ['700'],
'train_batch_size': ['4'],
'n_tokens': ['1', '2', '4'],
'seed': ['0'],
'debug': ['True'] # avoid sending the entire .rar file back after each training run (takes a long time)
}
#######################################################################################
#######################################################################################
output_filename = f"training_exp_{run_name}.sh"
# Create a set to hold the combinations that have already been run
scheduled_experiments = set()
# if output_filename exists, remove it:
os.remove(output_filename) if os.path.exists(output_filename) else None
# Open the shell script file
try_sampling_n_times = 500
with open(output_filename, "w") as f:
for exp_index in range(n_exp): # number of combinations you want to generate
resamples, combination = 0, None
while resamples < try_sampling_n_times:
experiment_settings = {name: random.choice(values) for name, values in hyperparameters.items()}
resamples += 1
min_distance = float('inf')
for str_experiment_settings in scheduled_experiments:
existing_experiment_settings = dict(sorted(ast.literal_eval(str_experiment_settings)))
distance = hamming_distance(experiment_settings, existing_experiment_settings)
min_distance = min(min_distance, distance)
if min_distance >= min_hamming_distance:
str_experiment_settings = str(sorted(experiment_settings.items()))
scheduled_experiments.add(str_experiment_settings)
# Write the experiment to the shell script file, if necessary.
break
if resamples == try_sampling_n_times:
print(f"\nCould not find a new experiment_setting after random sampling {try_sampling_n_times} times, dumping all experiment_settings to .sh script")
break
# extract the dataset name:
for name, value in sorted(experiment_settings.items()):
if name == "lora_training_urls":
dataset_name = value.split('/')[-1][:30]
# add experiment_settings to run_name:
run_name_exp = f"{run_name}_{dataset_name}_{exp_index:03d}_{int(time.time()*1000)%10000}"
f.write(f'cog predict \\\n')
f.write(f' -i run_name="{run_name_exp}" -i caption_prefix="{caption_prefix}" \\\n')
f.write(f' -i mask_target_prompts="{mask_target_prompts}" \\\n')
for name, value in sorted(experiment_settings.items()):
if type(value) == str:
f.write(f' -i {name}="{value}" \\\n')
else:
f.write(f' -i {name}={value} \\\n')
# Remove the last backslash and add a new line
f.seek(f.tell() - 3)
f.write(' \n\n')
print(f"\n\n---> Saved {len(scheduled_experiments)} experiment commands to {output_filename}")
print("To run this experiment:")
print(f"sudo sh {output_filename}")