Patrick Bateman

Patrick Bateman from American Psycho, trained on 36 short clips for [512] resolution bucket, and [24, 68] frame buckets. 5 repeats for 40 epochs. Trained with diffusion-pipe.

Trained on 2x 3090s. If you are training on 1x 3090, you can pipeline_stages=2

config.toml:

pipeline_stages = 2
output_dir = "/workspace/outputs/patrick-bateman"
dataset = "/workspace/configs/patrick-bateman/dataset.toml"
epochs = 150
micro_batch_size_per_gpu = 1
gradient_accumulation_steps = 4
gradient_clipping = 1
warmup_steps = 10
eval_every_n_epochs = 1
eval_before_first_step = true
eval_micro_batch_size_per_gpu = 1
eval_gradient_accumulation_steps = 1
save_every_n_epochs = 10
checkpoint_every_n_minutes = 120
activation_checkpointing = true
partition_method = "parameters"
save_dtype = "bfloat16"
caching_batch_size = 1
steps_per_print = 1
video_clip_mode = "single_middle"

[model]
type = "hunyuan-video"
transformer_path = "/workspace/models/hunyuan_video_720_cfgdistill_fp8_e4m3fn.safetensors"
vae_path = "/workspace/models/hunyuan_video_vae_fp32.safetensors"
llm_path = "/workspace/models/llava-llama-3-8b-text-encoder-tokenizer"
clip_path = "/workspace/models/clip-vit-large-patch14"
dtype = "bfloat16"
transformer_dtype = "float8"
timestep_sample_method = "logit_normal"

[adapter]
type = "lora"
rank = 32
dtype = "bfloat16"
only_double_blocks = false

[optimizer]
type = "adamw"
lr = 0.0001
betas = [ 0.9, 0.99,]
weight_decay = 0.01
eps = 1e-8

[monitoring]
enable_wandb = false

dataset.toml:

resolutions = [ 512, ]
enable_ar_bucket = true
min_ar = 0.5
max_ar = 2
num_ar_buckets = 7
frame_buckets = [ 24, 68,]
[[directory]]
path = "/workspace/datasets/patrick-bateman"
num_repeats = 5