用Colab Pro+和Google Cloud Storage低成本训练600M参数的模型

curl ipinfo.io

{
  "ip": "34.41.222.46",
  "hostname": "46.222.41.34.bc.googleusercontent.com",
  "city": "Council Bluffs",
  "region": "Iowa",
  "country": "US",
  "loc": "41.2619,-95.8608",
  "org": "AS396982 Google LLC",
  "postal": "51502",
  "timezone": "America/Chicago",
  "readme": "https://ipinfo.io/missingauth"
}

# 1. 用户授权
from google.colab import auth
auth.authenticate_user()

# 2. 安装 gcsfuse
!echo "deb https://packages.cloud.google.com/apt gcsfuse-`lsb_release -c -s` main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
!curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
!apt -qq update && apt -qq install gcsfuse

# 3. 挂载 bucket
# 将 "my-gcs-bucket1" 替换为你的bucket名称
gcs_bucket_name = "my-gcs-bucket1" 
local_mount_path = f"/mnt/gs/{gcs_bucket_name}"

!mkdir -p {local_mount_path}

# 使用优化参数挂载，提升数据加载性能
!gcsfuse --implicit-dirs --type-cache-max-size-mb=-1 --stat-cache-max-size-mb=-1 --kernel-list-cache-ttl-secs=-1 --metadata-cache-ttl-secs=-1 {gcs_bucket_name} {local_mount_path}

# 4. 验证挂载是否成功
!ls -lh {local_mount_path}

# 设置环境变量以优化tokenizer的并行处理
import os
os.environ["TOKENIZERS_PARALLELISM"] = "true"

# 克隆parler-tts代码库
%cd /content
!git clone https://github.com/huggingface/parler-tts.git

# 安装依赖
%cd parler-tts
!ls
!pip install -e .[train]
!pip install flash-attn --no-build-isolation

# 设置Huggingface缓存目录，避免预训练模型占满磁盘空间
%env HF_HOME=/mnt/gs/my-gcs-bucket1/parler-tts/huggingface_cache
!accelerate config

# 设置Huggingface缓存目录，避免预训练模型占满磁盘空间
%env HF_HOME=/mnt/gs/my-gcs-bucket1/parler-tts/huggingface_cache

# 运行初始化脚本
%cd /content/parler-tts/
!python helpers/model_init_scripts/init_model_600M.py \
    /mnt/gs/my-gcs-bucket1/parler-tts/parler-tts-untrained-600M \
    --text_model google/flan-t5-base \
    --audio_model parler-tts/dac_44khZ_8kbps

{
    "model_name_or_path": "/mnt/gs/my-gcs-bucket1/parler-tts/parler-tts-untrained-600M/parler-tts-untrained-600M/",
    "cache_dir": "/mnt/gs/my-gcs-bucket1/parler-tts/huggingface_cache/",
    "save_to_disk":  "/mnt/gs/my-gcs-bucket1/parler-tts/tmp_dataset_audio/",
    "temporary_save_to_disk": "/mnt/gs/my-gcs-bucket1/parler-tts/audio_code_tmp/",
    "resume_from_checkpoint": "/mnt/gs/my-gcs-bucket1/parler-tts/output_dir_training/checkpoint-200000-epoch-1/",

    "feature_extractor_name":"ylacombe/dac_44khZ_8kbps",
    "description_tokenizer_name":"google/flan-t5-base",
    "prompt_tokenizer_name":"google/flan-t5-base",

    "report_to": ["wandb"],
    "overwrite_output_dir": true,
    "output_dir": "/mnt/gs/my-gcs-bucket1/parler-tts/output_dir_training",

    "train_dataset_name": "blabble-io/libritts_r+blabble-io/libritts_r+blabble-io/libritts_r+parler-tts/mls_eng_10k",
    "train_metadata_dataset_name": "parler-tts/libritts_r_tags_tagged_10k_generated+parler-tts/libritts_r_tags_tagged_10k_generated+parler-tts/libritts_r_tags_tagged_10k_generated+parler-tts/mls-eng-10k-tags_tagged_10k_generated",
    "train_dataset_config_name": "clean+clean+other+default",
    "train_split_name": "train.clean.360+train.clean.100+train.other.500+train",

    "eval_dataset_name": "blabble-io/libritts_r+parler-tts/mls_eng_10k",
    "eval_metadata_dataset_name": "parler-tts/libritts_r_tags_tagged_10k_generated+parler-tts/mls-eng-10k-tags_tagged_10k_generated",
    "eval_dataset_config_name": "other+default",
    "eval_split_name": "test.other+test",

    "target_audio_column_name": "audio", 
    "description_column_name": "text_description",
    "prompt_column_name": "text",

    "max_eval_samples": 96,
    
    "max_duration_in_seconds": 30,
    "min_duration_in_seconds": 2.0,
    "max_text_length": 400,

    "group_by_length": true,

    "add_audio_samples_to_wandb": true,
    "id_column_name": "id",

    "preprocessing_num_workers": 1,

    "do_train": true,
    "num_train_epochs": 40,
    "gradient_accumulation_steps": 8,
    "gradient_checkpointing": false,
    "per_device_train_batch_size": 3,
    "learning_rate": 0.00095,
    "adam_beta1": 0.9,
    "adam_beta2": 0.99,
    "weight_decay": 0.01,

    "lr_scheduler_type": "constant_with_warmup",
    "warmup_steps":  20000,


    "logging_steps": 1000,
    "freeze_text_encoder": true,


    "do_eval": true, 
    "predict_with_generate": true,
    "include_for_metrics": true,
    "evaluation_strategy": "steps",
    "eval_steps": 10000,
    "save_steps": 10000,
    "save_total_limit": 10,

    "per_device_eval_batch_size": 6,

    "audio_encoder_per_device_batch_size": 8,
    "dtype": "bfloat16",
    "seed": 456,

    "dataloader_num_workers": 8,
    "attn_implementation": "sdpa"
}

!accelerate launch training/run_parler_tts_training.py helpers/training_configs/starting_point_0.01_colab.json

# 选择项目
gcloud projects list
gcloud config set project project_name

# 开始拷贝数据
gcloud transfer jobs create \
  gs://source_bucket \
  gs://destination_bucket

ERROR: (gcloud.transfer.jobs.create) FAILED_PRECONDITION: Failed to obtain the location of the GCS bucket my-gcs-bucket1 Additional details: project-xxxxxxxxxxxxx@storage-transfer-service.iam.gserviceaccount.com does not have storage.buckets.get access to the Google Cloud Storage bucket. Permission 'storage.buckets.get' denied on resource (or it may not exist).

gcloud transfer authorize --add-missing

# 监控本地磁盘
apt install sysstat
iostat -x 1

# 监控网络和本地磁盘
pip install dool
dool -cdngyltm --disk-util --disk-tps --fs 1

find /root/ -type f -size +100M -exec ls -lh {} \;

sudo du -h --max-depth=1 --exclude=/proc --exclude=/sys --exclude=/dev --exclude=/run --exclude=/mnt /