sgJ0dZddlZddlmZddlZddlZddlm Z ddl m Z ddl m Z mZmZmZerddlZej$eZdZe rerddlmZndd lmZGd d eZGd d eZdadZdZdZdZ dZ!ddZ"ddZ#y)z Integration with Deepspeed N) partialmethod)dep_version_check)is_accelerate_availableis_torch_availableis_torch_mlu_availableloggingctjjddu}|r7 trt j d}yt j d}yy#tj $rYywxYw)N deepspeedz deepspeed-mluTF) importlibutil find_specrimportlib_metadatametadataPackageNotFoundError)package_exists_s V/var/www/html/venv/lib/python3.12/site-packages/transformers/integrations/deepspeed.pyis_deepspeed_availabler#sp^^--k:$FN %'&//@"++K8A "66  sAAA21A2)HfDeepSpeedConfig)objectc"eZdZdZfdZxZS)raJ This object contains a DeepSpeed configuration dictionary and can be quickly queried for things like zero stage. A `weakref` of this object is stored in the module's globals to be able to access the config from areas where things like the Trainer object is not available (e.g. `from_pretrained` and `_get_resized_embeddings`). Therefore it's important that this object remains alive while the program is still running. [`Trainer`] uses the `HfTrainerDeepSpeedConfig` subclass instead. That subclass has logic to sync the configuration with values of [`TrainingArguments`] by replacing special placeholder values: `"auto"`. Without this special logic the DeepSpeed configuration is not modified in any way. Args: config_file_or_dict (`Union[str, Dict]`): path to DeepSpeed config file or dict. cft|tdtdt| |y)N accelerater )set_hf_deepspeed_configrsuper__init__selfconfig_file_or_dict __class__s rrzHfDeepSpeedConfig.__init__Ls)%,'+& ,-)__name__ __module__ __qualname____doc__r __classcell__r!s@rrr;s ..r"rcXeZdZdZfdZdZdZd dZeedZ d dZ d Z xZ S) HfTrainerDeepSpeedConfigz The `HfTrainerDeepSpeedConfig` object is meant to be created during `TrainingArguments` object creation and has the same lifespan as the latter. c@t||d|_g|_yN)rr_dtype mismatchesrs rrz!HfTrainerDeepSpeedConfig.__init__Zs ,- r"cH|j td|jS)Nz8trainer_config_process() wasn't called yet to tell dtype)r- ValueError)rs rdtypezHfTrainerDeepSpeedConfig.dtype_s" ;; WX X{{r"c4|j|}|y|dk(S)NFauto) get_value)r ds_key_longvals ris_autoz HfTrainerDeepSpeedConfig.is_autods"nn[) ;&= r"c |j|\}}|y|j|dk(r|||<y|sy|j|}|.||k7r(|jjd|d|d|d|yyy)a A utility method that massages the config file and can optionally verify that the values match. 1. Replace "auto" values with `TrainingArguments` value. 2. If it wasn't "auto" and `must_match` is true, then check that DS config matches Trainer config values and if mismatched add the entry to `self.mismatched` - will assert during `trainer_config_finalize` for one or more mismatches. Nr3z- ds =z vs hf )find_config_nodegetr.append)rr5hf_valhf_key must_matchconfigds_keyds_vals r fill_matchz#HfTrainerDeepSpeedConfig.fill_matchks..{; >  ::f  '#F6N  F#  &F"2 OO " "U;-qxqQWPX#Y Z#3 r"F)r?cn|j|jz|jz}|jd|jd| |jd|jd|jd|d| |jd|jd|jd|j d |jd |j |jgd |jd |jd |jd|jd|jdd|jd|j d |js |jr|jdk(rdnd}nd}|jrE|jj!di|jd<|j|jdd<|jd|jxs |jxr|dk(d|jd|dk(d|jd|j"d|jd|j$xs |j&d|j)drt*j,|_y|j1drt*j2|_yt*j4|_y) z Adjust the config with `TrainingArguments` values. This stage is run during `TrainingArguments` object creation. train_micro_batch_size_per_gpuper_device_train_batch_sizegradient_accumulation_stepstrain_batch_sizeztrain_batch_size (calculated)gradient_clipping max_grad_normzoptimizer.params.lr learning_ratezoptimizer.params.betaszadam_beta1+adam_beta2zoptimizer.params.eps adam_epsilonzoptimizer.params.weight_decay weight_decayzscheduler.params.warmup_min_lrrzscheduler.params.warmup_max_lrapexampN checkpointuse_node_local_storagez fp16.enabledz%fp16|fp16_full_eval+fp16_backend(amp)z amp.enabledzfp16+fp16_backend(apex)z amp.opt_levelfp16_opt_levelz bf16.enabledzbf16|bf16_full_eval) world_sizerFrGrCrJrK adam_beta1 adam_beta2rLrM fill_onlyfp16fp16_full_eval fp16_backendsave_on_each_noder@r;rRbf16bf16_full_evalis_truetorchbfloat16r-is_falsefloat32float16)rargsauto_find_batch_sizerHrYs rtrainer_config_processz/HfTrainerDeepSpeedConfig.trainer_config_processsL ??T-M-MMPTPpPpp  ,  , , )$ $   )  , , )    +$ $  +T-?-?Q -t/A/A?S  $ __doo . # .0A0A>R 79J9JN[ 7; 8$:L:Lo^ 99++%)%6%6&%@6eLL  ! !(,  b(IDKK %BFBXBXDKK %&> ?  ii.4.. ILE4I 3   |v'=?XY )<)<>NO )Id6I6ILab << '..DK ]]> *--DK--DKr"clgd}|Dcgc]}|j|s|}}t|dkDrt|jdr|jj}nt|jdr t |jj }nt|jdrAt|jjdr!|jjj}not|jdrJt|jjdr*t |jjj }ntd|d|jd||z|jr6|jd td |z|z|jd d |z|jd |d|jd|j|dt|jdkDr*dj|j}td|dycc}w)z This stage is run after we have the model and know num_training_steps. Now we can complete the configuration process. )$zero_optimization.reduce_bucket_size-zero_optimization.stage3_prefetch_bucket_size4zero_optimization.stage3_param_persistence_thresholdr hidden_size hidden_sizes text_configzThe model's config file has neither `hidden_size` nor `hidden_sizes` entry, therefore it's not possible to automatically fill out the following `auto` entries in the DeepSpeed config file: zb. You can fix that by replacing `auto` values for these keys with an integer value of your choice.rgrhg?ri z scheduler.params.total_num_stepsznum_training_steps (calculated)z!scheduler.params.warmup_num_steps warmup_steps z]Please correct the following DeepSpeed config values that mismatch TrainingArguments values: zF The easiest method is to set these DeepSpeed config values to 'auto'.N)r7lenhasattrr@rjmaxrkrlr0rVis_zero3intrCget_warmup_stepsr.join) rrcmodelnum_training_stepshidden_size_based_keysxhidden_size_auto_keysrjr.s rtrainer_config_finalizez0HfTrainerDeepSpeedConfig.trainer_config_finalizes"  -C VqdllSTo V V $ % )u||]3#ll66 ~6!%,,";";< }5'%,,BZBZ\i:j#ll66BB }5'%,,BZBZ\j:k!%,,":":"G"GH  55J4KLYY NNA;Q\C\ ]}}Ck)K78J$  .  -  /  ! !"4 5  t ! #4??3J'L(oq  $[!Ws H1H1)NTF) r#r$r%r&rr1r7rCrrVrer|r'r(s@rr*r*Ts8   ![4jU;IH(T@r"r*c.tj|ayr,)weakrefref_hf_deepspeed_config_weak_ref)hf_deepspeed_config_objs rrrs %,KK0G$H!r"cdayr,)rr"runset_hf_deepspeed_configr s %)!r"cTt"ttjSy)NF)rrsrr"ris_deepspeed_zero3_enabledr&s&$05R5T5`,.7799r"cLtttjSyr,)rr@rr"rdeepspeed_configr-s#$05R5T5`,.555r"cddlm}m}|j}d}d|vr!|jr t d||}n:|j rtjdj}d|d <d} d |vr ||} || fSt||rfd } ||| } || fSj| } || fS)zY A convenience wrapper that deals with optimizer and lr scheduler configuration. r) DummyOptimDummySchedulerN optimizerz|--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. Only one optimizer can be configured.)paramszDetected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the custom optimizer has both CPU and GPU implementation (except LAMB)Tzero_allow_untested_optimizer schedulercftj}d|_|j|}|S)Nrxr)copy lr_schedulercreate_scheduler)r trainer_copyrrxtrainers r_lr_scheduler_callablez5deepspeed_optim_sched.._lr_scheduler_callable_s=#yy1 -1 )+<<'9Y =  $#r")lr_scheduler_callabler) accelerate.utilsrrr@ adafactorr0 is_offloadloggerinfocreate_optimizer isinstancer) rhf_deepspeed_configrcrxmodel_parametersrrr@rrrs ` ` rdeepspeed_optim_schedr4s< ' 'FIf >>8 &67  ) ) + KKV ,,. 26./Lf%i0 & l ""# i , $*)KabL l ""#33GYen3oL l ""r"c ddlm}|j}|j}|jj j j}|j||||j|j|rH|js td|jd|jdd\}}d} ||fSd|_tt!d|j#} t%||||| \}}||fS) a Init DeepSpeed, after updating the DeepSpeed configuration with any relevant Trainer's args. If `resume_from_checkpoint` was passed then an attempt to resume from a previously saved checkpoint will be made. Args: trainer: Trainer object num_training_steps: per single gpu resume_from_checkpoint: path to a checkpoint if to resume from after normal DeepSpeedEngine load inference: launch in inference mode (no optimizer and no lr scheduler) auto_find_batch_size: whether to ignore the `train_micro_batch_size_per_gpu` argument as it's being set automatically by the auto batch size finder Returns: optimizer, lr_scheduler We may use `deepspeed_init` more than once during the life of Trainer, when we do - it's a temp hack based on: https://github.com/microsoft/DeepSpeed/issues/1394#issuecomment-937405374 until Deepspeed fixes a bug where it can't resume from a checkpoint after it did some stepping https://github.com/microsoft/DeepSpeed/issues/1612 r)rzMZeRO inference only makes sense with ZeRO Stage 3 - please adjust your configrr)NNNc|jSr,) requires_grad)ps rz deepspeed_init..s r")deepspeed.utilsrrwrc acceleratorstatedeepspeed_plugin hf_ds_configr|setLevelget_process_log_levelrsr0del_config_sub_treerlistfilter parametersr) rrx inference ds_loggerrwrcrrrrs rdeepspeed_initrqs *4 MME <rs /#9``   H %  !7!9O3..2}0}B!%I) :#z6#rPr"