
    %ɷi6                        d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlZddlmZ ddlmZmZmZmZmZmZmZmZmZmZmZ ddlmZ dd	lmZmZmZ dd
l m!Z! ddl"m#Z# ddl$m%Z%m&Z& g fdZ'd Z(de jR                  de*e+e+f   fdZ,de jR                  de-e.e+   e*e+e+f   f   fdZ/de jR                  de*e+e+f   fdZ0de jR                  de*de-e.e+   e*e+e+f   f   fdZ1de jR                  de-e.e+   e*e+e+f   f   fdZ2	 d!de jR                  de*e+e+f   de3de-e jR                  e*e+e+f   f   fdZ4de.e+   de*e+e+f   fdZ5dede jR                  de-e jR                  e*e+e
f   f   fdZ6d Z7 G d d       Z8y)"    N)literal_eval)which)Any   )SageMakerConfig)DynamoBackendPrecisionTypeis_fp8_availableis_hpu_availableis_mlu_availableis_musa_availableis_neuron_availableis_npu_availableis_sdaa_availableis_torch_xla_availableis_xpu_available)DEEPSPEED_MULTINODE_LAUNCHERS)get_free_portis_port_in_usemerge_dicts)compare_versions   )parse_flag_from_env)DistributedTypeSageMakerDistributedTypec                     |j                  |      \  }}t        |       j                         D ].  \  }}|t        |      j                         v s"t	        |||       0 |S )z4
    Filters out all `accelerate` specific args
    )parse_known_argsvarsitemskeyssetattr)argsparserdefault_argsnew_args_keyvalues          J/var/www/html/venv/lib/python3.12/site-packages/accelerate/utils/launch.py_filter_argsr*   /   s`     )),7KHa4j&&( *
U$x.%%''Hc5)* O    c                      dD  cg c]  } t        |       s|  }} t        |      dk(  rt        d      |d   }t        j                  |dg      }d|v r|dddd	fS |d
dddfS c c} w )a  
    Determines the executable and argument names for mpirun, based on the type of install. The supported MPI programs
    are: OpenMPI, Intel MPI, or MVAPICH.

    Returns: Program name and arg names for hostfile, num processes, and processes per node
    )mpirunmpiexecr   z\mpirun or mpiexec were not found. Ensure that Intel MPI, Open MPI, or MVAPICH are installed.z	--versions   Open MPI
--hostfilez-nz
--npernodez	--bind-toz-fz-ppn )r   lenOSError
subprocesscheck_output)xmpi_appsmpi_appmpirun_versions       r)   _get_mpirun_argsr9   :   s     1=aE!H=H=
8}tuu qkG,,g{-CDNn$dL+EE dFB.. >s
   A#A#r"   current_envc                 D   d}t        |       D ]  }|j                  d      st        | |      }|$|dk(  r=t        |d         ||dz   <   t        |d         ||dz   <   t        |d         ||d	z   <   ft        t        | |            || |j	                          <    |S )
z.
    Setup the FP8 environment variables.
    ACCELERATE_fp8_fp8_override_linear_precisionr   FP8_OVERRIDE_FPROPr   FP8_OVERRIDE_DGRADr   FP8_OVERRIDE_WGRAD)r   
startswithgetattrstrupper)r"   r:   prefixargr(   s        r)   setup_fp8_envrH   R   s     FDz 	T>>&!D#&E 99ADU1XK)= =>ADU1XK)= =>ADU1XK)= =><?c@R<SK6(399;- 89	T r+   returnc                 f	   g }| j                   r| j                  rt        d      t        | dd      }| j                  }| j
                  `t               \  }}}}}t        | dd      }	|r|rt        ||z        nd}
|||| j
                  ||
gz  }|r||t        |      gz  }|r|||	gz  }| j                   s<|j                  t        j                         | j                  r|j                  d       |j                  | j                         |j                  | j                         t        j                  j!                         }t        | j"                  xs | j$                        |d<   | j&                  rd	|d
<   | j(                  dk7  r| j(                  t+               r| j(                  |d<   nt-               r| j(                  |d<   nt/               r| j(                  |d<   nwt1               r| j(                  |d<   n]t3               r| j(                  |d<   nCt5               r| j(                  |d<   n)t7               r| j(                  |d<   n| j(                  |d<   |dkD  r&| j8                  J d       | j:                  J d       ||dkD  s|dkD  rC| j8                  | j8                  nd|d<   | j:                  t        | j:                        nd|d<   t=        |d   d      rd|d<   t        d      |d<   	 t?        | j@                  jC                               }t        |      |d"<   | j@                  jC                         d#k(  r!tG               stI        d$      tK        | |      }	 tM        | jN                  jQ                               }|jR                  |d&<   | jT                  |d'<   t        | jV                        |d(<   t        | jX                        |d)<   t        | jZ                        |d*<   t        | j\                        |d+<   | j^                  rd|d,<   ||fS # t        $ r= t        d| j@                  jC                          d t?        jD                          d!      w xY w# t        $ r= t        d%| jN                  jQ                          d tM        jD                          d!      w xY w)-zz
    Prepares and returns the command list and an environment with the correct simple launcher environment variables.
    0--module and --no_python cannot be used togethernum_processesNzbind-tosocket1z-mACCELERATE_USE_CPUtrueACCELERATE_DEBUG_MODEallZE_AFFINITY_MASKMLU_VISIBLE_DEVICESSDAA_VISIBLE_DEVICESMUSA_VISIBLE_DEVICESASCEND_RT_VISIBLE_DEVICESHABANA_VISIBLE_MODULESNEURON_RT_VISIBLE_CORESCUDA_VISIBLE_DEVICESr   zFWhen using multiple machines, you need to specify the main process IP.zHWhen using multiple machines, you need to specify the main process port.z	127.0.0.1MASTER_ADDR29500MASTER_PORTFzgranularity=fine,compact,1,0KMP_AFFINITYKMP_BLOCKTIMEUnknown mixed_precision mode: . Choose between .ACCELERATE_MIXED_PRECISIONfp8rFP8 is not available on this machine. Please ensure that either Transformer Engine, MSAMP or torchao is installed.Unknown dynamo backend: ACCELERATE_DYNAMO_BACKENDACCELERATE_DYNAMO_MODEACCELERATE_DYNAMO_USE_FULLGRAPHACCELERATE_DYNAMO_USE_DYNAMIC*ACCELERATE_DYNAMO_USE_REGIONAL_COMPILATIONOMP_NUM_THREADSACCELERATE_CPU_AFFINITY)0	no_pythonmodule
ValueErrorrC   num_machinesmpirun_hostfiler9   rD   appendsys
executabletraining_scriptextendtraining_script_argsosenvironcopycpuuse_cpudebuggpu_idsr   r   r   r   r   r   r   main_process_ipmain_process_portr   r	   mixed_precisionlowerlistr
   RuntimeErrorrH   r   dynamo_backendrE   r(   dynamo_modedynamo_use_fullgraphdynamo_use_dynamicdynamo_use_regional_compilationnum_cpu_threads_per_processenable_cpu_affinity)r"   cmdrL   rq   mpi_app_namehostfile_argnum_proc_argproc_per_node_argbind_to_argbind_tonproc_per_noder:   r   r   s                 r)   prepare_simple_launcher_cmd_envr   d   s    C~~$++KLLD/48M$$L'ScSePlL2C[$	84?LQ]]l:;cf  
 	
 L#m"455CK))C>>

3>>";;JJtJJt##$JJt(()**//#K(+DHH,D(EK$%zz/5+,||u!9.2llK*+15K-. 26,,K./ 26,,K./7;||K3448LLK01 "59\\K1226,,K./a##/ 	
T	
/ %%1 	
V	
1 	!ma&7L1<L=A=Q=Q=]T%9%9cnM"DHDZDZDfS)?)?%@lsM";';<eD&DN#'*1vO$
'(<(<(B(B(DE 14O0DK,-!!#u,! E  $D+6
&t':':'@'@'BC
 0>/C/CK+,,0,<,<K()589R9R5SK1236t7N7N3OK/0@CDDhDh@iK<=%()I)I%JK!"14-.9  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

s   ;#P! +#Q* !AQ'*AR0c                    | j                   dk(  rt               | _         n| j                   d| _         | j                  }| j                  }| j                  }| j                   }|dkD  rxt        ||z        | _        t        |      | _        t        | j                        | _
        t        | dd      r!t        |      | _        t        |      | _        n/| d| | _        n"t        |      | _        |t        |      | _        |dk  xs t        | j                        dk(  }|r@t        |      r5|dk  r!d| _        t#        j$                  d	| d
       nt'        d| d      | j(                  r| j*                  rt-        d      | j(                  rd| _        n| j*                  rd| _        t.        j0                  j3                         }| j4                  rd|d<   t        | dd      }|dk7  r| j6                  ut9               r||d<   net;               r||d<   nUt=               r||d<   nEt?               r||d<   n5tA               r||d<   n%tC               r||d<   ntE               r||d<   n||d<   | jF                  jI                         }	 tK        |      }t        |      |d<   | jF                  jI                         dk(  r!tO               stQ        d      tS        | |      }	 tU        | jV                  jY                               }	|	jZ                  |d!<   | j\                  |d"<   t        | j^                        |d#<   t        | j`                        |d$<   t        | jb                        |d%<   | jd                  r=d|d&<   | jf                  r| jh                  st-        d'      tk        | d(      rt        | jl                        nd)|d*<   t        | jn                        |d+<   t        | jp                        jI                         |d,<   t        | jr                        jI                         |d-<   t        | jt                        |d.<   | jv                  t        | jv                        |d/<   | jx                  t        | jx                        |d0<   | jz                  t        | jz                        |d1<   | j|                  t        | j|                        |d2<   t        | j~                        jI                         |d3<   t        | j                        jI                         |d4<   t        | jf                        jI                         |d5<   t        | jh                        jI                         |d6<   t        | j                        jI                         |d7<   t        | d8d      t        | j                        |d9<   | j                  rd:}
d|d;<   t        | j                        ||
d<z   <   t        | j                        ||
d=z   <   | j                  t        | j                        ||
d>z   <   | j                  t        | j                        ||
d?z   <   | j                  t        | j                        ||
d@z   <   | j                  t        | j                        ||
dAz   <   | j                  t        | j                        ||
dBz   <   | j                  t        | j                        ||
dCz   <   | j                  t        | j                        ||
dDz   <   t        | j                        ||
dEz   <   t        | j                        ||
dFz   <   | j                  t        | j                        ||
dGz   <   | j                  t        | j                        ||
dHz   <   | j                  t        | j                        ||
dIz   <   | j                  t        | j                        ||
dJz   <   | j                  t        | j                        ||
dKz   <   | j                  t        | j                        ||
dLz   <   | j                  t        | j                        ||
dMz   <   | j                  t        | j                        ||
dNz   <   | j                  t        | j                        ||
dOz   <   | j                  t        | j                        ||
dPz   <   | j                  t        | j                        ||
dQz   <   | j                  t        | j                        ||
dRz   <   | j                  t        | j                        ||
dSz   <   | j                  t        | j                        ||
dTz   <   | j                  t        | j                        ||
dUz   <   | j                  t        | j                        ||
dVz   <   t        | j                        |dW<   | j                  rd)|dX<   | j                  rt        | |      }|S # t,        $ r% t-        d| dtK        jL                          d      w xY w# t,        $ r= t-        d | jV                  jY                          dtU        jL                          d      w xY w)Yz_
    Prepares and returns an environment with the correct multi-GPU environment variables.
    r   N<s  r   same_networkF:TPort `  ` is already in use. Accelerate will attempt to launch in a standalone-like mode by finding an open port automatically for this session. If this current attempt fails, or for more control in future runs, please specify a different port (e.g., `--main_process_port <your_chosen_port>`) or use `--main_process_port 0` for automatic selection in your launch command or Accelerate config file.3Tried to launch distributed communication on port `$  `, but another process is utilizing it. Please specify a different port (such as using the `--main_process_port` flag or specifying a different `main_process_port` in your config file) and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`.rK   rP   rQ   r   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   ACCELERATE_USE_FSDPzWWhen using `--fsdp_cpu_ram_efficient_loading` set `--fsdp_sync_module_states` to `True`fsdp_versionrN   FSDP_VERSIONFSDP_SHARDING_STRATEGYFSDP_RESHARD_AFTER_FORWARDFSDP_OFFLOAD_PARAMSFSDP_MIN_NUM_PARAMSFSDP_AUTO_WRAP_POLICYFSDP_TRANSFORMER_CLS_TO_WRAPFSDP_BACKWARD_PREFETCHFSDP_STATE_DICT_TYPEFSDP_FORWARD_PREFETCHFSDP_USE_ORIG_PARAMSFSDP_CPU_RAM_EFFICIENT_LOADINGFSDP_SYNC_MODULE_STATESFSDP_ACTIVATION_CHECKPOINTINGfsdp_ignored_modulesFSDP_IGNORED_MODULESMEGATRON_LM_ACCELERATE_USE_MEGATRON_LM	TP_DEGREEUSE_CUSTOM_FSDPNO_LOAD_OPTIMEOD_MASK_LOSSNO_SAVE_OPTIMOPTIMIZER_CPU_OFFLOADUSE_PRECISION_AWARE_OPTIMIZEROVERLAP_CPU_OPTIMIZER_D2H_H2D DECODER_LAST_PIPELINE_NUM_LAYERS	PP_DEGREEGRADIENT_CLIPPINGNUM_MICRO_BATCHESSEQUENCE_PARALLELISMRECOMPUTE_ACTIVATIONSUSE_DISTRIBUTED_OPTIMIZERRECOMPUTE_GRANULARITYRECOMPUTE_METHODRECOMPUTE_NUM_LAYERSATTENTION_BACKENDEXPERT_MODEL_PARALLEL_SIZECONTEXT_PARALLEL_SIZEATTENTION_DROPOUTHIDDEN_DROPOUTATTENTION_SOFTMAX_IN_FP32EXPERT_TENSOR_PARALLEL_SIZECALCULATE_PER_TOKEN_LOSSUSE_ROTARY_POSITION_EMBEDDINGSrl   rm   )cr   r   rL   rq   r   rD   r   nnodesintmachine_rank	node_rankrC   master_addrmaster_portrdzv_endpointr   
standalonewarningswarnConnectionErrorro   rn   rp   ry   rz   r{   r~   r   r   r   r   r   r   r   r   r   r   r	   r   r
   r   rH   r   r   rE   r(   r   r   r   r   use_fsdpfsdp_cpu_ram_efficient_loadingfsdp_sync_module_stateshasattrr   fsdp_sharding_strategyfsdp_reshard_after_forwardfsdp_offload_paramsfsdp_min_num_paramsfsdp_auto_wrap_policy"fsdp_transformer_layer_cls_to_wrapfsdp_backward_prefetchfsdp_state_dict_typefsdp_forward_prefetchfsdp_use_orig_paramsfsdp_activation_checkpointingr   use_megatron_lmmegatron_lm_tp_degreemegatron_lm_use_custom_fsdpmegatron_lm_no_load_optimmegatron_lm_eod_mask_lossmegatron_lm_no_save_optim!megatron_lm_optimizer_cpu_offload)megatron_lm_use_precision_aware_optimizer)megatron_lm_overlap_cpu_optimizer_d2h_h2d,megatron_lm_decoder_last_pipeline_num_layersmegatron_lm_pp_degreemegatron_lm_gradient_clippingmegatron_lm_num_micro_batches megatron_lm_sequence_parallelism!megatron_lm_recompute_activations%megatron_lm_use_distributed_optimizer!megatron_lm_recompute_granularitymegatron_lm_recompute_method megatron_lm_recompute_num_layersmegatron_lm_attention_backend&megatron_lm_expert_model_parallel_size!megatron_lm_context_parallel_sizemegatron_lm_attention_dropoutmegatron_lm_hidden_dropout%megatron_lm_attention_softmax_in_fp32'megatron_lm_expert_tensor_parallel_size$megatron_lm_calculate_per_token_loss*megatron_lm_use_rotary_position_embeddingsr   r   use_parallelism_config%prepare_extend_env_parallelism_config)r"   rL   rq   r   r   need_port_checkr:   r   r   r   rF   s              r)   prepare_multi_gpu_envr     sF
   
 "!.				'!&&&M$$L**O..a!-<"?@,'T../4/"?3D"#45D$3#4A6G5H!ID!-0("#45D #a'F3t/@/@+AQ+FO>*;<1"DOMM*+ ,D D "EFWEX Y   {{t~~KLL		**//#Kzz/5+,dIu-G%DLL4.5K*+18K-. 29K./ 29K./7>K344;K01 "5<K1229K./**002Ou'8 14O0DK,-!!#u,! E  $D+6
&t':':'@'@'BC
 0>/C/CK+,,0,<,<K()589R9R5SK1236t7N7N3OK/0@CDDhDh@iK<=}}-3)*..t7S7Svww@Gn@]c$*;*;&<cfN# 14D4O4O0P,-478W8W4X4^4^4`01-01I1I-J-P-P-R)*-01I1I-J)*%%136t7Q7Q3RK/022>:=d>e>e:fK67&&2478S8S4TK01$$025d6O6O2PK.//243M3M/N/T/T/V+,.1$2K2K.L.R.R.T*+8;D<_<_8`8f8f8h4514T5Q5Q1R1X1X1Z-.7:4;];]7^7d7d7f344/6B25d6O6O2PK./4:01,/0J0J,KF[()25d6V6V2WF../))5478V8V4WK01))5478V8V4WK01))5478V8V4WK0111=<?@f@f<gK!88999EDGHvHvDwK!@@A99EDGHvHvDwK!@@A<<HGJAAHK!CCD -00J0J,KF[()478Z8Z4[F001--98;D<^<^8_K!44500<;>t?d?d;eK!77811=<?@f@f<gK!88955A@CDDnDn@oK!<<=11=<?@f@f<gK!889,,87:4;\;\7]K!33400<;>t?d?d;eK!778--98;D<^<^8_K!44566BADTEpEpAqK!==>11=<?@f@f<gK!889--98;D<^<^8_K!445**6589X9X5YK!11255A@CDDnDn@oK!<<=77CBEdFrFrBsK!>>?44@?B4ClCl?mK!;;<::FEH??FK!AAB &))I)I%JK!"14-."";D+Nm  u9/9JJ[\i\n\n\p[qqrsttu  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

s   
g9 "#h* 9.h'*Ai0c                    d}d|d<   t        | j                        ||dz   <   t        | j                        ||dz   <   t        | j                        ||dz   <   t        | j                        ||dz   <   t        | j
                        ||dz   <   t        | j                        ||d	z   <   t        | j                        ||d
z   <   | j                  dkD  rt        | j                        ||dz   <   | j                  dkD  rQt        | j                        ||dz   <   t        | j                        ||dz   <   t        | j                        ||dz   <   |S )zV
    Extends `current_env` with context parallelism env vars if any have been set
    PARALLELISM_CONFIG_rP   !ACCELERATE_USE_PARALLELISM_CONFIGDP_REPLICATE_SIZEDP_SHARD_SIZETP_SIZECP_SIZE
CP_BACKENDSP_SIZE
SP_BACKENDr   CP_COMM_STRATEGYSP_SEQ_LENGTHSP_SEQ_LENGTH_IS_VARIABLESP_ATTN_IMPLEMENTATION)rD   $parallelism_config_dp_replicate_size parallelism_config_dp_shard_sizeparallelism_config_tp_sizeparallelism_config_cp_sizeparallelism_config_cp_backendparallelism_config_sp_sizeparallelism_config_sp_backend#parallelism_config_cp_comm_strategy parallelism_config_sp_seq_length,parallelism_config_sp_seq_length_is_variable)parallelism_config_sp_attn_implementation)r"   r:   rF   s      r)   r  r    sX    #F7=K3403D4]4]0^K,,-,/0U0U,VK()&)$*I*I&JK"#&)$*I*I&JK"#),T-O-O)PK%&&)$*I*I&JK"#),T-O-O)PK%&&&*36t7_7_3`F//0&&*03D4Y4Y0ZF_,-<?@q@q<rF8899<T=k=k9lF556r+   c                    | j                   dk(  rt               | _         n| j                   d| _         | j                  }| j                  }| j                  }| j                   }d}| j
                  t        d   | _        |dkD  r| j
                  t        d   k7  rdg}|j                  dt        | j                        g       | j
                  dk(  r@t        ddd	      rt        d
      |j                  dt        | j                        dg       n'|j                  ddt        | j
                        g       | j                  '|j                  dt        | j                        g       nf| j                  '|j                  dt        | j                        g       n3|j                  dt        | j                  | j                  z        g       |r|j                  dt        |      g       |j                  dt        |      g       | j                  r| j                   rt        d      | j                  r|j#                  d       n| j                   r|j#                  d       |j#                  | j$                         |j                  | j&                         n|dkD  r| j
                  t        d   k(  rxt        ||z        | _        t        |      | _        t-        | j                        | _        t1        | dd      r!t        |      | _        t        |      | _        n/| d| | _        n"t        |      | _        |t        |      | _        |dk  xs t-        | j                        dk(  }|r@t9        |      r5|dk  r!d| _        t=        j>                  d| d       ntA        d| d      | j                  r| j                   rt        d      | j                  rd| _        n| j                   rd| _        tB        jD                  jG                         }| jH                  rd|d <   t1        | d!d"      }|d"k7  r| jJ                  utM               r||d#<   netO               r||d$<   nUtQ               r||d%<   nEtS               r||d&<   n5tU               r||d'<   n%tW               r||d(<   ntY               r||d)<   n||d*<   	 t[        | j\                  j_                               }	tc        d.tB        jd                  jg                  d-            |d.<   t        |	      |d/<   | j\                  j_                         d0k(  r!ti               stk        d1      tm        | |      }t        | jn                        j_                         |d2<   d|d3<   | jp                  t        | jp                        |d4<   | jr                  t        | jr                        |d5<   | jt                  &t        | jt                        j_                         |d6<   | jv                  &t        | jv                        j_                         |d7<   | jx                  &t        | jx                        j_                         |d8<   | jz                  &t        | jz                        j_                         |d9<   | j|                  &t        | j|                        j_                         |d:<   | j~                  t        | j~                        |d;<   | j                  rd<|d=<   | j                  t        | j                        |d><   | j                  rt        | |      }||fS # t        $ r= t        d+| j\                  j_                          d,t[        j`                          d-      w xY w)?zt
    Prepares and returns the command list and an environment with the correct DeepSpeed environment variables.
    r   Nr   r   	deepspeedr/   nossh<z0.14.5z+nossh launcher requires DeepSpeed >= 0.14.5z--node_rankz--no_sshz--no_local_rankz
--launcherz	--excludez	--includez
--num_gpusz--master_addrz--master_portrK   z--modulez--no_pythonr   Fr   Tr   r   r   r   rP   rQ   r   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r`   ra   rb   
PYTHONPATHrc   rd   re   ACCELERATE_CONFIG_DS_FIELDSACCELERATE_USE_DEEPSPEEDACCELERATE_DEEPSPEED_ZERO_STAGE&ACCELERATE_GRADIENT_ACCUMULATION_STEPSACCELERATE_GRADIENT_CLIPPING-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICEACCELERATE_DEEPSPEED_ZERO3_INIT+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODEL ACCELERATE_DEEPSPEED_CONFIG_FILErN   rm   (ACCELERATE_DEEPSPEED_MOE_LAYER_CLS_NAMES)Dr   r   rL   rq   r   deepspeed_multinode_launcherr   rw   rD   deepspeed_hostfiler   rp   r   deepspeed_exclusion_filterdeepspeed_inclusion_filterro   rn   rs   rv   rx   r   r   r   r   rC   r   r   r   r   r   r   r   r   ry   rz   r{   r~   r   r   r   r   r   r   r   r   r	   r   r   r   env_var_path_addpathabspathr
   r   rH   'deepspeed_fields_from_accelerate_config
zero_stagegradient_accumulation_stepsgradient_clippingoffload_optimizer_deviceoffload_param_devicezero3_init_flagzero3_save_16bit_modeldeepspeed_config_filer   deepspeed_moe_layer_cls_namesr  r  )
r"   rL   rq   r   r   r   r  r:   r   r   s
             r)   prepare_deepspeed_cmd_envrA    s   
 "!.				'!&&&M$$L**O..
C ((0,I!,L)aD==A^_`Aaam

L#d&=&=">?@,,7S(; !NOOJJs4+<+<'=zJKJJ)<T=^=^9_`a**6JJ778 ,,8JJ778 JJc$*<*<@Q@Q*Q&RSTJJ_)=>?

OS):%;<=;;4>>OPP[[JJz"^^JJ}%

4''(

4,,-		d??C`abCcc!-<"?@,'T../4/"?3D"#45D$3#4A6G5H!ID!-0("#45D #a'F3t/@/@+AQ+FO>*;<1"DOMM*+ ,D D "EFWEX Y   {{t~~KLL		**//#Kzz/5+,dIu-G%DLL4.5K*+18K-. 29K./ 29K./7>K344;K01 "5<K1229K./
'(<(<(B(B(DE !1rwws?S TK03O0DK,-!!#u,! E  $D+614T5a5a1b1h1h1jK-..4K*+"9<T__9M56''3@CDDdDd@e<=)69$:P:P6Q6W6W6Y23$$0GJ4KhKhGiGoGoGqCD  ,CFtG`G`CaCgCgCi?@'9<T=Q=Q9R9X9X9Z56"".EHIdIdEeEkEkEmAB!!-:=d>X>X:Y6714-.))5BEdFhFhBi>?"";D+NO  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

s   #] A^	podc                     | j                   dk(  r#t        d      r| j                  rd|d<   nd|d<   | j                  rd|d<   |r"| j                  | _        | j                  | _        | |fS )	zY
    Prepares and returns an environment with the correct TPU environment variables.
    bf16T)check_is_tpurN   XLA_DOWNCAST_BF16XLA_USE_BF16rP   rQ   )r   r   downcast_bf16r~   tpu_vmvmtpu_nametpu)r"   r:   rB  s      r)   prepare_tpurM  Q  sn     v%*@d*S/2K+,*-K'zz/5+,
++==r+   nargsc                 8   t        |       dk  ri S d }t        j                         }|j                  |       \  }}t	        |      D ]  \  }}|j                  d      sd }|dz   t        |      k  r"||dz      j                  d      rt        d      t        d      ||j                  ||       n|j                  ||        |j                  |       j                  j                         D 	ci c]  \  }}	||	dv rt        |	      n|	 c}	}S c c}	}w )	Nr   c                 f    	 t        |       } | dz  | k(  rt        |       S | S # t        $ r | cY S w xY w)Nr   )floatr   rp   )ss    r)   _infer_typez+_convert_nargs_to_dict.<locals>._infer_typej  s=    	aAAv{1vH 	H	s   " " 00)-z--r   ul   SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types)type)action)TrueFalse)r1   argparseArgumentParserr   	enumeraterB   rp   add_argument
parse_args__dict__r   r   )
rN  rS  r#   r&   unknownindexargumentrV  r'   r(   s
             r)   _convert_nargs_to_dictrb  e  s6   
5zA~	 $$&F((/JAw$W- =x{+Fqy3w<'519%00=$ G  ! C  ~##H;#?##HV#<#=* !++E2;;AACC 	U.?%?l5!UJ  s   6Dsagemaker_configc                 	   t        d       | j                  t        j                  d<   | j                  | j                  t        j                  d<   n^|j
                  G|j                  ;|j
                  t        j                  d<   |j                  t        j                  d<   nt        d      t        j                  j                  |j                        }|sd}t        j                  j                  |j                        }|j                  d      st        d	| d
      t        d       t        |j                        }	 t!        |j"                  j%                               }	 t)        |j*                  j-                               }dt/        |      |j0                  |j2                  t/        |j4                        t/        |j6                        t/        |j8                        | j:                  j0                  d}|j"                  j%                         dk(  r!t=               st?        d      tA        ||      }d }| j:                  tB        jD                  k(  rddddiii}d }	| jF                  t        d| jF                   d       i }	tI        | jF                        5 }
tK        |
      D ]5  \  }}|dk(  r|jM                  d      }|d   jO                         |	|d   <   7 	 d d d        t        d|	        d }| jP                  t        d| jP                   d       g }tI        | jP                        5 }
tK        |
      D ]F  \  }}|dk(  r|jM                  d      }|d   |d   jO                         d}|jS                  |       H 	 d d d        t        d|        t        d        | jT                  ||| jV                  | jX                  | jZ                  | j\                  | j^                  | j`                  | jb                  d!||||d"}| jd                  tg        | jd                  |      }||	fS # t        $ r= t        d|j"                  j%                          dt!        j&                          d      w xY w# t        $ r= t        d|j*                  j-                          dt)        j&                          d      w xY w# 1 sw Y   xY w# 1 sw Y   JxY w)#Nz(Configuring Amazon SageMaker environmentAWS_DEFAULT_REGIONAWS_PROFILEAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profilerb   z.pyz8Your training script should be a python script and not ""z'Converting Arguments to Hyperparametersr`   ra   rf   rP   )ACCELERATE_USE_SAGEMAKERrc   rg   rh   ri   rj   rk   %ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPErd   re   smdistributeddataparallelenabledTzLoading SageMaker Inputs from z filer   	r   zLoaded SageMaker Inputs: zLoading SageMaker Metrics from )NameRegexzLoaded SageMaker Metrics: zCreating EstimatorF)	image_urientry_point
source_dirroletransformers_versionpytorch_version
py_versionbase_job_nameinstance_countinstance_typedebugger_hook_configdistributionhyperparametersenvironmentmetric_definitions)4printregionry   rz   profileaws_access_key_idaws_secret_access_keyr2   r5  dirnamerv   basenameendswithrp   rb  rx   r	   r   r   r   r   r   rE   rD   r(   r   r   r   r   distributed_typer
   r   rH   r   DATA_PARALLELsagemaker_inputs_fileopenr[  splitstripsagemaker_metrics_filers   rr  iam_role_namerv  rw  rx  ry  rq   ec2_instance_typeadditional_argsr   )rc  r"   rt  rs  r~  r   r   r  r}  sagemaker_inputsfileilinelsagemaker_metricsmetric_dicts                   r)   prepare_sagemager_args_inputsr    s    

45'7'>'>BJJ#$ +$4$<$<

=!				+0J0J0V*.*@*@

&'.2.H.H

*+uvv !5!56J
''""4#7#78K&ST_S``abcc	
34,T-F-FGO
'(<(<(B(B(DE
&t':':'@'@'BC %+&)/&:%3%9%9"&"2"2+.t/H/H+I),T-D-D)E69$:^:^6_1A1R1R1X1X	K !!#u,! E  $D+6L((,D,R,RR'.9d:K)LM --9./?/U/U.VV[\]"889 	6T$T? 646JJt$)*1 1&	6	6 	)*:);<= ..:/0@0W0W/XX]^_"99: 		6d$T? 646JJt$aDqTZZ\ "((56		6 	*+<*=>? 

%//"  .. 0 E E+;;&11)77*77);; %$*"/D$ ''3+;;TB!!!m  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

@	6 	6		6 		6s4   #P? ,#R 'AS AS?ARASSS(c                     t         j                  j                  | d      j                  d      D cg c]  }t	        |      dkD  s| }}|j                  t        |             dj                  |      S c c}w )z
    Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the
    caller to set it in os.environ.
    r0   r   r   )ry   rz   getr  r1   rs   rD   join)env_var_namepath_to_addppathss       r)   r4  r4    sa    
 

|R8>>sCR1s1vPQzQRER	LL[!"88E? Ss   A7A7c                       e Zd ZdZddZd Zy)PrepareForLaunchai  
    Prepare a function that will launched in a distributed setup.

    Args:
        launcher (`Callable`):
            The function to launch.
        distributed_type ([`~state.DistributedType`]):
            The distributed type to prepare for.
        debug (`bool`, *optional*, defaults to `False`):
            Whether or not this is a debug launch.
    c                 @    || _         t        |      | _        || _        y )N)launcherr   r  r~   )selfr  r  r~   s       r)   __init__zPrepareForLaunch.__init__  s      /0@ A
r+   c                    | j                   rt        t        j                  j	                  d            }t        j                  j	                  d      }t
        j                  j                  d|t
        j                  j                  ||      |       n| j                  t        j                  t        j                  t        j                  t        j                  t        j                  t        j                   t        j"                  fv rt%        |      t        j                  d<   t        t        j                  j	                  dd            }t        t        j                  j	                  dd	            }t%        ||z  |z         t        j                  d
<   t%        d      t        j                  d<    | j&                  |  y )N
WORLD_SIZEACCELERATE_DEBUG_RDV_FILEgloo)rankstore
world_size
LOCAL_RANKNPROCr   	NODE_RANKr   RANKFORK_LAUNCHED)r~   r   ry   rz   r  torchdistributedinit_process_group	FileStorer  r   	MULTI_GPU	MULTI_MLU
MULTI_MUSA	MULTI_NPU	MULTI_XPU	MULTI_CPUMULTI_NEURONrD   r  )r  r`  r"   r  rdv_filenprocr   s          r)   __call__zPrepareForLaunch.__call__!  sB   ::RZZ^^L9:Jzz~~&ABH00''11(JG%	 1  ""%%%%&&%%%%%%(('
 
 (+5zBJJ|$

w23EBJJNN;:;I!$UY%6%>!?BJJv&)!f

?#tr+   N)NOF)__name__
__module____qualname____doc__r  r   r+   r)   r  r    s    

r+   r  )F)9rY  ry   r3   rt   r   astr   shutilr   typingr   r  commands.config.config_argsr   utilsr   r	   r
   r   r   r   r   r   r   r   r   utils.constantsr   utils.otherr   r   r   utils.versionsr   r0   r   dataclassesr   r   r*   r9   	NamespacedictrD   rH   tupler   r   r  r  rA  boolrM  rb  r  r4  r  r  r+   r)   <module>r     s
    	  
      9    < D D - ! B -/ /0** c3h $b(*<*< btCyRVWZ\_W_R`G`Aa bJF 2 2 FtCH~ FR


+/
49d38n$%6aH$6$6 a5cDQTVYQYNAZ;[ aJ HM


+/S>@D
8tCH~-.('$s) 'S#X 'Ts"%s"-5-?-?s"
8tCH~-.s"l, ,r+   