
    %ɷi^                        d dl Z d dlZd dlZd dlmZ d dlZd dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZmZ  ej:                  ej<                          e       Z G d dej@                  jB                        Z"d Z#d Z$d Z%d Z&d Z'd Z(d Z)d Z*d Z+e,dk(  r e#       \  Z-Z.Z/e/j`                  dkD  r	  ed      Z1e1je                         se1jg                  dd        e$e/e-      Z-e/ji                           e jj                           ee.e/e-e1       e/ji                           e(e-e1        e)e-e1        e*e-e1        e+e-e1       	 e/jn                  r ejp                  e1       e/ji                          e/js                          yyy# e6$ r  w xY w# e/jn                  r ejp                  e1       e/ji                          e/js                          w xY w)    N)Path)	load_file)ShardingStrategyStateDictType)
DataLoader)AcceleratorFullyShardedDataParallelPlugin)merge_commandmerge_command_parser)AcceleratorState)torch_device)RegressionDataset)merge_fsdp_weightspatch_environmentsave_fsdp_model)levelc                   $     e Zd Z fdZd Z xZS )	TinyModelc                 B   t         |           t        j                  j	                  dd      | _        t        j                  j                         | _        t        j                  j	                  dd      | _        t        j                  j                         | _
        y )N   )super__init__torchnnLinearlinear1ReLU
activationlinear2Softmaxsoftmax)self	__class__s    c/var/www/html/venv/lib/python3.12/site-packages/accelerate/test_utils/scripts/test_merge_weights.pyr   zTinyModel.__init__'   s_    xxr2.((--/xxr2.xx'')    c                 `    | j                  | j                  | j                  |                  S )N)r   r   r   )r"   xs     r$   forwardzTinyModel.forward.   s"    ||DOODLLO<==r%   )__name__
__module____qualname__r   r(   __classcell__)r#   s   @r$   r   r   &   s    *>r%   r   c                  f   t         j                  i k7  rt               j                          t        t        j
                  t        j                        } t               }t        d      5  | j                  |       d d d        t        |       }|j                  |      }|| |fS # 1 sw Y   +xY w)N)sharding_strategystate_dict_typeSIZE_BASED_WRAP)fsdp_auto_wrap_policy)fsdp_plugin)r   _shared_state_reset_stater	   r   
FULL_SHARDr   SHARDED_STATE_DICTr   r   set_auto_wrap_policyr   prepare)pluginmodelaccelerators      r$   setupr<   2   s    %%+'')+*55}GgGgF KE	1B	C +##E*+&1K&E&+%%	+ +s   +B''B0c                    t        dd      }t        |dd      }t        j                  j	                  |j                         d      }| j                  |||      \  }}}t        d	      D ]r  }|D ]k  }|j                           ||d
         }t        j                  j                  j                  ||d         }| j                  |       |j                          m t |S )N   *   )lengthseedr   F)
batch_sizeshuffleg?)lr   r'   y)r   r   r   optimSGD
parametersr8   range	zero_gradr   
functionalmse_lossbackwardstep)	r;   r:   	train_settrain_dl	optimizer_batchoutputlosss	            r$   mock_trainingrW   @   s    !26I)EBH 0 0 2s;I!,!4!4Xui!PHeY1X  	EOO5:&F88&&//c
CD  &NN	 Lr%   c                     t        |j                         |j                               D ];  \  }}| dk(  rt        j                  ||      r"J t        j                  ||      s;J  y )Nsame)zipvaluesr   allclose)	operationstate_1state_2weight_1weight_2s        r$   check_weightsrb   P   s[    !'.."2GNN4DE :(>>(H555~~h999	:r%   c                     t        | dz        }t               j                  t              }t	        d|j                         |j                                |j                  |       t	        d|j                         |j                                y )Nzmodel.safetensorsdiffrY   )r   r   tor   rb   
state_dictload_state_dict)pathr:   safe_state_dictsafe_loaded_models       r$   check_safetensors_weightsrk   X   so    ': :;O!|4&%**,.?.J.J.LM%%o6&%**,.?.J.J.LMr%   c                 8   t        j                  | dz  d      }t               j                  t              }t        d|j                         |j                                |j                  |       t        d|j                         |j                                y )Nzpytorch_model.binT)weights_onlyrd   rY   )r   loadr   re   r   rb   rf   rg   )rh   r:   nonsafe_state_dictnonsafe_loaded_models       r$   check_pytorch_weightsrq   `   sw    D+>$>TR$;>>,7&%**,.B.M.M.OP(();<&%**,.B.M.M.OPr%   c                 >    t        |dz  |d       t        ||        y )Npytorch_model_fsdp_0Tsafe_serialization)r   rk   r:   rh   s     r$   test_merge_weights_safetensorsrw   h   s    t44dtTdE*r%   c                     t         j                  t        |dz        t        |      g      }t        |       t	        ||        y )Nrs   )parser
parse_argsstrr
   rk   r:   rh   argss      r$   &test_merge_weights_command_safetensorsr~   n   s8    c$)?"?@#d)LMD$dE*r%   c                 >    t        |dz  |d       t        ||        y )Nrs   Frt   )r   rq   rv   s     r$   test_merge_weights_pytorchr   t   s    t44duU$&r%   c                     t         j                  t        |dz        t        |      dg      }t        |       t	        ||        y )Nrs   z--unsafe_serialization)ry   rz   r{   r
   rq   r|   s      r$   "test_merge_weights_command_pytorchr   z   s;    c$)?"?@#d)MefgD$$&r%   __main__   test_merge_weights_fsdp_weightsT)parentsexist_ok):gcloggingshutilpathlibr   r   safetensors.torchr   2torch.distributed.fsdp.fully_sharded_data_parallelr   r   torch.utils.datar   
accelerater   r	   accelerate.commands.merger
   r   accelerate.stater   accelerate.test_utilsr   accelerate.test_utils.trainingr   accelerate.utilsr   r   r   basicConfigINFOry   r   Moduler   r<   rW   rb   rk   rq   rw   r~   r   r   r)   r:   r9   r;   num_processesout_pathexistsmkdirwait_for_everyonecollect	Exceptionis_main_processrmtreeend_training r%   r$   <module>r      s   
     ' ^ ' B I - . < S S   ',, '			> 	>& :NQ++'' z!&E6;  1$	'=>H??$td; "+u5E))+BJJLFKA))+ +5(;25(C&uh7.uh?
 **h'))+$$&7 % .  		 **h'))+$$&s   BF+ +F22F5 5A G5