
    GJjLN                     r   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
mZ d dlmZ d dlmZ ddlmZmZmZmZmZmZ e G d d	             Ze G d
 d             Ze G d d             Zd"dZd Zd#dZd"dZ G d d      Zd Z d Z!d Z"d Z#d Z$d$dZ%d$dZ&d"dZ'd Z(d Z)d Z*d Z+d  Z,d! Z-y)%    N)defaultdict)	dataclass)DEVNULLrun)Optional   )HostHostfileOptionalBoolActionlog	log_errorlog_warningc                   (    e Zd ZU eed<   eed<   d Zy)SSHInfocan_sshhas_sudoc                     | j                   S N)r   )selfs    g/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx/_distributed_utils/config.py__bool__zSSHInfo.__bool__   s    ||    N)__name__
__module____qualname__bool__annotations__r    r   r   r   r      s    MNr   r   c                   2    e Zd ZU eed<   eed<   ee   ed<   y)ThunderboltPortifaceuuidconnected_toN)r   r   r   strr   r   r   r   r   r    r    "   s    J
I3-r   r    c                   (    e Zd ZU eed<   ee   ed<   y)ThunderboltHostnameportsN)r   r   r   r$   r   listr    r   r   r   r&   r&   )   s    
I  r   r&   c                    | D ]  }t        |d|j                         t        d|j                  dddgdd      j                  j	                         }|dk7  r|j
                  j                  |       pt        d|j                  ddd	gdd      j                  j	                         }|dk7  r|j
                  j                  |       t        d
|j                          y )NzGetting the ip fromsshipconfig	getifaddren0Tcapture_outputtext en1zCould not extract ip for)r   ssh_hostnamer   stdoutstripipsappendr   )hostsverbosehips       r   add_ipsr=   /   s    G*ANN;ANNJUC
 &	 	
 8EELLANNJUC
 &	 	
 8EELL.?) r   c                 H   | j                   rFt        | j                   d      5 }t        j                  |j	                         |d       d d d        y t        d       t        d       t        t        j                  |j	                         d             y # 1 sw Y   y xY w)Nw   )indentr
   z========)output_hostfileopenjsondumpto_jsonprintdumps)argshostfilefs      r   save_hostfilerL   H   su    $&&,IIh&&(!A6 -, 	jjdjj))+A67 -,s   'BB!c                    |st         nt        }d}| D ]  }t        |d|j                  d       t	        d|j                  dgdd      j
                  j                         j                         }|D cg c]  }|j                  d      s| }}|r ||j                  d	       d} |r& |         |d
        |         |d        |d       |r|rt        j                  d       | S c c}w )NFzChecking thatzsupports RDMAr+   ibv_devicesTr/   rdma_z"does not seem to have RDMA enabledzESome of the hosts don't have RDMA enabled or they don't support RDMA.zFSee https://ml-explore.github.io/mlx/build/html/usage/distributed.htmlz'for instructions on how to enable RDMA.r   )r   r   r   r4   r   r5   r6   split
startswithsysexit)r9   r:   strictlog_ffailedr;   	rdma_devsds           r   
check_rdmarY   R   s    %K9EFG_annoF6tRVWVEEGUW 	
 !*C	1Q\\'-BQ		C!.."FGF  UVVW78&: Ds   3C.
C.c                     t        d |D              }|sD|rBt        d       t        | |      D ](  \  }}|j                  rt        d|j                         * |S )Nc              3   4   K   | ]  }|j                     y wr   )r   ).0infos     r   	<genexpr>z!can_auto_setup.<locals>.<genexpr>p   s     5WT4==W   zOAutomatic setup requested but the following hosts do not have passwordless sudo - )allr   zipr   r4   )r9   sshinfo
auto_setupr   r;   is         r   can_auto_setuprf   o   sU    5W55H
]	
 w'DAq::E1>>2 ( Or   c                       e Zd Zd ZddZy)IPConfiguratorc                    t               }t        t              }d}d}t        |      D ]  \  }}	t        |	j                        D ]  \  }
}|j
                  s|j
                  |vr"||
f|v r)||j
                     \  }}d| d|dz    }d| d|dz    }|j                  }||   j                  |   j                  }|||f   j                  ||f       |||f   j                  ||f       |j                  ||
f       |j                  ||f       |dz  }|dkD  r|dz  }d}|dkD  st        d        || _
        || _        || _        y )	Nr   z192.168..r      r@      zRan out of available local IPs)setr   r)   	enumerater(   r#   r!   r8   add
ValueErrorr7   r9   tb_hosts)r   r9   rq   uuid_reverse_indexassignedr7   ip0ip1src_noder;   src_portpdst_nodedst_portip_srcip_dst	iface_src	iface_dsts                     r   __init__zIPConfigurator.__init__|   sx   5$$X.KHa(1!~~>>);;h'83%7%G"(#C5#'3#C5#'3GG	$X.44X>DD	Hh&'..	6/BCHh&'..	6/BCh12h12q91HCC9$%EFF5  2 /: 
 r   c                 $   d}t        t        | j                  | j                              D ]a  \  }\  }}d}|dz  }t	        t        | j                              D ]k  }||k(  s||f| j                  vrt        | j                  ||f   | j                  ||f         D ]&  \  \  }	}
\  }}|d|	 d|
 d| dz  }|d| d	|	 dz  }( m |rrt        d
|j                          |j                         j                  dd      }d|j                  |g}t        |t        j                  |             t        |       nGd|j                   }t        |       t        dt        |      z         t        |       t        d       t                d y )Nz255.255.255.252r2   zsudo ifconfig bridge0 down
zsudo ifconfig z inet z	 netmask 
zsudo route change z -interface zRunning auto setup for z ; r+   z
Setup for =zEnter to continue)rn   rb   r9   rq   rangelenr7   rG   r4   r6   replacer   shlexjoinr   input)r   r:   rd   netmaskre   r;   thcommandjr!   r<   _peermsgs                 r   setupzIPConfigurator.setup   sx   ##C

DMM$BCJAw2G55G3tzz?+6aV4883.1$((1a4.$((1a4..Q*KUB!TwfRD	'RTUUG!3D6eWBOOG /R , //?@A!--/11$> !..':GUZZ01G"1>>"23c
cCHn%g)*G+ Dr   N)FF)r   r   r   r   r   r   r   r   rh   rh   {   s    $!Lr   rh   c                     i }d }| j                  d      j                  d      D ]Q  }|j                  d      r|j                         dd  }(|j                  d      s:|j                         dd  ||<   d }S |S )Nzutf-8r   zHardware Port:   zDevice:   )decoderP   rQ   r6   )ports_stringr(   	port_namels       r   parse_hardware_portsr      sx    EI  )//5<<()	"#I\\)$ wwy}E)I 6 Lr   c                    g }| D ]a  }t        |d|j                         |j                  t        j                  t        d|j                  dddgd      j                               c g }| D ]V  }t        |d|j                         |j                  t        t        d|j                  d	d
gd      j                               X g }t        ||      D ]  \  }}d}g }	|d   D ]y  }
|
j                  d      }||
d   }|
d   d   }|
j                  dg       }|D cg c]	  }d|v s| }}|r|d   d   nd }|d|    }|	j                  t        |||             { |j                  t        |t        |	d                     i }t        |      D ]3  \  }}t        |j                        D ]  \  }}||f||j                  <    5 ||fS c c}w )NzGetting connectivity fromr+   system_profilerSPThunderboltDataTypez-jsonT)r0   zGetting interface names fromnetworksetupz-listallhardwareportsr2   domain_uuid_keydevice_name_keyreceptacle_1_tagreceptacle_id_key_itemsr   zThunderbolt c                     | j                   S r   )r!   xs    r   <lambda>z&extract_connectivity.<locals>.<lambda>   s    !''r   key)r   r4   r8   rD   loadsr   r5   r   rb   getr    r&   sortedrn   r(   r"   )r9   r:   thunderbolt_connectionsr;   interface_mapsrq   c	iface_mapr'   r(   tr"   tagitemsitemconnected_itemsr#   r!   rr   re   r   rx   s                         r   extract_connectivityr      s    G0!..A&&JJ)/ $(	 &	
   NG3Q^^D &/	 $( &
	
 " H3^D9*+A55*+D|&'D&'(;<CEE(B'E05S9Jd9RtOS9H"#45d  SE23ELLlCD , 	fU@Q.RST! E& (#1agg&DAq*+Qqvv& ' $ ''' Ts   	G'Gc                     g }t        |       D ]d  \  }}dgt        |       z  }|j                  D ]0  }|j                  |v s||j                     \  }}||xx   dz  cc<   2 |j	                  |       f |S )Nr   r   )rn   r   r(   r#   r8   )	rq   rr   connectivityre   r;   r   rx   r   r   s	            r   make_connectivity_matrixr   	  s|    L(#1C#h-A~~!33)!..91!	  	A $ r   c           
         g }t        t        |            D ]8  }d}|}	 |t        d|dz  z         z  }|dz  }|dk(  rn |j                  |       : t	        d       t	        d       t        |       D ]$  \  }}t	        d||    d|j                   d	       & t        |      D ]  \  }}|j                  D ]  }|j                  s|j                  |vr||j                     }	|	d   |k  r7t	        d||    d
||	d       d       t	        d|j                   d||	d      j                  |	d      j                   d         t	        d       y )Nr2   a      r   z	graph G {z  node [shape=rectangle];z  z	 [label="z"];z -- )end/r   z"]})
r   r   chrr8   rG   rn   r4   r(   r#   r!   )
r9   rq   rr   namesre   nr   r;   rx   dsts
             r   tb_connectivity_to_dotr     sf   E3x=!R!b&[!!A"HAAv	 
 	Q " 
+	
%&% 158*Iann%5S9: !(#1A>>~~%77$Q^^4C1vzBuQxjU3q6]O4"=IaggYaQ(8(>(>s1v(F(L(L'MRPQ  $ 
#Jr   c           
      z    g }t               }t                fdt              D ]~  } ||g t                     D ]f  t         fdt        t                    D              }t	        t                    }||vsC|j                  |f       |j                  |       h  t        |d       S )Nc              3     K   |j                  |       |j                  |       t              D ]2  }|   |   dk  r|| k(  r|d d   ||vs  | |||      E d {    4 |j                          |j	                  |       y 7 (wNr   )r8   ro   r   popremove)
start_nodenodepathvisitedr   r   dfs	num_nodess        r   r   zextract_rings.<locals>.dfs9  s     DDy!AD!!$)J1gz1dG<<< " 	
t =s   AB
B
B )B
c              3   X   K   | ]!  }|      |d z   t              z         # yw)r   Nr   )r\   re   r   rs     r   r^   z extract_rings.<locals>.<genexpr>H  s2     U}!l1Q4(AESV+;)<=}s   '*c                      t        | d          S r   r   r   s    r   r   zextract_rings.<locals>.<lambda>N  s    AaD	zr   r   )rm   r   r   mintupler   r8   ro   )	r   ringsexisting_ringsstartcntrkeyr   r   r   s	   `     @@@r   extract_ringsr   4  s    EUNL!I y!UE2su-AUuSQRV}UUC#D>)aX&""4( . " %122r   c           	      2   t        |      }t        |      D ]~  }t        |      D ]n  }||k(  r	||   |   dk  s|rVt        d| |   j                   d| |   j                          t                t        d       t	        j
                  d       m  y  y)Nr   zIncomplete mesh, z is not connected to /Try passing --dot to visualize the connectivityr   FT)r   r   r   r4   rR   rS   )r9   r   rT   r   re   r   s         r   check_valid_meshr   Q  s    L!I9y!AAvAq!Q&+E!H,A,A+BBWX]^_X`XmXmWno KOPHHQK  "  r   c                 `    t        |      dkD  xr t        |d   d         t               k(  }|r}|s{t        d       t                t        d       t        |      dkD  r8t        d       |D ](  }t        ddj                   fd|D                      * t        j                  d       |S )	Nr   zCould not find a full ring.r   zRings found:r`   ,c              3   <   K   | ]  }|   j                     y wr   r4   )r\   re   r9   s     r   r^   z#check_valid_ring.<locals>.<genexpr>m  s     (J1q)>)>s   r   )r   r   r   rR   rS   )r9   r   rT   has_ringr   s   `    r   check_valid_ringr   d  s    5zA~@#eAhqk"2c%j"@Hh/0CDu:>n%C(J(J JKLM Or   c           	         d gt        |       z  fd}t        |       D cg c](  \  }}t        j                  ||j                  |f      * }}}|D ]  }|j                           |D ]  }|j                           t              sf|sdt        d       t        |       D ]!  \  }}|   rt        d|j                         # t                t        d       t        j                  d       S c c}}w )Nc           
      
   t        dd      }||<   t        ddddd| ddgt        t              }|j                  d	k(  |_        |j                  sy t        ddddd| d
dgt        t              }|j                  d	k(  |_        y )NFr+   z-ozBatchMode=yeszConnectTimeout=5echosuccess)r5   stderrr   sudols)r   r   r   
returncoder   r   )hostnamere   r]   resultresultss       r   _checkz%check_ssh_connections.<locals>._checku  s    ue$
 "	 
 ((A-|| "	 
 ))Q.r   )targetrI   z%Could not ssh to the following hosts:z  - z0Maybe they are not set-up for password-less ssh?r   )r   rn   	threadingThreadr4   r   r   ra   r   rR   rS   )r9   ignore_unreachabler   re   r;   threadsr   r   s          @r   check_ssh_connectionsr   r  s    fs5z!G&/T e$$DAq 	anna-@A$   		 	  w< 29:e$DAq1:&!..1 % 	DEN%s   -C5c                 &   t        | j                  d       t        || j                         t        t	        |      D cg c]'  \  }}t        ||j                  |j                  g       ) c}}d| j                        }t        | |       y c c}}w )NzPreparing an ethernet hostfiler2   )
r   r:   r=   r
   rn   r	   r4   r7   envrL   )rI   r9   re   r;   rJ   s        r   prepare_ethernet_hostfiler     ss    68E4<< 8A%8HI8H1a	+8HI2txxH $! 	Js    ,B
c                    t        | j                  d       |\  }}g }t        |      D ]e  \  }}||   }	||dz
     }
|j                  t	        ||	j
                  t        |      D cg c]  }|j                  ||
f   |   d    c}g              g t        |d| j                        }t        ||| j                        }|j                  | j                  | j                  xr |       t        | |       y c c}w )NzPrepare a ring hostfiler   ringr:   rd   )r   r:   rn   r8   r	   r4   r   r7   r
   r   rf   rd   r   rL   )rI   r9   r7   r   rc   count
ring_hostsre   r   r;   r   r   rJ   r   s                 r   configure_ringr     s    /0KD%JT?4$KAE{1>>uU|#T|!CGGD$J$7$:1$=|#TVX	
 # 
FDHH5HeWdoo>HIIdllt/K8IL$! $Us   %C7c           	      h   t        | j                  d       t        || j                         g }t        |      D ]  \  }}g }t	        t        |            D ]B  }||k(  r|j                  d        |j                  d|j                  ||f   d   d           D |j                  t        ||j                  |j                  |              t        |d| j                        }	t        ||| j                        }
|j                  | j                  | j                  xr |
       t        | |	       y )NzPrepare a jaccl hostfilerO   r   jacclr   )r   r:   r=   rn   r   r   r8   r7   r	   r4   r
   r   rf   rd   r   rL   )rI   r9   r7   rc   jaccl_hostsre   r;   rdmar   rJ   r   s              r   configure_jacclr    s    01E4<< K% 1s5z"AAvD!eCGGAqDM!$4Q$7#89:	 #
 	41>>155$?@ ! Wdhh7HeWdoo>HIIdllt/K8IL$!r   c           	         t        | j                  d       t        || j                         g }t        |      }|\  }}t	        |      D ]  \  }}	||	   }
||dz
     }||dz   |z     }g }|D ]p  }|||fvr|j                  d        g }t        |      D ]+  }|j                  d|j                  |	|f   |   d           - |j                  |dk(  r|d   n|       r |j                  t        ||
j                  |
j                  |              t        |d| j                        }t        ||| j                        }|j                  | j                  | j                  xr |       t        | |       y )NzPrepare a jaccl-ring hostfiler   rO   r   
jaccl-ringr   )r   r:   r=   r   rn   r8   r   r7   r	   r4   r
   r   rf   rd   r   rL   )rI   r9   r7   r   rc   r   r   r   re   r   r;   	peer_left
peer_rightrdmasotherr  r   rJ   r   s                      r   configure_jaccl_ringr	    s_   56E4<< KE
IKD%T?4$KQK	1q5I-.
EY
33T"uAKK%e(<Q(?(B'C DE &
T!W=  	41>>155%@A # \488<HeWdoo>HIIdllt/K8IL$!r   c                    t        | j                  d       t        || j                        \  }}| j                  rt	        |||       y t        |||      }t        ||      }| j                  t        |      }t        ||d      }t        ||d      }	t        || j                  d      }
|	s.|s,t        d       t        d       t        j                  d       y |
r|rt        | |||       y |
r|	rt!        | |||d   |       y |	rt#        | |||d   |       y t        d       t        d       t        j                  d       y | j                  dk(  r*t        |      }t        ||       t#        | |||d   |       y | j                  d	k(  r1t        ||       t        || j                         t        | |||       y | j                  d
k(  r@t        |      }t        ||       t        || j                         t!        | |||d   |       y y )Nz,Preparing for communication over thunderboltFz(Neither thunderbolt mesh nor ring found.z>Perhaps run with --dot to generate a plot of the connectivity.r   r   z,RDMA is not available and ring is not found.r   r   r  )r   r:   r   dotr   rh   r   backendr   r   r   rY   r   rR   rS   r  r	  r   )rI   r9   rc   rq   rr   r7   r   r   has_meshr   has_rdmas              r   prepare_tb_hostfiler    s   DF#7t||#L H xxuh0BC
*<
=C+H6HIL||l+#E<?#E5%8eT\\59@AVWHHQK(D%g6( uc58WE4U1Xw? DEVWHHQK		l+&tUCq7;		 -5$,,'eS'2		%l+&5$,,'T5#uQxA	 
&r   c                     t        j                  d      } | j                  ddd       | j                  ddd	
       | j                  ddd       | j                  dd       | j                  dddgddd       | j                  dd       | j                  ddt        ddd        | j                  ddd       | j                  dg d d d!"       | j                  d#d$g d%&       | j	                         }|j
                  *t        j                  |j
                        j                  }n)t        j                  |j                        j                  }t        |j                  d'd(j                  d) |D                      t        ||j                        }t        ||      D cg c]
  \  }}|s	| }}}|D cg c]  }|s|	 }}|j                   dk(  rt#        ||       y t%        |||       y c c}}w c c}w )*Nz6Configure remote machines for use with MLX distributed)descriptionz	--verbose
store_truezPrint debug messages in stdout)actionhelpz--hostsz	127.0.0.1zA comma separated list of hosts)defaultr  z--ignore-unreachablez+Ignore hosts that are not reachable via sshz
--hostfilezThe file containing the hosts)r  z--overthunderboltethernetz&What type of connectivity to configureT)choicesr  r  requiredz--output-hostfilez+If provided, save the hostfile to this pathz--auto-setupz--no-auto-setupr   rd   )r  nargsdestr  z--dotz*Output the topology in DOT format and exitz	--backend)r   r   r  z&Which distributed backend to configure)r  r  r  z--envr8   z&Set environment variables for the jobs)r  r  r  zChecking for ssh access for z, c              3   4   K   | ]  }|j                     y wr   r   )r\   r;   s     r   r^   zmain.<locals>.<genexpr>s  s     0OAr_   )argparseArgumentParseradd_argumentr   
parse_argsrJ   r
   	from_filer9   	from_listr   r:   r   r   r   rb   overr   r  )parserrI   r9   rc   r   r;   s         r   mainr%  8  s?   $$LF L/O   ;-N   :  
 +JK

+5   "O   !   +W   /5	   5	   D}} ""4==177""4::.44 
&tyy0O0O'O&PQ $E4+B+BCGw.4.41a!Q.E4!''QQq'G' yyJ!$. 	D%1 5's   /
G=:G=HH)F)FT)T).r  rD   r   rR   r   collectionsr   dataclassesr   
subprocessr   r   typingr   mlx.corecoremxcommonr	   r
   r   r   r   r   r   r    r&   r=   rL   rY   rf   rh   r   r   r   r   r   r   r   r   r   r   r  r	  r  r%  r   r   r   <module>r.     s       
  # ! #             ! ! !
@28:	> >B	>(B	>3:&=@""("*":2BjH2r   