
    J5j3                        S r SSKJr  SSKrSSKJr  SSKJr  SSKJ	r	  SSK
JrJr  SrS	rS
rSrSS jrSS jrSS jr\\4SS jjrSS jr            SS jrSS jr\4         SS jjrg)u=   Extracción, chunking e indexación de documentos por agente.    )annotationsN)unescape)Path)Session)AgentFileChunkModelAgentFileModeli  x         c                    [         R                  " SSU =(       d    S5      n[        [         R                  " SSU5      5      R                  5       $ )N<[^>]+>  \s+)resubr   stripvaluetexts     XC:\Users\jesus\Desktop\proyecto_pie360\backend\app\backend\utils\agent_document_index.py
strip_htmlr      s9    66*c5;B/DBFF63-.4466    c                <   U (       a  U R                  5       (       d  gU n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " S	SU[        R                  S9n[        R                  " S
SU[        R                  S9n[        R                  " SSU[        R                  S9n[        R                  " SSU5      n[	        U5      n[        R                  " SSU5      n[        R                  " SSU5      n[        R                  " SSU5      nUR                  5       $ )uJ   Convierte HTML del rol a texto estructurado (títulos, listas, párrafos).r   z	<br\s*/?>
)flagsz</p>

z</div>z</li>z	</h[1-6]>z<h[1-6][^>]*>z

## z	<li[^>]*>z
- r   z[ \t]+\nz\n{3,}z +r   )r   r   r   
IGNORECASEr   r   s     r   role_instructions_to_textr      s   D66,d"--@D66'64r}}=D66)T4r}}=D66(D$bmm<D66,BMMBD66"It2==ID66,BMMBD66*b$'DD>D66+tT*D66)VT*D66%d#D::<r   c           
        U R                   R                  5       nUS;   a  U R                  SSS9$ US:X  a>  SS KnUR	                  U 5      n SR                  S U 5       5      UR                  5         $ US	:X  a9  SS
KJn  U" [        U 5      5      nSR                  S UR                   5       5      $ US:X  as  SS KnUR                  U S S9n/ nUR                  5        H8  u  pUR                  SU	 SU
R                  S5      R!                  SS9 35        M:     SR                  U5      $ g! UR                  5         f = f)N>   .md.csv.txtzutf-8ignore)encodingerrorsz.pdfr   r   c              3  B   #    U  H  oR                  S 5      v   M     g7f)r   N)get_text).0pages     r   	<genexpr>)extract_text_from_file.<locals>.<genexpr>7   s     Cst]]622ss   z.docx)Documentc              3  z   #    U  H1  oR                   R                  5       (       d  M#  UR                   v   M3     g 7fN)r   r   )r)   	paragraphs     r   r+   r,   ?   s'     g9LIP^P^PdPdPf9Ls   ";;z.xlsx)
sheet_namezHoja: r   F)indexr   )suffixlower	read_textfitzopenjoinclosedocxr-   str
paragraphspandas
read_excelitemsappendfillna	to_string)	file_pathr3   r6   docr-   documentpdsheetspartsr1   frames              r   extract_text_from_filerJ   ,   s(   ##%F((""GH"EEii	"	99CsCCIIK!C	N+yyg9L9LgggyT:!'JLL6*RR0@0J0JQV0J0W/XYZ "0{{5!!# IIKs   D. .E c                `   [         R                  " SSU =(       d    S5      R                  5       nU(       d  / $ / nSnU[        U5      :  ab  XQ-   nX5U R                  5       nU(       a  UR	                  U5        U[        U5      :  a   U$ [        Xb-
  US-   5      nU[        U5      :  a  Mb  U$ )Nr   r   r   r      )r   r   r   lenr@   max)r   
chunk_sizeoverlap
normalizedchunksstartendpieces           r   
chunk_textrV   M   s    TZR0668J	FE
#j/
! %++-MM% #j/!M CM519- #j/
! Mr   c                    U R                  [        5      R                  [        R                  U:H  [        R                  U:H  5      R                  SS9  g )NF)synchronize_session)queryr   filteragent_idfile_iddelete)dbr[   r\   s      r   delete_file_chunksr_   _   sD    HH !(($$0##w. ff'r   c           
         [        XU5        [        U5      nUR                  5       (       d  gSU S3n[        U5      n[	        U5       H$  u  pU R                  [        UUUU U	 3S95        M&     [        U5      $ )Nr   z	Archivo: r   )r[   r\   chunk_indexcontent)r_   rJ   r   rV   	enumerateaddr   rM   )
r^   r[   r\   rC   display_nameraw_textheaderrR   r2   rb   s
             r   index_agent_filerh   f   s     rW-%i0H>>b)F!F#F+
!!!(7),		
 , v;r   c                    [         R                  " SU R                  5       5       Vs/ s H  n[        U5      [        :  d  M  UPM     sn$ s  snf )Nz\w+)r   findallr4   rM   MIN_TERM_LEN)rY   terms     r   _query_termsrm      s8    ZZ>\>T#d)|B[D>\\\s   AAc                T  ^ [        U5      nU(       d  / $ U R                  [        [        R                  5      R                  [        [        R                  [        R                  :H  5      R                  [        R                  U:H  5      R                  5       n/ nU H  u  pxUR                  R                  5       n	Sn
U HN  mTU	;   a  XR                  T5      S-  -  n
[        U4S jU	R                  5        5       5      (       d  MI  U
S-  n
MP     U
S:  d  M}  UR!                  XU45        M     UR#                  S S9  / nUS [%        US5        HP  u  pnUR!                  UR                  UR                  UUR&                  [)        U
S5      UR                  S.5        MR     U$ )	Ng           c              3  .   >#    U  H
  nTU;   v   M     g 7fr/    )r)   wordrl   s     r   r+   )search_agent_knowledge.<locals>.<genexpr>   s     =,<D44<,<s   rL   r   c                ,    U S   * U S   R                   4$ )Nr   rL   )ra   )items    r   <lambda>(search_agent_knowledge.<locals>.<lambda>   s    47(DG,?,?!@r   )key)chunkIdfileIdfileName
chunkIndexscorerb   )rm   rY   r   r   re   r8   idr\   rZ   r[   allrb   r4   countanysplitr@   sortrN   ra   round)r^   r[   rY   top_ktermsrR   scoredchunkre   haystackr}   resultsrl   s               @r   search_agent_knowledger      sp    E	 	$n&A&AB	nn//3F3N3NN	O	#,,8	9		  <>F%==&&(Dx-11=HNN,<===
	 
 19MM567  & KK@KAG&,_s5!}&="l 88--(#//ua ==		
 '> Nr   )r   r;   returnr;   )rC   r   r   r;   )r   r;   rO   intrP   r   r   	list[str])r^   r   r[   r;   r\   r;   r   None)r^   r   r[   r;   r\   r;   rC   r   re   r;   r   r   )rY   r;   r   r   )
r^   r   r[   r;   rY   r;   r   r   r   z
list[dict])__doc__
__future__r   r   htmlr   pathlibr   sqlalchemy.ormr   app.backend.db.modelsr   r   
CHUNK_SIZECHUNK_OVERLAPrk   DEFAULT_TOP_Kr   r   rJ   rV   r_   rh   rm   r   rq   r   r   <module>r      s    C " 	   " E
7
(B -7} $(  	
  	4] 	*** * 	*
 *r   