
    :Ъi<e                        d Z ddlZddlZddlZddlZddlZddlZddlZddlZ	ddl
mZ ddlZddlmZ  e             ddlmZ ddlmZ ddlmZ ddlZddlmZ d	Z ed
           ej                            ej                            e                    Zej                            e          Zej                             de           ddl!m"Z"  eddd          Z#ddl$Z$dZ% ede%          Z&dZ' ed           i Z(g a)d Z*ddl+Z+ e+j,        e*d          -                                 d Z. e+j,        e.d          -                                 de/fdZ0de/de/de/fdZ1 G d de          Z2d Z3d e/d!e/fd"Z4g d#Z5d e/de/d!e/fd$Z6d%e/fd&Z7d%e/fd'Z8d%e/fd(Z9d%e/fd)Z:e#;                    d*          d+e2fd,            Z<dS )-a  
Backend API for Dr. Robert Young's semantic search Q&A system

This module provides a FastAPI application that:
1. Performs semantic search on scraped blog articles
2. Generates contextual answers using local LLM
3. Provides performance timing information
    N)deque)load_dotenv)FastAPI)StreamingResponse)	BaseModel)SentenceTransformerFzB[INFO] Groq has been removed from this version. Using only Ollama.)get_connectionz$Dr. Robert Young Semantic Search APIzBSemantic search and Q&A system for Dr. Robert Young's blog contentz1.0.0)titledescriptionversioncpuzall-MiniLM-L6-v2)devicez$[LLM] Using only Ollama (Local mode)c                     	 t          d           t                      } |                     d          }|                    d           |                                }|                                 |                                  g }|D ]k}	 t          j        t          j	        |d                             }|
                    |d         |d         |d         |d	           \# t          $ r Y hw xY w|at          d
t          t                     d           dS # t          $ r}t          d|            Y d}~dS d}~ww xY w)z:Load all article embeddings from DB into memory at startupz2
[CACHE] Loading article embeddings into memory...T
dictionaryz;SELECT id, title, url, embedding FROM dr_young_all_articles	embeddingidr
   url)r   r
   r   r   z[CACHE] Loaded z articles into memory!z[CACHE] Failed to load cache: N)printr	   cursorexecutefetchallclosenparrayastliteral_evalappend	Exceptionarticle_cachelen)conncurrows	new_cacherembes          KC:\Users\Aakash\Desktop\Scrapping\Chatbot-Dr.Young O Robert\backend\main.pyload_article_cacher*   K   s{   4CDDDkkTk**QRRR||~~		

	 
	 
	A	hs/+??@@  D'wZU8!$	" "        !JM 2 2JJJKKKKK 4 4 42q223333333334s=   B
D AC&%D &
C30D 2C33*D 
E)EET)targetdaemonc                  4   	 t          d           t          j        dddddd          } | j        d	k    rt          d
           dS t          d| j                    dS # t          $ r,}t          d|            t          d           Y d}~dS d}~ww xY w)z
    Warm up the Ollama model by making a simple request
    This pre-loads the model into memory to avoid delays on first user request
    z
[OLLAMA] Warming up model...#http://localhost:11434/api/generatetinyllama:latestHelloT)modelpromptstream<   )jsontimeout   z&[OLLAMA] Model warmed up successfully!z![OLLAMA] Warm-up returned status z[OLLAMA] Warm-up failed: zB[OLLAMA] Model will load on first request (may take 10-15 seconds)N)r   requestspoststatus_coder   )responser(   s     r)   warm_up_ollama_modelr<   m   s    
T.///=1+! 
 
 
 
 3& 	N:;;;;;Lh6JLLMMMMM T T T-!--...RSSSSSSSSSTs   AA! A! !
B+!BBconversation_idc                 ^    | t           vrt          d          t           | <   t           |          S )z%Get conversation history for given ID   )maxlen)conversation_memoryr   )r=   s    r)   get_conversation_historyrB      s.    11 ?/4AO,//    questionanswerc                 H   t          |           }|                    ||t          j                    d           t          d|  d           t          d|dd          d           t          d|dd          d           t          d	t	          |                      dS )
z'Add interaction to conversation history)rD   rE   	timestampz[SAVED] Session [z]:z   Question: Nr4   z...z   Answer: z   Total interactions: )rB   r   timer   r!   )r=   rD   rE   historys       r)   add_to_conversation_historyrJ      s    &77GNNY[[     

1o
1
1
1222	
,(3B3-
,
,
,---	
(ss
(
(
()))	
2CLL
2
233333rC   c                   ,    e Zd ZU dZeed<   dZeed<   dS )ChatRequestz
    Request model for chat endpoint
    
    Attributes:
        question (str): The user's question to be answered
        conversation_id (str): Optional conversation identifier to maintain context
    rD   defaultr=   N)__name__
__module____qualname____doc__str__annotations__r=    rC   r)   rL   rL      s7           MMM$OS$$$$$rC   rL   c                     t          j        | |          t           j                            |           t           j                            |          z  z  S )z
    Calculate cosine similarity between two vectors
    
    Args:
        a (numpy.ndarray): First vector
        b (numpy.ndarray): Second vector
        
    Returns:
        float: Cosine similarity score between 0 and 1
    )r   dotlinalgnorm)abs     r)   cosiner[      s:     6!Q<<29>>!,,ry~~a/@/@@AArC   textreturnc                     t          j        dd|           } t          j        dd|           } t          j        dd| t           j        t           j        z            } |                                 S )a"  
    Clean and preprocess context text for LLM consumption
    
    This function removes unwanted formatting elements that might confuse the LLM.
    
    Args:
        text (str): Raw text content to be cleaned
        
    Returns:
        str: Cleaned text ready for LLM processing
    z\n?\s*\d+[\.]\)\s* u   [•\-–▪]z\n\s*References?\s*:\s*$ )flags)resub	MULTILINE
IGNORECASEstrip)r\   s    r)   clean_contextrg      s`     6'd33D 6"C..D 6-r4r|bm?[\\\D::<<rC   )zph.dzm.sczd.sc
naturopathdisseminatedcoagulationdic	pathology	mechanismtheoreticalrobertyoungc                     |                                 t          fddD                       r	 dS t          fddD                       r	 dS | S )a  
    Sanitize answer based on question type
    
    This function applies specific response patterns for certain types of questions
    to ensure scientifically accurate and appropriately cautious responses.
    
    Args:
        text (str): Original answer text from LLM
        question (str): Original user question
        
    Returns:
        str: Potentially modified answer based on question type
    c              3       K   | ]}|v V  	d S NrT   .0kqs     r)   	<genexpr>z"sanitize_answer.<locals>.<genexpr>   s'      
@
@a16
@
@
@
@
@
@rC   )smallthreeprovestudyaT  Based on the information provided in the blog context, the study is exploratory and limited by its very small sample size. It cannot establish proof that the intervention removes toxins from the human body. The findings provide only initial observations, and larger, well-controlled studies would be required to draw definitive conclusions.c              3       K   | ]}|v V  	d S rs   rT   rt   s     r)   rx   z"sanitize_answer.<locals>.<genexpr>   s'      
G
Ga16
G
G
G
G
G
GrC   )	differentcomparemarketothera8  The blog context does not explicitly provide information about the study size, its stated purpose, or its methodological limitations in relation to this question. As a result, no conclusions can be drawn based on the available information. Further well-controlled studies would be required to address this topic.)lowerany)r\   rD   rw   s     @r)   sanitize_answerr      s     	A 
@
@
@
@?
@
@
@@@ 
&	
 	
 
G
G
G
GF
G
G
GGG 
W	
 	
 KrC   r2   c              #     K   	 t          j        dd| ddddddd	dd
          5 }|j        dk    rNt          |d          r|j        nd}t          d|j         d|            d|j         dV  	 ddd           dS d}|                                D ]}|s	 t          j        |	                    d                    }d|v r|d         rd}|d         V  |
                    d          r n=]# t          j        $ r Y nt          $ r}dt          |           V  Y d}~ nd}~ww xY w|sdV  ddd           dS # 1 swxY w Y   dS # t           j        j        $ r dV  Y dS t           j        j        $ r dV  Y dS t          $ rh}t          dt#          |          j         dt          |                      dt#          |          j         dt          |           V  Y d}~dS d}~ww xY w)a  
    Call locally running LLM via Ollama with streaming capability

    This function establishes a streaming connection to the Ollama service
    and yields response chunks as they become available, enabling real-time
    response delivery to the client.

    Args:
        prompt (str): Formatted prompt including context and question

    Yields:
        str: Response chunks from the LLM as they are generated

    Raises:
        Exception: If connection to Ollama fails or streaming encounters errors
    r.   r/   Tffffff??333333?  temperaturetop_prepeat_penaltynum_predictr1   r2   r3   options,  r5   r3   r6   r7   r\   zNo error detailsz[OLLAMA ERROR] Status z: z$[LLM ERROR]: Ollama returned status z?. The model may be loading. Please wait a moment and try again.NFutf-8r;   donez[PARSING ERROR]: zT[LLM ERROR]: No response received. The model may still be loading. Please try again.zZ[LLM ERROR]: Cannot connect to Ollama service. Is it running? Start it with 'ollama serve'za[LLM ERROR]: Ollama request timed out. The model may be loading or the prompt may be too complex.z[OLLAMA EXCEPTION]: [LLM ERROR]: z - )r8   r9   r:   hasattrr\   r   
iter_linesr5   loadsdecodegetJSONDecodeErrorr   rR   
exceptionsConnectionErrorTimeouttyperN   )r2   r&   
error_textreceived_responselinedatar(   s          r)   call_llama2_streamr      s     "A<]1+ #& &)#&	 	
 
 
 
 
 6	m }# '.q&'9'9QQVV?Q
Lq}LL
LLMMM LQ]  L  L  L  L  L  L-6	m 6	m 6	m 6	m 6	m 6	m 6	m 6	m2 !&    :dkk'&:&:;;D "T) /d:.> /,0)":.... xx''  +   H    6c!ff66666EEEEE
 % mllllm6	m 6	m 6	m 6	m 6	m 6	m 6	m 6	m 6	m 6	m 6	m 6	m 6	m 6	m 6	m 6	m 6	m 6	mp . k k kjjjjjjj& r r rqqqqqqq < < <AT!WW%5AAQAABBB;d1gg.;;3q66;;;;;;;;;;;<s   "E AE3E  EAC30E3D,E	D,D'!E'D,,	E5E EE 	E
E G5*G5	G5AG00G5c              #     K   	 t          j        dd| ddddddd	dd
          5 }|                                D ]Q}|st          j        |                    d                    }d|v r
|d         V  |                    d          r nRddd           dS # 1 swxY w Y   dS # t          $ r}dt          |           V  Y d}~dS d}~ww xY w)a  
    Call locally running LLM via Ollama with streaming capability for direct responses

    This function establishes a streaming connection to the Ollama service
    and yields response chunks as they become available.

    Args:
        prompt (str): Formatted prompt including context and question

    Yields:
        str: Response chunks from the LLM as they are generated

    Raises:
        Exception: If connection to Ollama fails or streaming encounters errors
    r.   r/   Tr   r   r   r   r   r   r   r   r   r;   r   Nz
[LLM ERROR]: )	r8   r9   r   r5   r   r   r   r   rR   )r2   r&   r   r   r(   s        r)   call_llama2_stream_directr   U  s      %)]1+ #& &)#&	 	
 
 
 
 
 	      z$++g"6"677 % +z**** 88F## E=	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	B  ) ) )(A((((((((((()s;   "B' A'BB' BB' !B"B' '
C1CCc              #      K   dV  dS zT
    This function is deprecated since Groq has been removed from this version.
    zD[ERROR]: Groq has been removed from this version. Using only Ollama.NrT   r2   s    r)   call_groq_streamr            QPPPPPrC   c              #      K   dV  dS r   rT   r   s    r)   call_groq_stream_directr     r   rC   z/chatrw   c           
          !"#$%&'()K   t          j                     )t           j                  }t          d j                    t          dt	          |           d           t          j                     }t
                               j        dd          }t          j                     |z
   t          j                     }t                      }|	                    d          }t          j                     |z
  g }t          j                     }t          D ]t          |d                   }	h d	((fd
 j                                                                        D             }
t          fd|
D                       r|	dz  }	|	dk    r|                    |	f           t!          |d d          dd         }t          j                     |z
  '|r=t          d|d         d         d         dd          d|d         d         d           nt          d j         d           |s_ j                                        %|st          d j         d           d!n
|r|r|d         ni }|r(|                    dd                                          nd}|r(|                    dd                                          nd}| d | }t%          |                                          }t%          %                                          }t	          |                    |                    }t	          |                    |                    }|dk    r||z  nd}|d!k    s5t          %fd"|                                dd#         D                       rt          d$|d%d& j         d           t          d'|dd          d(|dd          d)           |dd*          d+ j         d,"t          j                     #"# fd-}t+           |            d./          S t          d0|d%d& j         d           t          d1|dd          d(|dd          d)           d!!fd2}t+           |            d./          S t          j                     }g }g &|D ]\  }&                    d         d3         d4           |                    d5d6         f           |                                }|r2t1          |d7         dd8                   }|                    |           t          d9t	          |                     d:                    |          t          j                     |z
  dd;          d+ j         d,$t          j                     # #$ &')f
d<}t+           |            d./          S )=a  
    Main chat endpoint that processes user questions with session-based memory
    
    This endpoint performs semantic search on the blog database, maintains conversation
    context, and generates contextual answers using a local LLM.
    
    Args:
        q (ChatRequest): The user's question request with optional conversation ID
        
    Returns:
        StreamingResponse: Streaming response containing answer and references
    z[SESSION] CONVERSATION: z[HISTORY] LENGTH: z interactionsTr   )convert_to_numpyr   r   r   >   rY   iandoinisitmemyofortoandcanforhowthewhydoesfromgivethatthiswhatexampleexplainc                 B    g | ]}|vt          |          d k    |S )   )r!   )ru   w
skip_wordss     r)   
<listcomp>zchat.<locals>.<listcomp>  s8    hhh!QjEXh]`ab]c]cfg]ghAhhhrC   c              3   P   K   | ] }|d                                           v V  !dS )r
   N)r   )ru   wordarts     r)   rx   zchat.<locals>.<genexpr>  s8      IIts7|))+++IIIIIIrC   g?g?c                     | d         S )Nr   rT   )xs    r)   <lambda>zchat.<locals>.<lambda>  s
    !A$ rC   )keyreverseN   z[SEARCH] Top match: 'r   r
   2   z' score=z.3fz#[SEARCH] No match above 0.45 for: ''z?[CASE 4] TRIGGERED: No DB match + No conversation history for 'a#  I don't have reliable information about this specific topic in the available content. Could you please provide more details or rephrase your question? Alternatively, you might want to ask about related topics like general health principles, wellness practices, or preventive care approaches.rD   r`   rE   r_   g333333?c              3       K   | ]}|v V  	d S rs   rT   )ru   r   question_lowers     r)   rx   zchat.<locals>.<genexpr>  s(      ,n,nT^-C,n,n,n,n,n,nrC   
   z4[CASE 3] TRIGGERED: No DB match + Continuing topic (.2fz similarity) for 'z   Matching against: Question='z...' Answer='z...'r   z

Q: z
A:c               3     K   d} 	 t           } |          D ]}|                                r	| |z  } |V   d                    |                                           }t	          j        j        |           t          j                    z
  }t          d|dd           d S # t          $ r}dt          |           V  Y d }~d S d }~ww xY w)Nr`   r_   z[TIMING] CASE 3 LLM: r   sr   )r   rf   joinsplitrJ   r=   rD   rH   r   r   rR   )	full_answerllm_functionchunkclean_answerllm_timer(   
llm_prompt	llm_startrw   s	         r)   stream_case3_responsez#chat.<locals>.stream_case3_response  s     "$K7'@%1\*%=%= , ,E${{}} , +u 4&+ (+xx0A0A0C0C'D'D3A4EqzS_```#'9;;#:EhEEEEFFFFF$ 7 7 76c!ff666666666667s   BB' '
C1CCz
text/plain)
media_typez3[CASE 4] TRIGGERED: No DB match + Different topic (z   Context: Question='c               3      K    V  d S rs   rT   )general_answers   r)   stream_general_responsez%chat.<locals>.stream_general_response1  s            rC   r   )r
   r   z7SELECT content FROM dr_young_all_articles WHERE id = %sr   contenti  zCONTEXT LENGTH:z

i  c            	   3   f  
K   d} 	 t           } |          D ]3}|                                r| |z  } |V  t          j        d           4t	          j        d          }D ]/}|d         r%|d         |vr|                    |d                    0|rdV  |D ]	}| dV  
dV  t          d          D ]\  }}| d	|d
          dV  d                    | 	                                          }t          j        j        |           t          j                    z
  }t          j                    z
  }	t          d           t          ddddddd           t          ddd|dd|	dd           d S # t          $ r}
dt          |
           V  Y d }
~
d S d }
~
ww xY w)Nr`   g{Gz?zhttps?://[^\s\)]+r   z

See here for more info:

z
References:
r   z. r
   r_   z	[TIMING]:zEmbedding: r   zs | DB: zs | Search: r   z	Context: z	s | LLM: zs | Total: r   )r   rf   rH   sleeprb   findallr   	enumerater   r   rJ   r=   rD   r   r   rR   )r   r   r   content_urlsrefr   r   r   r   
total_timer(   contextcontext_timedb_time
embed_timer   r2   rw   
referencessearch_time
start_times              r)   stream_responsezchat.<locals>.stream_responseV  s_     %	+-L%f-- % %;;== %5(KKKKJt$$$ :&:GDDL! 4 4u: 4#e*L"@ 4 ''E
333 %5555' % %C ***$$$$ $####J22 / /3..c'l....... 88K$5$5$7$788L'(91:|TTTy{{Y.Hz1J+c
ccccccQ\ccccdddclccccccR\ccccddddd 	+ 	+ 	+*#a&&***********	+s   E>F 
F0F++F0)rH   rB   r=   r   r!   embed_modelencoderD   r	   r   r    r[   r   r   r   r   sortedr   setintersectionunionr   r   fetchonerg   r   )*rw   rI   embed_start	query_embdb_startr"   r#   scoredsearch_startscoremeaningful_wordslast_interactionlast_questionlast_answercombined_contextconversation_keywordscurrent_keywordsoverlaptotal_uniquesimilarity_ratior   r   context_startcontext_parts_content_rowcleanedr   r   r   r   r   r   r   r   r   r2   r   r   r   r   r   s*   `                           @@@@@@@@@@@@@@r)   chatr    s      J 'q'899G 

8Q%6
8
8999	
:s7||
:
:
:;;; )++K""	
 #  I
 {*J y{{HD
+++
&
&CikkH$G F 9;;L ( (y#k"233 L  L  L
hhhhqz'7'7'9'9'?'?'A'AhhhIIII8HIIIII 	TME 4< 	(MM5#,''' F===bqbAF)++,K  C\fQil7&;CRC&@\\&QR)TU,\\\]]]]AAJAAABBB  TU))++  J	aTUT^aaabbbE N  @	 /6=wr{{2L\d,00R@@FFHHHbdMHX`*..x<<BBDDD^`K #0??+??$'(8(>(>(@(@$A$A!">#7#7#9#9:: /<<=MNNOOG4::;KLLMML9E9IPw55q  #% .,n,n,n,nP`PfPfPhPhiljlilPm,n,n,n)n)n .  CM]  C  C  Cuvu  C  C  C  D  D  Docrc8JooYdehfhehYioooppp #.dsd"3  J  
 !IKK	7 7 7 7 7 7 7* ))>)>)@)@\ZZZZ   BL\  B  B  Btut~  B  B  B  C  C  Cf}SbS/AffP[\_]_\_P`fffgggI 	! 	! 	! 	! 	! !!8!8!:!:|TTTT IKKMMJ * *3\u:
 
 	 	 	
 	MPSTXPY|\\\llnn 	*#K	$:5D5$ABBG  )))	
S//000 kk-((G9;;.L %4%.  J  F
 	I*+ *+ *+ *+ *+ *+ *+ *+ *+ *+ *+ *+ *+ *+X    rC   )=rQ   sysosrH   rb   r   r5   
subprocessnumpyr   collectionsr   asynciodotenvr   fastapir   fastapi.responsesr   pydanticr   r8   sentence_transformersr   GROQ_AVAILABLEr   pathdirnameabspath__file__current_dirproject_rootinsertdatabase.dbr	   apptorchr   r   groq_clientrA   r    r*   	threadingThreadstartr<   rR   rB   rJ   rL   r[   rg   FORBIDDEN_TERMSr   r   r   r   r   r9   r  rT   rC   r)   <module>r7     s    


 				  				 



                              / / / / / /        5 5 5 5 5 5  J K K K goobgooh7788w{++ <       & & & & & & g
0T   	!!    , - - -   4 4 4:     	 *4 8 8 8 > > @ @ @T T T4 	 ,T : : : @ @ B B B0c 0 0 0 04 4 4S 4 4 4 4 	% 	% 	% 	% 	%) 	% 	% 	%B B B     2  ## # # # # # #LR<s R< R< R< R<j5)c 5) 5) 5) 5)pQS Q Q Q QQC Q Q Q Q 'i+ i i i i i irC   