
    LKi}K                        S SK Jr  S SKrS SKJrJr  S SKJrJrJ	r	  S SK
Jr  S SKJrJrJrJrJr  S SKJrJrJrJrJr  S SKJr  S S	KJrJrJr  S S
KJr  S SK J!r!  S SK"J#r#J$r$J%r%J&r&J'r'J(r(  S SK)J*r*  S SK+J,r,J-r-  S SK)r)S SK.r.S SK/J0r0  \.Rb                  " \25      r3\Rh                  S \Rj                  S\Rl                  S\Rn                  S0r8\8Rs                  5        V Vs0 s H  u  pX_M	     snn r:Sr; " S S\\\5      r<gs  snn f )    )cached_propertyN)ConfigurationParameter$EmbeddingsQueueConfigurationInternal)SqlDBParameterValueget_sql)BatchSizeExceededError)ProducerConsumerConsumerCallbackFndecode_vectorencode_vector)OperationRecord	LogRecordScalarEncodingSeqId	Operation)System)OpenTelemetryClientOpenTelemetryGranularitytrace_method)override)defaultdict)SequenceOptionalDictSetTuplecast)UUID)Table	functions)create_topic_name         Fc                   4  ^  \ rS rSr% Sr " S S5      r\\\\   4   \	S'   \
\   \	S'   \\	S'   \\	S'   S	rS
\4U 4S jjr\" S\R"                  5      \S<U 4S jj5       5       r\" S\R"                  5      \S\SS4S j5       5       r\" S\R"                  5      \S\SS4S j5       5       r\" S\R"                  5      \S\S\S\4S j5       5       r\" S\R"                  5      \S\S\\   S\\   4S j5       5       r\" S\R"                  5      \   S=S\S\S\
\   S\
\   S\
\   S\4S  jj5       5       r\" S!\R"                  5      \S"\SS4S# j5       5       r\S\4S$ j5       r\S\4S% j5       r \!\" S&\R"                  5      \S\4S' j5       5       5       r"\" S(\R"                  5      S\S\#\
\$   \
\   \
\   4   4S) j5       r%\" S*\R"                  5      S+\SS4S, j5       r&\" S-\R"                  5      S\
\   S\
\   S\#\\4   4S. j5       r'\" S/\R"                  5      S\4S0 j5       r(\" S1\R"                  5      S2\S\\)   SS4S3 j5       r*\" S4\R"                  5      S5\S\\)   SS4S6 j5       r+\,S\-4S7 j5       r.S8\-SS4S9 jr/S\4S: jr0S;r1U =r2$ )>SqlEmbeddingsQueue6   a  A SQL database that stores embeddings, allowing a traditional RDBMS to be used as
the primary ingest queue and satisfying the top level Producer/Consumer interfaces.

Note that this class is only suitable for use cases where the producer and consumer
are in the same process.

This is because notification of new embeddings happens solely in-process: this
implementation does not actively listen to the the database for new records added by
other processes.
c            
       f    \ rS rSr% \\S'   \\S'   \\S'   \\S'   \\S'   S\S\S\S\S\4
S jr	Sr
g	)
SqlEmbeddingsQueue.SubscriptionB   id
topic_namestartendcallbackc                 @    Xl         X l        X0l        X@l        XPl        g N)r-   r.   r/   r0   r1   )selfr-   r.   r/   r0   r1   s         e/var/www/html/dynamic-report/venv/lib/python3.13/site-packages/chromadb/db/mixins/embeddings_queue.py__init__(SqlEmbeddingsQueue.Subscription.__init__I   s     G(OJH$M    )r1   r0   r-   r/   r.   N)__name__
__module____qualname____firstlineno__r    __annotations__strintr   r6   __static_attributes__ r8   r5   Subscriptionr+   B   sQ    
$$	%	% 	% 		%
 	% )	%r8   rB   _subscriptions_max_batch_size_tenant_topic_namespace   systemc                   > [        [        5      U l        S U l        UR	                  [
        5      U l        UR                  R	                  S5      U l        UR                  R	                  S5      U l	        [        TU ]-  U5        g )N	tenant_idtopic_namespace)r   setrC   rD   requirer   _opentelemetry_clientsettingsrE   rF   superr6   )r4   rH   	__class__s     r5   r6   SqlEmbeddingsQueue.__init__^   se    )#.#%+^^4G%H"..{; & 7 78I J r8   zSqlEmbeddingsQueue.reset_statereturnNc                 p   > [         TU ]  5         [        [        5      U l         U ?g ! [         a     g f = fr3   )rP   reset_stater   rL   rC   configAttributeError)r4   rQ   s    r5   rU   SqlEmbeddingsQueue.reset_statef   s9     	)#.	 		s   ( 
55zSqlEmbeddingsQueue.delete_topiccollection_idc                    [        U R                  U R                  U5      n[        S5      nU R	                  5       R                  U5      R                  UR                  [        U5      :H  5      R                  5       nU R                  5        n[        X@R                  5       5      u  pgUR                  Xg5        S S S 5        g ! , (       d  f       g = f)Nembeddings_queue)r#   rE   rF   r!   querybuilderfrom_wheretopicr   deletetxr   parameter_formatexecute)r4   rY   r.   tqcursqlparamss           r5   
delete_logSqlEmbeddingsQueue.delete_logs   s     'LL$//

 $%U1XU177nZ889VX	 	
 WWY#!!%:%:%<=KCKK$ YYs   -C
CzSqlEmbeddingsQueue.purge_logc                 n   [        S5      nU R                  5       R                  U5      R                  [        R
                  " [        S5      R                  S5      5      R                  UR                  [        U R                  U5      5      :H  5      R                  [        S5      5      R                  UR                  [        S5      R                  :H  5      n[        U R                   U R"                  U5      nU R%                  5        n['        X0R)                  5       5      u  pgUR+                  Xg5        UR-                  5       nU(       a  [/        S U 5       5      n	O
 S S S 5        g [        S5      n
U R                  5       R                  U
5      R                  U
R                  [        U	5      :  5      R                  U
R0                  [        U5      :H  5      R3                  5       n['        XR)                  5       5      u  pgUR+                  Xg5        S S S 5        g ! , (       d  f       g = f)Nsegments
max_seq_idc              3   *   #    U  H	  oS    v   M     g7f)r   NrA   ).0rows     r5   	<genexpr>/SqlEmbeddingsQueue.purge_log.<locals>.<genexpr>   s      ;7CQ7s   r[   )r!   r\   r]   selectr"   Coalesceseq_idr^   
collectionr   
uuid_to_db	left_joinonr-   
segment_idr#   rE   rF   ra   r   rb   rc   fetchallminr_   r`   )r4   rY   
segments_tsegment_ids_qr.   rf   rg   rh   results
min_seq_idrd   re   s               r5   	purge_logSqlEmbeddingsQueue.purge_log   s   
 :&
U:
 VI&&u\':'A'A2FGU%%8V)WW Yu\*+R
|!4!?!??@ 	 'LL$//

 WWY#!-1F1F1HIKCKK$llnG  ;7 ;;
 Y ()A!!#qqxx."<<=qww."<<=  "!%:%:%<=KCKK$' YYs   AH&/B.H&&
H4z#SqlEmbeddingsQueue.submit_embedding	embeddingc                 d    U R                   (       d  [        S5      eU R                  X/5      S   $ )NComponent not runningr   )_runningRuntimeErrorsubmit_embeddings)r4   rY   r   s      r5   submit_embedding#SqlEmbeddingsQueue.submit_embedding   s/    
 }}677%%m[A!DDr8   z$SqlEmbeddingsQueue.submit_embeddings
embeddingsc                    U R                   (       d  [        S5      e[        U5      S:X  a  / $ [        U5      U R                  :  a(  [	        SU R                  S SU R                  S S35      eU R
                  n[        U R                  U R                  U5      n[        S5      nU R                  5       R                  U5      R                  UR                  UR                  UR                  UR                   UR"                  UR$                  5      n0 nU H  nU R'                  U5      u  n	n
nUR)                  [+        [,        US      5      [+        U5      [+        US	   5      [+        U	5      [+        U
5      [+        U5      5      n[        U5      XxS	   '   M     U R/                  5        n[1        X`R3                  5       5      u  pU S
3nUR5                  X5      R7                  5       n[9        [:        S 5      /[        U5      -  n/ nU HG  u  nnUUUU   '   X'U      n[=        U[?        UUS   US   US   US   S9S9nURA                  U5        MI     U RC                  UU5        U R
                  RE                  S5      RF                  (       a  U RI                  U5        UsS S S 5        $ ! , (       d  f       g = f)Nr   r   z)
                Cannot submit more than ,zf embeddings at once.
                Please submit your embeddings in batches of size
                z or less.
                r[   	operationr-   z RETURNING seq_id, idr   encodingmetadata)r-   r   r   r   r   
log_offsetrecordautomatically_purge)%r   r   lenmax_batch_sizer	   rV   r#   rE   rF   r!   r\   intocolumnsr   r_   r-   vectorr   r   !_prepare_vector_encoding_metadatainsertr   _operation_codesra   r   rb   rc   r|   r   r   r   r   append_notify_allget_parametervaluer   )r4   rY   r   _r.   rd   r   	id_to_idxr   embedding_bytesr   r   rf   rg   rh   r   seq_idsembedding_recordsrv   r-   submit_embedding_recordembedding_records                         r5   r   $SqlEmbeddingsQueue.submit_embeddings   s   
 }}677z?aIz?T000())-)<)<Q(? @$$Q' (  KK&LL$//

 $%T!WWQ[[!''1441::qzzR 	
 %'	#I
 66yA	]]/	+0FGHz*y//x(x(F *-YIo& $ WWY#!&*?*?*ABKC E./Ckk#.779GE4()C- G !#%
)/	"&*4r]*C' $-%*"9+"F!8!D!8!D"9+"F	$  "(()9: &  Z):;{{(()>?EE}-A YYs   /C:J33
KzSqlEmbeddingsQueue.subscribe
consume_fnr/   r0   r-   c                 f   U R                   (       d  [        S5      e[        U R                  U R                  U5      nU=(       d    [
        R                  " 5       nU R                  X45      u  p4U R                  XvX4U5      nU R                  U5        U R                  U   R                  U5        U$ )Nr   )r   r   r#   rE   rF   uuiduuid4_validate_rangerB   	_backfillrC   add)	r4   rY   r   r/   r0   r-   r.   subscription_idsubscriptions	            r5   	subscribeSqlEmbeddingsQueue.subscribe  s     }}677&LL$//

 ,

))%5
((Z

 	|$J'++L9r8   zSqlEmbeddingsQueue.unsubscriber   c                     U R                   R                  5        HN  u  p#U HC  nUR                  U:X  d  M  UR                  U5        [	        U5      S:X  a  U R                   U	     g    MP     g )Nr   )rC   itemsr-   remover   )r4   r   r.   subscriptionsr   s        r5   unsubscribeSqlEmbeddingsQueue.unsubscribe.  sc     *.)<)<)B)B)D%J -??o5!((6=)Q. //
; !. *Er8   c                     g)Nrn   rA   r4   s    r5   	min_seqidSqlEmbeddingsQueue.min_seqid9  s    r8   c                     g)Nl    rA   r   s    r5   	max_seqidSqlEmbeddingsQueue.max_seqid=  s    r8   z!SqlEmbeddingsQueue.max_batch_sizec                    U R                   c  U R                  5        nUR                  S5        UR                  5       nU H@  nSUS   ;   d  M  [	        US   R                  S5      S   5      U R                  -  U l         MB     U R                   c  SU R                  -  U l         S S S 5        U R                   $ U R                   $ ! , (       d  f       U R                   $ = f)NzPRAGMA compile_options;MAX_VARIABLE_NUMBERr   =r$   i  )rD   ra   rc   r|   r?   splitVARIABLES_PER_RECORD)r4   rf   compile_optionsoptions       r5   r   !SqlEmbeddingsQueue.max_batch_sizeA  s     'c56"%,,.-F,q	9/26!9??33G3J/K 550, . ''/ ,/$2K2K+KD(   ###t###!   ###s   .CAC
C z4SqlEmbeddingsQueue._prepare_vector_encoding_metadatac                     US   b/  [        [        US   5      nUR                  n[        US   U5      nOS nS nUS   (       a  [        R
                  " US   5      OS nXCU4$ )Nr   r   r   )r   r   r   r   jsondumps)r4   r   encoding_typer   r   r   s         r5   r   4SqlEmbeddingsQueue._prepare_vector_encoding_metadataX  sn     [!- :1FGM$**H+Ik,BMRO"OH8A*8M4::i
34SW(22r8   zSqlEmbeddingsQueue._backfillr   c                 ,   [        S5      nU R                  5       R                  U5      R                  UR                  [        UR                  5      :H  5      R                  UR                  [        UR                  5      :  5      R                  UR                  [        UR                  5      :*  5      R                  UR                  UR                  UR                  UR                  UR                  UR                  5      R!                  UR                  5      nU R#                  5        n[%        X0R'                  5       5      u  pVUR)                  XV5        UR+                  5       nU H  nUS   (       a  [-        US   5      n	[/        US   U	5      n
OSn	Sn
U R1                  U[3        US   [5        [6        US      US   U
U	US   (       a  [8        R:                  " US   5      OSS	9S
9/5        M     SSS5        g! , (       d  f       g= f)zMBackfill the given subscription with any currently matching records in the
DBr[   r&      Nr   r$   r%      )r   r-   r   r   r   r   )r!   r\   r]   r^   r_   r   r.   rv   r/   r0   rt   r   r-   r   r   r   orderbyra   r   rb   rc   r|   r   r   _notify_oner   r   _operation_codes_invr   loads)r4   r   rd   re   rf   rg   rh   rowsrq   r   r   s              r5   r   SqlEmbeddingsQueue._backfilli  s    $%U1XU177n\-D-DEEFU188n\-?-?@@AU188~l.>.>??@VAHHakk1441::qzzRWQXX 	
 WWY#!!%:%:%<=KCKK$<<>Dq6-c!f5H*3q68<F#H!F   !'*1v#2*>s1v*F#&q6*0)1?B1vCF);4$	 	 YYs   1CH
Hz"SqlEmbeddingsQueue._validate_rangec                    U=(       d    U R                  5       nU=(       d    U R                  5       n[        U[        5      (       a  [        U[        5      (       d  [	        S5      eX:  a  [        SU SU 35      eX4$ )zSValidate and normalize the start and end SeqIDs for a subscription using this
impl.z2SeqIDs must be integers for sql-based EmbeddingsDBzInvalid SeqID range: z to )_next_seq_idr   
isinstancer?   	TypeError
ValueError)r4   r/   r0   s      r5   r   "SqlEmbeddingsQueue._validate_range  ss     ,**,%T^^%%%%ZS-A-APQQ<4UG4uEFFzr8   zSqlEmbeddingsQueue._next_seq_idc                    [        S5      nU R                  5       R                  U5      R                  [        R
                  " UR                  5      5      nU R                  5        nUR                  UR                  5       5        [        UR                  5       S   5      S-   sSSS5        $ ! , (       d  f       g= f)z%Get the next SeqID for this database.r[   r   r$   N)r!   r\   r]   rt   r"   Maxrv   ra   rc   r   r?   fetchoner4   rd   re   rf   s       r5   r   SqlEmbeddingsQueue._next_seq_id  s     $%%%a(//	ahh0GHWWY#KK		$s||~a()A- YYs   (>B00
B>zSqlEmbeddingsQueue._notify_allr_   c                 v    U R                   (       a(  U R                  U    H  nU R                  X25        M     gg)z:Send a notification to each subscriber of the given topic.N)r   rC   r   )r4   r_   r   subs       r5   r   SqlEmbeddingsQueue._notify_all  s3     ==**51  1 2 r8   zSqlEmbeddingsQueue._notify_oner   c                    Sn/ nU H@  nUS   UR                   ::  a  M  US   UR                  :  a  Sn  OUR                  U5        MB      [        U5      S:  a  UR	                  U5        U(       a  U R                  UR                  5        gg! [         a^  n[        R                  SUR                  R                   3SUR                   S3-   [        U5      5        [        (       a  Ue SnAgSnAff = f)	z+Send a notification to a single subscriber.Fr   Tr   z6Exception occurred invoking consumer for subscription z	to topic z %sN)r/   r0   r   r   r1   r   r-   BaseExceptionloggererrorhexr.   r>   _called_from_test)r4   r   r   should_unsubscribefiltered_embeddingsr   es          r5   r   SqlEmbeddingsQueue._notify_one  s     # #I&#))3&0%)"&&y1 $	&'!+01!  ( " 	LLHUcnn-S12A
 !  !	s   AB 
C8AC33C8c                    [        S5      nU R                  5       R                  U5      R                  UR                  5      R                  S5      nU R                  5        nUR                  UR                  5       5        UR                  5       nS S S 5        Wc<  U R                  5       S:H  n[        [        SU5      /5      nU R                  U5        U$ [        R                  " US   5      $ ! , (       d  f       Nf= f)Nembeddings_queue_configr$   r   r   )r!   r\   r]   rt   config_json_strlimitra   rc   r   r   _get_wal_sizer   r   
set_configfrom_json_str)r4   rd   re   rf   resultis_fresh_systemrV   s          r5   rV   SqlEmbeddingsQueue.config  s    +,%%a(//0A0ABHHKWWY#KK		$\\^F  >"002a7O9'(=OPF OOF#M3AA&)LL Ys   #0C33
DrV   c                     U R                  5        nUR                  SSUR                  5       45        S S S 5         U ?g ! , (       d  f       N= f! [         a     g f = f)Nz
                INSERT OR REPLACE INTO embeddings_queue_config (id, config_json_str)
                VALUES (?, ?)
            r$   )ra   rc   to_json_strrV   rW   )r4   rV   rf   s      r5   r   SqlEmbeddingsQueue.set_config  s^    WWY#KK
 &&(	 	 Y  		s   #A A  
A
AAc                 h   [        S5      nU R                  5       R                  U5      R                  [        R
                  " S5      5      nU R                  5        nUR                  UR                  5       5        [        UR                  5       S   5      sS S S 5        $ ! , (       d  f       g = f)Nr[   *r   )r!   r\   r]   rt   r"   Countra   rc   r   r?   r   r   s       r5   r    SqlEmbeddingsQueue._get_wal_size  st    $%%%a(//	0DEWWY#KK		$s||~a() YYs   ;B##
B1)rD   rN   rC   rE   rF   )rS   N)NNN)3r9   r:   r;   r<   __doc__rB   r   r>   r   r=   r   r?   r   r   r6   r   r   ALLr   rU   r    ri   r   r   r   r   r   r   r   r   r   r   r   propertyr   r   bytesr   r   r   r   r   r   r   r   r   rV   r   r   r@   __classcell__)rQ   s   @r5   r(   r(   6   sm   	% %* c,//00c]"L!v ! 24L4P4PQ	  R	 35M5Q5QR% % %  S% 02J2N2NO)%t )% )%  P)%V 79Q9U9UVE!E.=E	E  WE 8:R:V:VWQ!Q/7/HQ	%Q  XQf 02J2N2NO
 "&#! ' 	
 e_ TN 
  P8 24L4P4PQ4 D   R 5   5   57O7S7ST$ $  U $( > $$3(3	xx}<	=3	3 02J2N2NO&l &t & P&P 68P8T8TUe_+3E?	sCx V 35M5Q5QR.c . S. 24L4P4PQ2 2(92E 2$ 2 R2 24L4P4PQ| )9L QU  R: M< M M$!E $ (*s * *r8   r(   )=	functoolsr   r   chromadb.api.configurationr   r   chromadb.db.baser   r   r   chromadb.errorsr	   chromadb.ingestr
   r   r   r   r   chromadb.typesr   r   r   r   r   chromadb.configr    chromadb.telemetry.opentelemetryr   r   r   	overridesr   collectionsr   typingr   r   r   r   r   r   r   r    pypikar!   r"   loggingchromadb.ingest.impl.utilsr#   	getLoggerr9   r   ADDUPDATEUPSERTDELETEr   r   r   r   r(   )kvs   00r5   <module>r     s    %  < ; 2   # 
  # = =  #   8 
		8	$ MM1aaa	  *:)?)?)AB)A)AB   E*( E* Cs   D 