+
   Y<3c                   s   ^ RI Ht ^ RI Ht ^ RI Ht ^ RI Ht ^ RI Ht ^ RI Ht ^ RI Ht ^ RIt^ RI	t	^ R	I
Ht ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt]P&                  ! R
4      t]! ^ 4      sRt ! R R]4      t ! R R4      tR# )    )Dict)List)Tuple)Optional)Any)Callable)IteratorN)contextmanagerzibus-typing-boosterz0.65c                   s    ] tR t^.tRtR# )DatabaseConnectionError N)__name__
__module____qualname____firstlineno____static_attributes__r
       4/usr/share/ibus-typing-booster/engine/tabsqlitedb.pyr	   r	   .   s    r   r	   c                   s  a  ] tR t^1t o RDV 3R lR llt]REV 3R lR ll4       t]V 3R lR l4       t]]RFV 3R lR ll4       4       t	]V 3R	 lR
 l4       t
V 3R lR ltV 3R lR ltV 3R lR ltV 3R lR ltRGV 3R lR lltV 3R lR ltV 3R lR ltRGV 3R lR lltV 3R lR ltV 3R lR ltV 3R  lR! ltRHV 3R" lR# lltV 3R$ lR% lt]V 3R& lR' l4       t]V 3R( lR) l4       tV 3R* lR+ ltRR]P8                  3V 3R, lR- lltRIV 3R. lR/ lltRJV 3R0 lR1 lltV 3R2 lR3 lt RHV 3R4 lR5 llt!V 3R6 lR7 lt"V 3R8 lR9 lt#V 3R: lR; lt$V 3R< lR= lt%V 3R> lR? lt&RKV 3R@ lRA llt'RBt(V t)RC# )LTabSqliteDbc                $   < V ^8  d   QhRS[ RR/# )   user_db_filereturnN)strZformat__classdict__   "r   __annotate__TabSqliteDb.__annotate__=   s     )$ )$S )$ )$r   c                s     \        \        \        P                  ! R 4      4      4      s\        ^8  d   \        P                  RV4       Wn	        V P                  R8w  d   \        P                  P                  V P                  4      V P                  8X  dM   \        P                  P                  \        P                  P                  R4      V P                  4      V n	        V P                  '       g   \        P                  R4       RV n	        V P                  R8w  d   \        P                  P                  \        P                  P                  V P                  4      4      '       g@   \        P                   ! \        P                  P                  V P                  4      RR7       . ROV n        . V n        \&        P(                  ! R	4      V n        V P-                  4        V P/                  4        V P1                  V P                  4      V n        V P5                  4        V P7                  4        V P9                  4        V P;                  4        R#   \
        \        3 d    \        ^ 4      s ELRi ; i)
ZIBUS_TYPING_BOOSTER_DEBUG_LEVELz'TabSqliteDb.__init__(user_db_file = %s):memory:z"~/.local/share/ibus-typing-boosterz&Falling back to ":memory:" for user.dbT)Zexist_okN)idinput_phrasephrasep_phrase	pp_phrase	user_freq	timestampr
   )intr   osZgetenvDEBUG_LEVEL	TypeErrorZ
ValueErrorLOGGERdebugr   pathZbasenamejoinZ
expanduserZisdirZdirnameZmakedirs_phrase_table_column_names_old_phraseshunspell_suggestZHunspellhunspell_obj_check_database_compatibility_check_database_readabilitysqlite3_connect_database_legacydatabasecreate_tables_restore_old_phrasescreate_indexesgenerate_userdb_desc)selfr   s   &&r   __init__ZTabSqliteDb.__init__=   s   	!c")),M"NOPK ?LL9<I(+  !2!23t7H7HH "RWW-?-?4.67;7H7H!JD   LLAB *D+bggood6G6G&HIIKK(9(9:TJ+' 9;,55b9**,((*<<T=N=NO!!#!!#K :& 	!a&K	!s   (I I=<I=c                s:   < V ^8  d   QhRS[ RS[ RS[R,          /# )r   suppress_errors
checkpointr   N)boolr   r   r   r   r   r   i   s-      !  )1r   c              #  s$  "    R x  V P                   P                  4        V'       d   V P                   P                  R4       R # R #   \        P                   d   p\
        P                  RTP                  P                  T4        T P                   P                  4        MT  \        P                   d=   p\
        P                  RTP                  P                  T4       \        R4      ThR p?ii ; iT'       g   h  R p?R # R p?ii ; i5i)N PRAGMA wal_checkpoint(TRUNCATE);z(Transaction failed, rolling back: %s: %szRollback failed! %s: %szRollback failed)r4   commitexecutesqlite3Errorr)   	exception	__class__r   ZrollbackZcriticalr	   )r9   r;   r<   errorZrollback_errors   &&&  r   transactionZTabSqliteDb.transactionh   s     	MM  " %%&HI  }} 	G"__55u>L &&(== L 9 % 8 8.J-.?@eKL # #	sY   DAA	 D	D+D
B%$D%C6:7C11C66
DDDDDc                s>   < V ^8  d   QhRS[ P                  RS[RR/# )r   database_connectiondb_filer   N)rA   
Connectionr   r   r   r   r   r      s+     5 5&-&8&85CF5KO5r   c                s    VP                  R 4       VP                  RV R24       VP                  R4      P                  4       p\        P	                  RV4       R Vn        R# )a{  
        PRAGMA encoding = "UTF-8";
        PRAGMA case_sensitive_like = true;
        PRAGMA page_size = 4096;
        PRAGMA cache_size = 20000;
        PRAGMA temp_store = MEMORY;
        PRAGMA journal_mode = WAL;
        PRAGMA journal_size_limit = 1000000;
        PRAGMA synchronous = NORMAL;
        PRAGMA auto_vacuum = FULL;
        PRAGMA busy_timeout = 5000;
        zATTACH DATABASE "z" AS user_db;zPRAGMA journal_modezJournal mode: %sc                 s(    V P                  R RR7      # )zutf-8replace)encodingZerrors)Zdecodex   &r   <lambda>Z8TabSqliteDb._setup_database_connection.<locals>.<lambda>   s    ahh   4r   N)executescriptr@   fetchoner)   r*   Ztext_factory)clsrG   rH   modes   &&& r   _setup_database_connectionZ&TabSqliteDb._setup_database_connection   sl     	)) + 	 	))y6	8"**!##+8: 	'.4 	(r   c                sP   < V ^8  d   QhRS[ RS[RS[S[P                  ,          /# )r   rH   r;   r   )r   r=   r   rA   rI   r   r   r   r   r      s5     P PP!P.6w7I7I.JPr   c              #  s  "   R p \         P                  ! V4      pV P                  W14       Vx  V'       d    VP                  4        R # R #   \         P                   d*   p\        P                  RY4       T'       g   h  R p?LVR p?ii ; i  \         P                   d7   p\        P                  RTP                  P                  T4        R p?R # R p?ii ; i  T'       d`    TP                  4        i   \         P                   d6   p\        P                  RTP                  P                  T4        R p?i R p?ii ; ii ; i5i)N#Database connection failed (%s): %sz"Failed to close connection: %s: %s)
rA   connectrT   rB   r)   rC   closewarningrD   r   )rR   rH   r;   rG   rE   close_error   &&&   r   sqlite3_connect_databaseZ$TabSqliteDb.sqlite3_connect_database   s     #	P")//'":**+>H%% #P'--/ # }} 	BGS" #	 }} PNN#G#.#8#8#A#A;P PP #P'--/}} PNN#G#.#8#8#A#A;P PP #s   E	+A E	B 
E	B#BC BC C$+CE	CE		E'C87E8E+D=8E=EEE	c                s:   < V ^8  d   QhRS[ RS[P                  /# r   rH   r   )r   rA   rI   r   r   r   r   r      s#     
 

")"4"4
r   c                s    \         P                  R V4        \        P                  ! V4      pV P	                  W!4       V#   \        P
                   d   p\         P                  RY4       h Rp?ii ; i)zConnect to the database %s.rU   N)r)   inforA   rV   rT   rB   rC   )rR   rH   rG   rE      &&  r   r3   Z+TabSqliteDb.sqlite3_connect_database_legacy   sd     	17;	")//'":**+>H&&}} 	BGS	s   (A A2A--A2c                    < V ^8  d   QhRS[ /# r   r   r=   r   r   r   r   r      s     " "d "r   c                s   V P                   '       g   R # \        P                  RV P                   4       . pV P                    FH  pVP                  RV^ ,          RV^,          RRRRRV^,          R\        P                  ! 4       /4       KJ  	  R	p V P                  4       ;_uu_ 4        V P                  P                  W14       R
R
R
4       R #   + '       g   i     R # ; i  \        P                   d7   p\        P                  RTP                  P                  T4        R
p?R# R
p?i\         d7   p\        P                  RTP                  P                  T4        R
p?R# R
p?ii ; i)TzRestoring old phrases: %sr   r    r!    r"   r#   r$   
        INSERT INTO user_db.phrases (input_phrase, phrase, p_phrase, pp_phrase, user_freq, timestamp)
        VALUES (:input_phrase, :phrase, :p_phrase, :pp_phrase, :user_freq, :timestamp)
        ;Nz,Database error restoring old phrases: %s: %szEUnexpected error inserting old phrases into the user database: %s: %sF)r.   r)   r]   appendtimerF   r4   executemanyrA   rB   rC   rD   r   	Exception)r9   sqlargsZophrasesqlstrrE      &    r   r6   Z TabSqliteDb._restore_old_phrases   s3   
    /1B1BC((GNN71:Rbgajdiik+, )	1!!##))&: $ $#}} 	1>((%1 1   	11((%1 1 	1sH   C$ *CC$ C!	C$ !C$ $E.9+D**E.7E.8+E))E.c                   < V ^8  d   QhRR/# r   r   Nr
   r   r   r   r   r      s     31 31t 31r   c                s   \         P                  R 4       V P                  R8X  d   \         P                  R4       R# \        P                  P                  V P                  4      '       g#   \         P                  RV P                  4       R#  V P                  V P                  4      pV'       dh   VR,          \        8X  dV   V P                  V P                  4      \        V P                  4      8X  d#   \         P                  RV P                  4       R# \         P                  RV P                  4       V'       g   \         P                  R4       MVR,          \        8w  d$   \         P                  R	\        VR,          4       MvV P                  V P                  4      \        V P                  4      8w  dD   \         P                  R
\        V P                  4      V P                  V P                  4      4       V P                  4       V n        V P                  4        R#   \         d7   p\         P                  RTP                   P"                  T4        Rp?R# Rp?ii ; i)z#Checking database compatibility ...r   zK:memory: database is always newly created  and therefore always compatible.NzgUser database %s does not exist yet and will be created. A newly created database is always compatible.versionzCompatible database %s found.z$User database %s seems incompatible.z&No version information in the databasezThe version of the database does not match (too old or too new?).ibus-typing-booster wants version=%s but the database actually has version=%s.zThe number of columns of the database does not match.ibus-typing-booster expectes %s columns but the database actually has %s columns.z8Unexpected error checking database compatibility: %s: %s)r)   r]   r   r&   r+   existsget_database_descUSER_DATABASE_VERSION%get_number_of_columns_of_phrase_tablelenr-   _extract_user_phrasesr.   '_rename_incompatible_or_broken_databaserg   rC   rD   r   )r9   descrE      &  r   r1   Z)TabSqliteDb._check_database_compatibility   s   9:
*KK45 ww~~d//00KKA!!# %	1))$*;*;<DY#88::4;L;LMt66783T5F5FHKK>))+ DEi$99 H 24	?	D
 <<T=N=NO889:8 778>>t?P?PQS !% : : <D88: 	1J((%1 1	1s,   "H* 3A%H* 'H* C&H* *I+5+I&&I+c                r_   r`   ra   r   r   r   r   r   '  s      T r   c                s   \         P                  R 4       V P                  R8X  d   \         P                  R4       R# \        P                  P                  V P                  4      '       g#   \         P                  RV P                  4       R# Rp V P                  P                  V P                  4      ;_uu_ 4       pVP                  R4      P                  4        RRR4       \         P                  R4       R#   + '       g   i     L'; i  \         dG   p\         P                  RTP                  P                  T4       T P                  4         Rp?R	# Rp?ii ; i)
z Checking database readability...r   zI:memory: database is always newly created  and therefore always readable.TzeUser database %s does not exist yet and will be created. A newly created database is always readable.NSELECT * FROM phrases;zDatabase seems readable.z+Error checking database readability: %s: %sF)r)   r]   r   r&   r+   rn   rD   r[   r@   fetchallrg   rC   r   rt   )r9   r4   rE   rv   r   r2   Z'TabSqliteDb._check_database_readability'  s    	67
*KK23 ww~~d//00KK?!!# 
	;88%%' '*2  !9:CCE' KK23	' '
  	;=((%1 88::	;s0   -D ? C>D >D		D E";EE"c                rk   rl   r
   r   r   r   r   r   E  s       r   c                s   \         P                  ! R 4      p\        P                  RV P                  V,           4       \
        P                  P                  V P                  4      '       d3   \
        P                  ! V P                  V P                  V,           4       \
        P                  P                  V P                  R,           4      '       dA   \
        P                  ! V P                  R,           V P                  R,           V,           4       \
        P                  P                  V P                  R,           4      '       dA   \
        P                  ! V P                  R,           V P                  R,           V,           4       \        P                  RV P                  4       V P                  P                  V P                  4      V n        R# )z-%Y-%m-%d_%H:%M:%Sz0Renaming incompatible or broken database to "%s"z-shmz-walz$Creating a new, empty database "%s".N)re   strftimer)   r]   r   r&   r+   rn   ZrenamerD   r3   r4   )r9   r$      & r   rt   Z3TabSqliteDb._rename_incompatible_or_broken_databaseE  s6    MM"67	F%%i/	177>>$++,,IId''''	1377>>$++F233IId''.''.y8:77>>$++F233IId''.''.y8:2D4E4E	GFFr   rb   c                >   < V ^8  d   QhRS[ RS[ RS[ RS[ RS[RS[/# r   r   r    r!   r"   r#   r   r   r%   r=   r   r   r   r   r   \  sG     - -- - 	-
 - - $(-r   c                s   V'       d	   V'       g   R # \         P                  ! VP                  4       4      p\        P                  ! \         P
                  V4      p\         P                  ! VP                  4       4      p\         P                  ! VP                  4       4      pRpRVRVRVRVRVR\        P                  ! 4       /p\        ^8  d-   \        P                  RV4       \        P                  R	V4        V P                  4       ;_uu_ 4        V P                  P                  Wg4       R
R
R
4       R #   + '       g   i     R # ; i  \        P                   d7   p\        P                  RTP                   P"                  T4        R
p?R# R
p?i\$         d7   p\        P                  RTP                   P"                  T4        R
p?R# R
p?ii ; i)Tz
        UPDATE user_db.phrases
        SET user_freq = :user_freq, timestamp = :timestamp
        WHERE input_phrase = :input_phrase
         AND phrase = :phrase AND p_phrase = :p_phrase AND pp_phrase = :pp_phrase
        r#   r   r    r!   r"   r$   	sqlstr=%s
sqlargs=%sNz&Database error updating phrase: %s: %sz3Unexpected error updating phrase in user_db: %s: %sF)itb_utilremove_accentslowerunicodedata	normalizeNORMALIZATION_FORM_INTERNALre   r'   r)   r*   rF   r4   r@   rA   rB   rC   rD   r   rg   )	r9   r   r    r!   r"   r#   ri   rh   rE   	   &&&&&&   r   update_phraseZTabSqliteDb.update_phrase\  ss    6..|/A/A/CD&&00&:**8>>+;<++IOO,=>	 	!<Vx			- ?LLf-LLw/	1!!##%%f6 $ $#}} 	18((%1 1 	  	1E((%1 1 		1sH   ?E D=3E =E	E E G&+FG$G%+GGc                rk   rl   r
   r   r   r   r   r     s     1 1D 1r   c                sb   \         P                  R 4        V P                  P                  4        V P                  P	                  R4       \         P                  R4       R#   \
        P                   d7   p\         P                  RTP                  P                  T4        Rp?R# Rp?ii ; i)z!commit and execute checkpoint ...r>   z#commit and execute checkpoint done.z.Unexpected error syncing user database: %s: %sN)
r)   r]   r4   r?   r@   rA   ZOperationalErrorrC   rD   r   r9   rE   rz   r   
sync_usrdbZTabSqliteDb.sync_usrdb  s    78		1MM  " MM!!"DEKK=>'' 	1@((%1 1	1s   A
A# #B.8+B))B.c                r_   r`   ra   r   r   r   r   r     s      t r   c                s   \         P                  R 4        V P                  4       ;_uu_ 4        V P                  P	                  R4       RRR4       \         P                  R4       R#   + '       g   i     L'; i  \
        P                   d7   p\         P                  RTP                  P                  T4        Rp?R# Rp?i\         d7   p\         P                  RTP                  P                  T4        Rp?R# Rp?ii ; i)zCreating tables...a7  
                CREATE TABLE IF NOT EXISTS user_db.phrases
                (id INTEGER PRIMARY KEY,
                input_phrase TEXT,
                phrase TEXT,
                p_phrase TEXT,
                pp_phrase TEXT,
                user_freq INTEGER,
                timestamp REAL)
                NzTables created.Tz&Database error creating tables: %s: %sz(Unexpected error creating tables: %s: %sF)r)   r]   rF   r4   r@   rA   rB   rC   rD   r   rg   r   rz   r   r5   ZTabSqliteDb.create_tables  s    
 	()	1!!##%% 	' 	 $ KK)* $# }} 	18((%1 1 	  	1:((%1 1 		1s@   A= A*A= *A:	5A= =D+CDD+DDc                r{   r|   r}   r   r   r   r   r     sG     > >> > 	>
 > > $(>r   c                sz   \         ^8  d   \        P                  RWV4       V'       d	   V'       g   R# \        P                  ! VP                  4       4      p\        P                  ! \        P                  V4      p\        P                  ! VP                  4       4      p\        P                  ! VP                  4       4      pRpRVRVRVRV/pV P                  P                  Wg4      P                  4       '       d   R# RpRVRVRVRVR	VR
\        P                  ! 4       /p	\         ^8  d-   \        P                  RV4       \        P                  RV	4        V P                  4       ;_uu_ 4        V P                  P                  W4       RRR4       R#   + '       g   i     R# ; i  \        P                   d7   p
\        P!                  RT
P"                  P$                  T
4        Rp
?
R# Rp
?
i\&         d7   p
\        P!                  RT
P"                  P$                  T
4        Rp
?
R# Rp
?
ii ; i)   z'input_phrase=%r phrase=%r user_freq=%s Tz
        SELECT * FROM user_db.phrases
        WHERE input_phrase = :input_phrase
        AND phrase = :phrase AND p_phrase = :p_phrase AND pp_phrase = :pp_phrase
        r   r    r!   r"   z
        INSERT INTO user_db.phrases
        (input_phrase, phrase, p_phrase, pp_phrase, user_freq, timestamp)
        VALUES (:input_phrase, :phrase, :p_phrase, :pp_phrase, :user_freq, :timestamp)
        r#   r$   zinsert_sqlstr=%szinsert_sqlargs=%sNz%Database error adding phrases: %s: %sz2Unexpected error adding phrase to database: %s: %sF)r'   r)   r*   r   r   r   r   r   r   r4   r@   rx   re   rF   rA   rB   rC   rD   r   rg   )r9   r   r    r!   r"   r#   Zselect_sqlstrZselect_sqlargsZinsert_sqlstrZinsert_sqlargsrE   s   &&&&&&     r   
add_phraseZTabSqliteDb.add_phrase  s    ?LL9i1 6..|/A/A/CD&&00&:**8>>+;<++IOO,=>	 Lf	$
 ==  ?HHJJ 
 ),"F$h%y%y%tyy{4 ?LL+];LL,n=	1!!##%%mD $ $#}} 	17((%1 1 	  	1D((%1 1 		1sH   F0 6FF0 F-	'F0 -F0 0H:+G66H:H:+H55H:c                r_   r`   ra   r   r   r   r   r     s       r   c                s   R p\         P                  R4        V P                  4       ;_uu_ 4        V P                  P	                  V4       RRR4       \         P                  R4       R#   + '       g   i     L'; i  \
        P                   d7   p\         P                  RTP                  P                  T4        Rp?R# Rp?i\         d7   p\         P                  RTP                  P                  T4        Rp?R# Rp?ii ; i)z
        CREATE INDEX IF NOT EXISTS user_db.phrases_index_p ON phrases
        (input_phrase, id ASC);
        CREATE INDEX IF NOT EXISTS user_db.phrases_index_i ON phrases
        (phrase)
        zCreating indexes...NzIndexes created.Tz'Database error creating indexes: %s: %sz)Unexpected error creating indexes: %s: %sF)r)   r]   rF   r4   rP   rA   rB   rC   rD   r   rg   )r9   ri   rE   rv   r   r7   ZTabSqliteDb.create_indexes  s    
 	)*	1!!##++F3 $KK*+ $# }} 	19((%1 1 	  	1;((%1 1 		1s@   A? A,A? ,A<	7A? ?D	+CD	D	+DD	c                sJ   < V ^8  d   QhRS[ RS[S[P                  ,          /# )r   r   r   r   r   r   ZPredictionCandidater   r   r   r   r     s(     ( (("&x'C'C"D(r   c                sN   \         P                  ! \        P                  V4      p\        ^8  d   \
        P                  RV4       / pRVR,           R\        P                  /pRp. p V P                  P                  WC4      P                  4       pV'       d   VP                  V4       \        P                   ! V4      p\        ^8  d   \
        P                  RV4       V#   \         d6   p\
        P                  RTP                  P                  T4        Rp?LRp?ii ; i)	r   zinput_phrase=%sr   Z%r#   zSELECT phrase, sum(user_freq) FROM user_db.phrases WHERE input_phrase LIKE :input_phrase AND user_freq >= :user_freq GROUP BY phrase;z>Unexpected error fetching user shortcuts from database: %s: %sNzbest_shortcut_candidates=%s)r   r   r   r   r'   r)   r*   SHORTCUT_USER_FREQr4   r@   rx   rg   rC   rD   r   updatebest_candidates)r9   r   phrase_frequenciesrh   ri   Zresults_shortcutsrE   Zbest_shortcut_candidates   &&      r   select_shortcutsZTabSqliteDb.select_shortcuts  s    #,,00,@?LL*L9/1!<##5 ; ;=% 46	2 $ 5 5!!!)  %%&78#+#;#;<N#O ?LL-/GI''  	27))52 2	2s   ))C$ $D$/+DD$c                sP   < V ^8  d   QhRS[ RS[ RS[S[P                  ,          /# )r   r!   r"   r   r   r   r   r   r   r   2  s4     9< 9<9< 9<  $H$@$@A9<r   c                s   \         ^8  d   \        P                  RW4       / pV'       d	   V'       g   \        P                  ! V4      # \        P
                  ! VP                  4       4      p\        P
                  ! VP                  4       4      pRVRV/pRpRp V P                  P                  WT4      P                  4       pV'       g   \        P                  ! V4      # Rp^ p V P                  P                  WT4      P                  4       ^ ,          ^ ,          pV'       g   \        P                  ! V4      # V F5  p	VP                  V	^ ,          V	^,          \        V4      ,          3.4       K7  	  \         ^8  d+   \        P                  R	\        P                  ! V4      4       \        P                  ! V4      #   \         d9   p\        P                  RTP                  P                  T4       Rp Rp?EL<Rp?ii ; i  \         d9   p\        P                  RTP                  P                  T4       ^ p Rp?EL(Rp?ii ; i)
r   zp_phrase=%s pp_phrase=%sr!   r"   zySELECT phrase, sum(user_freq) FROM user_db.phrases WHERE p_phrase = :p_phrase AND pp_phrase = :pp_phrase GROUP BY phrase;NzXUnexpected error getting phrases for empty input with given context from user_db: %s: %szaSELECT sum(user_freq) FROM user_db.phrases WHERE p_phrase = :p_phrase AND pp_phrase = :pp_phrase;z\Unexpected error getting total count for empty input with given context from user_db: %s: %sz5Best candidates for empty input with given context=%s)r'   r)   r*   r   r   r   r   r4   r@   rx   rg   rC   rD   r   r   float)
r9   r!   r"   r   rh   ri   resultsrE   count_pp_phrase_p_phraseresults
   &&&       r   select_words_empty_inputZ$TabSqliteDb.select_words_empty_input2  s    ?LL*HA/1y++,>??**8>>+;<++IOO,=>	xi@@ 	mm++F<EEGG ++,>??E 	 $% 	)'+}}'<'<(!!)A(//0(2$ (++,>??F%%)VAYu-E'FFGHJ  ?LLG(();<> ''(:;;=  	:((%1 G	  	):))52 ()$	)s0   )G )7H
 H-HH
I-IIc          	      sV   < V ^8  d   QhRS[ RS[ RS[ RS[S[P                  ,          /# )r   r   r!   r"   r   r   r   r   r   r   r   m  sJ     UN UNUN UN 	UN %))E)E$F	UNr   c           	     s   \         ^8  d   \        P                  RWV4       V'       g   V P                  W#4      # / p\        P
                  ! \        P                  V4      p\        P                  ! VP                  4       4      p\        P                  ! VP                  4       4      pVP                  4       pRV9  d+   VP                  V P                  P                  V4      4       \         ^8  d,   \        P                  R\        P                  ! WER7      4       \        P                  ! VP                  4       4      pVP                  RR4      P                  RR4      pV P                   P#                  R	4       R
V R2p V P                   P#                  V4       RVRV/p	Rp V P                   P#                  Wy4      P-                  4       p
X
'       g   \        P                  ! WER7      # Rp V P                   P#                  Wy4      P-                  4       ^ ,          ^ ,          pV
 F5  pVP                  V^ ,          V^,          \/        X4      ,          3.4       K7  	  \         ^8  d,   \        P                  R\        P                  ! WER7      4       V'       g   \        P                  ! WER7      # Rp V P                   P#                  Wy4      P-                  4       pX'       g   \        P                  ! WER7      # Rp V P                   P#                  Wy4      P-                  4       ^ ,          ^ ,          pV FW  pVP                  V^ ,          RV^,          ,          \/        X4      ,          RWO^ ,          ,          ,          ,           3.4       KY  	  \         ^8  d,   \        P                  R\        P                  ! WER7      4       V'       g   \        P                  ! WER7      # Rp V P                   P#                  Wy4      P-                  4       pX'       g   \        P                  ! WER7      # Rp V P                   P#                  Wy4      P-                  4       ^ ,          ^ ,          pV FX  pVP                  V^ ,          RV^,          ,          \/        X4      ,          RVV^ ,          ,          ,          ,           3.4       KZ  	  \         ^8  d,   \        P                  R\        P                  ! WER7      4       \        P                  ! WER7      #   \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELRp?ii ; i  \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELRp?ii ; i  \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELRp?ii ; i  \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELCRp?ii ; i  \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?EL.Rp?ii ; i  \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELRp?ii ; i  \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELRp?ii ; i) r   z(input_phrase=%s p_phrase=%s pp_phrase=%s zhunspell: best_candidates=%s)ZtitleZ rb   Z'z''z+DROP VIEW IF EXISTS like_input_phrase_view;z
        CREATE TEMPORARY VIEW IF NOT EXISTS like_input_phrase_view AS
        SELECT * FROM user_db.phrases
        WHERE input_phrase LIKE 'z%'
        ;z2Unexpected error in creating database view: %s: %sNr!   r"   zJSELECT phrase, sum(user_freq) FROM like_input_phrase_view GROUP BY phrase;u@   Unexpected error getting “unigram” data from user_db: %s: %sz2SELECT sum(user_freq) FROM like_input_phrase_view;zAUnexpected error getting total unigram count from user_db: %s: %szUnigram best_candidates=%szeSELECT phrase, sum(user_freq) FROM like_input_phrase_view WHERE p_phrase = :p_phrase GROUP BY phrase;u?   Unexpected error getting “bigram” data from user_db: %s: %szMSELECT sum(user_freq) FROM like_input_phrase_view WHERE p_phrase = :p_phrase;z@Unexpected error getting total bigram count from user_db: %s: %sg      ?zBigram best_candidates=%szSELECT phrase, sum(user_freq) FROM like_input_phrase_view WHERE p_phrase = :p_phrase AND pp_phrase = :pp_phrase GROUP BY phrase;u@   Unexpected error getting “trigram” data from user_db: %s: %szhSELECT sum(user_freq) FROM like_input_phrase_view WHERE p_phrase = :p_phrase AND pp_phrase = :pp_phrase;zAUnexpected error getting total trigram count from user_db: %s: %szTrigram best_candidates=%s)r'   r)   r*   r   r   r   r   r   r   r   Zistitler   r0   Zsuggestr   rJ   r4   r@   rg   rC   rD   r   rx   r   )r9   r   r!   r"   r   Z
title_caseZsql_escaped_input_phraseri   rE   rh   Zresults_uniZcountZ
result_uniZ
results_biZcount_p_phraseZ	result_biZresults_trir   Z
result_tris   &&&&               r   select_wordsZTabSqliteDb.select_wordsm  s    ?LL:	3 00EE/1",,00,@**8>>+;<++IOO,=>	!))+
l" %%d&7&7&?&?&MN?LL.(();NP  ..|/A/A/CD $0#7#7B$T* 	!KL" #;!; <

	1MM!!&)
 xi@ 		1 --//@IIKK  ++,>QQ
 F	2MM))&:CCEaHKE &J%%Q-AuU|!;<=? & ?LL,(();NP  ++,>QQ: 		1..v?HHJJ ++,>QQ* 		2!]]22!!)A//02N $I%%A,il"5#88)A,7789 :; $
 ?LL+(();NP  ++,>QQ@	2--//@IIKK ++,>QQE 		2'+}}'<'<(!!)A(//0(2$ &J%%Q-jm#E*B$CC)*Q-889: ;< &
 ?LL,(();NP ''(:MMa  	1D((%1 1	18  	1R((%1 1	1   	2'))52 2	20  	1'((%1 1	1  	2'))52 2	24  	2'))52 2	2   	2'))52 2	2s   
T' .)U+ 97V/ )W3 7X7 :)Y; 7Z? 'U(2+U##U(+V,6+V''V,/W0:+W++W03X4>+X//X47Y8+Y33Y8;Z<+Z77Z<?\ 
+[;;\ c                r_   r`   ra   r   r   r   r   r   D  s        d  r   c                s   \         P                  R 4        V P                  4       ;_uu_ 4        V P                  P	                  R4       V P                  P	                  RR\
        34       V P                  P	                  R4       RRR4       \         P                  R4       R#   + '       g   i     L'; i  \        P                   d7   p\         P                  RTP                  P                  T4        Rp?R
# Rp?i\         d7   p\         P                  R	TP                  P                  T4        Rp?R
# Rp?ii ; i)zGenerate userdb descriptionz{
                    CREATE TABLE IF NOT EXISTS user_db.desc
                    (name PRIMARY KEY, value)
                z2INSERT OR IGNORE INTO user_db.desc  VALUES (?, ?);rm   z
                    INSERT OR IGNORE INTO user_db.desc
                    VALUES ('create-time', DATETIME('now', 'localtime'))
                Nzuserdb description generatedTz3Database error generating description table: %s: %sz6Unexpected error adding description to user_db: %s: %sF)r)   r]   rF   r4   r@   rp   rA   rB   rC   rD   r   rg   r   rz   r   r8   Z TabSqliteDb.generate_userdb_descD  s	    	12	2!!##%% '  %%H 568 %% '  $ KK67 $# }} 	1E((%1 1 	  	2H))52 2 		2sA   B: AB'B: 'B7	2B: :E+D  EE+D??Ec                sL   < V ^8  d   QhRS[ RS[S[S[ S[ 3,          ,          /# r\   )r   r   r   r   r   r   r   r   g  s(       c3h0H r   c                s   \         P                  P                  V4      '       g   R # R p V P                  V4      ;_uu_ 4       p/ pVP	                  R4      P                  4        F  pV^,          W4^ ,          &   K  	  VuuR R R 4       #   + '       g   i     R # ; i  \        P                  \        3 d7   p\        P                  RTP                  P                  T4        R p?R # R p?i\        \        3 d7   p\        P                  RTP                  P                  T4        R p?R # R p?i\         d7   p\        P                  RTP                  P                  T4        R p?R # R p?ii ; i)NzSELECT * FROM descz-Database/OS error getting description: %s: %sz%Data format error in database: %s: %sz5Unexpected error getting database description: %s: %s)r&   r+   rn   r[   r@   rx   rA   rB   OSErrorr)   rC   rD   r   r(   
IndexErrorrg   )rR   rH   r4   ru   rowrE   s   &&    r   ro   ZTabSqliteDb.get_database_descf  s5    ww~~g&&	2--g66(#++,@AJJLC#&q6DQL M	 7666"  w' 	1?((%1 1  :& 	17((%1 1 	  	2G))52 2 		2sY   B <B 
B B	B B E4:+C++E4>E4?+D00E4=E4>+E//E4c                s6   < V ^8  d   QhRS[ RS[S[,          /# r\   )r   r   r%   r   r   r   r   r     s!     , ,,"*3-,r   c                s   \         P                  P                  V4      '       g   R # R p V P                  V4      ;_uu_ 4       pVP	                  R4      P                  4       pV'       d   V^ ,          ^ ,          '       g    R R R 4       ^ # RP                  V^ ,          ^ ,          P                  4       4      p\        P                  ! RV4      pV'       d6   VP                  ^4      P                  R4      p\        V4      uuR R R 4       #  R R R 4       ^ #   + '       g   i     ^ # ; i  \        P                  \        \         \"        3 d7   p\$        P'                  RTP(                  P*                  T4        R p?^ # R p?i\,         d7   p\$        P'                  RTP(                  P*                  T4        R p?^ # R p?ii ; i)Nz3select sql from sqlite_master where name='phrases';r   z
.*\((.*)\)Z,z'Error getting number of columns: %s: %sz>Unexpected error getting number of columns of database: %s: %s)r&   r+   rn   r[   r@   rx   r,   Z
splitlinesrematchZgroupZsplitrr   rA   rB   r   r   AttributeErrorr)   rC   rD   r   rg   )rR   rH   r4   Ztable_phrases_resultZstringr   Ztable_phrases_columnsrE   r   r   rq   Z1TabSqliteDb.get_number_of_columns_of_phrase_table  sm   $ ww~~g&&	2--g66('/'7'7I((* % ,3G3J13M3M 76 "6q"9!"<"G"G"IJ7,1KKN,@,@,E)45 76  7666.  w
NC 	19((%1 1   	2&))52 2 	2sl   D- =DDD- AD*D
D- DD- D*	$D- *D- -%G+FGG+GGc                sF   < V ^8  d   QhRS[ S[S[S[3,          ,          /# r`   )r   r   r   r   r   r   r   r     s!      T%S/%: r   c                s0   R pR\         P                  /p\        ^8  d-   \        P	                  RV4       \        P	                  RV4       V P
                  P                  W4      P                  4       p\        ^8  d   \        P	                  RV4       V# )z\
        SELECT input_phrase, phrase FROM user_db.phrases WHERE user_freq >= :freq
        ;Zfreqr~   r   z	result=%s)r   r   r'   r)   r*   r4   r@   rx   )r9   ri   rh   r   s   &   r   list_user_shortcutsZTabSqliteDb.list_user_shortcuts  su     8667?LLf-LLw/&&v7@@B?LLf-r   c                2   < V ^8  d   QhRS[ RS[ RS[RS[/# )r   r   r    r#   r   r}   r   r   r   r   r     s4     - -- - 	- >B	-r   c           	     sf   \         ^8  d   \        P                  RWV4       V'       d   V'       d   V\        P                  8  d#   \         ^8  d   \        P                  R4       R# \
        P                  ! \        P                  V4      p\
        P                  ! \        P                  V4      pRVRVRVR\        P                  ! 4       /pRp V P                  4       ;_uu_ 4        V P                  P                  WT4       R	R	R	4       R
#   + '       g   i     R
# ; i  \        P                   d7   p\        P                  RTP                  P                   T4        R	p?R# R	p?i\"         d7   p\        P                  RTP                  P                   T4        R	p?R# R	p?ii ; i)r   z&input_phrase=%s phrase=%s user_freq=%szNot defining shortcut.Fr   r    r#   r$   zINSERT INTO user_db.phrases (input_phrase, phrase, p_phrase, pp_phrase, user_freq, timestamp)VALUES (:input_phrase, :phrase, "", "", :user_freq, :timestamp);NTz(Database error defining shortcut: %s: %sz*Unexpected error defining shortcut: %s: %s)r'   r)   r*   r   r   r   r   r   re   rF   r4   r@   rA   rB   rC   rD   r   rg   )r9   r   r    r#   rh   ri   rE   s   &&&&   r   define_user_shortcutZ TabSqliteDb.define_user_shortcut  s\    ?LL8i1 F8666Q56&&00&: #,,00,@!<V			-
O 		1!!##%%f6 $ $#}} 	1:((%1 1 	  	1<((%1 1 		1sH   D& ,DD& D#	D& #D& &F0;+E,,F09F0:+F++F0c                r   )r   r   r    user_freq_incrementr   r}   r   r   r   r   r     s9     G GG G "%	G .2	Gr   c                s    \         ^8  d   \        P                  RWV4       VP                  4       pVP                  4       pV'       d	   V'       g   R# \        P
                  ! \        P                  V4      p\        P
                  ! \        P                  V4      pRVR\        P                  R\        P                  ! 4       /pRp. p V P                  P                  WT4      P                  4       pV'       g   R# V F  w  rW8X  d$   \         ^8  d   \        P                  R	WV	34       W,           p
\         ^8  d   \        P                  R
WV	3WV
34       WR&   WR&   Rp V P#                  4       ;_uu_ 4        V P                  P                  WT4       RRR4       K  	  R#   \         d6   p\        P                  RTP                  P                   T4        Rp?LRp?ii ; i  + '       g   i     LY; i  \$        P&                   d8   p\        P                  RTP                  P                   T4        Rp? R# Rp?i\         d8   p\        P                  RTP                  P                   T4        Rp? R# Rp?ii ; i)r   u<   input_phrase=“%s” phrase=“%s” user_freq_increment=%sFr    r#   r$   zgSELECT input_phrase, user_freq FROM user_db.phrases WHERE phrase = :phrase AND user_freq >= :user_freq;z;Unexpected error checking user shortcuts in user_db: %s: %sNzExactly matched shortcut: %sz(Increase shortcut frequency by: %s -> %sr   zUPDATE user_db.phrases SET user_freq = :user_freq, timestamp = :timestamp WHERE input_phrase = :input_phrase AND phrase = :phrase ;z(Database error updating shortcut: %s: %sz5Unexpected error updating shortcut in user_db: %s: %sT)r'   r)   r*   Zstripr   r   r   r   r   re   r4   r@   rx   rg   rC   rD   r   rF   rA   rB   )r9   r   r    r   rh   ri   r   rE   Zdb_input_phraseZdb_user_freqZnew_user_freqs   &&&&       r   #check_shortcut_and_update_frequencyZ/TabSqliteDb.check_shortcut_and_update_frequency  s;    ?LLN&9; $))+6&&00&:",,00,@V ; ;		-1 	2mm++F<EEGG
 /6+_.?LL6(,?A )>MQ>$l;$m<> $1K &5N#) 
5%%''MM))&: (- 07@ M  	2M))52 2	24 (' == 5  >OO,,e5 5 	  5  KOO,,e5 5 	5sZ   )F+ )HG.H+G+6+G&&G+.G>9HJ+IJJ+JJc                r{   )r   r   r    r!   r"   r   r   r}   r   r   r   r   r   6  sO     T+ T+T+ T+ 	T+
 T+ "%T+ .2T+r   c                s   \         ^8  d   \        P                  RWW4V4       V'       d   V P                  VVVR7      '       d   R# V'       g   TpV'       g   R# \        P
                  ! V4      p\        P
                  ! V4      p\        P                  ! \        P                  V4      p\        P                  ! VP                  4       4      p\        P                  ! VP                  4       4      p\        P                  ! VP                  4       4      pRpRVRVRVRV/p\         ^8  d-   \        P                  R	V4       \        P                  R
V4       V P                  P                  Wg4      P                  4       p\         ^8  d   \        P                  RV4       V'       d,   V P                  VVVVV^ ,          ^ ,          V,           R7      # V P                  VVVVVR7      # )r   ua   input_phrase=“%s” phrase=“%s” p_phrase=“%s” pp_phrase=“%s” user_freq_increment=%s)r   r    r   Tz
        SELECT max(user_freq) FROM user_db.phrases
        WHERE input_phrase = :input_phrase
        AND phrase = :phrase AND p_phrase = :p_phrase AND pp_phrase = :pp_phrase
        GROUP BY phrase
        ;r   r    r!   r"   z9TabSqliteDb.check_phrase_and_update_frequency() sqlstr=%sz:TabSqliteDb.check_phrase_and_update_frequency() sqlargs=%sz-check_phrase_and_update_frequency() result=%s)r   r    r!   r"   r#   )r'   r)   r*   r   r   Zstrip_tokenr   r   r   r   r   r4   r@   rx   r   r   )	r9   r   r    r!   r"   r   ri   rh   r   r   r   !check_phrase_and_update_frequencyZ-TabSqliteDb.check_phrase_and_update_frequency6  s    ?LL)h#	% 77!-!(; 8 = = !L++L9%%f-&&00&:**8>>+;<++IOO,=>	..|/A/A/CD "<Vx	+ ?LLK LLL &&v7@@B?LL?I %%)!# )A,':: & < < %)  + 	+r   c                &   < V ^8  d   QhRS[ RS[/# )r   r    r   )r   r%   r   r   r   r   r     s      C C r   c                sb   \         ^8  d   \        P                  RV4       V'       g   ^ # \        P                  ! \
        P                  V4      p V P                  P                  RV34      P                  4       pV'       d   V^ ,          e   \        V^ ,          4      # ^ #   \        P                   d7   p\        P                  RTP                  P                  T4        Rp?^ # Rp?i\          d7   p\        P                  RTP                  P                  T4        Rp?^ # Rp?ii ; i)r   	phrase=%rz;SELECT sum(user_freq) FROM user_db.phrases WHERE phrase = ?Nz,Error checking whether phrase exists: %s: %sz7Unexpected error checking whether phrase exists: %s: %s)r'   r)   r*   r   r   r   r   r4   r@   rQ   r%   rA   rB   rC   rD   r   rg   )r9   r    r   rE   r^   r   phrase_existsZTabSqliteDb.phrase_exists  s     ?LLf-&&00&:	2--''M	 hj  #&#a&*<3s1v;C!C}} 	1>((%1 1 	  	2I))52 2 		2s6   2B$ B$ "B$ $D.9+C**D.7D.8+D))D.c                s,   < V ^8  d   QhRS[ RS[ RS[/# )r   r   r    r   r   r=   r   r   r   r   r     s)     & && & "&&r   c                s   V'       d   \        V\        4      '       g   R # \        ^8  d   \        P	                  RV4       \
        P                  ! \        P                  V4      pV'       d&   \
        P                  ! \        P                  V4      pRV'       d   RMR R2pRVRV/p V P                  4       ;_uu_ 4        V P                  P                  W44       RRR4       R	#   + '       g   i     R	# ; i  \        P                   d7   p\        P                  R
TP                  P                   T4        Rp?R # Rp?i\"         d7   p\        P                  RTP                  P                   T4        Rp?R # Rp?ii ; i)Fr   z;
            DELETE FROM user_db.phrases
            WHERE z!input_phrase = :input_phrase AND rb   z&
            phrase = :phrase
        r   r    NTz&Database error removing phrase: %s: %sz(Unexpected error removing phrase: %s: %s)Z
isinstancer   r'   r)   r*   r   r   r   r   rF   r4   r@   rA   rB   rC   rD   r   rg   )r9   r   r    Zdelete_sqlstrZdelete_sqlargsrE   rZ   r   remove_phraseZTabSqliteDb.remove_phrase  s>    Z44?LLf-&&00&:&0044lDL:F6BO P	
 ),&I	2!!##%%mD $ $#}} 	18((%1 1 	  	2:))52 2 		2sH   +C= C)C= )C:	4C= :C= =F+EFF+FFc                sJ   < V ^8  d   QhRS[ S[S[S[S[3,          ,          /# r`   )r   r   r   r%   r   r   r   r   r     s$      tE#sC-,@'A r   c                sN   \         P                  R 4       Rp V P                  P                  V P                  4      ;_uu_ 4       pVP                  R4       VP                  R4      P                  4       pV Uu. uFc  p\        P                  ! \        P                  V^ ,          4      \        P                  ! \        P                  V^,          4      V^,          3NKe  	  upuuRRR4       # u upi   + '       g   i     . # ; i  \        P                  \        \        \        3 d7   p\         P!                  RTP                  P"                  T4        Rp?. # Rp?i\$         d7   p\         P!                  RTP                  P"                  T4        Rp?. # Rp?ii ; i)zTrying to recover phrases from old, incompatible databasePhrases successfully recovered can be used to initialize a newly created database.NPRAGMA wal_checkpoint;z
                    SELECT input_phrase, phrase, sum(user_freq)
                    FROM phrases GROUP BY phrase
                z%Error extracting user phrases: %s: %sz0Unexpected error extracting user phrases: %s: %s)r)   r]   rD   r[   r   r@   rx   r   r   r   r   rA   rB   r   r   r   rC   r   rg   )r9   r4   ZphrasesrM   rE   rj   r   rs   Z!TabSqliteDb._extract_user_phrases  sz   3	4 	288%%' '*2  !9:"** ,  XZ  % % !** <<adD ** <<adDqT	
 %' '' '. 	 w
NC 	17((%1 1 		  	2B))52 2 			2sZ   -D
 5C6;A)C1$C6&
D
 1C66D	D
 D
 
%F$/+E  F$-F$.+FF$c                r   )r   filenamer   r   r   r   r   r   r     s     f fS fT fr   c                s   \         P                  P                  V4      '       g1   VR ,          p\         P                  P                  V4      '       g   R# \        pVP	                  R 4      '       d   \
        P                  pV P                  P                  R4      P                  4       p\        VR R7      p\        P                  ! 4       p\        P                  ! 4       pV'       d!   V^ ,          ^,          pVR,          ^,          pVRWT,
          ,          ,           p\        P                  R\        P                  ! R\        P                  ! V4      4      4       \        P                  R\        P                  ! R\        P                  ! V4      4      4       \        P                  R\        P                  ! R\        P                  ! V4      4      4       R	pR	p/ p	V F  p
\        P                   ! V
^ ,          P#                  4       4      p\$        P&                  ! \        P(                  V
^,          4      p\        P                   ! V
^,          P#                  4       4      p\        P                   ! V
^,          P#                  4       4      pV	P+                  V
^ ,          V
^,          V
^,          V
^,          3R
VRVRVRVRV
^,          RV
^,          /3.4       EK  	  . p V! VRRR7      ;_uu_ 4       pVP-                  4        Uu. uF(  p\$        P&                  ! \        P(                  V4      NK*  	  ppRRR4       T EF.  p\        P6                  ! T4       EF  p\        P                   ! TP#                  4       4      T\        P                   ! TP#                  4       4      \        P                   ! TP#                  4       4      3pTT	9   d   T	T,          R;;,          ^,          uu&   MwR
\        P                   ! TP#                  4       4      RTR\        P                   ! TP#                  4       4      R\        P                   ! TP#                  4       4      R^RT/T	T&   TpTpEK  	  EK1  	  . pT	P9                  4        F  w  ppTP;                  T4       K  	  Rp T P                  P                  R4       T P                  P=                  4        T P                  P?                  TT4       T P                  P=                  4        T P                  P                  R4       R# u upi   + '       g   i     EL; i  \.         d7   p\        P1                  RTP2                  P4                  T4        Rp?R# Rp?ii ; i  \.         d7   p\        P1                  RTP2                  P4                  T4        Rp?R# Rp?ii ; i)z.gzFc                 s&    \        V ^,          4      # i   )r   rL   rN   r   rO   Z:TabSqliteDb.read_training_data_from_file.<locals>.<lambda>  s    U1Q4[r   keyg?z$Minimum timestamp in the database=%s%Y-%m-%d %H:%M:%Sz$Maximum timestamp in the database=%sz New timestamp in the database=%srb   r   r    r!   r"   r#   r$   ZrtzUTF-8)rS   rK   Nz8Unexpected error reading training data from file: %s: %src   DELETE FROM phrases;r   z:Unexpected error writing training data to database: %s: %sTzTSELECT input_phrase, phrase, p_phrase, pp_phrase, user_freq, timestamp FROM phrases;) r&   r+   ZisfileZopenZendswithgzipr4   r@   rx   sortedre   r)   r]   ry   gmtimer   r   r   r   r   r   r   Z	readlinesrg   rC   rD   r   ZtokenizeZitemsrd   r?   rf   )r9   r   Zopen_functionrowsZtime_minZtime_maxZtime_newZp_tokenZpp_tokenZdatabase_dictr   r   r    r!   r"   ZlinesZfile_handleZlinerE   Ztokenr   rh   Zvalueri   s   &&                      r   read_training_data_from_fileZ(TabSqliteDb.read_training_data_from_file  s    ww~~h''H77>>(++.2U## IIM}}$$344<HJ 	 d"9:99;99;AwqzHBx{Hdh&9:::MM"5t{{87LM	O:MM"5t{{87LM	O6MM"5t{{87LM	OIKC $223q6<<>BL **44c!f>F..s1v||~>H //A?I  CFCFCFCF#C$2L$,f$.$/$/Q$/Q$9#$ "% & " 	4'; ;>I !, 5 5 79 !8  )) <<dD 7  9; D!**40..u{{}=..w}}?..x~~/?@B -'!#&{3q83 '(?(?(N %"H$;$;GMMO$L#X%<%<X^^=M%N#Q#X*/M#& #! 1 $ '--/JCNN5! 0	MM!!"89 MM  "MM%%fg6MM  "MM!!":; a9; ; ;  	J))52 		N  	L))52 		sa   -U  T=.T8T=U 0BV 8T==U	U U V+VVW +WWc                r_   r`   ra   r   r   r   r   r   \  s      D r   c                s    V P                  4       ;_uu_ 4        V P                  P                  R 4       RRR4       R#   + '       g   i     R# ; i  \        P                   d7   p\
        P                  RTP                  P                  T4        Rp?R# Rp?i\         d7   p\
        P                  RTP                  P                  T4        Rp?R# Rp?ii ; i)r   NTz+Database error removing all phrases: %s: %sz;Unexpected error removing all phrases from database: %s: %sF)
rF   r4   r@   rA   rB   r)   rC   rD   r   rg   r   rz   r   remove_all_phrasesZTabSqliteDb.remove_all_phrases\  s    	2!!##%%&<= $ $#}} 	1=((%1 1 	  	2M))52 2 		2sE   A A A  A	A A C)+BC'C(+CCc                rk   rl   r
   r   r   r   r   r   q  s     ? ?t ?r   c                s     \         P                  R 4       V P                  P                  R4      P	                  4        F"  p\         P                  R\        V4      4       K$  	  \         P                  R4       V P                  P                  R4      P	                  4        F"  p\         P                  R\        V4      4       K$  	  R#   \         d7   p\         P                  RTP                  P                  T4        Rp?R# Rp?ii ; i)zSELECT * FROM desc;
zSELECT * FROM desc;z%szSELECT * FROM phrases;
rw   z)Unexpected error dumping database: %s: %sN)
r)   r*   r4   r@   rx   reprrg   rC   rD   r   )r9   r   rE   rv   r   dump_databaseZTabSqliteDb.dump_databaseq  s    
	?LL01}},,-BCLLNT49- OLL34}},,,..6hj9T49-9  	?H#oo66? ?	?s   CC D+DDc                r_   r`   )r%   r   r   r   r   r     s      C r   c                s     \        V P                  P                  R 4      P                  4       4      #   \         d8   p\
        P                  RTP                  P                  T4       Ru Rp?# Rp?ii ; i)rw   z8Unexpected error getting number of database rows: %s: %sNr   )	rr   r4   r@   rx   rg   r)   rC   rD   r   r   rz   r   number_of_rows_in_databaseZ&TabSqliteDb.number_of_rows_in_database  sf    	t}},,(**2(*6 6 	J((%1 I		s   14 A6,A1+A61A6c                r   )r   threadr   Nra   r   r   r   r   r     s#     eL eLt eLt eLr   c                s   V'       d)   V P                   R 8X  d   \        P                  R4       R# \        P                  R4       \        P                  ! 4       p. pRp V'       d   V P	                  V P                   4      pMV P
                  pVP                  R4      P                  4       pV'       g[    V'       d1   V'       d)    VP                  4        \        P                  R4       R# V'       g   \        P                  R4       R# R# \        VR R	7      p\        P                  R
\        V4      4       \        V4      pRp^ p. p	V F  p
V
^,          pWg8  d   V\        P                  8  dz   \        P                  R\!        V
4      \        P"                  ! R\        P$                  ! V
^,          4      4      4       V^,          pRpRV
^ ,          /p VP                  W4       MV	P-                  V
4       V^,          pK  	  \        P                  RV4       \        V	R R	7      p	\        V	4      p\        P                  RV4       \/        VR,          4      p\        P                  RV4       ^ p^ pV	 EFS  p
V
^,          pWo8  Ed7   V\        P                  8  Ed!   V^8X  dz   \        P                  R\!        V
4      \        P"                  ! R\        P$                  ! V
^,          4      4      4       V^,          pRpRV
^ ,          /p VP                  W4       M\        P                  R\!        V
4      \        P"                  ! R\        P$                  ! V
^,          4      4      4       V^,          pRpRV
^ ,          R\/        V^,          4      R\        P                  ! 4       /p VP                  VV4       V^,          pEKV  	  \        P                  R4       VP1                  4        VP                  R4       \        P                  R4       VP                  R 4       \        P                  R!VV,           4       \        P                  R"V4       \        P                  R#\        V4      4       \        P                  R$\        V	4      V,
          4       \        P                  R%\        P                  ! 4       V,
          4       \        P                  R&4        T'       d1   T'       d)    TP                  4        \        P                  R4       R# T'       g   \        P                  R4       R# R#   \         d"   p\        P                  RT4        Rp?R# Rp?ii ; i  \         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELRp?ii ; i  \         d7   p\        P'                  RTP(                  P*                  T4        Rp?EL0Rp?ii ; i  \         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELtRp?ii ; i  \         d   p\        P'                  R'TP(                  P*                  T4        Rp?T'       d`   T'       dX    TP                  4        \        P                  R4       R#   \         d"   p\        P                  RT4        Rp?R# Rp?ii ; iT'       g   \        P                  R4       R# R# Rp?ii ; i  \         d"   p\        P                  RT4        Rp?R# Rp?ii ; i  T'       d^   T'       dV    TP                  4        \        P                  R4       i   \         d!   p\        P                  RT4        Rp?i Rp?ii ; iT'       g   \        P                  R4       i i ; i)(r   z0Database cleanup not needed for memory database.NzDatabase cleanup starting ...rw   z'Closed thread-local database connectionzFailed to close database: %sz4Reused main thread database connection (not closing)c                 s6    V ^,          V ^,          V ^ ,          3# r   r
   rL   rN   r   rO   .TabSqliteDb.cleanup_database.<locals>.<lambda>  s    ddd+r   r   z)Total number of database rows to check=%siP  z1st pass: deleting %s %sr   z#DELETE from phrases WHERE id = :id;r   z61st pass: exception deleting row from database: %s: %sz61st pass: Number of rows deleted above maximum size=%sc                 s&    V ^,          V ^ ,          3# )i   r
   rL   rN   r   rO   r     s    #$Q4#$Q40!r   z 1st pass: Number of rows kept=%sg+?z2nd pass: Index for decay=%sz2nd pass: deleting %s %sz62nd pass: exception deleting row from database: %s: %sz2nd pass: decaying %s %sz
                        UPDATE phrases
                        SET user_freq = :user_freq, timestamp = :timestamp
                        WHERE id = :id
                        ;r#   r$   z62nd pass: exception decaying row from database: %s: %sz*Commit database and execute checkpoint ...r   z)Rebuild database using VACUUM command ...zVACUUM;z"Number of database rows deleted=%sz"Number of database rows decayed=%sz Number of rows before cleanup=%szNumber of rows remaining=%sz$Time for database cleanup=%s secondszDatabase cleanup finished.z)Exception when accessing database: %s: %s)r   r)   r]   re   r3   r4   r@   rx   rW   rg   rX   r*   r   rr   r   r   r   ry   r   rC   rD   r   rd   r%   r?   )r9   r   Ztime_nowr   r4   rY   ZindexZmax_rowsZnumber_delete_above_maxZ	rows_keptr   r#   Zsqlstr_deleteZsqlargs_deleterE   Zindex_decayZnumber_of_rows_to_decayZnumber_of_rows_to_deleteZsqlstr_updateZsqlargs_updates   &&                  r   cleanup_databaseZTabSqliteDb.cleanup_database  sU    d'':5KKJK3499;ACO	L  ??%%'  ==##$<=FFHDx (PNN$KK IJ JL C $!D KKCSYOIEH&'#JLIF	$!H$?$??KK : $S	 $.A.2kk#a&.A!CD ,q0+$IM&*CF^N> ((G $$S)
) * KKP/1$ y&!"I
 	NEKK:EBh./KKK6D&'#'($ F	'!H$?$?? A~$>$(I$(MM2E26++c!f2E%GH 1A50(M*.AB$,,]K $>$(I$(MM2E26++c!f2E%GH 014/)
 +/A*5s9Q;7G*5tyy{*DB$,,]NK 
S !T KKDEOO56KKCDY'KK<03KKMKK</1KK:CIFKK5I)AACKK>		h.0KK45 (PNN$KK IJ JL  ! PNN#A;OOPO % >((4"__55u> >>f  ) B",,!8!&!9!95B BB,  ) B",,!8!&!9!95B BB*  	H#oo66?(PNN$KK IJ  PNN#A;OOPJL 	 ! PNN#A;OOP	 (PNN$KK IJ  PNN#A;OOPJL s+   AY 	%U CY U<0D)Y W +BY :XD'Y %\ U9U44U9<V=+V82Y 8V==Y  X+W<6Y <XY Y+Y :Y  YY \+\
>] %Z: :[&[!![&
\] \>\99\>_%]98_9^$^_^$$
_/_)r.   r-   r4   r0   r   N)zuser.db)FT)F)rb   rb   rb   rb   r    )rb   rb   )rb   rb   r   )rb   rb   rb   rb   r   )T)*r   r   r   r   r:   r   rF   ZclassmethodrT   r[   r3   r6   r1   r2   rt   r   r   r5   r   r7   r   r   r   r8   ro   rq   r   r   r   r   r   r   r   r   rs   r   r   r   r   r   r   Z__classdictcell__)r   s   @r   r   r   1   s    )$ )$V   B 5 5< P P  P, 
 
" "H31 31j < .- -^1 1 <> >@ 6( (@9< 9<vUN UNn   D  2 , ,\ " !#%88	- -^G GRT+ T+l >& &P Bf fP *? ?$ eL eL eLr   r   )Ztypingr   r   r   r   r   r   r   r&   r   Z
contextlibr   rA   re   r   r   Zloggingr   r/   Z	getLoggerr)   r%   r'   rp   rg   r	   r   r
   r   r   <module>r      st   ,        	  %   	    			0	1!f <i <FL FLr   