+
   Y<3c                   s   R t ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^ RI	t	^ RI
t
^ R	IHt ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt]P(                  ! R
4      t]! ^ 4      sRt ! R R]4      t ! R R4      tR# )z@
Module for ibus-typing-booster to access the sqlite3 databases
)Dict)List)Tuple)Optional)Any)Callable)IteratorN)contextmanagerzibus-typing-boosterz0.65c                   s    ] tR t^.tRtRtR# )DatabaseConnectionErrorz1Custom exception for database connection failures N)__name__
__module____qualname____firstlineno____doc____static_attributes__r	       4/usr/share/ibus-typing-booster/engine/tabsqlitedb.pyr   r   .   s    ;r   r   c                   s  a  ] tR t^1t o RtREV 3R lR llt]RFV 3R lR ll4       t]V 3R lR l4       t	]]RGV 3R lR	 ll4       4       t
]V 3R
 lR l4       tV 3R lR ltV 3R lR ltV 3R lR ltV 3R lR ltRHV 3R lR lltV 3R lR ltV 3R lR ltRHV 3R lR lltV 3R lR ltV 3R lR  ltV 3R! lR" ltRIV 3R# lR$ lltV 3R% lR& lt]V 3R' lR( l4       t]V 3R) lR* l4       tV 3R+ lR, ltRR]P:                  3V 3R- lR. lltRJV 3R/ lR0 lltRKV 3R1 lR2 llt V 3R3 lR4 lt!RIV 3R5 lR6 llt"V 3R7 lR8 lt#V 3R9 lR: lt$V 3R; lR< lt%V 3R= lR> lt&V 3R? lR@ lt'RLV 3RA lRB llt(RCt)V t*RD# )MTabSqliteDbu_  Phrase databases for ibus-typing-booster

The phrases table in the database has columns with the names:

“id”, “input_phrase”, “phrase”, “p_phrase”, “pp_phrase”, “user_freq”, “timestamp”

It is a database where the phrases learned from the user are stored.
user_freq >= 1: The number of times the user has used this phrase
c                $   < V ^8  d   QhRS[ RR/# )   user_db_filereturnN)strZformat__classdict__   "r   __annotate__TabSqliteDb.__annotate__=   s     )$ )$S )$ )$r   c                s     \        \        \        P                  ! R 4      4      4      s\        ^8  d   \        P                  RV4       Wn	        V P                  R8w  d   \        P                  P                  V P                  4      V P                  8X  dM   \        P                  P                  \        P                  P                  R4      V P                  4      V n	        V P                  '       g   \        P                  R4       RV n	        V P                  R8w  d   \        P                  P                  \        P                  P                  V P                  4      4      '       g@   \        P                   ! \        P                  P                  V P                  4      RR7       . ROV n        . V n        \&        P(                  ! R	4      V n        V P-                  4        V P/                  4        V P1                  V P                  4      V n        V P5                  4        V P7                  4        V P9                  4        V P;                  4        R#   \
        \        3 d    \        ^ 4      s ELRi ; i)
ZIBUS_TYPING_BOOSTER_DEBUG_LEVELz'TabSqliteDb.__init__(user_db_file = %s):memory:z"~/.local/share/ibus-typing-boosterz&Falling back to ":memory:" for user.dbT)Zexist_okN)idinput_phrasephrasep_phrase	pp_phrase	user_freq	timestampr	   )intr   osZgetenvDEBUG_LEVEL	TypeErrorZ
ValueErrorLOGGERdebugr   pathZbasenamejoinZ
expanduserZisdirZdirnameZmakedirs_phrase_table_column_names_old_phraseshunspell_suggestZHunspellhunspell_obj_check_database_compatibility_check_database_readabilitysqlite3_connect_database_legacydatabasecreate_tables_restore_old_phrasescreate_indexesgenerate_userdb_desc)selfr   s   &&r   __init__ZTabSqliteDb.__init__=   s   	!c")),M"NOPK ?LL9<I(+  !2!23t7H7HH "RWW-?-?4.67;7H7H!JD   LLAB *D+bggood6G6G&HIIKK(9(9:TJ+' 9;,55b9**,((*<<T=N=NO!!#!!#K :& 	!a&K	!s   (I I=<I=c                s:   < V ^8  d   QhRS[ RS[ RS[R,          /# )r   suppress_errors
checkpointr   N)boolr   r   r   r   r   r   i   s-      !  )1r   c              #  s$  "    Rx  V P                   P                  4        V'       d   V P                   P                  R4       R# R#   \        P                   d   p\
        P                  RTP                  P                  T4        T P                   P                  4        MT  \        P                   d=   p\
        P                  RTP                  P                  T4       \        R4      ThRp?ii ; iT'       g   h  Rp?R# Rp?ii ; i5i)zContext manager for atomic database transactions

:param suppress_errors: If True, non-critical
                        exceptions are logged
                        but not raised
:param checkpoint: If True, do a wal_checkpoint
N PRAGMA wal_checkpoint(TRUNCATE);z(Transaction failed, rolling back: %s: %szRollback failed! %s: %szRollback failed)r4   commitexecutesqlite3Errorr)   	exception	__class__r
   ZrollbackZcriticalr   )r9   r;   r<   errorZrollback_errors   &&&  r   transactionZTabSqliteDb.transactionh   s     	MM  " %%&HI  }} 	G"__55u>L &&(== L 9 % 8 8.J-.?@eKL # #	sY   DAA	 D	D+D
B%$D%C6:7C11C66
DDDDDc                s>   < V ^8  d   QhRS[ P                  RS[RR/# )r   database_connectiondb_filer   N)rA   
Connectionr   r   r   r   r   r      s+     5 5&-&8&85CF5KO5r   c                s    VP                  R4       VP                  RV R24       VP                  R4      P                  4       p\        P	                  RV4       R Vn        R# )z(Shared configuration for all connectionsa{  
        PRAGMA encoding = "UTF-8";
        PRAGMA case_sensitive_like = true;
        PRAGMA page_size = 4096;
        PRAGMA cache_size = 20000;
        PRAGMA temp_store = MEMORY;
        PRAGMA journal_mode = WAL;
        PRAGMA journal_size_limit = 1000000;
        PRAGMA synchronous = NORMAL;
        PRAGMA auto_vacuum = FULL;
        PRAGMA busy_timeout = 5000;
        zATTACH DATABASE "z" AS user_db;zPRAGMA journal_modezJournal mode: %sc                 s(    V P                  R RR7      # )zutf-8replace)encodingZerrors)Zdecodex   &r   <lambda>Z8TabSqliteDb._setup_database_connection.<locals>.<lambda>   s    ahh   4r   N)executescriptr@   fetchoner)   r*   Ztext_factory)clsrG   rH   modes   &&& r   _setup_database_connectionZ&TabSqliteDb._setup_database_connection   sl     	)) + 	 	))y6	8"**!##+8: 	'.4 	(r   c                sP   < V ^8  d   QhRS[ RS[RS[S[P                  ,          /# )r   rH   r;   r   )r   r=   r   rA   rI   r   r   r   r   r      s5     P PP!P.6w7I7I.JPr   c              #  s  "   Rp \         P                  ! V4      pV P                  W14       Vx  V'       d    VP                  4        R# R#   \         P                   d*   p\        P                  RY4       T'       g   h  Rp?LVRp?ii ; i  \         P                   d7   p\        P                  RTP                  P                  T4        Rp?R# Rp?ii ; i  T'       d`    TP                  4        i   \         P                   d6   p\        P                  RTP                  P                  T4        Rp?i Rp?ii ; ii ; i5i)z-For short-lived connections (automatic close)N#Database connection failed (%s): %sz"Failed to close connection: %s: %s)
rA   connectrT   rB   r)   rC   closewarningrD   r
   )rR   rH   r;   rG   rE   close_error   &&&   r   sqlite3_connect_databaseZ$TabSqliteDb.sqlite3_connect_database   s     #	P")//'":**+>H%% #P'--/ # }} 	BGS" #	 }} PNN#G#.#8#8#A#A;P PP #P'--/}} PNN#G#.#8#8#A#A;P PP #s   E	+A E	B 
E	B#BC BC C$+CE	CE		E'C87E8E+D=8E=EEE	c                s:   < V ^8  d   QhRS[ RS[P                  /# r   rH   r   )r   rA   rI   r   r   r   r   r      s#     
 

")"4"4
r   c                s    \         P                  RV4        \        P                  ! V4      pV P	                  W!4       V#   \        P
                   d   p\         P                  RY4       h Rp?ii ; i)z)For long-lived connections (manual close)zConnect to the database %s.rU   N)r)   inforA   rV   rT   rB   rC   )rR   rH   rG   rE      &&  r   r3   Z+TabSqliteDb.sqlite3_connect_database_legacy   sd     	17;	")//'":**+>H&&}} 	BGS	s   (A A2A--A2c                    < V ^8  d   QhRS[ /# r   r   r=   r   r   r   r   r      s     " "d "r   c                s   V P                   '       g   R# \        P                  RV P                   4       . pV P                    FH  pVP                  RV^ ,          RV^,          RRRRRV^,          R	\        P                  ! 4       /4       KJ  	  R
p V P                  4       ;_uu_ 4        V P                  P                  W14       RRR4       R#   + '       g   i     R# ; i  \        P                   d7   p\        P                  RTP                  P                  T4        Rp?R# Rp?i\         d7   p\        P                  RTP                  P                  T4        Rp?R# Rp?ii ; i)zmRestore phrases recovered from old database into new database

:return: True if successful, False on failure
TzRestoring old phrases: %sr   r    r!    r"   r#   r$   
        INSERT INTO user_db.phrases (input_phrase, phrase, p_phrase, pp_phrase, user_freq, timestamp)
        VALUES (:input_phrase, :phrase, :p_phrase, :pp_phrase, :user_freq, :timestamp)
        ;Nz,Database error restoring old phrases: %s: %szEUnexpected error inserting old phrases into the user database: %s: %sF)r.   r)   r]   appendtimerF   r4   executemanyrA   rB   rC   rD   r
   	Exception)r9   sqlargsZophrasesqlstrrE      &    r   r6   Z TabSqliteDb._restore_old_phrases   s3   
    /1B1BC((GNN71:Rbgajdiik+, )	1!!##))&: $ $#}} 	1>((%1 1   	11((%1 1 	1sH   C$ *CC$ C!	C$ !C$ $E.9+D**E.7E.8+E))E.c                   < V ^8  d   QhRR/# r   r   Nr	   r   r   r   r   r      s     31 31t 31r   c                s   \         P                  R4       V P                  R8X  d   \         P                  R4       R# \        P                  P                  V P                  4      '       g#   \         P                  RV P                  4       R#  V P                  V P                  4      pV'       dh   VR,          \        8X  dV   V P                  V P                  4      \        V P                  4      8X  d#   \         P                  RV P                  4       R# \         P                  RV P                  4       V'       g   \         P                  R	4       MVR,          \        8w  d$   \         P                  R
\        VR,          4       MvV P                  V P                  4      \        V P                  4      8w  dD   \         P                  R\        V P                  4      V P                  V P                  4      4       V P                  4       V n        V P                  4        R#   \         d7   p\         P                  RTP                   P"                  T4        Rp?R# Rp?ii ; i)z(Check whether the database is compatiblez#Checking database compatibility ...r   zK:memory: database is always newly created  and therefore always compatible.NzgUser database %s does not exist yet and will be created. A newly created database is always compatible.versionzCompatible database %s found.z$User database %s seems incompatible.z&No version information in the databasezThe version of the database does not match (too old or too new?).ibus-typing-booster wants version=%s but the database actually has version=%s.zThe number of columns of the database does not match.ibus-typing-booster expectes %s columns but the database actually has %s columns.z8Unexpected error checking database compatibility: %s: %s)r)   r]   r   r&   r+   existsget_database_descUSER_DATABASE_VERSION%get_number_of_columns_of_phrase_tablelenr-   _extract_user_phrasesr.   '_rename_incompatible_or_broken_databaserg   rC   rD   r
   )r9   descrE      &  r   r1   Z)TabSqliteDb._check_database_compatibility   s   9:
*KK45 ww~~d//00KKA!!# %	1))$*;*;<DY#88::4;L;LMt66783T5F5FHKK>))+ DEi$99 H 24	?	D
 <<T=N=NO889:8 778>>t?P?PQS !% : : <D88: 	1J((%1 1	1s,   "H* 3A%H* 'H* C&H* *I+5+I&&I+c                r_   r`   ra   r   r   r   r   r   '  s      T r   c                s   \         P                  R4       V P                  R8X  d   \         P                  R4       R# \        P                  P                  V P                  4      '       g#   \         P                  RV P                  4       R# Rp V P                  P                  V P                  4      ;_uu_ 4       pVP                  R4      P                  4        RRR4       \         P                  R4       R#   + '       g   i     L'; i  \         dG   p\         P                  R	TP                  P                  T4       T P                  4         Rp?R
# Rp?ii ; i)zKCheck whether all rows from the phrases table of the
database are readable
z Checking database readability...r   zI:memory: database is always newly created  and therefore always readable.TzeUser database %s does not exist yet and will be created. A newly created database is always readable.NSELECT * FROM phrases;zDatabase seems readable.z+Error checking database readability: %s: %sF)r)   r]   r   r&   r+   rn   rD   r[   r@   fetchallrg   rC   r
   rt   )r9   r4   rE   rv   r   r2   Z'TabSqliteDb._check_database_readability'  s    	67
*KK23 ww~~d//00KK?!!# 
	;88%%' '*2  !9:CCE' KK23	' '
  	;=((%1 88::	;s0   -D ? C>D >D		D E";EE"c                rk   rl   r	   r   r   r   r   r   E  s       r   c                s   \         P                  ! R4      p\        P                  RV P                  V,           4       \
        P                  P                  V P                  4      '       d3   \
        P                  ! V P                  V P                  V,           4       \
        P                  P                  V P                  R,           4      '       dA   \
        P                  ! V P                  R,           V P                  R,           V,           4       \
        P                  P                  V P                  R,           4      '       dA   \
        P                  ! V P                  R,           V P                  R,           V,           4       \        P                  RV P                  4       V P                  P                  V P                  4      V n        R# )za
Creates a backup copy of an incompatible or broken database

Also creates a new empty database.
z-%Y-%m-%d_%H:%M:%Sz0Renaming incompatible or broken database to "%s"z-shmz-walz$Creating a new, empty database "%s".N)re   strftimer)   r]   r   r&   r+   rn   ZrenamerD   r3   r4   )r9   r$      & r   rt   Z3TabSqliteDb._rename_incompatible_or_broken_databaseE  s6    MM"67	F%%i/	177>>$++,,IId''''	1377>>$++F233IId''.''.y8:77>>$++F233IId''.''.y8:2D4E4E	GFFr   rb   c                >   < V ^8  d   QhRS[ RS[ RS[ RS[ RS[RS[/# r   r   r    r!   r"   r#   r   r   r%   r=   r   r   r   r   r   \  sG     - -- - 	-
 - - $(-r   c                s   V'       d	   V'       g   R# \         P                  ! VP                  4       4      p\        P                  ! \         P
                  V4      p\         P                  ! VP                  4       4      p\         P                  ! VP                  4       4      pRpRVRVRVRVRVR\        P                  ! 4       /p\        ^8  d-   \        P                  R	V4       \        P                  R
V4        V P                  4       ;_uu_ 4        V P                  P                  Wg4       RRR4       R#   + '       g   i     R# ; i  \        P                   d7   p\        P                  RTP                   P"                  T4        Rp?R# Rp?i\$         d7   p\        P                  RTP                   P"                  T4        Rp?R# Rp?ii ; i)zUUpdate the user frequency of a phrase

:return: True if successful, False on failure
Tz
        UPDATE user_db.phrases
        SET user_freq = :user_freq, timestamp = :timestamp
        WHERE input_phrase = :input_phrase
         AND phrase = :phrase AND p_phrase = :p_phrase AND pp_phrase = :pp_phrase
        r#   r   r    r!   r"   r$   	sqlstr=%s
sqlargs=%sNz&Database error updating phrase: %s: %sz3Unexpected error updating phrase in user_db: %s: %sF)itb_utilremove_accentslowerunicodedata	normalizeNORMALIZATION_FORM_INTERNALre   r'   r)   r*   rF   r4   r@   rA   rB   rC   rD   r
   rg   )	r9   r   r    r!   r"   r#   ri   rh   rE   	   &&&&&&   r   update_phraseZTabSqliteDb.update_phrase\  ss    6..|/A/A/CD&&00&:**8>>+;<++IOO,=>	 	!<Vx			- ?LLf-LLw/	1!!##%%f6 $ $#}} 	18((%1 1 	  	1E((%1 1 		1sH   ?E D=3E =E	E E G&+FG$G%+GGc                rk   rl   r	   r   r   r   r   r     s     1 1D 1r   c                sb   \         P                  R4        V P                  P                  4        V P                  P	                  R4       \         P                  R4       R#   \
        P                   d7   p\         P                  RTP                  P                  T4        Rp?R# Rp?ii ; i)zTrigger a checkpoint operation.z!commit and execute checkpoint ...r>   z#commit and execute checkpoint done.z.Unexpected error syncing user database: %s: %sN)
r)   r]   r4   r?   r@   rA   ZOperationalErrorrC   rD   r
   r9   rE   rz   r   
sync_usrdbZTabSqliteDb.sync_usrdb  s    78		1MM  " MM!!"DEKK=>'' 	1@((%1 1	1s   A
A# #B.8+B))B.c                r_   r`   ra   r   r   r   r   r     s      t r   c                s   \         P                  R4        V P                  4       ;_uu_ 4        V P                  P	                  R4       RRR4       \         P                  R4       R#   + '       g   i     L'; i  \
        P                   d7   p\         P                  RTP                  P                  T4        Rp?R# Rp?i\         d7   p\         P                  RTP                  P                  T4        Rp?R# Rp?ii ; i)	zSCreate table for the phrases

:return: True if tables were created, False on error
zCreating tables...a7  
                CREATE TABLE IF NOT EXISTS user_db.phrases
                (id INTEGER PRIMARY KEY,
                input_phrase TEXT,
                phrase TEXT,
                p_phrase TEXT,
                pp_phrase TEXT,
                user_freq INTEGER,
                timestamp REAL)
                NzTables created.Tz&Database error creating tables: %s: %sz(Unexpected error creating tables: %s: %sF)r)   r]   rF   r4   r@   rA   rB   rC   rD   r
   rg   r   rz   r   r5   ZTabSqliteDb.create_tables  s    
 	()	1!!##%% 	' 	 $ KK)* $# }} 	18((%1 1 	  	1:((%1 1 		1s@   A= A*A= *A:	5A= =D+CDD+DDc                r{   r|   r}   r   r   r   r   r     sG     > >> > 	>
 > > $(>r   c                sz   \         ^8  d   \        P                  RWV4       V'       d	   V'       g   R# \        P                  ! VP                  4       4      p\        P                  ! \        P                  V4      p\        P                  ! VP                  4       4      p\        P                  ! VP                  4       4      pRpRVRVRVRV/pV P                  P                  Wg4      P                  4       '       d   R# RpRVRVRVRVR	VR
\        P                  ! 4       /p	\         ^8  d-   \        P                  RV4       \        P                  RV	4        V P                  4       ;_uu_ 4        V P                  P                  W4       RRR4       R#   + '       g   i     R# ; i  \        P                   d7   p
\        P!                  RT
P"                  P$                  T
4        Rp
?
R# Rp
?
i\&         d7   p
\        P!                  RT
P"                  P$                  T
4        Rp
?
R# Rp
?
ii ; i)z
Add phrase to database
z'input_phrase=%r phrase=%r user_freq=%s Tz
        SELECT * FROM user_db.phrases
        WHERE input_phrase = :input_phrase
        AND phrase = :phrase AND p_phrase = :p_phrase AND pp_phrase = :pp_phrase
        r   r    r!   r"   z
        INSERT INTO user_db.phrases
        (input_phrase, phrase, p_phrase, pp_phrase, user_freq, timestamp)
        VALUES (:input_phrase, :phrase, :p_phrase, :pp_phrase, :user_freq, :timestamp)
        r#   r$   zinsert_sqlstr=%szinsert_sqlargs=%sNz%Database error adding phrases: %s: %sz2Unexpected error adding phrase to database: %s: %sF)r'   r)   r*   r   r   r   r   r   r   r4   r@   rx   re   rF   rA   rB   rC   rD   r
   rg   )r9   r   r    r!   r"   r#   Zselect_sqlstrZselect_sqlargsZinsert_sqlstrZinsert_sqlargsrE   s   &&&&&&     r   
add_phraseZTabSqliteDb.add_phrase  s    ?LL9i1 6..|/A/A/CD&&00&:**8>>+;<++IOO,=>	 Lf	$
 ==  ?HHJJ 
 ),"F$h%y%y%tyy{4 ?LL+];LL,n=	1!!##%%mD $ $#}} 	17((%1 1 	  	1D((%1 1 		1sH   F0 6FF0 F-	'F0 -F0 0H:+G66H:H:+H55H:c                r_   r`   ra   r   r   r   r   r     s       r   c                s   Rp\         P                  R4        V P                  4       ;_uu_ 4        V P                  P	                  V4       RRR4       \         P                  R4       R#   + '       g   i     L'; i  \
        P                   d7   p\         P                  RTP                  P                  T4        Rp?R# Rp?i\         d7   p\         P                  RTP                  P                  T4        Rp?R# Rp?ii ; i)	zXCreate indexes for the database.

:return: True if indexes were created, False on error
z
        CREATE INDEX IF NOT EXISTS user_db.phrases_index_p ON phrases
        (input_phrase, id ASC);
        CREATE INDEX IF NOT EXISTS user_db.phrases_index_i ON phrases
        (phrase)
        zCreating indexes...NzIndexes created.Tz'Database error creating indexes: %s: %sz)Unexpected error creating indexes: %s: %sF)r)   r]   rF   r4   rP   rA   rB   rC   rD   r
   rg   )r9   ri   rE   rv   r   r7   ZTabSqliteDb.create_indexes  s    
 	)*	1!!##++F3 $KK*+ $# }} 	19((%1 1 	  	1;((%1 1 		1s@   A? A,A? ,A<	7A? ?D	+CD	D	+DD	c                sJ   < V ^8  d   QhRS[ RS[S[P                  ,          /# )r   r   r   r   r   r   ZPredictionCandidater   r   r   r   r     s(     ( (("&x'C'C"D(r   c                sN   \         P                  ! \        P                  V4      p\        ^8  d   \
        P                  RV4       / pRVR,           R\        P                  /pRp. p V P                  P                  WC4      P                  4       pV'       d   VP                  V4       \        P                   ! V4      p\        ^8  d   \
        P                  RV4       V#   \         d6   p\
        P                  RTP                  P                  T4        Rp?LRp?ii ; i)	z4Get shortcuts from database completing input_phrase.zinput_phrase=%sr   Z%r#   zSELECT phrase, sum(user_freq) FROM user_db.phrases WHERE input_phrase LIKE :input_phrase AND user_freq >= :user_freq GROUP BY phrase;z>Unexpected error fetching user shortcuts from database: %s: %sNzbest_shortcut_candidates=%s)r   r   r   r   r'   r)   r*   SHORTCUT_USER_FREQr4   r@   rx   rg   rC   rD   r
   updatebest_candidates)r9   r   phrase_frequenciesrh   ri   Zresults_shortcutsrE   Zbest_shortcut_candidates   &&      r   select_shortcutsZTabSqliteDb.select_shortcuts  s    #,,00,@?LL*L9/1!<##5 ; ;=% 46	2 $ 5 5!!!)  %%&78#+#;#;<N#O ?LL-/GI''  	27))52 2	2s   ))C$ $D$/+DD$c                sP   < V ^8  d   QhRS[ RS[ RS[S[P                  ,          /# )r   r!   r"   r   r   r   r   r   r   r   2  s4     9< 9<9< 9<  $H$@$@A9<r   c                s   \         ^8  d   \        P                  RW4       / pV'       d	   V'       g   \        P                  ! V4      # \        P
                  ! VP                  4       4      p\        P
                  ! VP                  4       4      pRVRV/pRpRp V P                  P                  WT4      P                  4       pV'       g   \        P                  ! V4      # Rp^ p V P                  P                  WT4      P                  4       ^ ,          ^ ,          pV'       g   \        P                  ! V4      # V F5  p	VP                  V	^ ,          V	^,          \        V4      ,          3.4       K7  	  \         ^8  d+   \        P                  R	\        P                  ! V4      4       \        P                  ! V4      #   \         d9   p\        P                  RTP                  P                  T4       Rp Rp?EL<Rp?ii ; i  \         d9   p\        P                  RTP                  P                  T4       ^ p Rp?EL(Rp?ii ; i)
zGet phrases from database which occured previously with
any input after the given context

Returns a list of matches where each match is a tuple in the
form of (phrase, user_freq), i.e. returns something like
[(phrase, user_freq), ...]

zp_phrase=%s pp_phrase=%sr!   r"   zySELECT phrase, sum(user_freq) FROM user_db.phrases WHERE p_phrase = :p_phrase AND pp_phrase = :pp_phrase GROUP BY phrase;NzXUnexpected error getting phrases for empty input with given context from user_db: %s: %szaSELECT sum(user_freq) FROM user_db.phrases WHERE p_phrase = :p_phrase AND pp_phrase = :pp_phrase;z\Unexpected error getting total count for empty input with given context from user_db: %s: %sz5Best candidates for empty input with given context=%s)r'   r)   r*   r   r   r   r   r4   r@   rx   rg   rC   rD   r
   r   float)
r9   r!   r"   r   rh   ri   resultsrE   count_pp_phrase_p_phraseresults
   &&&       r   select_words_empty_inputZ$TabSqliteDb.select_words_empty_input2  s    ?LL*HA/1y++,>??**8>>+;<++IOO,=>	xi@@ 	mm++F<EEGG ++,>??E 	 $% 	)'+}}'<'<(!!)A(//0(2$ (++,>??F%%)VAYu-E'FFGHJ  ?LLG(();<> ''(:;;=  	:((%1 G	  	):))52 ()$	)s0   )G )7H
 H-HH
I-IIc          	      sV   < V ^8  d   QhRS[ RS[ RS[ RS[S[P                  ,          /# )r   r   r!   r"   r   r   r   r   r   r   r   m  sJ     UN UNUN UN 	UN %))E)E$F	UNr   c           	     s   \         ^8  d   \        P                  RWV4       V'       g   V P                  W#4      # / p\        P
                  ! \        P                  V4      p\        P                  ! VP                  4       4      p\        P                  ! VP                  4       4      pVP                  4       pRV9  d+   VP                  V P                  P                  V4      4       \         ^8  d,   \        P                  R\        P                  ! WER7      4       \        P                  ! VP                  4       4      pVP                  RR4      P                  RR4      pV P                   P#                  R	4       R
V R2p V P                   P#                  V4       RVRV/p	Rp V P                   P#                  Wy4      P-                  4       p
X
'       g   \        P                  ! WER7      # Rp V P                   P#                  Wy4      P-                  4       ^ ,          ^ ,          pV
 F5  pVP                  V^ ,          V^,          \/        X4      ,          3.4       K7  	  \         ^8  d,   \        P                  R\        P                  ! WER7      4       V'       g   \        P                  ! WER7      # Rp V P                   P#                  Wy4      P-                  4       pX'       g   \        P                  ! WER7      # Rp V P                   P#                  Wy4      P-                  4       ^ ,          ^ ,          pV FW  pVP                  V^ ,          RV^,          ,          \/        X4      ,          RWO^ ,          ,          ,          ,           3.4       KY  	  \         ^8  d,   \        P                  R\        P                  ! WER7      4       V'       g   \        P                  ! WER7      # Rp V P                   P#                  Wy4      P-                  4       pX'       g   \        P                  ! WER7      # Rp V P                   P#                  Wy4      P-                  4       ^ ,          ^ ,          pV FX  pVP                  V^ ,          RV^,          ,          \/        X4      ,          RVV^ ,          ,          ,          ,           3.4       KZ  	  \         ^8  d,   \        P                  R\        P                  ! WER7      4       \        P                  ! WER7      #   \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELRp?ii ; i  \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELRp?ii ; i  \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELRp?ii ; i  \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELCRp?ii ; i  \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?EL.Rp?ii ; i  \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELRp?ii ; i  \$         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELRp?ii ; i) z
Get phrases from database completing input_phrase.

Returns a list of matches where each match is a tuple in the
form of (phrase, user_freq), i.e. returns something like
[(phrase, user_freq), ...]
z(input_phrase=%s p_phrase=%s pp_phrase=%s zhunspell: best_candidates=%s)ZtitleZ rb   Z'z''z+DROP VIEW IF EXISTS like_input_phrase_view;z
        CREATE TEMPORARY VIEW IF NOT EXISTS like_input_phrase_view AS
        SELECT * FROM user_db.phrases
        WHERE input_phrase LIKE 'z%'
        ;z2Unexpected error in creating database view: %s: %sNr!   r"   zJSELECT phrase, sum(user_freq) FROM like_input_phrase_view GROUP BY phrase;u@   Unexpected error getting “unigram” data from user_db: %s: %sz2SELECT sum(user_freq) FROM like_input_phrase_view;zAUnexpected error getting total unigram count from user_db: %s: %szUnigram best_candidates=%szeSELECT phrase, sum(user_freq) FROM like_input_phrase_view WHERE p_phrase = :p_phrase GROUP BY phrase;u?   Unexpected error getting “bigram” data from user_db: %s: %szMSELECT sum(user_freq) FROM like_input_phrase_view WHERE p_phrase = :p_phrase;z@Unexpected error getting total bigram count from user_db: %s: %sg      ?zBigram best_candidates=%szSELECT phrase, sum(user_freq) FROM like_input_phrase_view WHERE p_phrase = :p_phrase AND pp_phrase = :pp_phrase GROUP BY phrase;u@   Unexpected error getting “trigram” data from user_db: %s: %szhSELECT sum(user_freq) FROM like_input_phrase_view WHERE p_phrase = :p_phrase AND pp_phrase = :pp_phrase;zAUnexpected error getting total trigram count from user_db: %s: %szTrigram best_candidates=%s)r'   r)   r*   r   r   r   r   r   r   r   Zistitler   r0   Zsuggestr   rJ   r4   r@   rg   rC   rD   r
   rx   r   )r9   r   r!   r"   r   Z
title_caseZsql_escaped_input_phraseri   rE   rh   Zresults_uniZcountZ
result_uniZ
results_biZcount_p_phraseZ	result_biZresults_trir   Z
result_tris   &&&&               r   select_wordsZTabSqliteDb.select_wordsm  s    ?LL:	3 00EE/1",,00,@**8>>+;<++IOO,=>	!))+
l" %%d&7&7&?&?&MN?LL.(();NP  ..|/A/A/CD $0#7#7B$T* 	!KL" #;!; <

	1MM!!&)
 xi@ 		1 --//@IIKK  ++,>QQ
 F	2MM))&:CCEaHKE &J%%Q-AuU|!;<=? & ?LL,(();NP  ++,>QQ: 		1..v?HHJJ ++,>QQ* 		2!]]22!!)A//02N $I%%A,il"5#88)A,7789 :; $
 ?LL+(();NP  ++,>QQ@	2--//@IIKK ++,>QQE 		2'+}}'<'<(!!)A(//0(2$ &J%%Q-jm#E*B$CC)*Q-889: ;< &
 ?LL,(();NP ''(:MMa  	1D((%1 1	18  	1R((%1 1	1   	2'))52 2	20  	1'((%1 1	1  	2'))52 2	24  	2'))52 2	2   	2'))52 2	2s   
T' .)U+ 97V/ )W3 7X7 :)Y; 7Z? 'U(2+U##U(+V,6+V''V,/W0:+W++W03X4>+X//X47Y8+Y33Y8;Z<+Z77Z<?\ 
+[;;\ c                r_   r`   ra   r   r   r   r   r   D  s        d  r   c                s   \         P                  R4        V P                  4       ;_uu_ 4        V P                  P	                  R4       V P                  P	                  RR\
        34       V P                  P	                  R4       RRR4       \         P                  R4       R#   + '       g   i     L'; i  \        P                   d7   p\         P                  R	TP                  P                  T4        Rp?R# Rp?i\         d7   p\         P                  R
TP                  P                  T4        Rp?R# Rp?ii ; i)z
Add a description table to the user database

This adds the database version and  the create time

:return: True on success, False on failure.
zGenerate userdb descriptionz{
                    CREATE TABLE IF NOT EXISTS user_db.desc
                    (name PRIMARY KEY, value)
                z2INSERT OR IGNORE INTO user_db.desc  VALUES (?, ?);rm   z
                    INSERT OR IGNORE INTO user_db.desc
                    VALUES ('create-time', DATETIME('now', 'localtime'))
                Nzuserdb description generatedTz3Database error generating description table: %s: %sz6Unexpected error adding description to user_db: %s: %sF)r)   r]   rF   r4   r@   rp   rA   rB   rC   rD   r
   rg   r   rz   r   r8   Z TabSqliteDb.generate_userdb_descD  s	    	12	2!!##%% '  %%H 568 %% '  $ KK67 $# }} 	1E((%1 1 	  	2H))52 2 		2sA   B: AB'B: 'B7	2B: :E+D  EE+D??Ec                sL   < V ^8  d   QhRS[ RS[S[S[ S[ 3,          ,          /# r\   )r   r   r    r   r   r   r   r   g  s(       c3h0H r   c                s   \         P                  P                  V4      '       g   R# Rp V P                  V4      ;_uu_ 4       p/ pVP	                  R4      P                  4        F  pV^,          W4^ ,          &   K  	  VuuRRR4       #   + '       g   i     R# ; i  \        P                  \        3 d7   p\        P                  RTP                  P                  T4        Rp?R# Rp?i\        \        3 d7   p\        P                  RTP                  P                  T4        Rp?R# Rp?i\         d7   p\        P                  RTP                  P                  T4        Rp?R# Rp?ii ; i)z#Get the description of the databaseNzSELECT * FROM descz-Database/OS error getting description: %s: %sz%Data format error in database: %s: %sz5Unexpected error getting database description: %s: %s)r&   r+   rn   r[   r@   rx   rA   rB   OSErrorr)   rC   rD   r
   r(   
IndexErrorrg   )rR   rH   r4   ru   rowrE   s   &&    r   ro   ZTabSqliteDb.get_database_descf  s5    ww~~g&&	2--g66(#++,@AJJLC#&q6DQL M	 7666"  w' 	1?((%1 1  :& 	17((%1 1 	  	2G))52 2 		2sY   B <B 
B B	B B E4:+C++E4>E4?+D00E4=E4>+E//E4c                s6   < V ^8  d   QhRS[ RS[S[,          /# r\   )r   r   r%   r   r   r   r   r     s!     , ,,"*3-,r   c                s   \         P                  P                  V4      '       g   R# Rp V P                  V4      ;_uu_ 4       pVP	                  R4      P                  4       pV'       d   V^ ,          ^ ,          '       g    RRR4       ^ # RP                  V^ ,          ^ ,          P                  4       4      p\        P                  ! RV4      pV'       d6   VP                  ^4      P                  R4      p\        V4      uuRRR4       #  RRR4       ^ #   + '       g   i     ^ # ; i  \        P                  \        \         \"        3 d7   p\$        P'                  RTP(                  P*                  T4        Rp?^ # Rp?i\,         d7   p\$        P'                  RTP(                  P*                  T4        Rp?^ # Rp?ii ; i)a  
        Get the number of columns in the 'phrases' table in
        the database in db_file.

        Determines the number of columns by parsing this:

        sqlite> select sql from sqlite_master where name='phrases';
CREATE TABLE phrases (id INTEGER PRIMARY KEY, input_phrase TEXT, phrase TEXT, p_phrase TEXT, pp_phrase TEXT, user_freq INTEGER, timestamp REAL)
        sqlite>

        This result could be on a single line, as above, or on multiple
        lines.
        Nz3select sql from sqlite_master where name='phrases';r   z
.*\((.*)\)Z,z'Error getting number of columns: %s: %sz>Unexpected error getting number of columns of database: %s: %s)r&   r+   rn   r[   r@   rx   r,   Z
splitlinesrematchZgroupZsplitrr   rA   rB   r   r   AttributeErrorr)   rC   rD   r
   rg   )rR   rH   r4   Ztable_phrases_resultZstringr   Ztable_phrases_columnsrE   r   r   rq   Z1TabSqliteDb.get_number_of_columns_of_phrase_table  sm   $ ww~~g&&	2--g66('/'7'7I((* % ,3G3J13M3M 76 "6q"9!"<"G"G"IJ7,1KKN,@,@,E)45 76  7666.  w
NC 	19((%1 1   	2&))52 2 	2sl   D- =DDD- AD*D
D- DD- D*	$D- *D- -%G+FGG+GGc                sF   < V ^8  d   QhRS[ S[S[S[3,          ,          /# r`   )r   r   r   r   r   r   r   r     s!      T%S/%: r   c                s0   RpR\         P                  /p\        ^8  d-   \        P	                  RV4       \        P	                  RV4       V P
                  P                  W4      P                  4       p\        ^8  d   \        P	                  RV4       V# )zIReturns a list of user defined shortcuts from the user database.
        z\
        SELECT input_phrase, phrase FROM user_db.phrases WHERE user_freq >= :freq
        ;Zfreqr~   r   z	result=%s)r   r   r'   r)   r*   r4   r@   rx   )r9   ri   rh   r   s   &   r   list_user_shortcutsZTabSqliteDb.list_user_shortcuts  su     8667?LLf-LLw/&&v7@@B?LLf-r   c                2   < V ^8  d   QhRS[ RS[ RS[RS[/# )r   r   r    r#   r   r}   r   r   r   r   r     s4     - -- - 	- >B	-r   c           	     sf   \         ^8  d   \        P                  RWV4       V'       d   V'       d   V\        P                  8  d#   \         ^8  d   \        P                  R4       R# \
        P                  ! \        P                  V4      p\
        P                  ! \        P                  V4      pRVRVRVR\        P                  ! 4       /pRp V P                  4       ;_uu_ 4        V P                  P                  WT4       R	R	R	4       R
#   + '       g   i     R
# ; i  \        P                   d7   p\        P                  RTP                  P                   T4        R	p?R# R	p?i\"         d7   p\        P                  RTP                  P                   T4        R	p?R# R	p?ii ; i)zI
Defines a new user shortcut

:return: True on success, False on failure
z&input_phrase=%s phrase=%s user_freq=%szNot defining shortcut.Fr   r    r#   r$   zINSERT INTO user_db.phrases (input_phrase, phrase, p_phrase, pp_phrase, user_freq, timestamp)VALUES (:input_phrase, :phrase, "", "", :user_freq, :timestamp);NTz(Database error defining shortcut: %s: %sz*Unexpected error defining shortcut: %s: %s)r'   r)   r*   r   r   r   r   r   re   rF   r4   r@   rA   rB   rC   rD   r
   rg   )r9   r   r    r#   rh   ri   rE   s   &&&&   r   define_user_shortcutZ TabSqliteDb.define_user_shortcut  s\    ?LL8i1 F8666Q56&&00&: #,,00,@!<V			-
O 		1!!##%%f6 $ $#}} 	1:((%1 1 	  	1<((%1 1 		1sH   D& ,DD& D#	D& #D& &F0;+E,,F09F0:+F++F0c                r   )r   r   r    user_freq_incrementr   r}   r   r   r   r   r     s9     G GG G "%	G .2	Gr   c                s    \         ^8  d   \        P                  RWV4       VP                  4       pVP                  4       pV'       d	   V'       g   R# \        P
                  ! \        P                  V4      p\        P
                  ! \        P                  V4      pRVR\        P                  R\        P                  ! 4       /pRp. p V P                  P                  WT4      P                  4       pV'       g   R# V F  w  rW8X  d$   \         ^8  d   \        P                  R	WV	34       W,           p
\         ^8  d   \        P                  R
WV	3WV
34       WR&   WR&   Rp V P#                  4       ;_uu_ 4        V P                  P                  WT4       RRR4       K  	  R#   \         d6   p\        P                  RTP                  P                   T4        Rp?LRp?ii ; i  + '       g   i     LY; i  \$        P&                   d8   p\        P                  RTP                  P                   T4        Rp? R# Rp?i\         d8   p\        P                  RTP                  P                   T4        Rp? R# Rp?ii ; i)z
Check whether there are user defined shortcuts expanding to phrase.

If yes, increase the frequency of each of them by user_freq_increment
and return True, else return False
u<   input_phrase=“%s” phrase=“%s” user_freq_increment=%sFr    r#   r$   zgSELECT input_phrase, user_freq FROM user_db.phrases WHERE phrase = :phrase AND user_freq >= :user_freq;z;Unexpected error checking user shortcuts in user_db: %s: %sNzExactly matched shortcut: %sz(Increase shortcut frequency by: %s -> %sr   zUPDATE user_db.phrases SET user_freq = :user_freq, timestamp = :timestamp WHERE input_phrase = :input_phrase AND phrase = :phrase ;z(Database error updating shortcut: %s: %sz5Unexpected error updating shortcut in user_db: %s: %sT)r'   r)   r*   Zstripr   r   r   r   r   re   r4   r@   rx   rg   rC   rD   r
   rF   rA   rB   )r9   r   r    r   rh   ri   r   rE   Zdb_input_phraseZdb_user_freqZnew_user_freqs   &&&&       r   #check_shortcut_and_update_frequencyZ/TabSqliteDb.check_shortcut_and_update_frequency  s;    ?LLN&9; $))+6&&00&:",,00,@V ; ;		-1 	2mm++F<EEGG
 /6+_.?LL6(,?A )>MQ>$l;$m<> $1K &5N#) 
5%%''MM))&: (- 07@ M  	2M))52 2	24 (' == 5  >OO,,e5 5 	  5  KOO,,e5 5 	5sZ   )F+ )HG.H+G+6+G&&G+.G>9HJ+IJJ+JJc                r{   )r   r   r    r!   r"   r   r   r}   r   r   r   r   r   6  sO     T+ T+T+ T+ 	T+
 T+ "%T+ .2T+r   c                s   \         ^8  d   \        P                  RWW4V4       V'       d   V P                  VVVR7      '       d   R# V'       g   TpV'       g   R# \        P
                  ! V4      p\        P
                  ! V4      p\        P                  ! \        P                  V4      p\        P                  ! VP                  4       4      p\        P                  ! VP                  4       4      p\        P                  ! VP                  4       4      pRpRVRVRVRV/p\         ^8  d-   \        P                  R	V4       \        P                  R
V4       V P                  P                  Wg4      P                  4       p\         ^8  d   \        P                  RV4       V'       d,   V P                  VVVVV^ ,          ^ ,          V,           R7      # V P                  VVVVVR7      # )z
Check whether input_phrase and phrase are already in database. If
they are in the database, increase the frequency by 1, if not
add them.
ua   input_phrase=“%s” phrase=“%s” p_phrase=“%s” pp_phrase=“%s” user_freq_increment=%s)r   r    r   Tz
        SELECT max(user_freq) FROM user_db.phrases
        WHERE input_phrase = :input_phrase
        AND phrase = :phrase AND p_phrase = :p_phrase AND pp_phrase = :pp_phrase
        GROUP BY phrase
        ;r   r    r!   r"   z9TabSqliteDb.check_phrase_and_update_frequency() sqlstr=%sz:TabSqliteDb.check_phrase_and_update_frequency() sqlargs=%sz-check_phrase_and_update_frequency() result=%s)r   r    r!   r"   r#   )r'   r)   r*   r   r   Zstrip_tokenr   r   r   r   r   r4   r@   rx   r   r   )	r9   r   r    r!   r"   r   ri   rh   r   r   r   !check_phrase_and_update_frequencyZ-TabSqliteDb.check_phrase_and_update_frequency6  s    ?LL)h#	% 77!-!(; 8 = = !L++L9%%f-&&00&:**8>>+;<++IOO,=>	..|/A/A/CD "<Vx	+ ?LLK LLL &&v7@@B?LL?I %%)!# )A,':: & < < %)  + 	+r   c                &   < V ^8  d   QhRS[ RS[/# )r   r    r   )r   r%   r   r   r   r   r     s      C C r   c                sb   \         ^8  d   \        P                  RV4       V'       g   ^ # \        P                  ! \
        P                  V4      p V P                  P                  RV34      P                  4       pV'       d   V^ ,          e   \        V^ ,          4      # ^ #   \        P                   d7   p\        P                  RTP                  P                  T4        Rp?^ # Rp?i\          d7   p\        P                  RTP                  P                  T4        Rp?^ # Rp?ii ; i)a  
Checks if an entry for phrase already exists in the user database

:param phrase: The phrase to check whether it is already recorded
:return: 0 if not in the database. > 0 if in the database.
         The value > 0 gives the sum(user_freq) in the database
         for phrase.
	phrase=%rz;SELECT sum(user_freq) FROM user_db.phrases WHERE phrase = ?Nz,Error checking whether phrase exists: %s: %sz7Unexpected error checking whether phrase exists: %s: %s)r'   r)   r*   r   r   r   r   r4   r@   rQ   r%   rA   rB   rC   rD   r
   rg   )r9   r    r   rE   r^   r   phrase_existsZTabSqliteDb.phrase_exists  s     ?LLf-&&00&:	2--''M	 hj  #&#a&*<3s1v;C!C}} 	1>((%1 1 	  	2I))52 2 		2s6   2B$ B$ "B$ $D.9+C**D.7D.8+D))D.c                s,   < V ^8  d   QhRS[ RS[ RS[/# )r   r   r    r   r   r=   r   r   r   r   r     s)     & && & "&&r   c                s   V'       d   \        V\        4      '       g   R# \        ^8  d   \        P	                  RV4       \
        P                  ! \        P                  V4      pV'       d&   \
        P                  ! \        P                  V4      pRV'       d   RMR R2pRVRV/p V P                  4       ;_uu_ 4        V P                  P                  W44       R	R	R	4       R
#   + '       g   i     R
# ; i  \        P                   d7   p\        P                  RTP                  P                   T4        R	p?R# R	p?i\"         d7   p\        P                  RTP                  P                   T4        R	p?R# R	p?ii ; i)u   
Remove all rows matching “input_phrase” and “phrase” from database.
Or, if “input_phrase” is '', remove all rows matching “phrase”
no matter for what input phrase from the database.

:return: True if successful, False on failure
Fr   z;
            DELETE FROM user_db.phrases
            WHERE z!input_phrase = :input_phrase AND rb   z&
            phrase = :phrase
        r   r    NTz&Database error removing phrase: %s: %sz(Unexpected error removing phrase: %s: %s)Z
isinstancer   r'   r)   r*   r   r   r   r   rF   r4   r@   rA   rB   rC   rD   r
   rg   )r9   r   r    Zdelete_sqlstrZdelete_sqlargsrE   rZ   r   remove_phraseZTabSqliteDb.remove_phrase  s>    Z44?LLf-&&00&:&0044lDL:F6BO P	
 ),&I	2!!##%%mD $ $#}} 	18((%1 1 	  	2:))52 2 		2sH   +C= C)C= )C:	4C= :C= =F+EFF+FFc                sJ   < V ^8  d   QhRS[ S[S[S[S[3,          ,          /# r`   )r   r   r   r%   r   r   r   r   r     s$      tE#sC-,@'A r   c                sN   \         P                  R4       Rp V P                  P                  V P                  4      ;_uu_ 4       pVP                  R4       VP                  R4      P                  4       pV Uu. uFc  p\        P                  ! \        P                  V^ ,          4      \        P                  ! \        P                  V^,          4      V^,          3NKe  	  upuuRRR4       # u upi   + '       g   i     . # ; i  \        P                  \        \        \        3 d7   p\         P!                  RTP                  P"                  T4        Rp?. # Rp?i\$         d7   p\         P!                  RTP                  P"                  T4        Rp?. # Rp?ii ; i)z"extract user phrases from databasezTrying to recover phrases from old, incompatible databasePhrases successfully recovered can be used to initialize a newly created database.NPRAGMA wal_checkpoint;z
                    SELECT input_phrase, phrase, sum(user_freq)
                    FROM phrases GROUP BY phrase
                z%Error extracting user phrases: %s: %sz0Unexpected error extracting user phrases: %s: %s)r)   r]   rD   r[   r   r@   rx   r   r   r   r   rA   rB   r   r   r   rC   r
   rg   )r9   r4   ZphrasesrM   rE   rj   r   rs   Z!TabSqliteDb._extract_user_phrases  sz   3	4 	288%%' '*2  !9:"** ,  XZ  % % !** <<adD ** <<adDqT	
 %' '' '. 	 w
NC 	17((%1 1 		  	2B))52 2 			2sZ   -D
 5C6;A)C1$C6&
D
 1C66D	D
 D
 
%F$/+E  F$-F$.+FF$c                r   )r   filenamer   r   r   r   r   r   r     s     f fS fT fr   c                s   \         P                  P                  V4      '       g1   VR,          p\         P                  P                  V4      '       g   R# \        pVP	                  R4      '       d   \
        P                  pV P                  P                  R4      P                  4       p\        VR R7      p\        P                  ! 4       p\        P                  ! 4       pV'       d!   V^ ,          ^,          pVR,          ^,          pVRWT,
          ,          ,           p\        P                  R\        P                  ! R\        P                  ! V4      4      4       \        P                  R\        P                  ! R\        P                  ! V4      4      4       \        P                  R	\        P                  ! R\        P                  ! V4      4      4       R
pR
p/ p	V F  p
\        P                   ! V
^ ,          P#                  4       4      p\$        P&                  ! \        P(                  V
^,          4      p\        P                   ! V
^,          P#                  4       4      p\        P                   ! V
^,          P#                  4       4      pV	P+                  V
^ ,          V
^,          V
^,          V
^,          3RVRVRVRVRV
^,          RV
^,          /3.4       EK  	  . p V! VRRR7      ;_uu_ 4       pVP-                  4        Uu. uF(  p\$        P&                  ! \        P(                  V4      NK*  	  ppRRR4       T EF.  p\        P6                  ! T4       EF  p\        P                   ! TP#                  4       4      T\        P                   ! TP#                  4       4      \        P                   ! TP#                  4       4      3pTT	9   d   T	T,          R;;,          ^,          uu&   MwR\        P                   ! TP#                  4       4      RTR\        P                   ! TP#                  4       4      R\        P                   ! TP#                  4       4      R^RT/T	T&   TpTpEK  	  EK1  	  . pT	P9                  4        F  w  ppTP;                  T4       K  	  Rp T P                  P                  R4       T P                  P=                  4        T P                  P?                  TT4       T P                  P=                  4        T P                  P                  R4       R# u upi   + '       g   i     EL; i  \.         d7   p\        P1                  RTP2                  P4                  T4        Rp?R# Rp?ii ; i  \.         d7   p\        P1                  RTP2                  P4                  T4        Rp?R# Rp?ii ; i)zk
Read data to train the prediction from a text file.

:param filename: Full path of the text file to read.
z.gzFc                 s&    \        V ^,          4      # i   )r   rL   rN   r   rO   Z:TabSqliteDb.read_training_data_from_file.<locals>.<lambda>  s    U1Q4[r   keyg?z$Minimum timestamp in the database=%s%Y-%m-%d %H:%M:%Sz$Maximum timestamp in the database=%sz New timestamp in the database=%srb   r   r    r!   r"   r#   r$   ZrtzUTF-8)rS   rK   Nz8Unexpected error reading training data from file: %s: %src   DELETE FROM phrases;r   z:Unexpected error writing training data to database: %s: %sTzTSELECT input_phrase, phrase, p_phrase, pp_phrase, user_freq, timestamp FROM phrases;) r&   r+   ZisfileZopenZendswithgzipr4   r@   rx   sortedre   r)   r]   ry   gmtimer   r   r   r   r   r   r   Z	readlinesrg   rC   rD   r
   ZtokenizeZitemsrd   r?   rf   )r9   r   Zopen_functionrowsZtime_minZtime_maxZtime_newZp_tokenZpp_tokenZdatabase_dictr   r   r    r!   r"   ZlinesZfile_handleZlinerE   Ztokenr   rh   Zvalueri   s   &&                      r   read_training_data_from_fileZ(TabSqliteDb.read_training_data_from_file  s    ww~~h''H77>>(++.2U## IIM}}$$344<HJ 	 d"9:99;99;AwqzHBx{Hdh&9:::MM"5t{{87LM	O:MM"5t{{87LM	O6MM"5t{{87LM	OIKC $223q6<<>BL **44c!f>F..s1v||~>H //A?I  CFCFCFCF#C$2L$,f$.$/$/Q$/Q$9#$ "% & " 	4'; ;>I !, 5 5 79 !8  )) <<dD 7  9; D!**40..u{{}=..w}}?..x~~/?@B -'!#&{3q83 '(?(?(N %"H$;$;GMMO$L#X%<%<X^^=M%N#Q#X*/M#& #! 1 $ '--/JCNN5! 0	MM!!"89 MM  "MM%%fg6MM  "MM!!":; a9; ; ;  	J))52 		N  	L))52 		sa   -U  T=.T8T=U 0BV 8T==U	U U V+VVW +WWc                r_   r`   ra   r   r   r   r   r   \  s      D r   c                s    V P                  4       ;_uu_ 4        V P                  P                  R4       RRR4       R#   + '       g   i     R# ; i  \        P                   d7   p\
        P                  RTP                  P                  T4        Rp?R# Rp?i\         d7   p\
        P                  RTP                  P                  T4        Rp?R# Rp?ii ; i)z
Remove all phrases from the database, i.e. delete all the
data learned from user input or text files.

:return: True if successful, False on failure
r   NTz+Database error removing all phrases: %s: %sz;Unexpected error removing all phrases from database: %s: %sF)
rF   r4   r@   rA   rB   r)   rC   rD   r
   rg   r   rz   r   remove_all_phrasesZTabSqliteDb.remove_all_phrases\  s    	2!!##%%&<= $ $#}} 	1=((%1 1 	  	2M))52 2 		2sE   A A A  A	A A C)+BC'C(+CCc                rk   rl   r	   r   r   r   r   r   q  s     ? ?t ?r   c                s     \         P                  R4       V P                  P                  R4      P	                  4        F"  p\         P                  R\        V4      4       K$  	  \         P                  R4       V P                  P                  R4      P	                  4        F"  p\         P                  R\        V4      4       K$  	  R#   \         d7   p\         P                  RTP                  P                  T4        Rp?R# Rp?ii ; i)z?
Dump the contents of the database to the log

(For debugging)
zSELECT * FROM desc;
zSELECT * FROM desc;z%szSELECT * FROM phrases;
rw   z)Unexpected error dumping database: %s: %sN)
r)   r*   r4   r@   rx   reprrg   rC   rD   r
   )r9   r   rE   rv   r   dump_databaseZTabSqliteDb.dump_databaseq  s    
	?LL01}},,-BCLLNT49- OLL34}},,,..6hj9T49-9  	?H#oo66? ?	?s   CC D+DDc                r_   r`   )r%   r   r   r   r   r     s      C r   c                s     \        V P                  P                  R4      P                  4       4      #   \         d8   p\
        P                  RTP                  P                  T4       Ru Rp?# Rp?ii ; i)zD
Return the current number of rows in the database

(For debugging)
rw   z8Unexpected error getting number of database rows: %s: %sNr   )	rr   r4   r@   rx   rg   r)   rC   rD   r
   r   rz   r   number_of_rows_in_databaseZ&TabSqliteDb.number_of_rows_in_database  sf    	t}},,(**2(*6 6 	J((%1 I		s   14 A6,A1+A61A6c                r   )r   threadr   Nra   r   r   r   r   r     s#     eL eLt eLt eLr   c                s   V'       d)   V P                   R8X  d   \        P                  R4       R# \        P                  R4       \        P                  ! 4       p. pRp V'       d   V P	                  V P                   4      pMV P
                  pVP                  R4      P                  4       pV'       g[    V'       d1   V'       d)    VP                  4        \        P                  R4       R# V'       g   \        P                  R4       R# R# \        VR	 R
7      p\        P                  R\        V4      4       \        V4      pRp^ p. p	V F  p
V
^,          pWg8  d   V\        P                  8  dz   \        P                  R\!        V
4      \        P"                  ! R\        P$                  ! V
^,          4      4      4       V^,          pRpRV
^ ,          /p VP                  W4       MV	P-                  V
4       V^,          pK  	  \        P                  RV4       \        V	R R
7      p	\        V	4      p\        P                  RV4       \/        VR,          4      p\        P                  RV4       ^ p^ pV	 EFS  p
V
^,          pWo8  Ed7   V\        P                  8  Ed!   V^8X  dz   \        P                  R\!        V
4      \        P"                  ! R\        P$                  ! V
^,          4      4      4       V^,          pRpRV
^ ,          /p VP                  W4       M\        P                  R\!        V
4      \        P"                  ! R\        P$                  ! V
^,          4      4      4       V^,          pRpRV
^ ,          R\/        V^,          4      R\        P                  ! 4       /p VP                  VV4       V^,          pEKV  	  \        P                  R4       VP1                  4        VP                  R4       \        P                  R 4       VP                  R!4       \        P                  R"VV,           4       \        P                  R#V4       \        P                  R$\        V4      4       \        P                  R%\        V	4      V,
          4       \        P                  R&\        P                  ! 4       V,
          4       \        P                  R'4        T'       d1   T'       d)    TP                  4        \        P                  R4       R# T'       g   \        P                  R4       R# R#   \         d"   p\        P                  RT4        Rp?R# Rp?ii ; i  \         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELRp?ii ; i  \         d7   p\        P'                  RTP(                  P*                  T4        Rp?EL0Rp?ii ; i  \         d7   p\        P'                  RTP(                  P*                  T4        Rp?ELtRp?ii ; i  \         d   p\        P'                  R(TP(                  P*                  T4        Rp?T'       d`   T'       dX    TP                  4        \        P                  R4       R#   \         d"   p\        P                  RT4        Rp?R# Rp?ii ; iT'       g   \        P                  R4       R# R# Rp?ii ; i  \         d"   p\        P                  RT4        Rp?R# Rp?ii ; i  T'       d^   T'       dV    TP                  4        \        P                  R4       i   \         d!   p\        P                  RT4        Rp?i Rp?ii ; iT'       g   \        P                  R4       i i ; i))a  Cleanup user database by expiring entries which have not been
used for a long time.

:param thread: Whether this function is called in a different thread or not.

Usually it is called in a thread started when Typing Booster
has just finished starting up and is ready for input.

But in unittest test cases it may be called with an in memory
database without starting a thread. In that case a new
database connection is not needed.

r   z0Database cleanup not needed for memory database.NzDatabase cleanup starting ...rw   z'Closed thread-local database connectionzFailed to close database: %sz4Reused main thread database connection (not closing)c                 s6    V ^,          V ^,          V ^ ,          3# r   r	   rL   rN   r   rO   .TabSqliteDb.cleanup_database.<locals>.<lambda>  s    ddd+r   r   z)Total number of database rows to check=%siP  z1st pass: deleting %s %sr   z#DELETE from phrases WHERE id = :id;r   z61st pass: exception deleting row from database: %s: %sz61st pass: Number of rows deleted above maximum size=%sc                 s&    V ^,          V ^ ,          3# )i   r	   rL   rN   r   rO   r     s    #$Q4#$Q40!r   z 1st pass: Number of rows kept=%sg+?z2nd pass: Index for decay=%sz2nd pass: deleting %s %sz62nd pass: exception deleting row from database: %s: %sz2nd pass: decaying %s %sz
                        UPDATE phrases
                        SET user_freq = :user_freq, timestamp = :timestamp
                        WHERE id = :id
                        ;r#   r$   z62nd pass: exception decaying row from database: %s: %sz*Commit database and execute checkpoint ...r   z)Rebuild database using VACUUM command ...zVACUUM;z"Number of database rows deleted=%sz"Number of database rows decayed=%sz Number of rows before cleanup=%szNumber of rows remaining=%sz$Time for database cleanup=%s secondszDatabase cleanup finished.z)Exception when accessing database: %s: %s)r   r)   r]   re   r3   r4   r@   rx   rW   rg   rX   r*   r   rr   r   r   r   ry   r   rC   rD   r
   rd   r%   r?   )r9   r   Ztime_nowr   r4   rY   ZindexZmax_rowsZnumber_delete_above_maxZ	rows_keptr   r#   Zsqlstr_deleteZsqlargs_deleterE   Zindex_decayZnumber_of_rows_to_decayZnumber_of_rows_to_deleteZsqlstr_updateZsqlargs_updates   &&                  r   cleanup_databaseZTabSqliteDb.cleanup_database  sU    d'':5KKJK3499;ACO	L  ??%%'  ==##$<=FFHDx (PNN$KK IJ JL C $!D KKCSYOIEH&'#JLIF	$!H$?$??KK : $S	 $.A.2kk#a&.A!CD ,q0+$IM&*CF^N> ((G $$S)
) * KKP/1$ y&!"I
 	NEKK:EBh./KKK6D&'#'($ F	'!H$?$?? A~$>$(I$(MM2E26++c!f2E%GH 1A50(M*.AB$,,]K $>$(I$(MM2E26++c!f2E%GH 014/)
 +/A*5s9Q;7G*5tyy{*DB$,,]NK 
S !T KKDEOO56KKCDY'KK<03KKMKK</1KK:CIFKK5I)AACKK>		h.0KK45 (PNN$KK IJ JL  ! PNN#A;OOPO % >((4"__55u> >>f  ) B",,!8!&!9!95B BB,  ) B",,!8!&!9!95B BB*  	H#oo66?(PNN$KK IJ  PNN#A;OOPJL 	 ! PNN#A;OOP	 (PNN$KK IJ  PNN#A;OOPJL s+   AY 	%U CY U<0D)Y W +BY :XD'Y %\ U9U44U9<V=+V82Y 8V==Y  X+W<6Y <XY Y+Y :Y  YY \+\
>] %Z: :[&[!![&
\] \>\99\>_%]98_9^$^_^$$
_/_)r.   r-   r4   r0   r   N)zuser.db)FT)F)rb   rb   rb   rb   i    )rb   rb   )rb   rb      )rb   rb   rb   rb   r   )T)+r
   r   r   r   r   r:   r   rF   ZclassmethodrT   r[   r3   r6   r1   r2   rt   r   r   r5   r   r7   r   r   r   r8   ro   rq   r   r   r   r   r   r   r   r   rs   r   r   r   r   r   r   Z__classdictcell__)r   s   @r   r   r   1   s    )$ )$V   B 5 5< P P  P, 
 
" "H31 31j < .- -^1 1 <> >@ 6( (@9< 9<vUN UNn   D  2 , ,\ " !#%88	- -^G GRT+ T+l >& &P Bf fP *? ?$ eL eL eLr   r   )r   Ztypingr    r   r   r   r   r   r   r&   r   Z
contextlibr   rA   re   r   r   Zloggingr   r/   Z	getLoggerr)   r%   r'   rp   rg   r   r   r	   r   r   <module>r      sy   $        	  %   	    			0	1!f <i <FL FLr   