+
    i}&  c                   s    R t ^ RIt^ RIt^ RIt^ RIt^ RItR.t]P                  ! RR4      t	R t
R t ! R R4      t ! R R	4      t ! R
 R4      tR# )a  robotparser.py

Copyright (C) 2000  Bastian Kleineidam

You can choose between two licenses when using this package:
1) GNU GPLv2
2) PSF license for Python 2.2

The robots.txt Exclusion Protocol is implemented as specified in
http://www.robotstxt.org/norobots-rfc.txt
NRobotFileParserRequestRatezrequests secondsc                 s    \         P                  P                  V R R7      p\         P                  P                  VR R7      # )surrogateescape)Zerrors)urllibparseZunquoteZquote)pathZunquoted   & +/usr/lib64/python3.14/urllib/robotparser.py	normalizer      s7    ||##D1B#CH<<h/@AA    c                 s    V P                  R 4      w  rp\        V 4      p V'       d*   \        P                  ! RR V4      pV R V,           ,          p V # )Z?z[^=&]+c                 s&    \        V ^ ,          4      # )i    )r   )Zm   &r   Z<lambda>Z normalize_path.<locals>.<lambda>    s    IadOr	   )Z	partitionr   reZsub)r   ZsepZquerys   &  r   normalize_pathr      sG    ~~c*DuT?D
y";UCeKr	   c                   sp   a  ] tR t^%t o RtRR ltR tR tR tR t	R t
R tR	 tR
 tR tR tR tRtV tR# )r    zjThis class provides a set of methods to read, parse and answer
questions about a single robots.txt file.

c                s|    . V n         . V n        R V n        RV n        RV n        V P                  V4       ^ V n        R # )NF)entriessitemapsdefault_entrydisallow_all	allow_allset_urllast_checkedselfurl   &&r   __init__ZRobotFileParser.__init__+   s;    !!Sr	   c                s    V P                   # )zReturns the time the robots.txt file was last fetched.

This is useful for long-running web spiders that need to
check for new robots.txt files periodically.

)r   r   r
   r   mtimeZRobotFileParser.mtime4   s        r	   c                s8    ^ RI pVP                  4       V n        R# )zISets the time the robots.txt file was last fetched to the
current time.

N)timer   )r   r   r   r   modifiedZRobotFileParser.modified=   s    
 	 IIKr	   c                sx    Wn         \        P                  P                  V4      R,          w  V n        V n        R# )z,Sets the URL referring to a robots.txt file.:i   i   NN)r   r   r   Zurlsplithostr   r   r   r   r   ZRobotFileParser.set_urlE   s)    %||44S9#>	49r	   c                s    \         P                  P                  V P                  4      pVP	                  4       pV P                  VP                  RR4      P                  4       4       R#   \         P                  P                   d^   pTP                  R9   d	   RT n        M)TP                  R8  d   TP                  R8  d   RT n        TP                  4         Rp?R# Rp?ii ; i)z4Reads the robots.txt URL and feeds it to the parser.zutf-8r   Ti  i  N)i  i  )r   ZrequestZurlopenr   readr   ZdecodeZ
splitlinesZerrorZ	HTTPErrorZcoder   r   Zclose)r   ZfZrawZerr   &   r   r   ZRobotFileParser.readJ   s    
	L&&txx0A &&(CJJszz'+<=HHJK ||%% 	xx:%$(!SSXX^!%IIKK	s   )A, ,C(AC##C(c                s    R VP                   9   d   V P                  f	   Wn        R# R# V P                  P                  V4       R# )*N)
useragentsr   r   append)r   entryr   r   
_add_entryZRobotFileParser._add_entryX   s;    %"""!!)%*" * LL&r	   c                s   ^ p\        4       pV P                  4        V EF(  pV'       g9   V^8X  d   \        4       p^ pM$V^8X  d   V P                  V4       \        4       p^ pVP                  R4      pV^ 8  d   VRV pVP	                  4       pV'       g   K{  VP                  R^4      p\        V4      ^8X  g   K  V^ ,          P	                  4       P                  4       V^ &   V^,          P	                  4       V^&   V^ ,          R8X  dJ   V^8X  d   V P                  V4       \        4       pVP                  P                  V^,          4       ^pEK8  V^ ,          R8X  d<   V^ 8w  d2   VP                  P                  \        V^,          R4      4       ^pEK~  EK  V^ ,          R8X  d<   V^ 8w  d2   VP                  P                  \        V^,          R4      4       ^pEK  EK  V^ ,          R	8X  dR   V^ 8w  dH   V^,          P	                  4       P                  4       '       d   \        V^,          4      Vn        ^pEK&  EK)  V^ ,          R
8X  d   V^ 8w  d   V^,          P                  R4      p\        V4      ^8X  d   V^ ,          P	                  4       P                  4       '       d]   V^,          P	                  4       P                  4       '       d2   \        \        V^ ,          4      \        V^,          4      4      Vn        ^pEK  EK  V^ ,          R8X  g   EK  V P"                  P                  V^,          4       EK+  	  V^8X  d   V P                  V4       R# R# )z|Parse the input lines from a robots.txt file.

We allow that a user-agent: line is not preceded by
one or more blank lines.
Z#NZ:z
user-agentZdisallowFZallowTzcrawl-delayzrequest-rate/Zsitemap)Entryr   r$   ZfindZstripsplitZlenlowerr!   r"   	rulelinesRuleLineZisdigitZintdelayr   req_rater   )r   ZlinesZstater#   lineZiZnumberss   &&     r   r   ZRobotFileParser.parsea   s    DA:!GEEaZOOE*!GEE		#AAvBQx::<D::c1%D4yA~q'--///1Qq'--/Q7l*z. %$$++DG4E!W
*z..xQ/GH ! " !W'z..xQ/FG ! " !W-z  7==?2244*-d1g,EK ! " !W.z"&q'--"4LA-'!*2B2B2D2L2L2N2N '
 0 0 2 : : < <-8WQZ#gVWj/-ZEN ! " !W	)
 MM((a1o p A:OOE" r	   c                s   V P                   '       d   R# V P                  '       d   R# V P                  '       g   R# \        P                  P                  VR4      p\        P                  P                  ! RR.VR,          O5!  p\        V4      pV'       g   RpV P                   F-  pVP                  V4      '       g   K  VP                  V4      u # 	  V P                  '       d   V P                  P                  V4      # R# )z=using the parsed robots.txt decide if useragent can fetch urlFT N:i   NNr%   )r   r   r   r   r   Z	_urlsplitZ_urlunsplitr   r   
applies_to	allowancer   )r   	useragentr   Z
parsed_urlr#   s   &&&  r   	can_fetchZRobotFileParser.can_fetch   s    >>>
    
 \\++C4
ll&&tTCJrNCS!C\\E	**s++ " %%//44r	   c                    V P                  4       '       g   R # V P                   F(  pVP                  V4      '       g   K  VP                  u # 	  V P                  '       d   V P                  P                  # R # N)r   r   r/   r+   r   r   r1   r#      && r   crawl_delayZRobotFileParser.crawl_delay   s\    zz||\\E	**{{" " %%+++r	   c                r3   r4   )r   r   r/   r,   r   r5   r6   r   request_rateZRobotFileParser.request_rate   s\    zz||\\E	**~~% " %%...r	   c                sB    V P                   '       g   R # V P                   # r4   )r   r   r
   r   	site_mapsZRobotFileParser.site_maps   s    }}}}}r	   c                s    V P                   pV P                  e   WP                  .,           pRP                  \        \        V4      4      # )Nz

)r   r   joinmapstr)r   r   r   r   __str__ZRobotFileParser.__str__   s>    ,,)!3!3 44G{{3sG,--r	   )	r   r   r   r   r   r   r   r   r   N)r.   )__name__
__module____qualname____firstlineno____doc__r   r   r   r   r   r$   r   r2   r7   r8   r9   r=   __static_attributes____classdictcell__Z__classdict__   @r   r    r    %   sN     
!(?
L'G#R<
. .r	   c                   s6   a  ] tR t^t o RtR tR tR tRtV t	R# )r*   zhA rule line is a single "Allow:" (allowance==True) or "Disallow:"
(allowance==False) followed by a path.c                sT    VR 8X  d   V'       g   Rp\        V4      V n        W n        R# )r.   TN)r   r   r0   )r   r   r0   s   &&&r   r   ZRuleLine.__init__   s!    2:iI"4(	"r	   c                sf    V P                   R 8H  ;'       g    VP                  V P                   4      # )r    )r   Z
startswith)r   filenamer   r   r/   ZRuleLine.applies_to   s(    yyCAA8#6#6tyy#AAr	   c                s^    V P                   '       d   R MRR,           V P                  ,           # )ZAllowZDisallowz: r0   r   r   r
   r   r=   ZRuleLine.__str__   s     >>>zTADIIMMr	   rH   N)
r>   r?   r@   rA   rB   r   r/   r=   rC   rD   rE   rF   r   r*   r*      s!     1#BN Nr	   r*   c                   s<   a  ] tR t^t o RtR tR tR tR tRt	V t
R# )r&   z?An entry has one or more user-agents and zero or more rulelinesc                s>    . V n         . V n        R V n        R V n        R # r4   )r!   r)   r+   r,   r   r
   r   r   ZEntry.__init__   s    
r	   c                s   . pV P                    F  pVP                  R V 24       K  	  V P                  e   VP                  RV P                   24       V P                  e8   V P                  pVP                  RVP                   RVP
                   24       VP                  \        \        V P                  4      4       RP                  V4      # )zUser-agent: zCrawl-delay: zRequest-rate: r%   Z
)r!   r"   r+   r,   ZrequestsZsecondsZextendr;   r<   r)   r:   )r   ZretagentZrater   r   r=   ZEntry.__str__   s    __EJJeW-. %::!JJtzzl34==$==DJJa~FG

3sDNN+,yy~r	   c                s    VP                  R4      ^ ,          P                  4       pV P                   F&  pVR8X  d    R# VP                  4       pW!9   g   K%   R# 	  R# )z2check if this entry applies to the specified agentr%   r    TF)r'   r(   r!   )r   r1   rI   r6   r   r/   ZEntry.applies_to  sQ     OOC(+113	__E|KKME! % r	   c                sv    V P                    F(  pVP                  V4      '       g   K  VP                  u # 	  R# )zJPreconditions:
- our agent applies to this entry
- filename is URL encodedT)r)   r/   r0   )r   rG   r-   r6   r   r0   ZEntry.allowance  s0     NNDx((~~% # r	   )r+   r,   r)   r!   N)r>   r?   r@   rA   rB   r   r=   r/   r0   rC   rD   rE   rF   r   r&   r&      s!     I
 r	   r&   )rB   Zcollectionsr   Zurllib.errorr   Zurllib.parseZurllib.requestZ__all__Z
namedtupler   r   r   r    r*   r&   ) r	   r   <module>rJ      sd   
  	   
$$]4FGB@. @.DN N"( (r	   