
    i                     ~   S r SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSK	r	SSK
r
SSKrSSKJr  SSKJr  SSKJr  SSKJrJrJrJrJrJrJr  \
R4                  R6                  r " S	 S
5      r " S S\5      r " S S\5      r " S S\5      r " S S5      r  " S S5      r! " S S\5      r" " S S\5      r#S r$\R0                  " SSS9S 5       r%0 r&0 r'Sr(Sr)Sr*S r+S!r,\)\)\*\+\+\,S".r-S#r.S$r/\.\.\.\/\/\/S".r0 S% \
Rb                  l2         \Rh                  Rj                  SLr6 \Rh                  Rp                  r9\Rt                  " SS'S(5      r;\;(       a.   SSK<r<\<Rz                  " \;\R|                  R~                  5        C;gC;g! \3 a     Nf = f! \7 a    S&r6 Nxf = f! \7 a    S&r9 Nof = f! \3 a     C;gf = f))z7Common classes and constants used by extractor modules.    N)ElementTree)HTTPAdapter   )Message   )configoutputtextutildtcache	exceptionc                   (   \ rS rSrSrSrSrSrSrSr	Sr
SrSrSrSrSrSrSrSrSrSrSrSr\R0                  rS	rS	rS
rS	rS r\S 5       r S r!S r"S r#S r$S r%S3S jr&S\RN                  4S jr(S\RN                  \)" 5       4S jr*S r+S3S jr,S3S jr-S r.   S4S jr/S r0S r1S5S jr2\Rf                  r4  S6S jr5S r6S5S jr7S7S  jr8S3S! jr9S" r:S# r;S$ r<S% r=S& r>S' r?S7S( jr@S) rAS8S* jrBS+ rCS, rDS3S- jrES9S. jrFS9S/ jrG\S0 5       rHS5S1 jrIS2rJg):	Extractor    F)
{category}z{filename}.{extension}r   NT        g      N@c                 L   [         R                  " U R                  5      U l        UR                  U l        Xl        UR                  5       U l        0 U l        U R                  [        ;   aT  U R                   SU R                   3nU[        ;   a  [        U   u  U l        U l
        O[        U R                     U l        [        R                  U l        [        R                  U l        [        R                   U l        SU R                  U R                  4U l        SU l        g )N:	extractorr   )logging	getLoggercategorylogstringurlmatchgroupskwdictCATEGORY_MAPsubcategoryr   parseparse_datetime	parse_isoparse_datetime_isoparse_tsparse_timestamp_cfgpath
_parentdir)selfr   catsubs      H/app/mltbenv/lib/python3.13/site-packages/gallery_dl/extractor/common.py__init__Extractor.__init__8   s    $$T]]3<<
lln==L(a(8(8'9:F%2>v2F/t/ ,T]] ; hh"$,,!{{$dmmT5E5EF    c                     [        U R                  [        5      (       a%  [        R                  " U R                  5      U l        U R                  R                  U5      nU(       a  U " U5      $ S $ N)
isinstancepatternstrr   
re_compiler   )clsr   r   s      r-   from_urlExtractor.from_urlM   sL    ckk3''//#++6CK!!#&"s5z,,r0   c                 B    U R                  5         U R                  5       $ r2   )
initializeitemsr+   s    r-   __iter__Extractor.__iter__T   s    zz|r0   c                 P   U R                  5         U R                  cI  U R                  5         U R                  R                  U l        U R                  b  U R                  5         OU R                  R                  U l        U R                  5         [        R                  U l	        g r2   )
_init_optionssession_init_sessioncookiescookies_domain_init_cookies_initr   noopr;   r=   s    r-   r;   Extractor.initializeX   ss    << <<//DL"".""$<<//DL

))r0   c                     g r2    r=   s    r-   finalizeExtractor.finalizef       r0   c              #      #    g 7fr2   rK   r=   s    r-   r<   Extractor.itemsi   s     s   c                     g)Nr   rK   )r+   nums     r-   skipExtractor.skipm   s    r0   c                 D    [         R                  " U R                  X5      $ r2   )r   interpolater)   r+   keydefaults      r-   r   Extractor.configp   s    !!$-->>r0   c                 R    U R                  X5      nXTLa  U$ U R                  X#5      $ r2   r   )r+   rX   key2rY   sentinelvalues         r-   config2Extractor.config2s   s*    C* L{{4))r0   c                     U R                  X$5      nXdLa4  X%;  a-  UR                  U5        U R                  R                  SX!5        UnU R                  X5      nXdLa  U$ U$ )Nz%'%s' is deprecated. Use '%s' instead.)r   addr   warning)r+   rX   
deprecatedrY   r^   historyr_   s          r-   config_deprecatedExtractor.config_deprecatedy   sc    J1 (J'  !H!+2GC* Lr0   c                 D    [         R                  " U R                  U5      $ r2   )r   
accumulater)   )r+   rX   s     r-   config_accumulateExtractor.config_accumulate   s      44r0   c                     U$ r2   rK   rW   s      r-   config_instanceExtractor.config_instance   s    r0   c                 F    [         R                  " SU R                  X5      $ )Nr   )r   interpolate_commonr)   rW   s      r-   _config_sharedExtractor._config_shared   s     ((DMM39 	9r0   c                    SnSnU R                    Hl  nU(       a  Sn[        R                  " X4-   U5      nM'  [        R                  " X4S   5      =n(       d  MI  [        R                  " U R                  4XS9WS S& Mn     W$ )NTrq   Fr   )conf)r)   r   rj   getr"   )r+   rX   firstextrpathvaluesrv   s          r-   _config_shared_accumulate#Extractor._config_shared_accumulate   sy    MMD**4;<Dq'222#..%%'9r
 " r0   c                 6   Uc  U R                   nUc  U R                  nUc  U R                  nSU;  a  U R                  US'   SU;  a  U R                  US'   SU;  a  U R
                  US'   SU;   aW  US   =nbO  [        R                  " U5      R                  5       US'   US	 UR                  S5      =n(       a  SUS'   OSS0US'   S =pS	nU R                  (       aV  U(       aO  U R                  5       [        R                  " 5       [        R                  -
  -
  nUS
:  a  U R                  US5          UR                  " X!40 UD6nUR                   nU R"                  (       a  U R%                  U5        US:  d!  UU;   d  US:  a  U(       d  US:w  d  Ub  USL a5  U	(       a  Xl        UU(       a  [        R                  " 5       [        l        $ $ U
bd  US:X  a^  U
SL a  U R(                  R*                  n
U =R,                  [.        R0                  R2                  -  sl        [.        R0                  " U
5      eSU SUR4                   SUR6                   S3n[        R8                  " U5      nUb  U R:                  R=                  U5        US:X  aB  U R?                  U5      (       a,   U(       d  GM  [        R                  " 5       [        l        GM  US:X  a  U R@                  (       a  O3UU;  a-  US:  a'   U(       a  [        R                  " 5       [        l        OU(       a  [        R                  " 5       [        l        U R:                  R]                  SUUUS	-   5        UU:  a  OUnU R                  (       a  U R                  5       nUU:  a  UnUS:X  a:  U R@                  (       a)  U RA                  5       nUU:  a  UnU R_                  USS9  OU R                  US5        US	-  nGM  U(       a  USL a2  U R:                  R=                  U5        [        R`                  " UU5      $ Uc  [.        Rb                  " UU5      nO[.        Rd                  " X5      nU =R,                  UR2                  -  sl        Ue! [B        RD                  RF                   a  n URH                  S   R4                  nUR(                  RJ                  n[M        URH                  S   5      RO                  S5      u  nnnSU SU=(       d    URQ                  5        3nO! [R         a    Un Of = fSn S nAGNS nAf[B        RD                  RT                  [B        RD                  RV                  [B        RD                  RX                  4 a  nUnSn S nAGNsS nAf[B        RD                  RZ                   a6  nUn S nAU(       d  GM  [        R                  " 5       [        l        GM  S nAff = f! U(       a  [        R                  " 5       [        l        f f = f)Nproxiestimeoutverifyjsondataheaderszapplication/jsonContent-Typer   r   requestTi  i  i  .i  ' z' for 'r   r   z: z
%s (%s/%s)z429 Too Many Requests)secondsreasonretry)3rB   _retries_retry_codes_proxies_timeout_verifyr   
json_dumpsencoderw   	_intervaltimer   request_timestampsleepr   status_code_write_pages_dump_responseencoding	__class__r"   statusr   NotFoundErrorcoder   r   detect_challenger   rd   _handle_429_interval_429requests
exceptionsConnectionErrorargs__name__r5   	partitionlstrip	ExceptionTimeoutChunkedEncodingErrorContentDecodingErrorRequestExceptiondebugwaitNullResponse	HttpErrorChallengeError)r+   r   methodrB   fatalretriesretry_codesexpectedintervalr   notfoundkwargsr   r   response	challengetriesr   r   msgexcr   r7   pre_errss                              r-   r   Extractor.request   s    ?llG?mmG++KF" $F9F" $F96!#||F8Vv&3!%!6!=!=!?v6N$jj3373.@GN+)79K(LF9%##>>h~~'		i&A&AACG}

7I.6>"??6A&A&  ++$$''13JH$3J!dckU]SL,4)#* 26))+I/ ) 'DCK4'#'>>#=#=KK9#:#:#?#??K#11(;;$q 1aH 11(;	(HH$$Y/3;4#3#3H#=#= 826))+I/ S[T%7%7, 26))+I/ 26))+I/HHNN<eWQY?wG~~NN$Q;Gs{t11&&(Q;G		'2I	J

7G,QJES V HHS!$$S#..%%c84C**9?Csxx	c &&66  XXa[//F **33C"%fkk"o"6"@"@"EKCCcU"cjS%8%8%:$;<C  C''//''<<''<<>  ''88 F 826))+I/MJ 26))+I/ s   8R A#W0 CW0 $W0 W-$A9TT8T-*T8,T--T82W0 8AW-VW0 !W-7W(9W0 (W--W0 0(Xc                     UR                  SS5        UR                  SS5        UR                  SS5        U R                  " U40 UD6R                  R                  SS5      $ )Nr   HEADallow_redirectsFr   locationr   )
setdefaultr   r   rw   )r+   r   r   s      r-   request_locationExtractor.request_location  sZ    (F++U3*e,||C*6*2266z2FFr0   c                    U R                   " U40 UD6n [        R                  " UR                  5      $ ! [         a  nUR                  SS5      nU(       a  USL aq  [        R                  " U5      =n(       a  U R                  R                  U5        O1U R                  R                  SUR                  R                  U5        0 s S nA$ e S nAff = f)Nr   T.%s: %s)r   r   
json_loadsr
   r   rw   r   r   rd   r   r   )r+   r   r   r   r   r   r   s          r-   request_jsonExtractor.request_json  s    <<.v.
	??8==11 	JJw-EESL $ 5 5h ??9?HH$$Y/HH$$Xs}}/E/EsK		s!   5 
CB	CCCCc                 b   U R                   " U40 UD6nU(       a  UR                  nOUR                  R                  SS5      n[        R                  " 5       n UR                  U5        UR                  5       $ ! [         a  nUR                  SS5      nU(       a  USL a  [        R                  " U5      =n	(       a  U R                  R                  U	5        O1U R                  R                  SUR                  R                  U5        [        R                  " S5      s S nA$ e S nAff = f)Nz xmlns=z ns=r   T.r   r   )r   r
   replacer   	XMLParserfeedcloser   rw   r   r   r   rd   r   r   Element)
r+   r   xmlnsr   r   r
   parserr   r   r   s
             r-   request_xmlExtractor.request_xml-  s    <<.v.==D==((F;D&&(	KK<<>! 	JJw-EESL $ 5 5h ??9?HH$$Y/HH$$Xs}}/E/EsK"**2..	s%    A; ;
D.BD)"D.(D))D.c                 D   [         R                   " 5       nU(       a  [        U5      nXQ-   nOXU(       aF  [        U[        R                  5      (       a  [        R
                  " U5      nO[        U5      nX%-
  nO[        S5      eX-  nUS::  a  g U(       at  [        R                  R                  U5      R                  5       nUR                  S SUR                  S SUR                  S 3nU R                  R                  SXt5        [         R                  " U5        g )Nz'Either 'seconds' or 'until' is requiredr   02r   zWaiting until %s (%s))r   floatr3   r   datetimeto_ts
ValueErrorfromtimestamphourminutesecondr   infor   )r+   r   untiladjustr   nowtisotimes           r-   r   Extractor.waitE  s    iikGnGME%--ekGFGGc>))%0557A1QXXbM188B-@GHHMM17C

7r0   c                 h    U R                   R                  SX5        [        R                  " U5        g )NzSleeping %.2f seconds (%s))r   r   r   r   )r+   r   r   s      r-   r   Extractor.sleep`  s$    3	(

7r0   c                     U R                  U5        U(       a   [        U5      $ [        R                  " U5      $ ! [        [        4 a     g f = fr2   )_check_input_allowedinputEOFErrorOSErrorgetpass)r+   promptechos      r-   r   Extractor.inpute  sL    !!&)V}$ ??6** g& s   
; AAc                     U R                  S5      nUc  [        R                  nU(       d)  [        R                  " SUR                  S5       S35      eg )Nr   zUser input required (z :))r   r	   	TTY_STDINr   AbortExtractionstrip)r+   r   r   s      r-   r   Extractor._check_input_allowedp  sQ    G$=$$E++'T(:';1=? ? r0   c                 >   U R                  S5      nU(       d  U(       aA  U R                  S5      nU(       d&  U R                  S5        [        R                  " 5       nX!4$ U R                  SS5      (       a7   [        R                  " 5       R                  U R                  5      nUu  p$nX!4$ X!4$ ! [        [        R                  4 a(  nU R                  R                  SU5         SnAX!4$ SnAf[         a     U R                  R                  S5         X!4$ f = f)z?Return authentication information as (username, password) tupleusernamepasswordnetrcFz	netrc: %sNznetrc: No authentication info)r   r   r   
LazyPromptr  authenticatorsr   r   NetrcParseErrorr   error	TypeErrorrd   )r+   r   r   r   r   r   s         r-   _get_auth_infoExtractor._get_auth_infox  s    ;;z*x{{:.H))*5??, !! [[%((B{{}33DMMB(,%X !!x!! U223 1{C00 !!  B  !@A!!Bs   93B2 2DC//(DDc                     g r2   rK   r=   s    r-   rG   Extractor._init  rN   r0   c                    U R                  SS5      U l        U R                  S5      U l        U R                  SS5      U l        U R                  SS5      U l        U R                  SS	5      U l        [        R                  " U R                  S
5      U R                  5      U l	        [        R                  " U R                  SU R                  5      U R                  5      U l        [        R                  " U R                  SU R                  5      5      U l        U R                  S:  a  [!        S5      U l        U R                  (       d  SU l        g g )Nzwrite-pagesFzretry-codesr      r      r   Tproxyzsleep-requestz	sleep-429r   infrK   )r   r   r   r   r   r   r   build_proxy_mapr   r   build_duration_funcrequest_intervalrequest_interval_minr   request_interval_429r   r   r=   s    r-   rA   Extractor._init_options  s    KKu= KK6Iq1Ir2{{8T2,,T[[-A488L11KK)>)>?%%
 "55KKT%>%>?
 ==1!%LDM   "D !r0   c                 B   [         R                  " 5       =U l        nUR                  nUR	                  5         S=p4U R                  SS5      (       a  SOSUl        U R                  S5      nUc  U R                  nU(       Ga3  [        U[        5      (       Ga  UR                  5       R                  S5      u  pVnU(       a  US:X  a  [        R                  (       a  SOS	nOUS
:X  a  SnOUS:X  a  S	nOUS:X  a  SnUS:X  a)  UR                  S5      (       a  UR                  SS5      nOSn[         U    H,  u  pU	(       a  SU	;   a  U	R                  SU5      X('   M(  XU'   M.     U["        R$                  ["        R&                  -  ["        R(                  -  ["        R*                  -  -  n[,        U   nO;U R.                  US'   SUS'   SUS'   U R0                  nUb  U[,        ;   a	  [,        U   n[2        (       a  SUS'   OSUS'   [4        (       a  US==   S-  ss'   U R                  SU R6                  5      =n
(       a=  [        U
[        5      (       a  XS'   O#U R8                  (       a  U R8                  S-   US'   U R                  S 5      nU(       a  US:X  a  GOUS:X  a  [;        S 5      US'   OUS   S!:X  a  [;        US"S  5      US'   OUS   S#:X  a  US"S  R                  5       nUS$;   a  [        R<                  US'   OUS%;   a  [        R>                  US'   OUS&;   a  [        R@                  US'   OuUS';   a  S(US'   OiU RB                  RE                  S)U5        OLU R.                  [F        R.                  L a  U R                  (       a  U[
        RH                  " S*S 5      La  XS'   U R                  S+5      =n(       aX  [        U[        5      (       a2  U[         ;   a
  [         U   nOU RB                  RK                  S,U5        S-nURM                  U5        U R                  S.5      =n(       a=  [        U[N        5      (       a  SRQ                  U5      nOU[,        ;   a
  [,        U   nOUnU R                  S/5      =n(       a$  [        U[        5      (       a  US4nO
US   US"   4nU R                  S05      nUc  U RR                  nU(       d.  U["        RT                  -  nU RB                  RW                  S15        U R                  S25      (       a   SS3K,J-n  OS n[c        X4UU5      nURe                  S5U5        URe                  S6U5        g ! [\         a=  nU RB                  RK                  S4UR^                  R`                  U5        S n S nANtS nAff = f)7Nr   z	proxy-envTFbrowserr   autozWindows NT 10.0; Win64; x64zX11; Linux x86_64windowslinuxmacoszMacintosh; Intel Mac OS X 15.5chrome	Macintosh.r   firefoxz{}
User-Agentz*/*Accepten-US,en;q=0.5Accept-Languagezgzip, deflate, brAccept-Encodingzgzip, deflatez, zstdrefererReferer/z
user-agent@r   +>   ffr  >   crr  >   gdl	gallerydl
gallery-dl>   bot	googlebot
google-botzGooglebot-Image/1.0z"Unsupported User-Agent preset '%s'rq   r   zInvalid 'headers' value '%s'rK   cipherszsource-addresstls12zTLS 1.2 disabled.
truststore)
SSLContextr   zhttps://http://)3r   SessionrB   r   clearr   	trust_envr  r3   r5   lowerr   r   WINDOWS
startswithr   HEADERSsslOP_NO_SSLv2OP_NO_SSLv3OP_NO_TLSv1OP_NO_TLSv1_1CIPHERS	useragentr2  BROTLIZSTDr%  root_browser_useragentUSERAGENT_FIREFOXUSERAGENT_CHROMEUSERAGENT_GALLERYDLr   rd   r   rw   r  updatelistjoinr3  OP_NO_TLSv1_2r   r4  r5  ImportErrorr   r   _build_requests_adaptermount)r+   rB   r   ssl_optionsssl_ciphersr  r   platformrX   r_   r%  	custom_uacustom_headerscustom_cipherssource_addressr3  ssl_ctxr   adapters                      r-   rC   Extractor._init_session  s   !)!1!1!33w//$%% %)KKT$B$BD++i(?llGz'3//#*==?#<#<S#A Gx61#|| :1D Y&8W$.W$;("&&{33'//S9H#%g.
TU]#(==x#@GL#(CL	 / COOcoo=OO,.1.?.?@ AK!'*K$(NNGL! %GH)9G%&,,K&;'+A%k26)<G%&)8G%&4%&(2&kk)T\\::7:'3''%,	"%)YY_	"KK-	I/)#$6t$<GL!q\S $6y}$EGL!q\S !!"++-I--(,(>(>%..(,(=(=%@@(,(@(@%@@(=%  8)E^^y2224<<NL!II$-L!![[33>3.#..!W,%,^%<NHHNN#A#13%'NNN>*![[33>3.$//!hh~67*%n5,![[)9::>:.#.."0!!4"0"3^A5F!GG$=JJE3,,,KHHNN./;;|$$<
 G)g~?j'*i)  x)?)?Es   U 
V!3VVc                 (   U R                  S5      =n(       az  U R                  S5      =n(       aP  US:X  a4  XR                  [        U5      -     n[        =R                  S-  sl        O[        R
                  " U5      nU R                  U5        gg)z Populate the session's cookiejarrD   zcookies-selectrotater   N)r   cookies_indexlenr   randomchoicecookies_load)r+   rD   selects      r-   rF   Extractor._init_cookies'  s|    kk),,7,%566v6X%%&8&83w<&GHG++q0+$mmG4Gg& -r0   c                    [        U[        5      (       a  U R                  XR                  5        g [        U[        5      (       a  [
        R                  " U5      n [        USS9 n[
        R                  " U5      nS S S 5        U R                  R                  SU5        U R                  R                  nW H  nU" U5        M     X l        g [        U[&        [(        45      (       a  [)        U5      n[*        R-                  U5      nUc  SSKJn	   U	" U5      nU[*        U'   OU R                  R                  S
U5        U R                  R                  nU H  nU" U5        M     g U R                  R1                  SUR                   R"                  U5        g ! , (       d  f       GN&= f! [         a;  nU R                  R                  SXR                   R"                  U5         S nAg S nAf[$         a;  nU R                  R                  SXR                   R"                  U5         S nAg S nAff = f! [$         a)  nU R                  R                  SU5        S	n S nAGN%S nAff = f)Nutf-8r   z"cookies: Loading cookies from '%s'z8cookies: Invalid Netscape cookies.txt file '%s' (%s: %s)z%cookies: Failed to load '%s' (%s: %s)r   )load_cookieszcookies: %srK   z%cookies: Using cached cookies from %sz\cookies: Expected 'dict', 'list', or 'str' value for 'cookies' option, got '%s' instead (%r))r3   dictcookies_update_dictrE   r5   r   expand_pathopencookiestxt_loadr   r   rD   
set_cookiecookies_filer   rd   r   r   r   rM  tupleCACHE_COOKIESrw   ri  r  )
r+   cookies_sourcerz   fprD   ro  cookier   rX   ri  s
             r-   rc  Extractor.cookies_load2  s   nd++$$^5H5HI,,##N3D)$1R"2226G 2 C-/!\\44
%Fv& &$(!u66'C#'',G21*>:G
 *1M#&FL00J!6" " HHNN:((11>CM 21 N   "1!/1G1GN N  N  !H!/1G1GN NN& ! !HH$$]C8 G!sT   
F0 (F?F0 H; 
F-(F0 0
H8:1G00H8=1H33H8;
I.I))I.c                 
   U R                  SS5      nU(       d  g[        U[        5      (       a  [        R                  " U5      nOU R
                  nU(       d  gUS-   n [        USSS9 n[        R                  " X@R                  5        SSS5        [        R                  " X25        g! , (       d  f       N%= f! [         a;  nU R                  R                  SX%R                  R                  U5         SnAgSnAff = f)	z1Store the session's cookies in a cookies.txt filezcookies-updateTNz.tmpwrg  rh  z)cookies: Failed to write to '%s' (%s: %s))r   r3   r5   r   rl  rp  rm  cookiestxt_storerD   osr   r   r   r  r   r   )r+   exportrz   path_tmprt  r   s         r-   cookies_storeExtractor.cookies_stored  s    -t4fc""##F+D$$D&=	Jhg6"%%b,,7 7JJx& 76  	JHHNN &'+]]-C-CSJ J	Js0   !B= ,!B,B= ,
B:6B= =
D1C==Dc                    [        U[        5      (       a%  U R                  X=(       d    U R                  5        gU R                  R
                  n [        U5      nU H  nU" U5        M     g! [         a    U" U5         gf = f)z-Update the session's cookiejar with 'cookies'N)r3   rj  rk  rE   rD   ro  iterr  )r+   rD   domainro  ru  s        r-   cookies_updateExtractor.cookies_updatez  sr    gt$$$$W.K8K8KL00J'w- &Fv& &  $7#$s   A/ /BBc                 p    U R                   R                  nUR                  5        H  u  pEU" XEUS9  M     g)z2Update cookiejar with name-value pairs from a dict)r  N)rD   setr<   )r+   
cookiedictr  ro  namer_   s         r-   rk  Extractor.cookies_update_dict  s0    \\%%
%++-KDt62 .r0   c           	         U R                   (       d  gUc  U R                  n[        U5      n[        R                  " 5       nU R                    GHm  nUR                  U;  a  M  U(       a  UR
                  U:X  a  O)U(       a   UR
                  R                  U5      (       d  MW  UR                  (       a  [        UR                  U-
  5      nUS::  aj  U R                  R                  SUR
                  R                  S5      UR                  [        R                  R                  UR                  5      5        M  US::  aS  US-  nU R                  R                  SUR
                  R                  S5      UR                  US-   U(       a  S	OS
5        UR                  UR                  5        U(       a  GMn    g   g)z;Check if all 'cookies_names' are in the session's cookiejarFr   zcookies: %s/%s expired at %sr  Q i  z1cookies: %s/%s will expire in less than %s hour%sr   r   r   T)rD   rE   r  r   r  r  endswithexpiresintr   rd   r   r   r   r   discard)	r+   cookies_namesr  
subdomainsnamesr   ru  diffhourss	            r-   cookies_checkExtractor.cookies_check  sL   ||>((FM"iikllF{{%'V]]f4v}}'='=f'E'E~~6>>C/019HH$$6,,S16;;11&..AC U] DLEHH$$K,,S16;;	%3R9
 MM&++&59 #: r0   c                     [         R                  " [        R                  " USS5      =(       d    [        R                  " USS5      5      $ )Nz#<script type="application/ld+json">	</script>z#<script type='application/ld+json'>r   r   r
   ry   r+   pages     r-   _extract_jsonldExtractor._extract_jsonld  s@    IIdA!# #IIdA!#$ 	$r0   c                     [         R                  " [        R                  " USS5      =(       d    [        R                  " USS5      5      $ )Nz, id="__NEXT_DATA__" type="application/json">r  z, id='__NEXT_DATA__' type='application/json'>r  r  s     r-   _extract_nextdataExtractor._extract_nextdata  s@    IIdJ!# #IIdJ!#$ 	$r0   c                 0    [         R                  " XU5      $ r2   )r   DatabaseCacheDecoratorr+   funcmaxagekeyargs       r-   _cacheExtractor._cache  s    ++D&AAr0   c                 ,    [         R                  " 5       $ r2   )r   Memcacher  s       r-   _cache_memoryExtractor._cache_memory  s    ~~r0   c                 Z   ^ ^ UU 4S jnT R                  S5      mU" SU5      U" SU5      4$ )z:Retrieve and parse 'date-min' and 'date-max' config valuesc                 f  > TR                  X5      n[        U[        5      (       a  Tc  [        R                  " U5      O[        R
                  " UT5      nU[        R                  L a'  TR                  R                  SU Tc  SOSU5        UnU$ [        [        R                  " U5      5      nU$ )Nz,Unable to parse '%s': Invalid %s string '%s'	isoformatdate)r   r3   r5   r   r%   r#   NONEr   rd   r  r   )rX   rY   tsdt_objfmtr+   s       r-   rw   (Extractor._get_date_min_max.<locals>.get  s    S*B"c""-0[b)bhhr3>ORWW$HH$$FCK[VRI !B I RXXf-.BIr0   zdate-formatzdate-minzdate-maxr\   )r+   dmindmaxrw   r  s   `   @r-   _get_date_min_maxExtractor._get_date_min_max  s1    	 kk-(:t$c*d&;;;r0   c                 0    [         R                  " USSS9  g )NFr   )ensure_asciiindent)r   	dump_json)r7   objs     r-   _dumpExtractor._dump  s    sq9r0   c                    U(       a%  UR                    H  nU R                  US5        M     [        [        S5      (       a  [        =R                  S-  sl        O4S[        l        [
        R                  " S5      R                  [        l        [        R                  S S[        R                  SUR                  5       3n[
        R                  (       a#  [        R                  R                  U5      SS nOUSS	 n [        US
-   S5       n[
        R                  " XU R                   S;   U R                   S:g  S9  SSS5        U R"                  R%                  SUR                  US
-   5        g! , (       d  f       N9= f! [&         a;  nU R"                  R)                  SUR*                  R,                  U5         SnAgSnAff = f)zWrite the response content to a .txt file in the current directory.

The file name is derived from the response url,
replacing special characters with "_"
F_dump_indexr   z[\\\\|/<>:\"?*&=#]+z>02r   N      z.txtwb)allALLr  )r   	hide_authzWriting '%s' response to '%s'z$Failed to dump HTTP request (%s: %s))rf   r   hasattrr   r  r   r6   sub_dump_sanitizer   r;  rz  rz   abspathrm  dump_responser   r   r   r   rd   r   r   )r+   r   rf   respfnamerz   rt  es           r-   r   Extractor._dump_response  su     ((##D%0 ) 9m,,!!Q&!$%I!'+&(((+ $ ))#.a,,S(,,?@B <<77??5)$3/D#;D	6dVmT*b""!...@#00E9 + HHMM9",,v7 +*  	6HHC[[1116 6	6s0   F 1E52F 5
F?F 
G1GG)r)   r   r   r*   r   r   r   r   r   r   r   rD   rp  r   r;   r    r   r   r$   r&   r(   rB   r"   r   r2   )	GETNTNNrK   TNN)T)NNg      ?z
rate limit)r   )NFNN)Kr   
__module____qualname____firstlineno__r   r"   basecategorybasesubcategorycategorytransferparentdirectory_fmtfilename_fmtarchive_fmtr   rG  rp  r_  rE   rB   r%  r2  r3  r  r   rI  rD  r  r  r  r   r.   classmethodr8   r>   r;   rL   r<   rS   r   SENTINELr`   r  rg   rk   rn   rs   r|   r   r   r   r   falser   r   r   r   r   r  rG   rA   rC   rF   rc  r}  r  rk  r  r  r  r  r  r  r  r   __static_attributes__rK   r0   r-   r   r      s   HKLOF#M+LKFDLMNGGGEG&&I* - -$? *. * :>#'==#%59 >BFJ(,vpG, **K47 6
	+?"*#(A*F	'0CdJ,'3'R$$B <" : :$6r0   r   c                   P    \ rS rSrSrSrSrSrSrSS jr	S	 r
S
 rS rS rS rSrg)GalleryExtractori  galleryz-{category}_{gallery_id}_{num:>03}.{extension})r   z{gallery_id} {title}z{gallery_id}_{num}rR   Nc                     [         R                  X5        Uc4  U R                  S   =n(       a  US   S:X  a  U R                  U-   U l        g X l        g )Nr   r'  )r   r.   r   rG  page_urlr+   r   r   rz   s       r-   r.   GalleryExtractor.__init__  sF    4';DKKN2D2Q3 II,DMMr0   c              #   >  #    U R                  5         U R                  (       a/  U R                  U R                  U R                  S9R                  nOS nU R                  U5      nU R                  U5      nU R                  U5      nSU;   aO  U R                  S5      (       a  [        R                  " USUS   5      nOa[        [        SUS   S-   5      U5      nOD[        n [        U5      US'   U R                  S5      (       a  [        R                  nU" US5      n[         R"                  SU4v   U R$                  nU(       aS  U HM  nUS   n	UR'                  U5        SX'   SU;  a  [        R(                  " X5        [         R*                  X4v   MO     U Hi  u  X''   u  pU
(       a.  UR'                  U
5        SU
;  a  [        R(                  " X5        O[        R(                  " X5        [         R*                  X4v   Mk     g ! [         a     GNf = f7f)	Nr   countzpage-reverser   r   r   r   	extension)loginr  r   r"   r
   metadataimagesassetsr   r   enumerate_reversedziprange	enumerater`  r  r   	DirectoryenumrL  nameext_from_urlUrl)r+   r  r   imgsr  r  r  enum_keyassetr   imgdatas              r-   r<   GalleryExtractor.items  s    

==<<(8(8   ::>$  D}}T"{{4 T"d?{{>**00q$w-H!T']1_-
 D3 #D	W ;;~..22D$]FT))99ElT""#e+))#5kk3--   /5*DNNSG$g-))#4%%c0++s(( /5'  s+   C!H$H 2DH
HHHHc                     gzLogin and set necessary cookiesNrK   r=   s    r-   r  GalleryExtractor.loginM      r0   c                     g)z#Return a dict with general metadataNrK   r  s     r-   r  GalleryExtractor.metadataP  r   r0   c                     g)z=Return a list or iterable of all (image-url, metadata)-tuplesNrK   r  s     r-   r  GalleryExtractor.imagesS  r   r0   c                     g)zrReturn an iterable of additional gallery assets

Each asset must be a 'dict' containing at least 'url' and 'type'
NrK   r  s     r-   r  GalleryExtractor.assetsV  r   r0   )r  r2   )r   r  r  r  r"   r  r  r  r  r.   r<   r  r  r  r  r  rK   r0   r-   r  r    s;    KBL:M&KD 3)j.2Lr0   r  c                   (    \ rS rSrSrSrSrSrSrSr	g)	ChapterExtractori]  chapter)r   z{manga}z?{volume:?v/ />02}c{chapter:>03}{chapter_minor:?//}{title:?: //}z@{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}z'{manga}_{chapter}{chapter_minor}_{page}r  rK   N)
r   r  r  r  r"   r  r  r  r  r  rK   r0   r-   r  r  ]  s)    KKM 	K  	2 Dr0   r  c                   @    \ rS rSrSrSrSrSrS
S jrS r	S r
S rS	rg)MangaExtractorij  mangaTNc                    [         R                  X5        Uc4  U R                  S   =n(       a  US   S:X  a  U R                  U-   U l        OX l        U R                  SS5      (       a  U R                  (       + U l        g g )Nr   r'  zchapter-reverseF)r   r.   r   rG  r  r   reverser  s       r-   r.   MangaExtractor.__init__q  sk    4';DKKN2D2Q3 II,DMM;;(%00#||+DL 1r0   c              #   p  #    U R                  5         U R                  (       a/  U R                  U R                  U R                  S9R                  nOS nU R                  U5      nU R                  (       a  UR                  5         U H(  u  p4U R                  US'   [        R                  X44v   M*     g 7f)Nr  
_extractor)
r  r  r   r"   r
   chaptersr  chapterclassr   Queue)r+   r  r  r	  r   s        r-   r<   MangaExtractor.items|  s     

==<<8H8H<INNDD==&<<%MG!%!2!2D--.. &s   B4B6c                     gr  rK   r=   s    r-   r  MangaExtractor.login  r   r0   c                     g)z3Return a list of all (chapter-url, metadata)-tuplesNrK   r  s     r-   r  MangaExtractor.chapters  r   r0   )r  r  r2   )r   r  r  r  r"   r  r  r  r.   r<   r  r  r  rK   r0   r-   r  r  j  s,    KLG	,/ .Br0   r  c                   b    \ rS rSrSrSr\R                  r\R                  rS r	S r
SS jrSrg)	Dispatchi  userNc                 "    U R                  5       $ r2   )r<   r=   s    r-   r>   Dispatch.__iter__  s    zz|r0   c                     g r2   rK   r=   s    r-   r;   Dispatch.initialize  rN   r0   rK   c                 &   U Vs0 s H  nUS   R                   U_M     nnUb"  U H  u  pgnUc  XV   XW'   M  XV   S   U4XW'   M     U R                  SU5      =(       d    Sn	U	S:X  a  Un	O6[        U	[        5      (       a!  U	R	                  SS5      R                  S5      n	/ n
U	 H.  n X[   u  pU
R                  [        R                  USU045        M0     [        U
5      $ s  snf ! [         a     U R                  R                  S	U5         Mk  f = f)
Nr   includerK   r  r   r   ,r  zInvalid include '%s')r"   r   r3   r5   r   splitappendr   r  KeyErrorr   rd   r  )r+   extractor_datarY   altr   
extractorsr  sub_altr   r"  resultsr   ry   s                r-   _dispatch_extractorsDispatch._dispatch_extractors  s.    '
& G%& 	 

 ?%(!c;*4/J'+5?1+=s*CJ'	 &) ++i17Re G%%ooc2.44S9GHK&0	 s\44HIJ   G}5
,  C  !7BCs   C!(C&&&DD)rK   N)r   r  r  r  r"   rE   r   rL   rS   r>   r;   r,  r  rK   r0   r-   r  r    s/    KN!!H>>Dr0   r  c                   $    \ rS rSrSrS rS rSrg)AsynchronousMixini  z(Run info extraction in a separate threadc              #   |  #    U R                  5         [        R                  " S5      n[        R                  " U R
                  U4SS9nUR                  5          UR                  5       nUc  UR                  5         g [        U[        5      (       a  UR                  5         UeUv   UR                  5         Ma  7f)N   T)targetr   daemon)r;   queuer  	threadingThreadasync_itemsstartrw   rN  r3   r   	task_done)r+   messagesthreadr   s       r-   r>   AsynchronousMixin.__iter__  s     ;;q>!!##
 	,,.C{#y))	I  s   B:B<c                      U R                  5        H  nUR                  U5        M     UR                  S 5        g ! [         a  nUR                  U5         S nAN2S nAff = fr2   )r<   putr   )r+   r:  r   r   s       r-   r7  AsynchronousMixin.async_items  sR    	zz|S! $ 	T  	LL	s   (< 
A!AA!rK   N)r   r  r  r  __doc__r>   r7  r  rK   r0   r-   r/  r/    s    2!,r0   r/  c                   4    \ rS rSrSrS rS r\S 5       rSr	g)BaseExtractori  rK   c                 r    U R                   (       d  U R                  U5        [        R                  X5        g r2   )r   _init_categoryr   r.   )r+   r   s     r-   r.   BaseExtractor.__init__  s%    }}&4'r0   c                 `   [        UR                  5       5       H  u  p#Uc  M
  U(       aa  U R                  US-
     u  U l        U l        nU R                  (       d  [
        R                  " US   5      U l        UR                  U l        OX0l        UR                  S5      S   U l          g    g )Nr   r   z://r   )
r  r   	instancesr   rG  r
   root_from_urlrw   rn   r   )r+   r   indexgroupr   s        r-   rD  BaseExtractor._init_category  s    %elln5LE 59^^E!G5L2DM49d99$($6$6uQx$@	+/88D( %I$)OOE$:1$=DM 6r0   c                 j   [         R                  " SU R                  5      =n(       a<  UR                  5        H(  u  p4[	        U[
        5      (       d  M  SU;   d  M$  XAU'   M*     / n/ =o`l        UR                  5        H  u  p4US   =n(       a  UR                  S5      nUR                  X7U45        UR                  S5      nU(       d*  [        R                  " XwR                  S5      S-   S  5      nUR                  US-   5        M     SU R                  -   S	-   S
R                  U5      -   S-   $ )Nrq   rG  r'  r4   r      z()z(?:z%:(https?://[^/?#]+)|(?:https?://)?(?:|z)))r   rw   r  r<   r3   rj  rG  rstripr%  reescaperI  rN  )	r7   rG  extra_instancesr   r   pattern_listinstance_listrG  r4   s	            r-   rL  BaseExtractor.update  s   $jj9I9IJJ?J"1"7"7"9dD))fn*.h' #: (**'oo/NHF|#t#{{3'  ($!78hhy)G))DC1)<)=$>?$/ 0 S%%% )! !#&88L#9:<AB	
r0   )r   rn   rG  N)
r   r  r  r  rG  r.   rD  r  rL  r  rK   r0   r-   rB  rB    s%    I(
 
 
r0   rB  c                   *    \ rS rSrSS jrS rS rSrg)RequestsAdapteri  Nc                 H    Xl         X l        [        R                  " U 5        g r2   )ssl_contextrY  r   r.   )r+   rY  rY  s      r-   r.   RequestsAdapter.__init__  s    &,T"r0   c                 r    U R                   US'   U R                  US'   [        R                  " U /UQ70 UD6$ NrY  rY  )rY  rY  r   init_poolmanagerr+   r   r   s      r-   r]   RequestsAdapter.init_poolmanager  s>     $ 0 0}#'#6#6 ++DB4B6BBr0   c                 r    U R                   US'   U R                  US'   [        R                  " U /UQ70 UD6$ r\  )rY  rY  r   proxy_manager_forr^  s      r-   ra  !RequestsAdapter.proxy_manager_for  s>     $ 0 0}#'#6#6 ,,TCDCFCCr0   )rY  rY  r  )r   r  r  r  r.   r]  ra  r  rK   r0   r-   rW  rW    s    #
C
Dr0   rW  c                    XX#4n [         U   $ ! [         a     Of = fU (       d  U(       d  U(       Ga#  Uci  [        R                  R	                  U =(       d    S US9n[
        R                  S:  d-  UR                  [
        R                  R                  5       5        O[        R                  R                  R                  n U[        R                  R                  l        [        R                  R	                  U =(       d    S US9nU[        R                  R                  l        O$! U[        R                  R                  l        f = fSUl        OS n[        XS5      =n[         U'   U$ )N)optionsr2  z2.32F)CACHE_ADAPTERSr&  urllib3
connectioncreate_urllib3_contextr   __version__load_verify_locationscertswherer   ssl_r5  check_hostnamerW  )rS  rT  rZ  rY  rX   rY  ssl_ctx_origr[  s           r-   rQ  rQ    s4    W
=Cc""  kW?!,,CC#+t[ D BK''&011(..2F2F2HI"<<,,77L</6!!,%00GG'/4 H F 0<!!,|!!,%*"$3%% %GnS!Ns    
AD, ,!Er  )r  r  c                   ^  SSK n UR                  T 5      R                  nSSKnUR                  UR                  UR                  5      nUR                  UR                  UR                  S5        UR                  S5        UR                  S5        UR                  5       u  pgU" SU SU S35        UR                  5       S   nUR!                  5         UR#                  S	5      R%                  S
5       HK  n	U	R'                  S5      u  pnU
R)                  5       R+                  5       S:X  d  M;  UR)                  5       n  O   SnUR-                  SU-   5        UR!                  5         UR/                  5       $ ! UR                   a/    T (       d  e SSKnUR                  T 5      =m (       d  e U 4S jn GNf = f)z*Get User-Agent header from default browserr   Nc                 b   > [         R                  " TU 4[         R                  (       a  SOSS9  g )NFT)start_new_session)r   Popenr;  )r   r  s    r-   rm   _browser_useragent.<locals>.openM  s!    JJ~26,,DJr0   r   )z	127.0.0.1r   r6  r   z/user-agenti   s   
   :s
   user-agentr0   s   HTTP/1.1 200 OK

)
webbrowserrw   rm  ErrorshutilwhichsocketAF_INETSOCK_STREAM
setsockopt
SOL_SOCKETSO_REUSEADDRbindlistengetsocknameacceptr   recvr$  r   r   r:  senddecode)r  rv  rm  rx  rz  serverhostportclientlinerX   r   r_   rD  s   `             r-   rH  rH  @  s    J~~g&++ ]]6>>6+=+=>F
f'')<)<a@
KK !
MM!##%JD74&${	+,]]_QF
LLND!''0t,99;-/I	 1 	
KK*Y67
LLNE  	J!<<000	J 	J	Js   E> >;F=<F=rK   ))r   z7Mozilla/5.0 ({}; rv:140.0) Gecko/20100101 Firefox/140.0)r!  z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8r#  r"  r$  N
Connectionz
keep-alive)r   N)zContent-LengthNr&  N)OriginNCookieNzSec-Fetch-Destempty)Sec-Fetch-ModecorszSec-Fetch-Sitezsame-originTEtrailers))r   z7Mozilla/5.0 ({}; rv:128.0) Gecko/20100101 Firefox/128.0)r!  zmtext/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8r  r  r  r  zUpgrade-Insecure-Requests1r  r  r  zno-corsr  r  )r  )z	sec-ch-uaz'"Not)A;Brand";v="8", "Chromium";v="138")zsec-ch-ua-mobilez?0)zsec-ch-ua-platformz"Linux"r  )r   zVMozilla/5.0 ({}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36r!  ztext/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7r  r  r  r  r  r#  zen-US,en;q=0.9)r  r  )r   zVMozilla/5.0 ({}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36r  r  r  r  r  r  r  )ru  N)zcontent-lengthN)r  zfirefox/140zfirefox/128r  z
chrome/138z
chrome/111a  TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA:AES256-SHAa\  TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA:AES256-SHAc                     g r2   rK   )r   s    r-   <lambda>r    s    r0   FwarningsrY   )@r@  rz  rP  r>  r   r  r4  ra  r   r   r   r5  	xml.etreer   requests.adaptersr   messager   r   r   r	   r
   r   r   r   r   packagesrf  r   r  r  r  r  r/  rB  rW  rQ  rH  re  rr  r!   HEADERS_FIREFOX_140HEADERS_FIREFOX_128HEADERS_CHROMIUM_138HEADERS_CHROMIUM_111r=  CIPHERS_FIREFOXCIPHERS_CHROMIUMrC  sessionsget_netrc_authr   r   brotlirE  AttributeErrorHAS_ZSTDrF  rw   actionr  simplefilterr   HTTPWarningrK   r0   r-   <module>r     s0   > 	 	 
         ! )  ? ? ?



#
#h6 h6VRy Rj
' 
&BY &BR' 'T D+
I +
\Dk D$B E!$' %'T  "  & $ '&&''' ( $ #""###	'5H$
$$D0F
$$D
 
B
I	.	fg&8&8&D&DE /  		  F  D  
sH   =F
 F )F$ +F2 
FFF! F!$F/.F/2F<;F<