Re~D DdZddlZddlZddlZddlZddlZddlZddlZddlZ ddl Z ddl Z ddl mZddlmZmZmZmZmZmZmZmZddlmZmZddlmZddlmZmZddl m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+ddl,m-Z-m.Z.ddl/m0Z0ddl1m2Z2m3Z3m4Z4ej5e6Z7e j8j9j:Z;eeGdde?Z@deddfdZAGdde?ZBdeZVGd?d@eZWGdAdBZXdS)EzO The main purpose of this module is to expose LinkCollector.collect_sources(). N)Values)CallableIterableListMutableMapping NamedTupleOptionalSequenceUnion)html5librequests)Response) RetryErrorSSLError)NetworkConnectionError)Link) SearchScope) PipSession)raise_for_status)is_archive_file)pairwiseredact_auth_from_url)vcs)CandidatesFromPage LinkSource build_sourceurlreturnctjD]D}||r|t |dvr|cSEdS)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rschemeslower startswithlen)rschemes /builddir/build/BUILDROOT/alt-python311-pip-21.3.1-3.el8.x86_64/opt/alt/python311/lib/python3.11/site-packages/pip/_internal/index/collector.py_match_vcs_schemer'0sW + 99;; ! !& ) ) c#f++.>$.F.FMMM 4c,eZdZdededdffd ZxZS)_NotHTML content_type request_descrNcht||||_||_dSN)super__init__r+r,)selfr+r, __class__s r&r0z_NotHTML.__init__<s3 |444((r()__name__ __module__ __qualname__strr0 __classcell__)r2s@r&r*r*;sR)S))))))))))))r(r*responsec|jdd}|dst ||jjdS)zCheck the Content-Type header to ensure the response contains HTML. Raises `_NotHTML` if the content type is not text/html. Content-Type text/htmlN)headersgetr"r#r*requestmethod)r8r+s r&_ensure_html_headerrABs^ #'';;L      * *; 7 7>|X%5%<===>>r(ceZdZdS)_NotHTTPN)r3r4r5r(r&rCrCLsDr(rCsessionctj|\}}}}}|dvrt||d}t |t |dS)zSend a HEAD request to the URL, and ensure the response contains HTML. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotHTML` if the content type is not text/html. >httphttpsT)allow_redirectsN)urllibparseurlsplitrCheadrrA)rrEr%netlocpathqueryfragmentresps r&_ensure_html_responserSPsq -3L,A,A#,F,F)FFD% &&&jj <<T< 2 2DTr(c*tt|jrt||tdt |||ddd}t|t||S)aAccess an HTML page with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotHTML` if it is not HTML. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got HTML, and raise `_NotHTML` otherwise. rEzGetting page %sr<z max-age=0)Acceptz Cache-Control)r=) rrfilenamerSloggerdebugrr>rrA)rrErRs r&_get_html_responserZ`stCyy)**4c73333 LL"$8$=$=>>> ;; !)     D(T Kr(r=cd|r-d|vr)tj|d\}}d|vr|dSdS)z=Determine if we have any encoding information in our headers.r:charsetN)cgi parse_header)r=r+paramss r&_get_encoding_from_headersr`sI%>W,,"/0GHH f   )$ $ 4r(documentpage_urlcl|dD]}|d}||cS|S)aDetermine the HTML document's base URL. This looks for a ```` tag in the HTML document. If present, its href attribute denotes the base URL of anchor tags in the document. If there is no such tag (or if it does not have a valid href attribute), the HTML file's URL is used as the base URL. :param document: An HTML document representation. The current implementation expects the result of ``html5lib.parse()``. :param page_url: The URL of the HTML document. z.//basehref)findallr>)rarbbaserds r&_determine_base_urlrgsK  ++xx  KKK  Or(partcztjtj|S)zP Clean a "part" of a URL path (i.e. after splitting on "@" characters). )rJrKquoteunquoterhs r&_clean_url_path_partrms* <  fl22488 9 99r(cztjtj|S)z Clean the first part of a URL path that corresponds to a local filesystem path (i.e. the first part after splitting on "@" characters). )rJr? pathname2url url2pathnamerls r&_clean_file_url_pathrqs* > & &v~'B'B4'H'H I IIr(z(@|%2F)rO is_local_pathcb|rt}nt}t|}g}t t j|dgD]J\}}|||||Kd |S)z* Clean the path portion of a URL. r;) rqrm_reserved_chars_resplitr itertoolschainappendupperjoin)rOrr clean_funcparts cleaned_partsto_cleanreserveds r&_clean_url_pathrs*) )   $ $T * *EM&yurd'C'CDD//(ZZ11222X^^--.... 77= ! !!r(ctj|}|j }t |j|}tj||S)z Make sure a link is fully quoted. For example, if ' ' occurs in the URL, it will be replaced with "%20", and without double-quoting other characters. )rr)rO)rJrKurlparserNrrO urlunparse_replace)rresultrrrOs r& _clean_linkrs[\ " "3 ' 'F %M 6;m D D DD < " "6???#=#= > >>r(anchorbase_urlc |d}|sdSttj||}|d}|d}t ||||}|S)zJ Convert an anchor element in a simple repository page to a Link. rdNzdata-requires-pythonz data-yanked) comes_fromrequires_python yanked_reason)r>rrJrKurljoinr)rrbrrdr pyrequirerlinks r&_create_link_from_elementrs ::f  D t fl**8T:: ; ;C 122IJJ}--M  !#    D Kr(c2eZdZd dZdedefdZdefdZdS) CacheablePageContentpageHTMLPagerNc&|jsJ||_dSr.)cache_link_parsingr)r1rs r&r0zCacheablePageContent.__init__s&&&& r(othercpt|t|o|jj|jjkSr.) isinstancetyperr)r1rs r&__eq__zCacheablePageContent.__eq__ s*%d,,P%*.1PPr(c4t|jjSr.)hashrrr1s r&__hash__zCacheablePageContent.__hash__sDIM"""r()rrrN) r3r4r5r0objectboolrintrrDr(r&rrskQFQtQQQQ########r(rfnrctjddtdttffd tjdddttffd }|S) z Given a function that parses an Iterable[Link] from an HTMLPage, cache the function's result (keyed by CacheablePageContent), unless the HTMLPage `page` has `page.cache_link_parsing == False`. N)maxsizecacheable_pagerc>t|jSr.)listr)rrs r&wrapperz'with_cached_html_pages..wrappersBB~*++,,,r(rrcr|jrt|St|Sr.)rrr)rrrs r&wrapper_wrapperz/with_cached_html_pages..wrapper_wrappers;  " 77/5566 6BBtHH~~r() functools lru_cacherrrwraps)rrrs` @r&with_cached_html_pagesrs&&&- 4-d-----'&-_RjT$Z r(rc#Ktj|j|jd}|j}t ||}|dD]}t|||}||VdS)zP Parse an HTML document, and yield its anchor elements as Link objects. F)transport_encodingnamespaceHTMLElementsz.//a)rbrN)r rKcontentencodingrrgrer)rrarrrrs r& parse_linksr(s ~ =#H (C"8S11H""6**(     <  r(c JeZdZdZ d dedeedededdf d Zdefd Z dS) rz'Represents one page, along with its URLTrrrrrNc>||_||_||_||_dS)am :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. N)rrrr)r1rrrrs r&r0zHTMLPage.__init__Cs'   "4r(c*t|jSr.)rrrs r&__str__zHTMLPage.__str__Vs#DH---r(T) r3r4r5__doc__bytesr r6rr0rrDr(r&rr@s11$( 5553-5 5 ! 5  5555&.......r(rreasonmeth).Nc<| tj}|d||dS)Nz%Could not fetch URL %s: %s - skipping)rXrY)rrrs r&_handle_get_page_failrZs+  ||D 0$?????r(Trcdt|j}t|j||j|S)N)rrr)r`r=rrr)r8rrs r&_make_html_pagerds:)(*:;;H  L-    r(c|td|jddd}t|}|rtd||dSt j|\}}}}}}|dkrtj t j |rU|ds|dz }t j|d}td | t#|| }t%||j S#t($rtd |Yn t*$r1}td ||j|jYd}~nd}~wt0$r}t3||Yd}~nd}~wt4$r}t3||Yd}~nd}~wt6$r:}d} | t9|z } t3|| tjYd}~nRd}~wt<j$r}t3|d|Yd}~n(d}~wt<j $rt3|dYnwxYwdS)Nz?_get_html_page() missing 1 required keyword argument: 'session'#rrzICannot look at %s URL %s because it does not support lookup as web pages.file/z index.htmlz# file: URL is directory, getting %srU)rz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.ziSkipping page %s because the %s request got Content-Type: %s.The only supported Content-Type is text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: z timed out)! TypeErrorrrur'rXwarningrJrKrosrOisdirr?rpendswithrrYrZrrrCr*r,r+rrrrr6infor ConnectionErrorTimeout) rrEr vcs_schemer%_rOrRexcrs r&_get_html_pagerns M    (..a  #C#3''J W     t &|44S99FAtQ1 BGMM&.*E*Ed*K*KLL||C    3JCl""3 55 :C@@@Q!#w7778t8OPPPP7     1          ;               ")))dC(((((((( )))dC(((((((( >>>G#c((dF=========  #@@@d$>$>$>????????  111dK000001 4sT E%I9/ I98'F$$ I91G I9G(( I950H**I9<I"I98I9cTeZdZUeeeed<eeeed<dS)CollectedSources find_links index_urlsN)r3r4r5r r r__annotations__rDr(r&rrs>*-....*-......r(rc eZdZdZdededdfdZe ddeded e ddfd Z e de e fd Zd edeefd Zde dedefdZdS) LinkCollectorz Responsible for collecting Link objects from all configured locations, making network requests as needed. The class's main method is its collect_sources() method. rE search_scoperNc"||_||_dSr.)rrE)r1rErs r&r0zLinkCollector.__init__s ) r(Foptionssuppress_no_indexc |jg|jz}|jr<|s:tddd|Dg}|jpg}tj||}t||}|S)z :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. zIgnoring indexes: %s,c34K|]}t|VdSr.)r).0rs r& z'LinkCollector.create..s+IIs-c22IIIIIIr(rr)rEr) index_urlextra_index_urlsno_indexrXrYrzrrcreater)clsrErrrrrlink_collectors r&rzLinkCollector.creates'(7+CC   $5  LL&IIjIIIII   J'-2 ")!!   '%   r(c|jjSr.)rrrs r&rzLinkCollector.find_linkss ++r(locationc.t||jS)z> Fetch an HTML page containing package links. rU)rrE)r1rs r& fetch_pagezLinkCollector.fetch_pagesh ====r( project_namecandidates_from_pagecttjfdj|D}tjfdjD}t tj redtj ||D}t|d|dg|z}t d|tt!|t!|S)Nc3TK|]"}t|jjddV#dS)Frpage_validator expand_dirrNrrEis_secure_originrlocrr1s r&rz0LinkCollector.collect_sources..s\ 4 4  %9#|< #(     4 4 4 4 4 4 r(c3TK|]"}t|jjddV#dS)TrNrrs r&rz0LinkCollector.collect_sources..s\ 5 5  %9#|<#'     5 5 5 5 5 5 r(c4g|]}||j d|jS)Nz* )r)rss r& z1LinkCollector.collect_sources.. s6=QV%7QV %7%7%7r(z' location(s) to search for versions of : r) collections OrderedDictrget_index_urls_locationsvaluesrrX isEnabledForloggingDEBUGrvrwr$rYrzrr)r1rrindex_url_sourcesfind_links_sourcesliness` ` r&collect_sourceszLinkCollector.collect_sourcess (3 4 4 4 4 4 (AA,OO 4 4 4    &(( )4 5 5 5 5 5  5 5 5    &((    w} - - +");=NOOE u::33#/333E LL5)) * * *.//-..    r()F)r3r4r5rrrr0 classmethodrrrpropertyrr6rrr rrrrr rDr(r&rrs!"   #(    [@,DI,,,X,>4>HX,>>>>> * * 1*   * * * * * * r(rr.r)Yrr]rrrvrrre urllib.parserJurllib.requestxml.etree.ElementTreexmloptparsertypingrrrrrr r r pip._vendorr r pip._vendor.requestsrpip._vendor.requests.exceptionsrrpip._internal.exceptionsrpip._internal.models.linkr!pip._internal.models.search_scoperpip._internal.network.sessionrpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrrpip._internal.vcsrsourcesrrr getLoggerr3rXetree ElementTreeElement HTMLElementr6ResponseHeadersr' Exceptionr*rArCrSrZr`rgrmrqcompile IGNORECASErtrrrrrrrrrrrrrrDr(r&r(s9                      +*******))))))@@@@@@@@;;;;;;******999999444444888888999999CCCCCCCC!!!!!!AAAAAAAAAA  8 $ $i#+  c*38C=)))))y)))>(>t>>>>     y    s Z D     /C/*/////dHSM+&:s:s:::: Js Js J J J J RZ 2=99"#"d"s"""", ?S ?S ? ? ? ? d^ 4 # # # # # # # #*x~-. zlDJ&',jXd^.........:+/@@ @ #y. !@ 8I& '@ @@@@hDH15<< <!*-< j<<<<~/////z/// f f f f f f f f f f r(