Kui?dZddlmZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl mZmZmZddlmZddlmZddlmZddlmZmZmZdd lmZdd lmZdd lm Z m!Z!dd l"m#Z#dd l$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5m6Z6m7Z7e jpe9Z:ee;e;fZZ?d4dZ@Gdde>ZAd5dZBd6dZCd7dZDGd d!ZEGd"d#eZFd8d$ZGeGd9d&ZHed'(Gd)d%ZIGd*d+eZJ d: d;d,ZK d< d=d-ZLd>d.ZMGd/d0eZNGd1d2ZOy)?zO The main purpose of this module is to expose LinkCollector.collect_sources(). ) annotationsN)IterableMutableMappingSequence) dataclass) HTMLParser)Values)Callable NamedTupleProtocol)requests)Response) RetryErrorSSLError)NetworkConnectionError)Link) SearchScope) PipSession)raise_for_status)is_archive_fileredact_auth_from_url) url_to_path)vcs)CandidatesFromPage LinkSource build_sourcectjD]6}|jj|s#|t |dvs4|cSy)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rschemeslower startswithlen)urlschemes g/root/.openclaw/workspace/chatterbox_venv/lib/python3.12/site-packages/pip/_internal/index/collector.py_match_vcs_schemer'.sE ++ 99; ! !& )c#f+.>$.FM c eZdZdfd ZxZS)_NotAPIContentcBt|||||_||_yN)super__init__ content_type request_desc)selfr/r0 __class__s r&r.z_NotAPIContent.__init__:s" |4((r()r/strr0r3returnNone)__name__ __module__ __qualname__r. __classcell__r2s@r&r*r*9s ))r(r*c|jjdd}|j}|jdryt ||j j )z Check the Content-Type header to ensure the response contains a Simple API Response. Raises `_NotAPIContent` if the content type is not a valid content-type. Content-TypeUnknown)z text/htmlz#application/vnd.pypi.simple.v1+html#application/vnd.pypi.simple.v1+jsonN)headersgetr!r"r*requestmethod)responser/content_type_ls r&_ensure_api_headerrE@s[##'' BL!'')N     x'7'7'>'> ??r(c eZdZy)_NotHTTPN)r6r7r8r(r&rGrGVsr(rGctjj|\}}}}}|dvr t|j |d}t |t |y)z Send a HEAD request to the URL, and ensure the response contains a simple API Response. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotAPIContent` if the content type is not a valid content type. >httphttpsT)allow_redirectsN)urllibparseurlsplitrGheadrrE)r$sessionr%netlocpathqueryfragmentresps r&_ensure_api_responserWZsV-3LL,A,A#,F)FFD% &&j <<T< 2DTtr(ctt|jr t||tj dt ||j|djgddd}t|t|tj dt ||jjd d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. rQzGetting page %sz, )r>z*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z max-age=0)Acceptz Cache-Control)r?zFetched page %s as %sr<r=) rrfilenamerWloggerdebugrr@joinrrEr?)r$rQrVs r&_get_simple_responser_lstCy))*S'2 LL"$8$=> ;; ii()+   D4Tt LLS! 3 Kr(c|rHd|vrDtjj}|d|d<|jd}|r t |Sy)z=Determine if we have any encoding information in our headers.r<z content-typecharsetN)emailmessageMessage get_paramr3)r?mras r&_get_encoding_from_headersrgsK>W, MM ! ! ##N3.++i( w<  r(c$eZdZddZddZddZy)CacheablePageContentc.|jsJ||_yr,)cache_link_parsingpager1rls r&r.zCacheablePageContent.__init__s&&&& r(ct|t|xr-|jj|jjk(Sr,) isinstancetyperlr$)r1others r&__eq__zCacheablePageContent.__eq__s-%d,P%**..1PPr(c@t|jjSr,)hashrlr$r1s r&__hash__zCacheablePageContent.__hash__sDIIMM""r(N)rl IndexContentr4r5)rqobjectr4bool)r4int)r6r7r8r.rrrvrHr(r&ririsQ#r(riceZdZddZy) ParseLinkscyr,rHrms r&__call__zParseLinks.__call__sr(Nrlrwr4zIterable[Link])r6r7r8r~rHr(r&r|r|sAr(r|cztjdfd tjdfd }|S)z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. c:t|jSr,)listrl)cacheable_pagefns r&wrapperz*with_cached_index_content..wrappersB~**+,,r(c`|jrt|St|Sr,)rkrir)rlrrs r&wrapper_wrapperz2with_cached_index_content..wrapper_wrappers+  " "/56 6BtH~r()rrir4 list[Link])rlrwr4r) functoolscachewraps)rrrs` @r&with_cached_index_contentrs@__--__R r(rwc#TK|jj}|jdr^tj|j }|j dgD])}tj||j}|&|+yt|j}|jxsd}|j|j j||j}|jxs|}|jD]!} tj | ||}||#yw)z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. r>filesNzutf-8)page_urlbase_url)r/r!r"jsonloadscontentr@r from_jsonr$HTMLLinkParserencodingfeeddecoderanchors from_element) rlrDdatafilelinkparserrr$ranchors r& parse_linksrs &&,,.N  !FGzz$,,'HHWb) D>>$1D|J    DHH %F}}'H KK ##H-. ((C%#H..  #I <  sD&D(T)frozencPeZdZUdZded<ded<ded<ded<d Zd ed <dd Zy )rwaRepresents one response (or page), along with its URL. :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. bytesrr3r/ str | Nonerr$Tryrkc,t|jSr,)rr$rus r&__str__zIndexContent.__str__s#DHH--r(N)r4r3)r6r7r8__doc____annotations__rkrrHr(r&rwrws/N H##.r(c4eZdZdZdfd ZddZddZxZS)rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. cPt|d||_d|_g|_y)NT)convert_charrefs)r-r.r$rr)r1r$r2s r&r.zHTMLLinkParser.__init__s( $/$( 46 r(c|dk(r(|j|j|}|||_yy|dk(r%|jjt |yy)Nbasea)rget_hrefrappenddict)r1tagattrshrefs r&handle_starttagzHTMLLinkParser.handle_starttagsT &=T]]2=='D $  CZ LL  U ,r(c*|D]\}}|dk(s |cSy)NrrH)r1rnamevalues r&rzHTMLLinkParser.get_href!s&  KD%v~  r()r$r3r4r5)rr3rlist[tuple[str, str | None]]r4r5)rrr4r)r6r7r8rr.rrr9r:s@r&rr s 7-r(rc<|tj}|d||y)Nz%Could not fetch URL %s: %s - skipping)r\r])rreasonmeths r&_handle_get_simple_failr(s  ||| 0$?r(ct|j}t|j|jd||j|S)Nr<)rr$rk)rgr?rwrr$)rCrkrs r&_make_index_contentr2sE*(*:*:;H ( LL-  r(c$|jjddd}t|}|rtj d||y|j drtt jjt|rL|jds|dz }tjj|d}tjd| t|| }t!||j" S#t$$rtj d |Yyt&$r6}tj d ||j(|j*Yd}~yd}~wt,$r}t/||Yd}~yd}~wt0$r}t/||Yd}~yd}~wt2$r6}d }|t5|z }t/||tj6Yd}~yd}~wt8j:$r}t/|d|Yd}~yd}~wt8j<$rt/|dYywxYw)N#rrzICannot look at %s URL %s because it does not support lookup as web pages.zfile:/z index.htmlz# file: URL is directory, getting %srY)rkz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: z timed out)r$splitr'r\warningr"osrSisdirrendswithrMrNurljoinr]r_rrkrGr*r0r/rrrrr3infor ConnectionErrorTimeout)rrQr$ vcs_schemerVexcrs r&_get_index_contentr?s ((..a  #C#3'J W    ~~g277==S1A#B||C 3JC ll""3 5 :C@U#C9:#4Dz'LinkCollector.create..sIs-c2Is)rrno_index)rQr) index_urlextra_index_urlsrr\r]r^rrcreater)clsrQoptionssuppress_no_indexrrrlink_collectors r&rzLinkCollector.creates''(7+C+CC   $5 LL&IjII J''-2 "))!!%% '% r(c.|jjSr,)rrrus r&rzLinkCollector.find_linkss  +++r(c0t||jS)z> Fetch an HTML page containing package links. rY)rrQ)r1locations r&fetch_responsezLinkCollector.fetch_responses"(DLLAAr(ctjfdjjDj }tjfdj Dj }t jtjrwtj||Dcgc]}||jd|j!}}t|ddg|z}t jdj|t!t#|t#|Scc}w)Nc 3hK|])}t|jjdd+yw)Fcandidates_from_pagepage_validator expand_dirrk project_nameNrrQis_secure_originrlocrrr1s r&rz0LinkCollector.collect_sources..sA 4  %9#||<< #()    4 /2c 3hK|])}t|jjdd+yw)TrNrrs r&rz0LinkCollector.collect_sources..sA 5  %9#||<<#')    5 rz* z' location(s) to search for versions of : )rr) collections OrderedDictrget_index_urls_locationsvaluesrr\ isEnabledForloggingDEBUG itertoolschainrr#r]r^rr)r1rrindex_url_sourcesfind_links_sourcessliness``` r&collect_sourceszLinkCollector.collect_sourcess2 (33 4 ((AA,O 4   &( )44 5  5   &(    w}} -#);=NO=QVV%7QVVH E u:,##/.3E LL5) *./-.  s>$D<N)rQrrrr4r5)F)rQrrr rryr4r)r4z list[str])rrr4IndexContent | None)rr3rrr4r) r6r7r8rr. classmethodrpropertyrrrrHr(r&rrs"   #(    B,,B , , 1,   , r(r)r$r3r4r)rCrr4r5)r$r3rQrr4r5)r$r3rQrr4r)r?ResponseHeadersr4r)rr|r4r|rr,)rrrzstr | ExceptionrzCallable[..., None] | Noner4r5)T)rCrrkryr4rw)rrrQrr4r)Pr __future__rr email.messagerbrrrrr urllib.parserMcollections.abcrrr dataclassesr html.parserroptparser typingr r r pip._vendorr pip._vendor.requestsrpip._vendor.requests.exceptionsrrpip._internal.exceptionsrpip._internal.models.linkr!pip._internal.models.search_scoperpip._internal.network.sessionrpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrpip._internal.utils.urlsrpip._internal.vcsrsourcesrrr getLoggerr6r\r3rr' Exceptionr*rErGrWr_rgrir|rrrwrrrrrrrHr(r&r s#  >>!" !)@;*948990!AA   8 $ c*)Y)@, y $<~ # #BB(8 $...(Z>(,@ @ @ %@ @48  ,0  9x,z, h h r(