o ~ReΛ@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl ZddlZddlZddlZddlmZddlZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+dd l,m-Z-e.d Z/e.d ej0Z1e.d Z2e.d ej0j3Z4d5Z6gdZ7dZ8dZ9e9j:dj:ej;edZddZ?dBddZ@dBddZAdBddZBdedfd d!ZCd"d#ZDe.d$ej0ZEeDd%d&ZFGd'd(d(ZGGd)d*d*eGZHGd+d,d,eZIe.d-jJZKd.d/ZLd0d1ZMdCd2d3ZNd4d5ZOGd6d7d7ZPGd8d9d9e jQZRejSjTfd:d;ZUdd?ZWd@dAZXdS)Dz#PyPI and direct package downloadingNwraps) CHECKOUT_DIST Distribution BINARY_DISTnormalize_path SOURCE_DIST Environmentfind_distributions safe_name safe_version to_filename Requirement DEVELOP_DISTEGG_DIST)log)DistutilsError) translate)Wheelunique_everseenz^egg=([-A-Za-z0-9_.+!]+)$zhref\s*=\s*['"]?([^'"> ]+)z([^<]+)\n\s+\(md5\)z([-+.a-z0-9]{2,}):z.tar.gz .tar.bz2 .tar .zip .tgz) PackageIndexdistros_for_urlparse_bdist_wininstinterpret_distro_namezsz(interpret_distro_name..Nr3) py_versionrArS)r7anyrangerPrjoin)rJrQrFr]rArSr9rZr!r!r"rs $ rcstfdd}|S)zs Wrap a function returning an iterable such that the resulting iterable only ever yields unique items. cst|i|SNr)argskwargsfuncr!r"wrapperszunique_values..wrapperr)rerfr!rdr" unique_valuessrgz(<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>c cst|D]2}|\}}tttj|d}d|vs#d|vr8t |D]}t j |t |dVq(qdD] }||}|dkr[t ||}|r[t j |t |dVq;dS)zEFind rel="homepage" and rel="download" links in `page`, yielding URLs,Zhomepager0r3)z Home PagezDownload URLr/N)RELfinditergroupssetmapstrstripr'r7HREFr4rurljoin htmldecoderEfindsearch)r8pagerDtagrelZrelsposr!r!r"find_external_linkss    ryc@(eZdZdZddZddZddZdS) ContentCheckerzP A null content checker that defines the interface for checking content cCdS)z3 Feed a block of data to the hash. Nr!selfblockr!r!r"feedzContentChecker.feedcCr|)zC Check the hash. Return False if validation fails. Tr!r~r!r!r"is_validrzContentChecker.is_validcCr|)zu Call reporter with information about the checker (hash name) substituted into the template. Nr!)r~reportertemplater!r!r"reportszContentChecker.reportN)__name__ __module__ __qualname____doc__rrrr!r!r!r"r{s  r{c@sBeZdZedZddZeddZddZ dd Z d d Z d S) HashCheckerzK(?Psha1|sha224|sha384|sha256|sha512|md5)=(?P[a-f0-9]+)cCs||_t||_||_dSra) hash_namehashlibnewhashexpected)r~rrr!r!r"__init__s  zHashChecker.__init__cCsBtj|d}|s tS|j|}|stS|di|S)z5Construct a (possibly null) ContentChecker from a URLr/Nr!)r4rr5r{patternrt groupdict)clsr8r?rDr!r!r"from_urls zHashChecker.from_urlcCs|j|dSra)rupdater}r!r!r"rzHashChecker.feedcCs|j|jkSra)r hexdigestrrr!r!r"rrzHashChecker.is_validcCs||j}||Sra)r)r~rrmsgr!r!r"rs zHashChecker.reportN) rrrrXcompilerr classmethodrrrrr!r!r!r"rs   rcsLeZdZdZ  dLddZdMd d ZdMd d ZdMd dZddZddZ ddZ ddZ ddZ dNddZ ddZdNfdd Zdd Zd!d"Zd#d$Zd%d&Zd'd(Z  dOd)d*ZdPd+d,Zd-d.Zd/Zd0d1Zd2d3ZdNd4d5Zd6d7Zd8d9Zd:d;Zdd?Z e!dMd@dAZ"dBdCZ#dDdEZ$dFdGZ%dHdIZ&dJdKZ'Z(S)Qrz;A distribution index that scans web pages for download URLshttps://pypi.org/simple/*NTcOsrtj|g|Ri||dd|d |_i|_i|_i|_td t t |j |_ g|_tjj|_dS)Nr.|)r rr( index_url scanned_urls fetched_urls package_pagesrXrr`rmrrDallowsto_scanr4requesturlopenopener)r~rhostsZ ca_bundleZ verify_sslrbkwr!r!r"rszPackageIndex.__init__Fc Cs||jvr |s dSd|j|<t|s||dStt|}|r.||s(dS|d||s7|r7||jvrAtt|j |dS||sMd|j|<dS| d|d|j|<d}| |||}|durhdSt |t jjr||jdkr|| d|jd|j|j<d|jd d vr|dS|j}|}t |tst |t jjrd }n|jd pd }||d }|t|D]} t j|t| d} |!| q|"|j#rt$|dddkr|%||}dSdSdS)z.)filterrUr<rr itertoolsstarmap scan_egg_link)r~ search_pathdirsZ egg_linksr!r!r"scan_egg_links|s zPackageIndex.scan_egg_linkscCsttj||}ttdttj|}Wdn1swYt |dkr,dS|\}}t tj||D]}tjj|g|R|_ t |_ ||q9dS)Nr\)openrUr<r`rrrmrnrorPr rJrrAr)r~r<rZ raw_lineslinesZegg_pathZ setup_pathrGr!r!r"rs  zPackageIndex.scan_egg_linkcCsd}||js |Stttjj|t|jdd}t|dks)d|dvr+|St |d}t |d}d|j | i|<t|t|fS)N)NNr.r\r2r3rT)r)rrrmr4rr6rPr7r r r setdefaultr'r )r~rZNO_MATCH_SENTINELr9pkgverr!r!r"_scans   zPackageIndex._scanc Cst|D]}z|tj|t|dWqty!Yqw||\}}|s-dSt ||D]$}t |\}}| drQ|sQ|rL|d||f7}n| || |q2tdd|S)z#Process the contents of a PyPI pager3r.pyz #egg=%s-%scSsd|dddS)Nz%sr3r\)rE)mr!r!r"sz,PackageIndex.process_index..)rprjrr4rrqrrrErryr@r(need_version_infoscan_urlPYPI_MD5sub) r~r8rurDrrnew_urlr+fragr!r!r"rs&"    zPackageIndex.process_indexcCs|d|dS)NzPPage at %s links to .py file(s) without version info; an index scan is required.)scan_allr~r8r!r!r"rszPackageIndex.need_version_infocGs<|j|jvr|r|j|g|R|d||jdS)Nz6Scanning index of all packages (this may take a while))rrrrrr~rrbr!r!r"rs zPackageIndex.scan_allcCsz||j|jd|j|js||j|jd|j|js)||t|j|jdD]}||q3dS)Nr.r!) rr unsafe_namerrkeyrKnot_found_in_indexr)r~ requirementr8r!r!r" find_packagess  zPackageIndex.find_packagescsR|||||jD]}||vr|S|d||qtt|||S)Nz%s does not match %s)prescanrrrsuperrobtain)r~r installerrG __class__r!r"rs zPackageIndex.obtaincCsL||jd||s$|t|td|jjtj |fdS)z- checker is a ContentChecker zValidating %%s checksum for %sz7%s validation failed for %s; possible download problem?N) rrrrrUunlinkrrr*r<rQ)r~checkerrVtfpr!r!r" check_hashs zPackageIndex.check_hashcCsN|D]"}|jdust|r|dstt|r||q|j|qdS)z;Add `urls` to the list that will be prescanned for searchesNfile:)rrr)rrrappend)r~urlsr8r!r!r"add_find_linkss   zPackageIndex.add_find_linkscCs"|jr tt|j|jd|_dS)z7Scan urls scheduled for prescanning (e.g. --find-links)N)rrrmrrr!r!r"r s zPackageIndex.prescancCs<||jr |jd}}n|jd}}|||j|dS)Nz#Couldn't retrieve index page for %rz3Couldn't find index page for %r (maybe misspelled?))rrrrr)r~rmethrr!r!r"rs   zPackageIndex.not_found_in_indexcCs~t|ts5t|}|r)||d||}t|\}}|dr'||||}|Stj |r1|St |}t | ||ddS)aLocate and/or download `spec` to `tmpdir`, returning a local path `spec` may be a ``Requirement`` object, or a string containing a URL, an existing local filename, or a project/version requirement spec (i.e. the string form of a ``Requirement`` object). If it is the URL of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is automatically created alongside the downloaded file. If `spec` is a ``Requirement`` object or a string containing a project/version requirement spec, this method returns the location of a matching distribution (possibly after downloading it to `tmpdir`). If `spec` is a locally existing file or directory name, it is simply returned unchanged. If `spec` is a URL, it is downloaded to a subpath of `tmpdir`, and the local filename is returned. Various errors may be raised if a problem occurs during downloading. r3rrJN)rrr _download_urlrEr@r( gen_setuprUr<rr#rfetch_distribution)r~rtmpdirr:foundr+r?r!r!r"r0s    zPackageIndex.downloadc sd|id}d fdd }|r$|||}|s/|dur/|||}|dur@jdur<||}|durO|sO|||}|dur`drZdp[d|dSd||j|jd S) a|Obtain a distribution suitable for fulfilling `requirement` `requirement` must be a ``pkg_resources.Requirement`` instance. If necessary, or if the `force_scan` flag is set, the requirement is searched for in the (online) package index as well as the locally installed packages. If a distribution matching `requirement` is found, the returned distribution's ``location`` is the value you would have gotten from calling the ``download()`` method with the matching distribution's URL or filename. If no matching distribution is found, ``None`` is returned. If the `source` flag is set, only source distributions and source checkout links will be considered. Unless the `develop_ok` flag is set, development and system eggs (i.e., those using the ``.egg-info`` format) will be ignored. zSearching for %sNcs|dur}||jD];}|jtkr#s#|vr"d|d|<q ||vo.|jtkp. }|rF|j}||_tj |jrF|Sq dS)Nz&Skipping development or system egg: %sr3) rrArrrr0rJdownload_locationrUr<r)reqenvrGtestloc develop_okr~skippedsourcer r!r"rsUs(z-PackageIndex.fetch_distribution..findz:No local packages or working download links found for %s%sza source distribution of rzBest match: %s)rJra)rrrrrcloner ) r~rr  force_scanrrZ local_indexrGrsr!rr"r=s2         zPackageIndex.fetch_distributioncCs"|||||}|dur|jSdS)a3Obtain a file suitable for fulfilling `requirement` DEPRECATED; use the ``fetch_distribution()`` method now instead. For backward compatibility, this routine is identical but returns the ``location`` of the downloaded distribution instead of a distribution object. N)rrJ)r~rr rrrGr!r!r"fetchszPackageIndex.fetchc Cs t|}|rddt||ddDpg}t|dkrxtj|}tj||krEtj ||}ddl m }|||sEt |||}ttj |dd} | d|dj|djtj|dfWd|S1sqwY|S|rtd ||ftd ) NcSsg|]}|jr|qSr!)rL)rYdr!r!r" s z*PackageIndex.gen_setup..r3r)samefilezsetup.pywzIfrom setuptools import setup setup(name=%r, version=%r, py_modules=[%r]) zCan't unambiguously interpret project/version identifier %r; any dashes in the name or version should be escaped using underscores. %rzpCan't process plain .py files without an '#egg=name-version' suffix to enable automatic setup script generation.)rCrDrrErPrUr<rQdirnamer`Zsetuptools.command.easy_installrshutilcopy2rwriterKrLsplitextr) r~rVr?r rDrrQdstrrr!r!r"rsF          zPackageIndex.gen_setupi c Cs2|d|d}zt|}||}t|tjjr%td||j |j f|}d}|j }d}d|vrI| d} t tt| }||||||t|d1} ||} | ro|| | | |d 7}||||||nnqP|||| Wdn1swY|W|r|SS|r|ww) NzDownloading %szCan't download %s: %s %srr/zcontent-lengthzContent-LengthwbTr3)rrrrrr4rrrrr dl_blocksizeget_allmaxrmint reporthookrrrrrr) r~r8rVfprrblocknumbssizesizesrrr!r!r" _download_tosF            zPackageIndex._download_tocCsdSrar!)r~r8rVr(Zblksizer*r!r!r"r&zPackageIndex.reporthookc Cs|dr t|Szt||jWSttjjfyD}z$ddd|j D}|r0| ||n t d||f|WYd}~dSd}~wt j jyY}z|WYd}~Sd}~wt j jy}z|rl| ||jn t d||jf|WYd}~dSd}~wtjjy}z|r| ||jn t d||jf|WYd}~dSd}~wtjjtj fy}z|r| ||n t d||f|WYd}~dSd}~ww)Nr cSsg|]}t|qSr!)rn)rYargr!r!r"rsz)PackageIndex.open_url..z%s %szDownload error for %s: %sz;%s returned a bad status line. The server might be down, %s)r) local_openopen_with_authrrhttpclient InvalidURLr`rbrrr4rrURLErrorreason BadStatusLineline HTTPExceptionsocket)r~r8warningvrr!r!r"rs^ zPackageIndex.open_urlcCst|\}}|rd|vr|dddd}d|vs nd}|dr(|dd}tj||}|dks8|d r>|||S|d ksG|d rM|||S|d rX| ||S|d krht j t j |dS||d|||S)Nz...\_Z__downloaded__rHr&svnzsvn+gitzgit+zhg+rr\T)r@replacer(rUr<r`r) _download_svn _download_git _download_hgr4r url2pathnamerr5r_attempt_download)r~r:r8r r*r?rVr!r!r"r s(         zPackageIndex._download_urlcCs||ddS)NT)rrr!r!r"r)rzPackageIndex.scan_urlcCs2|||}d|ddvr||||S|S)Nrrr)r,rr'_download_html)r~r8rVrr!r!r"rG,s zPackageIndex._attempt_downloadcCsjt|}|D]}|r%td|r#|t||||Snq|t|td|)Nz ([^- ]+ - )?Revision \d+:zUnexpected HTML page found at ) r���ro���rX���rt���r���rU���r���rC��r���)r~���r8���r���rV���r���r8��r!���r!���r"���rH��3��s���    zPackageIndex._download_htmlc�����������������C���s��t�dt�|ddd�}d}|�drqd|v�rqtj|\}}}}}} |sq|drqd |d d��v�rq|d d��d d\}}t |\} } | rqd | v�r]| d d\} } d | | f�}nd | �}| }|||||| f}tj |}|� d||�t d|||f��|S�)Nz"SVN download support is deprecatedr2���r3���r���r���zsvn:@z//r.���r\���:z --username=%s --password=%sz --username=z'Doing subversion checkout from %s to %szsvn checkout%s -q %s %s)warningsr��� UserWarningr7���r'���r)���r4���r���r5��� _splituser urlunparser���rU���system)r~���r8���rV���credsr:���netlocr<���rZ���qr���authhostuserpwr9���r!���r!���r"���rC��B��s&���   zPackageIndex._download_svnc�����������������C���sp���t�j|�\}}}}}|ddd�}|ddd�}d�}d|v�r)|dd\}}t�j||||df}�|�|fS�)N+r3���r/���r2���r���rI��r���)r4���r���urlsplitr7���rsplit urlunsplit)r8��� pop_prefixr:���rQ��r<���r>���r���revr!���r!���r"���_vcs_split_rev_from_urlX��s���z$PackageIndex._vcs_split_rev_from_urlc�����������������C���l���|�ddd�}|�j|dd\}}|�d||�td||f��|d�ur4|�d|�td ||f��|S�) Nr2���r3���r���Tr[��zDoing git clone from %s to %szgit clone --quiet %s %szChecking out %szgit -C %s checkout --quiet %sr7���r]��r���rU���rO��r~���r8���rV���r\��r!���r!���r"���rD��j����� zPackageIndex._download_gitc�����������������C���r^��) Nr2���r3���r���Tr_��zDoing hg clone from %s to %szhg clone --quiet %s %szUpdating to %szhg --cwd %s up -C -r %s -qr`��ra��r!���r!���r"���rE��z��rb��zPackageIndex._download_hgc�����������������G������t�j|g|R���d�S�ra���)r���r���r���r!���r!���r"���r��������zPackageIndex.debugc�����������������G���rc��ra���)r���r���r���r!���r!���r"���r�����rd��zPackageIndex.infoc�����������������G���rc��ra���)r���r���r���r!���r!���r"���r�����rd��zPackageIndex.warn)r���r���NT)Fra���)FFFN)FF))r���r���r���r���r���r���r���r���r���r���r���r���r���r���r���r���r���r��r���r���r0���r��r��r��r"��r,��r&��r���r��r���rG��rH��rC�� staticmethodr]��rD��rE��r���r���r��� __classcell__r!���r!���r���r"���r�����sT����  5       # L )$ # r���z!&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?c�����������������C���s���|��d}t|S�)Nr���)rE���r���unescape)rD���whatr!���r!���r"��� decode_entity��s���  ri��c�����������������C���s ���t�t|�S�)a�� Decode HTML entities in the given text. >>> htmldecode( ... 'https://../package_name-0.1.2.tar.gz' ... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz') 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz' ) entity_subri��)textr!���r!���r"���rr�����s��� rr���c��������������������s����fdd}|S�)Nc��������������������s����fdd}|S�)Nc��������������� ������s:���t��}t��z �|�i�|W�t�|�S�t�|�w�ra���)r:��getdefaulttimeoutsetdefaulttimeout)rb���rc���Z old_timeout)re���timeoutr!���r"���_socket_timeout��s ��� z@socket_timeout.<locals>._socket_timeout.<locals>._socket_timeoutr!���)re���ro��rn��rd���r"���ro����s���z'socket_timeout.<locals>._socket_timeoutr!���)rn��ro��r!���rp��r"���socket_timeout��s���  rq��c�����������������C���s2���t�j|�}|�}t|}|�}|ddS�)a9�� Encode auth from a URL suitable for an HTTP header. >>> str(_encode_auth('username%3Apassword')) 'dXNlcm5hbWU6cGFzc3dvcmQ=' Long auth strings should not cause a newline to be inserted. >>> long_auth = 'username:' + 'password'*10 >>> chr(10) in str(_encode_auth(long_auth)) False  r���)r4���r���r6���encodebase64 b64encoder���rB��)rS��Zauth_sZ auth_bytesZ encoded_bytesencodedr!���r!���r"��� _encode_auth��s ���   rw��c�������������������@���rz���) Credentialz: A username/password pair. Use like a namedtuple. c�����������������C���s���||�_�||�_d�S�ra���usernamepassword)r~���rz��r{��r!���r!���r"���r�����s��� zCredential.__init__c�����������������c���s����|�j�V��|�jV��d�S�ra���ry��r���r!���r!���r"���__iter__��s��� zCredential.__iter__c�����������������C���s ���dt�|��S�)Nz%(username)s:%(password)s)varsr���r!���r!���r"���__str__��s��� zCredential.__str__N)r���r���r���r���r���r|��r~��r!���r!���r!���r"���rx����s ���� rx��c�������������������@���s0���e�Zd�Zdd�Zedd�Zdd�Zdd�Zd S�) PyPIConfigc�����������������C���sR���t�g�dd}tj|�|�tjtjdd}tj |r'|� |�dS�dS�)z% Load from ~/.pypirc )rz��r{�� repositoryr���~z.pypircN) dictfromkeys configparserRawConfigParserr���rU���r<���r`��� expanduserr���r���)r~���defaultsrcr!���r!���r"���r�����s ��� zPyPIConfig.__init__c��������������������s&����fdd���D�}tt�j|S�)Nc��������������������s ���g�|�] }��|d��r|qS�)r��)r���ro���)rY���sectionr���r!���r"���r����s ����z2PyPIConfig.creds_by_repository.<locals>.<listcomp>)sectionsr��rm���_get_repo_cred)r~���Zsections_with_repositoriesr!���r���r"���creds_by_repository��s��� zPyPIConfig.creds_by_repositoryc�����������������C���s6���|��|d�}|t|��|d�|��|d�fS�)Nr��rz��r{��)r���ro���rx��)r~���r��repor!���r!���r"���r����s ���zPyPIConfig._get_repo_credc�����������������C���s*���|�j��D�] \}}||r|��S�qdS�)z If the URL indicated appears to be a repository defined in this config, return the credential for that repository. N)r��itemsr)���)r~���r8���r��credr!���r!���r"���find_credential��s ��� zPyPIConfig.find_credentialN)r���r���r���r���propertyr��r��r��r!���r!���r!���r"���r����s ����  r��c�����������������C���s:��t�j|�}|\}}}}}}|drtjd|dv�r$t|\} } nd} | sBt� |�} | rBt | } | j |�f} t j dg| R���| redt| �} || ||||f} t�j| }t�j|}|d| �nt�j|�}|dt�||}| rt�j|j\}}}}}}||kr|| kr||||||f} t�j| |_|S�) z4Open a urllib2 request, handling HTTP authenticationrJ��znonnumeric port: '')r2��httpsNz*Authenticating as %s for %s (from .pypirc)zBasic Authorizationz User-Agent)r4���r���r5���r(���r2��r3��r4��rM��r��r��rn���rz��r���r���rw��rN��r���Request add_header user_agentr8���)r8���r���parsedr:���rQ��r<���paramsr>���r���rS��addressr��r���r9���r���r���r'��s2h2Zpath2Zparam2Zquery2Zfrag2r!���r!���r"���r1����s8���          r1��c�����������������C���s$���|��d\}}}�|r||�fS�d|�fS�)zNsplituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.rI��N) rpartition)rT��rU��delimr!���r!���r"���rM��4��s���rM��c�����������������C���s���|�S�ra���r!���)r8���r!���r!���r"��� fix_sf_url?��r-��r��c�������������� ���C���s*��t�j|�\}}}}}}t�j|}tj|rt�j|�S�| dr}tj |r}g�}t |D�];} tj || } | dkrXt | d } | �} W�d���n1�sQw���Y���n tj | rb| d7�} |dj| d�q0d} | j|�d |d } d \}}nd \}}} d d i}t| }t�j|�||||S�)z7Read a local path, with special support for directoriesr.���z index.htmlrNz<a href="{name}">{name}</a>)r*���zB<html><head><title>{url}{files}rr)r8files)OK)rzPath not foundz Not foundrz text/html)r4rr5rrFrUr<isfilerr(rrr`rrrformatioStringIOrr)r8r:r;r<paramr>rrVrrfilepathr'bodyrstatusmessagerZ body_streamr!r!r"r0Cs0         r0ra)r)YrsysrUrXrrr:rtrrrKrr http.clientr2 urllib.parser4urllib.request urllib.error functoolsrr pkg_resourcesrrrrrr r r r r rrr distutilsrdistutils.errorsrfnmatchrZsetuptools.wheelrZ setuptools.extern.more_itertoolsrrrCIrprrDrr7rO__all__Z_SOCKET_TIMEOUTZ_tmplr version_inforr#rr@rrBrWrrgriryr{rrrrjrirrrqrwrxrrrrr1rMrr0r!r!r!r"s <          ! #  !  &/