a DOge@s>dZdZddlmZddlmZmZddlZddl m Z ddl Z ddl Z ddlZddlTddlmZe d e jZe d e jZddlZejgd Z[Gd d d ed dZddZddZddZdZdZeedeeeZdZdZdZ dZ!dZ"eee e!e"Z#dZ$eddee$Z%d e$Z&ee%e&Z'ed!e'd"Z(ee(e'e#Z)d#d$Z*d%d&Z+ee*Z,d'Z-d(Z.d)Z/d*Z0ee,d+e,d,Z1ee,d-e,d.Z2ee3e j4e5ed/d0Z6ed1e6Z7ee)e7e2eZ8ee8Z9ee,d2ed3de,d4ed5dZ:ed6ee1Z;eee;e)e7e:eZe-e=e>d3<e.e=e>d5<e/e=e>d+<e0e=e>d,<q"e?Z@e?ZAe*D]JZBeBd5eBd3fD]ZCe@DeCqeBd,eBd+fD]ZCeADeCqqld7ZEGd8d9d9eFZGGd:d;d;eFZHGdd?ZJd@dAZKdBdCZLdDdEZdFdGZMdHdIZNdJdKZOdLdMZPeQdNkr:ePdS)OzKa-Ping Yee zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord)open)lookupBOM_UTF8N) TextIOWrapper)*)EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$))tokenizegenerate_tokensdetect_encoding untokenize TokenInfoc@s eZdZddZeddZdS)r cCs$d|jt|jf}d|j|dS)Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r tok_name_replace)selfannotated_typer-/opt/alt/python39/lib64/python3.9/tokenize.py__repr__.s zTokenInfo.__repr__cCs(|jtkr|jtvrt|jS|jSdSN)r OPstringrrrrr exact_type3s zTokenInfo.exact_typeN)__name__ __module__ __qualname__rpropertyrrrrrr -sr ztype string start end linecGsdd|dS)N(|))joinchoicesrrrgroup:r$cGs t|dS)Nrr$r"rrrany;r%r'cGs t|dS)N?r&r"rrrmaybe<r%r)z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]cCsVgd}dh}|D]>}t|D].}tjdd|DD]}|d|q8q q|S)N)brufbrfrcSsg|]}||fqSr)upper).0crrr ^r%z(_all_string_prefixes..) _itertools permutationsproductaddr!)_valid_string_prefixesresultprefixtr,rrr_all_string_prefixesSsr=cCst|tjSr)recompileUNICODE)exprrrr_compilebsrBz[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"T)reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z \\\r?\n|\Zc@s eZdZdS) TokenErrorNrrrrrrrrGr%rGc@s eZdZdS)StopTokenizingNrHrrrrrIr%rIc@s,eZdZddZddZddZddZd S) UntokenizercCsg|_d|_d|_d|_dS)Nr)tokensprev_rowprev_colencodingrrrr__init__szUntokenizer.__init__cCs|\}}||jks&||jkr>||jkr>td|||j|j||j}|rb|jd|d|_||j}|r|jd|dS)Nz+start ({},{}) precedes previous end ({},{})\ r )rMrN ValueErrorformatrLappend)rstartrowcol row_offset col_offsetrrradd_whitespaces   zUntokenizer.add_whitespacec Cs6t|}g}d}|D]}t|dkr8|||q*|\}}}} } |tkrV||_q|tkrdq*|tkrz||qnl|tkr| | \|_ |_ qnL|t t fvrd}n:|r|r|d} |dt| kr|j| t| |_ d}|||j|| \|_ |_ |t t fvr|j d7_ d|_ qd|jS)NFTrKrr0)iterlencompatENCODINGrO ENDMARKERINDENTrUDEDENTpoprMrNNEWLINENLrLr[r!) riterableitindents startliner<tok_typetokenrVendlineindentrrrr sF            zUntokenizer.untokenizec Csg}|jj}|dttfv}d}t|g|D]}|dd\}} |tkrR| |_q.|tt fvrf| d7} |t kr|rzd| } d}nd}|t kr|| q.n>|t kr| q.n*|ttfvrd}n|r|r||dd}|| q.dS)NrFr\rRTr])rLrUrfrgr5chainrarONAMENUMBERSTRINGrcrdre) rrmrhrj toks_appendrk prevstringtoktoknumtokvalrrrr`s8    zUntokenizer.compatN)rrrrPr[r r`rrrrrJs %rJcCs*t}||}|jdur&||j}|Sr)rJr rOencode)rhutoutrrrr s    r cCsH|dddd}|dks*|dr.dS|dvs@|drDdS|S) N _-utf-8zutf-8-)zlatin-1 iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-r)lowerreplace startswith)orig_encencrrr_get_normal_namesrcsz jjWnty"dYn0dd}d}fdd}fdd}|}|trnd|dd}d }|sz|gfS||}|r||gfSt|s||gfS|}|s||gfS||}|r|||gfS|||gfS) NFrcs"zWStyYdS0dSNr%) StopIterationrreadlinerr read_or_stop?s z%detect_encoding..read_or_stopcsz|d}Wn2ty@d}dur4d|}t|Yn0t|}|sTdSt|d}z t|}Wn8t ydurd|}n d|}t|Yn0r|dkrԈdurd}n d}t||d 7}|S) Nrz'invalid or missing encoding declarationz {} for {!r}rKzunknown encoding: zunknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig) decodeUnicodeDecodeErrorrT SyntaxError cookie_rematchrr$r LookupError)ro line_stringmsgrrOcodec) bom_foundfilenamerr find_cookieEs8       z$detect_encoding..find_cookieT utf-8-sig)__self__nameAttributeErrorrrblank_rer)rrOdefaultrrfirstsecondr)rrrrr 's8    &       r cCsXt|d}z2t|j\}}|dt||dd}d|_|WS|Yn0dS)NrbrT)line_bufferingr+) _builtin_openr rseekrmodeclose)rbufferrOlinestextrrrrs  rcCs6t|\}}td}t|t|d|}t|j|Sr)r r5repeatrqr^ _tokenize__next__)rrOconsumedemptyrl_genrrrrs  rccszd}}}d}d\}}d}dg} |durH|dkr6d}tt|dddVd} d} z| } |} Wntytd} Yn0|dur| |} |d 7}dt| } } |rn| std ||| }|r|d} }tt|| d||||f|| Vd\}}d}nf|rX| d dd krX| d ddkrXtt || ||t| f|Vd}d}qPn|| }|| }qPn|dkr|s| sqd}| | kr| | dkr|d 7}n8| | dkr|t d t }n| | dkrd}nq| d 7} q| | krq| | dvr| | dkr\| | d d}tt ||| f|| t|f| V| t|7} tt | | d|| f|t| f| VqP|| dkr| |tt| d| |df|| f| V|| dkr,|| vrtdd|| | f| dd} ttd|| f|| f| Vqn| s(td|dfd}| | krPtt| | }|r|d \}}||f||f|}}} ||kr~q,| ||| |}}||vs|dkr|dkr|dkrtt|||| Vq|dvr|dkrtt |||| Vntt|||| Vq|dkr0tt |||| Vq|tvrtt|}|| | }|r|d} | || }tt|||| f| Vn||f}| |d}| }qPq|tvs|ddtvs|ddtvrD|ddkr0||f}tt|pt|d pt|d}| |dd }}| }qPntt|||| Vnf|rbtt|||| VnH|dkrrd }n8|dvr|d 7}n|d vr|d 8}tt|||| Vn*tt | | || f|| d f| V| d 7} q,qP| r.| ddvr.| ds.ttd|d t| f|d t| d fdV| d dD] }ttd|df|dfdVq:tt d|df|dfdVdS)!Nr 0123456789)r0rrr)rrr0r%rKzEOF in multi-line stringrQz\ rR  z# #z r]z3unindent does not match any outer indentation levelz zEOF in multi-line statement.z...r\r \z([{z)]})!r rarrr_rGrrnrt ERRORTOKENtabsizerstripCOMMENTrgrUrcIndentationErrorrdrB PseudoTokenspanrsrf triple_quotedendpats single_quotedget isidentifierrrrstriprrb)rrOlnumparenlev continuednumcharscontstrneedcontcontlinerj last_lineroposmaxstrstartendprogendmatchrncolumn comment_token pseudomatchrVsposeposrminitialrprrrrs&       *        "                        $.rcCs t|dSr)rrrrrr dsr c s@ddl}dddfdd }|jdd}|jdd d d d |jd ddddd|}z|jr|j}t|d}tt|j}Wdq1s0Ynd}t t j jd}|D]>}|j }|j r|j}d|j|j} td| t||jfqWn@tyL} z8| jddd\} } || jd|| | fWYd} ~ nd} ~ 0ty} z0| jd\} } || jd|| | fWYd} ~ nd} ~ 0ty} z|| |WYd} ~ nd} ~ 0ty} z|| WYd} ~ nVd} ~ 0tytdYn4ty:} zd| WYd} ~ n d} ~ 00dS)NrcSstj|tjddS)Nr)sysstderrwrite)messagerrrperrorps zmain..perrorcsR|r"|f||f}d|n"|r8d||fn d|tddS)Nz%s:%d:%d: error: %sz %s: error: %sz error: %srK)rexit)rrlocationargsrrrerrorts zmain..errorzpython -m tokenize)progrr(z filename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact store_truez(display token names using the exact type)ractionrrzz %d,%d-%d,%d:z%-20s%-15s%-15rrKrz interrupted zunexpected error: %s)NN)argparseArgumentParser add_argument parse_argsrrlistrrrrstdinr rrrVrnprintrrrrrGrOSErrorKeyboardInterrupt Exception) rrparserrrr-rLrm token_type token_rangeerrrorrrrmainlsT   .**  r__main__)R __author__ __credits__builtinsrrcodecsrr collectionsior itertoolsr5r>rrmrr?ASCIIrr__all__ namedtupler r$r'r) WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent PointfloatExpfloat Floatnumber ImagnumberNumberr=rB StringPrefixSingleDoubleSingle3Double3TripleStringmapescapesortedSpecialFunny PlainTokenTokenContStr PseudoExtrasrr_prefixsetrrr<r,r8rrrGrIrJr rr rrr rrrrrrs              _ ]8=