o =?hAe@sR dZdZddlmZddlmZmZddlZddl Z ddl m Z ddl Z ddlZddlZddlTddlmZed ejZed ejZddlZejgd Z[Gd d d ed dZddZddZddZdZdZeedeeeZdZdZ dZ!dZ"dZ#ee e!e"e#Z$dZ%eddee%Z&d e%Z'ee&e'Z(ed!e(d"Z)ee)e(e$Z*d#d$Z+e j,d%d&Z-ee+Z.d'Z/d(Z0d)Z1d*Z2ee.d+e.d,Z3ee.d-e.d.Z4ee5ej6e7ed/d0Z8ed1e8Z9ee*e9e4eZ:ee:Z;ee.d2ed3de.d4ed5dZiZ?e+D]Z@e/e?e@d3<e0e?e@d5<e1e?e@d+<e2e?e@d,<qeAZBeAZCe+D]%ZDeDd5eDd3fD]ZEeBFeEqHeDd,eDd+fD]ZEeCFeEqYq>d7ZGGd8d9d9eHZIGd:d;d;eHZJGdd?ZLd@dAZMdBdCZNdDdEZdFdGZOdHdIZPdJdKZQdLdMZReSdNkreRdSdS)OzKa-Ping Yee zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord)open)lookupBOM_UTF8N) TextIOWrapper)*)EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$))tokenizegenerate_tokensdetect_encoding untokenize TokenInfoc@s eZdZddZeddZdS)r cCs$d|jt|jf}d|j|dS)Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r tok_name_replace)selfannotated_typer//opt/alt/python310/lib64/python3.10/tokenize.py__repr__/s zTokenInfo.__repr__cCs$|jtkr|jtvrt|jS|jSN)r OPstringrrrrr exact_type4s zTokenInfo.exact_typeN)__name__ __module__ __qualname__rpropertyrrrrrr .sr ztype string start end linecGsdd|dS)N(|))joinchoicesrrrgroup;sr$cG t|dS)Nrr$r"rrrany< r'cGr%)N?r&r"rrrmaybe=r(r*z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]cCsVgd}dh}|D]}t|D]}tjdd|DD] }|d|qqq |S)N)brufbrfrcSsg|]}||fqSr)upper).0crrr _sz(_all_string_prefixes..) _itertools permutationsproductaddr!)_valid_string_prefixesresultprefixtr-rrr_all_string_prefixesTsr>cCst|tjSr)recompileUNICODE)exprrrr_compilecsrCz[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"T)reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z \\\r?\n|\Zc@ eZdZdS) TokenErrorNrrrrrrrrIr(rIc@rH)StopTokenizingNrJrrrrrKr(rKc@s,eZdZddZddZddZddZd S) UntokenizercCsg|_d|_d|_d|_dS)Nr)tokensprev_rowprev_colencodingrrrr__init__s zUntokenizer.__init__cCs|\}}||jks||jkr||jkrtd|||j|j||j}|r1|jd|d|_||j}|rB|jd|dSdS)Nz+start ({},{}) precedes previous end ({},{})\ r )rOrP ValueErrorformatrNappend)rstartrowcol row_offset col_offsetrrradd_whitespaces  zUntokenizer.add_whitespacec Cs,t|}g}d}|D]}t|dkr|||nv|\}}}} } |tkr)||_q |tkr/na|tkr9||q |tkrH| | \|_ |_ q |t t fvrQd}n|rn|rn|d} |dt| krl|j| t| |_ d}|||j|| \|_ |_ |t t fvr|j d7_ d|_ q d|jS)NFTrMrr1)iterlencompatENCODINGrQ ENDMARKERINDENTrWDEDENTpoprOrPNEWLINENLrNr]r!) riterableitindents startliner=tok_typetokenrXendlineindentrrrr sH            zUntokenizer.untokenizec Csg}|jj}|dttfv}d}t|g|D]W}|dd\}} |tkr)| |_q|tt fvr3| d7} |t kr@|r=d| } d}nd}|t krL|| q|t krU| q|ttfvr^d}n |rj|rj||dd}|| qdS)NrFr^rTTr_)rNrWrhrir6chainrcrQNAMENUMBERSTRINGrerfrg) rrorjrl toks_appendrm prevstringtoktoknumtokvalrrrrbs:     zUntokenizer.compatN)rrrrRr]r rbrrrrrLs  %rLcCs, t}||}|jdur||j}|Sr)rLr rQencode)rjutoutrrrr s    r cCsJ |dddd}|dks|drdS|dvs!|dr#dS|S) N _-utf-8zutf-8-)zlatin-1 iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-r)lowerreplace startswith)orig_encencrrr_get_normal_namesrcs zjjWn tydYnwdd}d}fdd}fdd}|}|tr8d|dd}d }|s>|gfS||}|rI||gfSt|sS||gfS|}|s]||gfS||}|ri|||gfS|||gfS) NFrcszWStyYdSwN) StopIterationrreadlinerr read_or_stopAs  z%detect_encoding..read_or_stopcsz|d}Wntyd}durd|}t|wt|}|s(dSt|d}zt|}Wnt yRdurHd|}t|d|}t|wrp|dkrldurcd}t|d}t||d 7}|S) Nrz'invalid or missing encoding declarationz {} for {!r}rMzunknown encoding: zunknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig) decodeUnicodeDecodeErrorrV SyntaxError cookie_rematchrr$r LookupError)rq line_stringmsgrrQcodec) bom_foundfilenamerr find_cookieGs@       z$detect_encoding..find_cookieT utf-8-sig)__self__nameAttributeErrorrrblank_rer)rrQdefaultrrfirstsecondr)rrrrr )s<   &        r cCsP t|d}zt|j\}}|dt||dd}d|_|WS|)NrbrT)line_bufferingr,) _builtin_openr rseekrmodeclose)rbufferrQlinestextrrrrs  rcCs8 t|\}}td}t|t|d|}t|j|Sr)r r6repeatrsr` _tokenize__next__)rrQconsumedemptyrl_genrrrrs    rccsld}}}d}d\}}d}dg} |dur%|dkrd}tt|dddVd} d} z| } |} Wn ty<d} Ynw|durF| |} |d 7}dt| } } |r| sZtd ||| }|r|d} }tt|| d||||f|| Vd\}}d}n|r| d dd kr| dddkrtt || ||t| f|Vd}d}q)|| }|| }q)|dkr|s| sn)d}| | kr| | dkr|d 7}n| | dkr|t d t }n | | dkrd}nn| d 7} | | ks| | krn| | dvr8| | dkr$| | d d}tt ||| f|| t|f| V| t|7} tt | | d|| f|t| f| Vq)|| dkrU| |tt| d| |df|| f| V|| dkr|| vrjtdd|| | f| dd} ttd|| f|| f| V|| dks\n | std|dfd}| | krtt| | }|r|d \}}||f||f|}}} ||krq| ||| |}}||vs|dkr|dkr|dkrtt|||| Vn|dvr|dkrtt |||| Vntt|||| Vn|dkrtt |||| Vn|tvrItt|}|| | }|r<|d} | || }tt|||| f| Vn||f}| |d}| }n|tvs`|ddtvs`|ddtvr|ddkr||f}tt|pt|d pt|d}| |dd }}| }nXtt|||| VnI|rtt|||| Vn:|dkrd }n2|d vr|d 7}n |d!vr|d 8}tt|||| Vntt | | || f|| d f| V| d 7} | | ksq*| r| ddvr| dsttd|d t| f|d t| d fdV| d dD]}ttd|df|dfdVqtt d|df|dfdVdS)"Nr 0123456789)r1rrr)rrr1rTrMzEOF in multi-line stringrSz\ rT  z# #z r_z3unindent does not match any outer indentation levelz zEOF in multi-line statement.z...r^r \z([{z)]})!r rcrrrarIrrprv ERRORTOKENtabsizerstripCOMMENTrirWreIndentationErrorrfrC PseudoTokenspanrurh triple_quotedendpats single_quotedget isidentifierrtrstriprrd)rrQlnumparenlev continuednumcharscontstrneedcontcontlinerl last_linerqposmaxstrstartendprogendmatchrpcolumn comment_token pseudomatchrXsposeposroinitialrrrrrrs*      $        "                          $#.rcCs t|dSr)rrrrrr fs r c s2ddl}dddfdd }|jdd}|jdd d d d |jd ddddd|}zQ|jrP|j}t|d}tt|j}Wdn1sJwYn d}t t j jd}|D]}|j }|j rf|j}d|j|j} td| t||jfq[WdSty} z| jddd\} } || jd|| | fWYd} ~ dSd} ~ wty} z| jd\} } || jd|| | fWYd} ~ dSd} ~ wty} z || |WYd} ~ dSd} ~ wty} z || WYd} ~ dSd} ~ wtytdYdSty} zd| d} ~ ww)NrcSstj|tjddS)Nr)sysstderrwrite)messagerrrperrorrs zmain..perrorcsR|r|f||f}d|n|rd||fnd|tddS)Nz%s:%d:%d: error: %sz %s: error: %sz error: %srM)rexit)rrlocationargsrrrerrorvs zmain..errorzpython -m tokenize)progrr)z filename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact store_truez(display token names using the exact type)ractionrrzz %d,%d-%d,%d:z%-20s%-15s%-15rrMrz interrupted zunexpected error: %s)NN)argparseArgumentParser add_argument parse_argsrrlistrrrrstdinr rrrXrpprintrrrrrIrOSErrorKeyboardInterrupt Exception) rrparserrrr.rNro token_type token_rangeerrrqrrrrmainnsf   $$ r__main__)T __author__ __credits__builtinsrrcodecsrr collections functoolsior itertoolsr6r?rrorr@ASCIIrr__all__ namedtupler r$r'r* WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent PointfloatExpfloat Floatnumber ImagnumberNumberr> lru_cacherC StringPrefixSingleDoubleSingle3Double3TripleStringmapescapesortedSpecialFunny PlainTokenTokenContStr PseudoExtrasrr_prefixsetrrr=r-r9rrrIrKrLr rr rrr rrrrrrs               _ ]8 =