B sa;/@sdZdZddlmZddlmZmZddlZddl m Z ddl m Z ddl Z ddlZddlZddlTed ejZed ejZddlZejd d d dgZ[eeeeeeeeeee e!e"e#e$e%e&e'e(e)e*e+e,e-e.e/e0e1e2e3e4e5e6e7e8e9e:e;ee?e@eAeBeCd.ZDGdddeEddZFddZGddZHddZIdZJdZKeJeHdeJeIeKZLdZMdZNdZOdZPdZQeGeNeOePeQZRd ZSeGd!d"eIeSZTd#eSZUeGeTeUZVeGd$eVd%ZWeGeWeVeRZXd&d'ZYd(d)ZZeGeYZ[d*Z\d+Z]d,Z^d-Z_eGe[d.e[d/Z`eGe[d0e[d1ZaeGd2d3d4d5d6d7d8d9Zbd:ZceGd;deGd?de[d@eGdAdZheGdBeKe`ZieJeGeieXeeeheMZjiZkx@eYD]6Zle\ekeld?<e]ekeldA<e^ekeld.<e_ekeld/<qWemZnemZox\eYD]RZpx$epdAepd?fD]ZqenreqqWx$epd/epd.fD]Zqeoreqq$WqWdCZsGdDdEdEetZuGdFdGdGetZvGdHdIdIZwdJd ZxdKdLZydMd ZzdNdOZdPd Z{dQdRZ|dSdTZ}dUdVZ~edWkre~dS)XzKa-Ping Yee zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord)open)lookupBOM_UTF8N) TextIOWrapper)chain)*z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$)tokenizedetect_encoding untokenize TokenInfo).()[]:,;+-r/|&<>=.%{}z==z!=z<=z>=~^z<>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z...z->@z@=c@s eZdZddZeddZdS)r cCs$d|jt|jf}d|j|dS)Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r"tok_name_replace)selfannotated_typer'+/opt/alt/python37/lib/python3.7/tokenize.py__repr___szTokenInfo.__repr__cCs(|jtkr|jtkrt|jS|jSdS)N)r"OPstringEXACT_TOKEN_TYPES)r%r'r'r( exact_typeds zTokenInfo.exact_typeN)__name__ __module__ __qualname__r)propertyr-r'r'r'r(r ^sztype string start end linecGsdd|dS)Nr rr )join)choicesr'r'r(groupkr4cGs t|dS)Nr)r4)r3r'r'r(anylr5r6cGs t|dS)N?)r4)r3r'r'r(maybemr5r8z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]cCsjddddddg}dh}xN|D]F}x@t|D]2}x,tjdd |DD]}|d|qFWq,WqW|S) NbrufbrfrcSsg|]}||fqSr')upper).0cr'r'r( sz(_all_string_prefixes..) _itertools permutationsproductaddr2)_valid_string_prefixesresultprefixtr;r'r'r(_all_string_prefixess rLcCst|tjS)N)recompileUNICODE)exprr'r'r(_compilesrQz[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z!=z//=?z->z[+\-*/%&@|^=<>]=?rz[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z \\\r?\n|\Zc@s eZdZdS) TokenErrorN)r.r/r0r'r'r'r(rUsrUc@s eZdZdS)StopTokenizingN)r.r/r0r'r'r'r(rVsrVc@s,eZdZddZddZddZddZd S) UntokenizercCsg|_d|_d|_d|_dS)Nr)tokensprev_rowprev_colencoding)r%r'r'r(__init__szUntokenizer.__init__cCs|\}}||jks&||jkr>||jkr>td|||j|j||j}|rb|jd|d|_||j}|r|jd|dS)Nz+start ({},{}) precedes previous end ({},{})z\ r )rZr[ ValueErrorformatrYappend)r%startrowcol row_offset col_offsetr'r'r(add_whitespaces  zUntokenizer.add_whitespacec Cs4t|}g}d}x|D] }t|dkr8|||P|\}}}} } |tkrV||_q|tkr`P|tkrv||qnl|tkr| | \|_ |_ qnL|t t fkrd}n:|r|r|d} |dt| kr|j| t| |_ d}|||j|| \|_ |_ |t t fkr|j d7_ d|_ qWd|jS)NFTrXrr?)iterlencompatENCODINGr\ ENDMARKERINDENTraDEDENTpoprZr[NEWLINENLrYrgr2) r%iterableitindents startlinerKtok_typetokenrbendlineindentr'r'r(r sF            zUntokenizer.untokenizec Csg}|jj}|dttfk}d}xt|g|D]}|dd\}} |tkrR| |_q.|ttfkrf| d7} |t kr|rzd| } d}nd}|t kr|| q.n>|t kr| q.n*|ttfkrd}n|r|r||dd}|| q.WdS)NrFrhr^Tri) rYrarrrsrrmr\NAMENUMBERSTRINGrorprq) r%ryrtrv toks_appendrw prevstringtoktoknumtokvalr'r'r(rls8    zUntokenizer.compatN)r.r/r0r]rgr rlr'r'r'r(rWs %rWcCs*t}||}|jdk r&||j}|S)N)rWr r\encode)rtutoutr'r'r(r 8s    cCsH|dddd}|dks*|dr.dS|dks@|drDdS|S) N _rzutf-8zutf-8-)zlatin-1z iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-z iso-8859-1)lowerreplace startswith)orig_encencr'r'r(_get_normal_nameSs rcsy jjWntk r$dYnXdd}d}fdd}fdd}|}|trpd|dd}d }|s||gfS||}|r||gfSt|s||gfS|}|s||gfS||}|r|||gfS|||gfS) NFzutf-8cs yStk rdSXdS)Nr5) StopIterationr')readliner'r( read_or_stopvsz%detect_encoding..read_or_stopcsy|d}Wn4tk rBd}dk r6d|}t|YnXt|}|sVdSt|d}y t|}Wn:t k rdkrd|}n d|}t|YnXr|dkr؈dkrd}n d}t||d 7}|S) Nzutf-8z'invalid or missing encoding declarationz {} for {!r}rXzunknown encoding: zunknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig) decodeUnicodeDecodeErrorr` SyntaxError cookie_rematchrr4r LookupError)r{ line_stringmsgrr\codec) bom_foundfilenamer'r( find_cookie|s6     z$detect_encoding..find_cookieTz utf-8-sig)__self__nameAttributeErrorrrblank_rer)rr\defaultrrfirstsecondr')rrrr(r ^s8   &       cCsVt|d}y0t|j\}}|dt||dd}d|_|S|YnXdS)NrbrT)line_bufferingr:) _builtin_openr rseekrmodeclose)rbufferr\linestextr'r'r(rs  rcCsBddlm}m}t|\}}t|d}|d}t||||j|S)Nr)rrepeatr5) itertoolsrrr rj _tokenize__next__)rrrr\consumedrl_genemptyr'r'r(rs   ccsd}}}d}d\}}d}dg} |dk rH|dkr6d}tt|dddVd} d} xy| } |} Wntk rzd} YnX|dk r| |} |d 7}dt| } } |rt| std ||| }|r |d} }tt|| d||||f|| Vd\}}d}nf|r^| d dd kr^| d ddkr^tt || ||t| f|Vd}d}qTn|| }|| }qTn|dkr|s| sPd}xf| | kr| | dkr|d 7}n6| | dkr|t d t }n| | dkrd}nP| d 7} qW| | krP| | dkr| | dkr`| | d d}tt ||| f|| t|f| V| t|7} tt | | d|| f|t| f| VqT|| dkr| |tt| d| |df|| f| Vxp|| dkr|| krtdd|| | f| dd} ttd|| f|| f| VqWn| s0td|dfd}x| | krtt| | }|r|d \}}||f||f|}}} ||krq8| ||| |}}||ks|dkr|dkr|dkrtt|||| Vq|dkr|dkrtt |||| Vntt|||| Vq|dkr>tt |||| Vq|tkrtt|}|| | }|r|d} | || }tt|||| f| Vn||f}| |d}| }Pq|tks|ddtks|ddtkrR|ddkr>||f}tt|p t|d p t|d}| |dd }}| }Pntt|||| Vnf|rptt|||| VnH|dkrd }n8|dkr|d 7}n|d kr|d 8}tt|||| Vn*tt | | || f|| d f| V| d 7} q8WqTW| r0| ddkr0ttd|d t| f|d t| d fdVx0| d dD] }ttd|df|dfdVq>Wttd|df|dfdVdS)!Nr 0123456789)r?rz utf-8-sigzutf-8)rrr?r5rXzEOF in multi-line stringz\ z\ r^  z# #z riz3unindent does not match any outer indentation levelz zEOF in multi-line statementrz...rhr \z([{z)]})r rmrrrkrUrrzr ERRORTOKENtabsizerstripCOMMENTrsraroIndentationErrorrprQ PseudoTokenspanr~rr triple_quotedendpats single_quotedget isidentifierr}r*rn)rr\lnumparenlev continuednumcharscontstrneedcontcontlinerv last_liner{posmaxstrstartendprogendmatchrzcolumn comment_token pseudomatchrbsposeposryinitialr|r'r'r(rs       *      "  "                  . rcCs t|dS)N)r)rr'r'r(generate_tokenssrc s(ddl}dddfdd }|jdd}|jdd d d d |jd ddddd|}y|jr|j}t|d}tt|j}WdQRXnd}t t j jd}xF|D]>}|j }|j r|j}d|j|j} td| t||jfqWWn8tk r:} z0| jddd\} } || jd|| | fWdd} ~ XYntk r} z(| jd\} } || jd|| | fWdd} ~ XYntk r} z|| |Wdd} ~ XYnxtk r} z || Wdd} ~ XYnNtk rtdYn2tk r"} zd| Wdd} ~ XYnXdS)NrcSst|tjddS)N)file)printsysstderr)messager'r'r(perrorszmain..perrorcsR|r"|f||f}d|n"|r8d||fn d|tddS)Nz%s:%d:%d: error: %sz %s: error: %sz error: %srX)rexit)rrlocationargs)rr'r(errors zmain..errorzpython -m tokenize)progrr7z filename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact store_truez(display token names using the exact type)ractionrrzz %d,%d-%d,%d:z%-20s%-15s%-15rrXrz interrupted zunexpected error: %s)NN)argparseArgumentParser add_argument parse_argsrrlistrrrrstdinr"rr-rbrzrr#r+rrrUrOSErrorKeyboardInterrupt Exception) rrparserrrr<rYry token_type token_rangeerrr{rr')rr(mainsN    &&  r__main__) __author__ __credits__builtinsrrcodecsrr collectionsiorrrrDrMrryrNASCIIrr__all__LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL LESSEQUAL GREATEREQUALTILDE CIRCUMFLEX LEFTSHIFT RIGHTSHIFT DOUBLESTAR PLUSEQUALMINEQUAL STAREQUAL SLASHEQUAL PERCENTEQUAL AMPEREQUAL VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUAL DOUBLESLASHDOUBLESLASHEQUALELLIPSISRARROWATATEQUALr, namedtupler r4r6r8 WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent PointfloatExpfloat Floatnumber ImagnumberNumberrLrQ StringPrefixSingleDoubleSingle3Double3TripleStringOperatorBracketSpecialFunny PlainTokenTokenContStr PseudoExtrasrr_prefixsetrrrKr;rGrrrUrVrWr rr rrrrr.r'r'r'r(s                   _ ]:<