test_tokenize 1,0-1,35: COMMENT "# Tests for the 'tokenize' module.\012" 2,0-2,43: COMMENT '# Large bits stolen from test_grammar.py. \012' 3,0-3,1: NL '\012' 4,0-4,11: COMMENT '# Comments\012' 5,0-5,3: STRING '"#"' 5,3-5,4: NEWLINE '\012' 6,0-6,3: COMMENT "#'\012" 7,0-7,3: COMMENT '#"\012' 8,0-8,3: COMMENT '#\\\012' 9,7-9,9: COMMENT '#\012' 10,4-10,10: COMMENT '# abc\012' 11,0-12,4: STRING "'''#\012#'''" 12,4-12,5: NEWLINE '\012' 13,0-13,1: NL '\012' 14,0-14,1: NAME 'x' 14,2-14,3: OP '=' 14,4-14,5: NUMBER '1' 14,7-14,8: COMMENT '#' 14,8-14,9: NEWLINE '\012' 15,0-15,1: NL '\012' 16,0-16,25: COMMENT '# Balancing continuation\012' 17,0-17,1: NL '\012' 18,0-18,1: NAME 'a' 18,2-18,3: OP '=' 18,4-18,5: OP '(' 18,5-18,6: NUMBER '3' 18,6-18,7: OP ',' 18,8-18,9: NUMBER '4' 18,9-18,10: OP ',' 18,10-18,11: NL '\012' 19,2-19,3: NUMBER '5' 19,3-19,4: OP ',' 19,5-19,6: NUMBER '6' 19,6-19,7: OP ')' 19,7-19,8: NEWLINE '\012' 20,0-20,1: NAME 'y' 20,2-20,3: OP '=' 20,4-20,5: OP '[' 20,5-20,6: NUMBER '3' 20,6-20,7: OP ',' 20,8-20,9: NUMBER '4' 20,9-20,10: OP ',' 20,10-20,11: NL '\012' 21,2-21,3: NUMBER '5' 21,3-21,4: OP ']' 21,4-21,5: NEWLINE '\012' 22,0-22,1: NAME 'z' 22,2-22,3: OP '=' 22,4-22,5: OP '{' 22,5-22,8: STRING "'a'" 22,8-22,9: OP ':' 22,9-22,10: NUMBER '5' 22,10-22,11: OP ',' 22,11-22,12: NL '\012' 23,2-23,5: STRING "'b'" 23,5-23,6: OP ':' 23,6-23,7: NUMBER '6' 23,7-23,8: OP '}' 23,8-23,9: NEWLINE '\012' 24,0-24,1: NAME 'x' 24,2-24,3: OP '=' 24,4-24,5: OP '(' 24,5-24,8: NAME 'len' 24,8-24,9: OP '(' 24,9-24,10: OP '`' 24,10-24,11: NAME 'y' 24,11-24,12: OP '`' 24,12-24,13: OP ')' 24,14-24,15: OP '+' 24,16-24,17: NUMBER '5' 24,17-24,18: OP '*' 24,18-24,19: NAME 'x' 24,20-24,21: OP '-' 24,22-24,23: NAME 'a' 24,23-24,24: OP '[' 24,24-24,25: NL '\012' 25,3-25,4: NUMBER '3' 25,5-25,6: OP ']' 25,6-25,7: NL '\012' 26,3-26,4: OP '-' 26,5-26,6: NAME 'x' 26,7-26,8: OP '+' 26,9-26,12: NAME 'len' 26,12-26,13: OP '(' 26,13-26,14: OP '{' 26,14-26,15: NL '\012' 27,3-27,4: OP '}' 27,4-27,5: NL '\012' 28,4-28,5: OP ')' 28,5-28,6: NL '\012' 29,2-29,3: OP ')' 29,3-29,4: NEWLINE '\012' 30,0-30,1: NL '\012' 31,0-31,37: COMMENT '# Backslash means line continuation:\012' 32,0-32,1: NAME 'x' 32,2-32,3: OP '=' 32,4-32,5: NUMBER '1' 33,0-33,1: OP '+' 33,2-33,3: NUMBER '1' 33,3-33,4: NEWLINE '\012' 34,0-34,1: NL '\012' 35,0-35,55: COMMENT '# Backslash does not means continuation in comments :\\\012' 36,0-36,1: NAME 'x' 36,2-36,3: OP '=' 36,4-36,5: NUMBER '0' 36,5-36,6: NEWLINE '\012' 37,0-37,1: NL '\012' 38,0-38,20: COMMENT '# Ordinary integers\012' 39,0-39,4: NUMBER '0xff' 39,5-39,7: OP '<>' 39,8-39,11: NUMBER '255' 39,11-39,12: NEWLINE '\012' 40,0-40,4: NUMBER '0377' 40,5-40,7: OP '<>' 40,8-40,11: NUMBER '255' 40,11-40,12: NEWLINE '\012' 41,0-41,10: NUMBER '2147483647' 41,13-41,15: OP '!=' 41,16-41,28: NUMBER '017777777777' 41,28-41,29: NEWLINE '\012' 42,0-42,1: OP '-' 42,1-42,11: NUMBER '2147483647' 42,11-42,12: OP '-' 42,12-42,13: NUMBER '1' 42,14-42,16: OP '!=' 42,17-42,29: NUMBER '020000000000' 42,29-42,30: NEWLINE '\012' 43,0-43,12: NUMBER '037777777777' 43,13-43,15: OP '!=' 43,16-43,17: OP '-' 43,17-43,18: NUMBER '1' 43,18-43,19: NEWLINE '\012' 44,0-44,10: NUMBER '0xffffffff' 44,11-44,13: OP '!=' 44,14-44,15: OP '-' 44,15-44,16: NUMBER '1' 44,16-44,17: NEWLINE '\012' 45,0-45,1: NL '\012' 46,0-46,16: COMMENT '# Long integers\012' 47,0-47,1: NAME 'x' 47,2-47,3: OP '=' 47,4-47,6: NUMBER '0L' 47,6-47,7: NEWLINE '\012' 48,0-48,1: NAME 'x' 48,2-48,3: OP '=' 48,4-48,6: NUMBER '0l' 48,6-48,7: NEWLINE '\012' 49,0-49,1: NAME 'x' 49,2-49,3: OP '=' 49,4-49,23: NUMBER '0xffffffffffffffffL' 49,23-49,24: NEWLINE '\012' 50,0-50,1: NAME 'x' 50,2-50,3: OP '=' 50,4-50,23: NUMBER '0xffffffffffffffffl' 50,23-50,24: NEWLINE '\012' 51,0-51,1: NAME 'x' 51,2-51,3: OP '=' 51,4-51,23: NUMBER '077777777777777777L' 51,23-51,24: NEWLINE '\012' 52,0-52,1: NAME 'x' 52,2-52,3: OP '=' 52,4-52,23: NUMBER '077777777777777777l' 52,23-52,24: NEWLINE '\012' 53,0-53,1: NAME 'x' 53,2-53,3: OP '=' 53,4-53,35: NUMBER '123456789012345678901234567890L' 53,35-53,36: NEWLINE '\012' 54,0-54,1: NAME 'x' 54,2-54,3: OP '=' 54,4-54,35: NUMBER '123456789012345678901234567890l' 54,35-54,36: NEWLINE '\012' 55,0-55,1: NL '\012' 56,0-56,25: COMMENT '# Floating-point numbers\012' 57,0-57,1: NAME 'x' 57,2-57,3: OP '=' 57,4-57,8: NUMBER '3.14' 57,8-57,9: NEWLINE '\012' 58,0-58,1: NAME 'x' 58,2-58,3: OP '=' 58,4-58,8: NUMBER '314.' 58,8-58,9: NEWLINE '\012' 59,0-59,1: NAME 'x' 59,2-59,3: OP '=' 59,4-59,9: NUMBER '0.314' 59,9-59,10: NEWLINE '\012' 60,0-60,18: COMMENT '# XXX x = 000.314\012' 61,0-61,1: NAME 'x' 61,2-61,3: OP '=' 61,4-61,8: NUMBER '.314' 61,8-61,9: NEWLINE '\012' 62,0-62,1: NAME 'x' 62,2-62,3: OP '=' 62,4-62,8: NUMBER '3e14' 62,8-62,9: NEWLINE '\012' 63,0-63,1: NAME 'x' 63,2-63,3: OP '=' 63,4-63,8: NUMBER '3E14' 63,8-63,9: NEWLINE '\012' 64,0-64,1: NAME 'x' 64,2-64,3: OP '=' 64,4-64,9: NUMBER '3e-14' 64,9-64,10: NEWLINE '\012' 65,0-65,1: NAME 'x' 65,2-65,3: OP '=' 65,4-65,9: NUMBER '3e+14' 65,9-65,10: NEWLINE '\012' 66,0-66,1: NAME 'x' 66,2-66,3: OP '=' 66,4-66,9: NUMBER '3.e14' 66,9-66,10: NEWLINE '\012' 67,0-67,1: NAME 'x' 67,2-67,3: OP '=' 67,4-67,9: NUMBER '.3e14' 67,9-67,10: NEWLINE '\012' 68,0-68,1: NAME 'x' 68,2-68,3: OP '=' 68,4-68,9: NUMBER '3.1e4' 68,9-68,10: NEWLINE '\012' 69,0-69,1: NL '\012' 70,0-70,18: COMMENT '# String literals\012' 71,0-71,1: NAME 'x' 71,2-71,3: OP '=' 71,4-71,6: STRING "''" 71,6-71,7: OP ';' 71,8-71,9: NAME 'y' 71,10-71,11: OP '=' 71,12-71,14: STRING '""' 71,14-71,15: OP ';' 71,15-71,16: NEWLINE '\012' 72,0-72,1: NAME 'x' 72,2-72,3: OP '=' 72,4-72,8: STRING "'\\''" 72,8-72,9: OP ';' 72,10-72,11: NAME 'y' 72,12-72,13: OP '=' 72,14-72,17: STRING '"\'"' 72,17-72,18: OP ';' 72,18-72,19: NEWLINE '\012' 73,0-73,1: NAME 'x' 73,2-73,3: OP '=' 73,4-73,7: STRING '\'"\'' 73,7-73,8: OP ';' 73,9-73,10: NAME 'y' 73,11-73,12: OP '=' 73,13-73,17: STRING '"\\""' 73,17-73,18: OP ';' 73,18-73,19: NEWLINE '\012' 74,0-74,1: NAME 'x' 74,2-74,3: OP '=' 74,4-74,32: STRING '"doesn\'t \\"shrink\\" does it"' 74,32-74,33: NEWLINE '\012' 75,0-75,1: NAME 'y' 75,2-75,3: OP '=' 75,4-75,31: STRING '\'doesn\\\'t "shrink" does it\'' 75,31-75,32: NEWLINE '\012' 76,0-76,1: NAME 'x' 76,2-76,3: OP '=' 76,4-76,32: STRING '"does \\"shrink\\" doesn\'t it"' 76,32-76,33: NEWLINE '\012' 77,0-77,1: NAME 'y' 77,2-77,3: OP '=' 77,4-77,31: STRING '\'does "shrink" doesn\\\'t it\'' 77,31-77,32: NEWLINE '\012' 78,0-78,1: NAME 'x' 78,2-78,3: OP '=' 78,4-83,3: STRING '"""\012The "quick"\012brown fox\012jumps over\012the \'lazy\' dog.\012"""' 83,3-83,4: NEWLINE '\012' 84,0-84,1: NAME 'y' 84,2-84,3: OP '=' 84,4-84,63: STRING '\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\'' 84,63-84,64: NEWLINE '\012' 85,0-85,1: NAME 'y' 85,2-85,3: OP '=' 85,4-90,3: STRING '\'\'\'\012The "quick"\012brown fox\012jumps over\012the \'lazy\' dog.\012\'\'\'' 90,3-90,4: OP ';' 90,4-90,5: NEWLINE '\012' 91,0-91,1: NAME 'y' 91,2-91,3: OP '=' 91,4-96,1: STRING '"\\n\\\012The \\"quick\\"\\n\\\012brown fox\\n\\\012jumps over\\n\\\012the \'lazy\' dog.\\n\\\012"' 96,1-96,2: OP ';' 96,2-96,3: NEWLINE '\012' 97,0-97,1: NAME 'y' 97,2-97,3: OP '=' 97,4-102,1: STRING '\'\\n\\\012The \\"quick\\"\\n\\\012brown fox\\n\\\012jumps over\\n\\\012the \\\'lazy\\\' dog.\\n\\\012\'' 102,1-102,2: OP ';' 102,2-102,3: NEWLINE '\012' 103,0-103,1: NAME 'x' 103,2-103,3: OP '=' 103,4-103,9: STRING "r'\\\\'" 103,10-103,11: OP '+' 103,12-103,17: STRING "R'\\\\'" 103,17-103,18: NEWLINE '\012' 104,0-104,1: NAME 'x' 104,2-104,3: OP '=' 104,4-104,9: STRING "r'\\''" 104,10-104,11: OP '+' 104,12-104,14: STRING "''" 104,14-104,15: NEWLINE '\012' 105,0-105,1: NAME 'y' 105,2-105,3: OP '=' 105,4-107,6: STRING "r'''\012foo bar \\\\\012baz'''" 107,7-107,8: OP '+' 107,9-108,6: STRING "R'''\012foo'''" 108,6-108,7: NEWLINE '\012' 109,0-109,1: NAME 'y' 109,2-109,3: OP '=' 109,4-111,3: STRING 'r"""foo\012bar \\\\ baz\012"""' 111,4-111,5: OP '+' 111,6-112,3: STRING "R'''spam\012'''" 112,3-112,4: NEWLINE '\012' 113,0-113,1: NL '\012' 114,0-114,14: COMMENT '# Indentation\012' 115,0-115,2: NAME 'if' 115,3-115,4: NUMBER '1' 115,4-115,5: OP ':' 115,5-115,6: NEWLINE '\012' 116,0-116,4: INDENT ' ' 116,4-116,5: NAME 'x' 116,6-116,7: OP '=' 116,8-116,9: NUMBER '2' 116,9-116,10: NEWLINE '\012' 117,0-117,0: DEDENT '' 117,0-117,2: NAME 'if' 117,3-117,4: NUMBER '1' 117,4-117,5: OP ':' 117,5-117,6: NEWLINE '\012' 118,0-118,8: INDENT ' ' 118,8-118,9: NAME 'x' 118,10-118,11: OP '=' 118,12-118,13: NUMBER '2' 118,13-118,14: NEWLINE '\012' 119,0-119,0: DEDENT '' 119,0-119,2: NAME 'if' 119,3-119,4: NUMBER '1' 119,4-119,5: OP ':' 119,5-119,6: NEWLINE '\012' 120,0-120,4: INDENT ' ' 120,4-120,9: NAME 'while' 120,10-120,11: NUMBER '0' 120,11-120,12: OP ':' 120,12-120,13: NEWLINE '\012' 121,0-121,5: INDENT ' ' 121,5-121,7: NAME 'if' 121,8-121,9: NUMBER '0' 121,9-121,10: OP ':' 121,10-121,11: NEWLINE '\012' 122,0-122,11: INDENT ' ' 122,11-122,12: NAME 'x' 122,13-122,14: OP '=' 122,15-122,16: NUMBER '2' 122,16-122,17: NEWLINE '\012' 123,5-123,5: DEDENT '' 123,5-123,6: NAME 'x' 123,7-123,8: OP '=' 123,9-123,10: NUMBER '2' 123,10-123,11: NEWLINE '\012' 124,0-124,0: DEDENT '' 124,0-124,0: DEDENT '' 124,0-124,2: NAME 'if' 124,3-124,4: NUMBER '0' 124,4-124,5: OP ':' 124,5-124,6: NEWLINE '\012' 125,0-125,2: INDENT ' ' 125,2-125,4: NAME 'if' 125,5-125,6: NUMBER '2' 125,6-125,7: OP ':' 125,7-125,8: NEWLINE '\012' 126,0-126,3: INDENT ' ' 126,3-126,8: NAME 'while' 126,9-126,10: NUMBER '0' 126,10-126,11: OP ':' 126,11-126,12: NEWLINE '\012' 127,0-127,8: INDENT ' ' 127,8-127,10: NAME 'if' 127,11-127,12: NUMBER '1' 127,12-127,13: OP ':' 127,13-127,14: NEWLINE '\012' 128,0-128,10: INDENT ' ' 128,10-128,11: NAME 'x' 128,12-128,13: OP '=' 128,14-128,15: NUMBER '2' 128,15-128,16: NEWLINE '\012' 129,0-129,1: NL '\012' 130,0-130,12: COMMENT '# Operators\012' 131,0-131,1: NL '\012' 132,0-132,0: DEDENT '' 132,0-132,0: DEDENT '' 132,0-132,0: DEDENT '' 132,0-132,0: DEDENT '' 132,0-132,3: NAME 'def' 132,4-132,7: NAME 'd22' 132,7-132,8: OP '(' 132,8-132,9: NAME 'a' 132,9-132,10: OP ',' 132,11-132,12: NAME 'b' 132,12-132,13: OP ',' 132,14-132,15: NAME 'c' 132,15-132,16: OP '=' 132,16-132,17: NUMBER '1' 132,17-132,18: OP ',' 132,19-132,20: NAME 'd' 132,20-132,21: OP '=' 132,21-132,22: NUMBER '2' 132,22-132,23: OP ')' 132,23-132,24: OP ':' 132,25-132,29: NAME 'pass' 132,29-132,30: NEWLINE '\012' 133,0-133,3: NAME 'def' 133,4-133,8: NAME 'd01v' 133,8-133,9: OP '(' 133,9-133,10: NAME 'a' 133,10-133,11: OP '=' 133,11-133,12: NUMBER '1' 133,12-133,13: OP ',' 133,14-133,15: OP '*' 133,15-133,19: NAME 'rest' 133,19-133,20: OP ',' 133,21-133,23: OP '**' 133,23-133,27: NAME 'rest' 133,27-133,28: OP ')' 133,28-133,29: OP ':' 133,30-133,34: NAME 'pass' 133,34-133,35: NEWLINE '\012' 134,0-134,1: NL '\012' 135,0-135,1: OP '(' 135,1-135,2: NAME 'x' 135,2-135,3: OP ',' 135,4-135,5: NAME 'y' 135,5-135,6: OP ')' 135,7-135,9: OP '<>' 135,10-135,11: OP '(' 135,11-135,12: OP '{' 135,12-135,15: STRING "'a'" 135,15-135,16: OP ':' 135,16-135,17: NUMBER '1' 135,17-135,18: OP '}' 135,18-135,19: OP ',' 135,20-135,21: OP '{' 135,21-135,24: STRING "'b'" 135,24-135,25: OP ':' 135,25-135,26: NUMBER '2' 135,26-135,27: OP '}' 135,27-135,28: OP ')' 135,28-135,29: NEWLINE '\012' 136,0-136,1: NL '\012' 137,0-137,13: COMMENT '# comparison\012' 138,0-138,2: NAME 'if' 138,3-138,4: NUMBER '1' 138,5-138,6: OP '<' 138,7-138,8: NUMBER '1' 138,9-138,10: OP '>' 138,11-138,12: NUMBER '1' 138,13-138,15: OP '==' 138,16-138,17: NUMBER '1' 138,18-138,20: OP '>=' 138,21-138,22: NUMBER '1' 138,23-138,25: OP '<=' 138,26-138,27: NUMBER '1' 138,28-138,30: OP '<>' 138,31-138,32: NUMBER '1' 138,33-138,35: OP '!=' 138,36-138,37: NUMBER '1' 138,38-138,40: NAME 'in' 138,41-138,42: NUMBER '1' 138,43-138,46: NAME 'not' 138,47-138,49: NAME 'in' 138,50-138,51: NUMBER '1' 138,52-138,54: NAME 'is' 138,55-138,56: NUMBER '1' 138,57-138,59: NAME 'is' 138,60-138,63: NAME 'not' 138,64-138,65: NUMBER '1' 138,65-138,66: OP ':' 138,67-138,71: NAME 'pass' 138,71-138,72: NEWLINE '\012' 139,0-139,1: NL '\012' 140,0-140,9: COMMENT '# binary\012' 141,0-141,1: NAME 'x' 141,2-141,3: OP '=' 141,4-141,5: NUMBER '1' 141,6-141,7: OP '&' 141,8-141,9: NUMBER '1' 141,9-141,10: NEWLINE '\012' 142,0-142,1: NAME 'x' 142,2-142,3: OP '=' 142,4-142,5: NUMBER '1' 142,6-142,7: OP '^' 142,8-142,9: NUMBER '1' 142,9-142,10: NEWLINE '\012' 143,0-143,1: NAME 'x' 143,2-143,3: OP '=' 143,4-143,5: NUMBER '1' 143,6-143,7: OP '|' 143,8-143,9: NUMBER '1' 143,9-143,10: NEWLINE '\012' 144,0-144,1: NL '\012' 145,0-145,8: COMMENT '# shift\012' 146,0-146,1: NAME 'x' 146,2-146,3: OP '=' 146,4-146,5: NUMBER '1' 146,6-146,8: OP '<<' 146,9-146,10: NUMBER '1' 146,11-146,13: OP '>>' 146,14-146,15: NUMBER '1' 146,15-146,16: NEWLINE '\012' 147,0-147,1: NL '\012' 148,0-148,11: COMMENT '# additive\012' 149,0-149,1: NAME 'x' 149,2-149,3: OP '=' 149,4-149,5: NUMBER '1' 149,6-149,7: OP '-' 149,8-149,9: NUMBER '1' 149,10-149,11: OP '+' 149,12-149,13: NUMBER '1' 149,14-149,15: OP '-' 149,16-149,17: NUMBER '1' 149,18-149,19: OP '+' 149,20-149,21: NUMBER '1' 149,21-149,22: NEWLINE '\012' 150,0-150,1: NL '\012' 151,0-151,17: COMMENT '# multiplicative\012' 152,0-152,1: NAME 'x' 152,2-152,3: OP '=' 152,4-152,5: NUMBER '1' 152,6-152,7: OP '/' 152,8-152,9: NUMBER '1' 152,10-152,11: OP '*' 152,12-152,13: NUMBER '1' 152,14-152,15: OP '%' 152,16-152,17: NUMBER '1' 152,17-152,18: NEWLINE '\012' 153,0-153,1: NL '\012' 154,0-154,8: COMMENT '# unary\012' 155,0-155,1: NAME 'x' 155,2-155,3: OP '=' 155,4-155,5: OP '~' 155,5-155,6: NUMBER '1' 155,7-155,8: OP '^' 155,9-155,10: NUMBER '1' 155,11-155,12: OP '&' 155,13-155,14: NUMBER '1' 155,15-155,16: OP '|' 155,17-155,18: NUMBER '1' 155,19-155,20: OP '&' 155,21-155,22: NUMBER '1' 155,23-155,24: OP '^' 155,25-155,26: OP '-' 155,26-155,27: NUMBER '1' 155,27-155,28: NEWLINE '\012' 156,0-156,1: NAME 'x' 156,2-156,3: OP '=' 156,4-156,5: OP '-' 156,5-156,6: NUMBER '1' 156,6-156,7: OP '*' 156,7-156,8: NUMBER '1' 156,8-156,9: OP '/' 156,9-156,10: NUMBER '1' 156,11-156,12: OP '+' 156,13-156,14: NUMBER '1' 156,14-156,15: OP '*' 156,15-156,16: NUMBER '1' 156,17-156,18: OP '-' 156,19-156,20: OP '-' 156,20-156,21: OP '-' 156,21-156,22: OP '-' 156,22-156,23: NUMBER '1' 156,23-156,24: OP '*' 156,24-156,25: NUMBER '1' 156,25-156,26: NEWLINE '\012' 157,0-157,1: NL '\012' 158,0-158,11: COMMENT '# selector\012' 159,0-159,6: NAME 'import' 159,7-159,10: NAME 'sys' 159,10-159,11: OP ',' 159,12-159,16: NAME 'time' 159,16-159,17: NEWLINE '\012' 160,0-160,1: NAME 'x' 160,2-160,3: OP '=' 160,4-160,7: NAME 'sys' 160,7-160,8: OP '.' 160,8-160,15: NAME 'modules' 160,15-160,16: OP '[' 160,16-160,22: STRING "'time'" 160,22-160,23: OP ']' 160,23-160,24: OP '.' 160,24-160,28: NAME 'time' 160,28-160,29: OP '(' 160,29-160,30: OP ')' 160,30-160,31: NEWLINE '\012' 161,0-161,1: NL '\012' 162,0-162,0: ENDMARKER ''