update windows build to Python 3.7

Lib/site-packages/pycparser/ply/__init__.py

@@ -1,4 +1,5 @@
 # PLY package
 # Author: David Beazley (dave@dabeaz.com)
 
+__version__ = '3.9'
 __all__ = ['lex','yacc']

Lib/site-packages/pycparser/ply/cpp.py

@@ -2,12 +2,19 @@
 # cpp.py
 #
 # Author:  David Beazley (http://www.dabeaz.com)
-# Copyright (C) 2007
+# Copyright (C) 2017
 # All rights reserved
 #
-# This module implements an ANSI-C style lexical preprocessor for PLY. 
+# This module implements an ANSI-C style lexical preprocessor for PLY.
 # -----------------------------------------------------------------------------
-from __future__ import generators
+import sys
+
+# Some Python 3 compatibility shims
+if sys.version_info.major < 3:
+    STRING_TYPES = (str, unicode)
+else:
+    STRING_TYPES = str
+    xrange = range
 
 # -----------------------------------------------------------------------------
 # Default preprocessor lexer definitions.   These tokens are enough to get
@@ -15,7 +22,7 @@ from __future__ import generators
 # -----------------------------------------------------------------------------
 
 tokens = (
-   'CPP_ID','CPP_INTEGER', 'CPP_FLOAT', 'CPP_STRING', 'CPP_CHAR', 'CPP_WS', 'CPP_COMMENT', 'CPP_POUND','CPP_DPOUND'
+   'CPP_ID','CPP_INTEGER', 'CPP_FLOAT', 'CPP_STRING', 'CPP_CHAR', 'CPP_WS', 'CPP_COMMENT1', 'CPP_COMMENT2', 'CPP_POUND','CPP_DPOUND'
 )
 
 literals = "+-*/%|&~^<>=!?()[]{}.,;:\\\'\""
@@ -34,7 +41,7 @@ t_CPP_ID = r'[A-Za-z_][\w_]*'
 
 # Integer literal
 def CPP_INTEGER(t):
-    r'(((((0x)|(0X))[0-9a-fA-F]+)|(\d+))([uU]|[lL]|[uU][lL]|[lL][uU])?)'
+    r'(((((0x)|(0X))[0-9a-fA-F]+)|(\d+))([uU][lL]|[lL][uU]|[uU]|[lL])?)'
     return t
 
 t_CPP_INTEGER = CPP_INTEGER
@@ -55,11 +62,21 @@ def t_CPP_CHAR(t):
     return t
 
 # Comment
-def t_CPP_COMMENT(t):
-    r'(/\*(.|\n)*?\*/)|(//.*?\n)'
-    t.lexer.lineno += t.value.count("\n")
+def t_CPP_COMMENT1(t):
+    r'(/\*(.|\n)*?\*/)'
+    ncr = t.value.count("\n")
+    t.lexer.lineno += ncr
+    # replace with one space or a number of '\n'
+    t.type = 'CPP_WS'; t.value = '\n' * ncr if ncr else ' '
     return t
-    
+
+# Line comment
+def t_CPP_COMMENT2(t):
+    r'(//.*?(\n|$))'
+    # replace with '/n'
+    t.type = 'CPP_WS'; t.value = '\n'
+    return t
+
 def t_error(t):
     t.type = t.value[0]
     t.value = t.value[0]
@@ -73,8 +90,8 @@ import os.path
 
 # -----------------------------------------------------------------------------
 # trigraph()
-# 
-# Given an input string, this function replaces all trigraph sequences. 
+#
+# Given an input string, this function replaces all trigraph sequences.
 # The following mapping is used:
 #
 #     ??=    #
@@ -244,7 +261,7 @@ class Preprocessor(object):
     # ----------------------------------------------------------------------
     # add_path()
     #
-    # Adds a search path to the preprocessor.  
+    # Adds a search path to the preprocessor.
     # ----------------------------------------------------------------------
 
     def add_path(self,path):
@@ -288,7 +305,7 @@ class Preprocessor(object):
 
     # ----------------------------------------------------------------------
     # tokenstrip()
-    # 
+    #
     # Remove leading/trailing whitespace tokens from a token list
     # ----------------------------------------------------------------------
 
@@ -314,7 +331,7 @@ class Preprocessor(object):
     # argument.  Each argument is represented by a list of tokens.
     #
     # When collecting arguments, leading and trailing whitespace is removed
-    # from each argument.  
+    # from each argument.
     #
     # This function properly handles nested parenthesis and commas---these do not
     # define new arguments.
@@ -326,7 +343,7 @@ class Preprocessor(object):
         current_arg = []
         nesting = 1
         tokenlen = len(tokenlist)
-    
+
         # Search for the opening '('.
         i = 0
         while (i < tokenlen) and (tokenlist[i].type in self.t_WS):
@@ -360,7 +377,7 @@ class Preprocessor(object):
             else:
                 current_arg.append(t)
             i += 1
-    
+
         # Missing end argument
         self.error(self.source,tokenlist[-1].lineno,"Missing ')' in macro arguments")
         return 0, [],[]
@@ -372,9 +389,9 @@ class Preprocessor(object):
     # This is used to speed up macro expansion later on---we'll know
     # right away where to apply patches to the value to form the expansion
     # ----------------------------------------------------------------------
-    
+
     def macro_prescan(self,macro):
-        macro.patch     = []             # Standard macro arguments 
+        macro.patch     = []             # Standard macro arguments
         macro.str_patch = []             # String conversion expansion
         macro.var_comma_patch = []       # Variadic macro comma patch
         i = 0
@@ -421,7 +438,7 @@ class Preprocessor(object):
         rep = [copy.copy(_x) for _x in macro.value]
 
         # Make string expansion patches.  These do not alter the length of the replacement sequence
-        
+
         str_expansion = {}
         for argnum, i in macro.str_patch:
             if argnum not in str_expansion:
@@ -439,7 +456,7 @@ class Preprocessor(object):
         # Make all other patches.   The order of these matters.  It is assumed that the patch list
         # has been sorted in reverse order of patch location since replacements will cause the
         # size of the replacement sequence to expand from the patch point.
-        
+
         expanded = { }
         for ptype, argnum, i in macro.patch:
             # Concatenation.   Argument is left unexpanded
@@ -476,7 +493,7 @@ class Preprocessor(object):
                 if t.value in self.macros and t.value not in expanded:
                     # Yes, we found a macro match
                     expanded[t.value] = True
-                    
+
                     m = self.macros[t.value]
                     if not m.arglist:
                         # A simple macro
@@ -508,7 +525,7 @@ class Preprocessor(object):
                                     else:
                                         args[len(m.arglist)-1] = tokens[j+positions[len(m.arglist)-1]:j+tokcount-1]
                                         del args[len(m.arglist):]
-                                        
+
                                 # Get macro replacement text
                                 rep = self.macro_expand_args(m,args)
                                 rep = self.expand_macros(rep,expanded)
@@ -521,13 +538,13 @@ class Preprocessor(object):
                 elif t.value == '__LINE__':
                     t.type = self.t_INTEGER
                     t.value = self.t_INTEGER_TYPE(t.lineno)
-                
+
             i += 1
         return tokens
 
-    # ----------------------------------------------------------------------    
+    # ----------------------------------------------------------------------
     # evalexpr()
-    # 
+    #
     # Evaluate an expression token sequence for the purposes of evaluating
     # integral expressions.
     # ----------------------------------------------------------------------
@@ -574,14 +591,14 @@ class Preprocessor(object):
                 tokens[i].value = str(tokens[i].value)
                 while tokens[i].value[-1] not in "0123456789abcdefABCDEF":
                     tokens[i].value = tokens[i].value[:-1]
-        
+
         expr = "".join([str(x.value) for x in tokens])
         expr = expr.replace("&&"," and ")
         expr = expr.replace("||"," or ")
         expr = expr.replace("!"," not ")
         try:
             result = eval(expr)
-        except StandardError:
+        except Exception:
             self.error(self.source,tokens[0].lineno,"Couldn't evaluate expression")
             result = 0
         return result
@@ -599,7 +616,7 @@ class Preprocessor(object):
 
         if not source:
             source = ""
-            
+
         self.define("__FILE__ \"%s\"" % source)
 
         self.source = source
@@ -614,10 +631,11 @@ class Preprocessor(object):
             if tok.value == '#':
                 # Preprocessor directive
 
+                # insert necessary whitespace instead of eaten tokens
                 for tok in x:
-                    if tok in self.t_WS and '\n' in tok.value:
+                    if tok.type in self.t_WS and '\n' in tok.value:
                         chunk.append(tok)
-                
+
                 dirtokens = self.tokenstrip(x[i+1:])
                 if dirtokens:
                     name = dirtokens[0].value
@@ -625,7 +643,7 @@ class Preprocessor(object):
                 else:
                     name = ""
                     args = []
-                
+
                 if name == 'define':
                     if enable:
                         for tok in self.expand_macros(chunk):
@@ -685,7 +703,7 @@ class Preprocessor(object):
                                     iftrigger = True
                     else:
                         self.error(self.source,dirtokens[0].lineno,"Misplaced #elif")
-                        
+
                 elif name == 'else':
                     if ifstack:
                         if ifstack[-1][0]:
@@ -771,7 +789,7 @@ class Preprocessor(object):
     # ----------------------------------------------------------------------
 
     def define(self,tokens):
-        if isinstance(tokens,(str,unicode)):
+        if isinstance(tokens,STRING_TYPES):
             tokens = self.tokenize(tokens)
 
         linetok = tokens
@@ -855,7 +873,7 @@ class Preprocessor(object):
     def parse(self,input,source=None,ignore={}):
         self.ignore = ignore
         self.parser = self.parsegen(input,source)
-        
+
     # ----------------------------------------------------------------------
     # token()
     #
@@ -885,14 +903,3 @@ if __name__ == '__main__':
         tok = p.token()
         if not tok: break
         print(p.source, tok)
-
-
-
-
-    
-
-
-
-
-
-

Lib/site-packages/pycparser/ply/ctokens.py

@@ -9,10 +9,10 @@
 
 tokens = [
     # Literals (identifier, integer constant, float constant, string constant, char const)
-    'ID', 'TYPEID', 'ICONST', 'FCONST', 'SCONST', 'CCONST',
+    'ID', 'TYPEID', 'INTEGER', 'FLOAT', 'STRING', 'CHARACTER',
 
     # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=)
-    'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD',
+    'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MODULO',
     'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT',
     'LOR', 'LAND', 'LNOT',
     'LT', 'LE', 'GT', 'GE', 'EQ', 'NE',
@@ -22,7 +22,7 @@ tokens = [
     'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL',
 
     # Increment/decrement (++,--)
-    'PLUSPLUS', 'MINUSMINUS',
+    'INCREMENT', 'DECREMENT',
 
     # Structure dereference (->)
     'ARROW',
@@ -74,7 +74,7 @@ t_LSHIFTEQUAL      = r'<<='
 t_RSHIFTEQUAL      = r'>>='
 t_ANDEQUAL         = r'&='
 t_OREQUAL          = r'\|='
-t_XOREQUAL         = r'^='
+t_XOREQUAL         = r'\^='
 
 # Increment/decrement
 t_INCREMENT        = r'\+\+'

Lib/site-packages/pycparser/ply/ygen.py

@@ -0,0 +1,74 @@
+# ply: ygen.py
+#
+# This is a support program that auto-generates different versions of the YACC parsing
+# function with different features removed for the purposes of performance.
+#
+# Users should edit the method LParser.parsedebug() in yacc.py.   The source code 
+# for that method is then used to create the other methods.   See the comments in
+# yacc.py for further details.
+
+import os.path
+import shutil
+
+def get_source_range(lines, tag):
+    srclines = enumerate(lines)
+    start_tag = '#--! %s-start' % tag
+    end_tag = '#--! %s-end' % tag
+
+    for start_index, line in srclines:
+        if line.strip().startswith(start_tag):
+            break
+
+    for end_index, line in srclines:
+        if line.strip().endswith(end_tag):
+            break
+
+    return (start_index + 1, end_index)
+
+def filter_section(lines, tag):
+    filtered_lines = []
+    include = True
+    tag_text = '#--! %s' % tag
+    for line in lines:
+        if line.strip().startswith(tag_text):
+            include = not include
+        elif include:
+            filtered_lines.append(line)
+    return filtered_lines
+
+def main():
+    dirname = os.path.dirname(__file__)
+    shutil.copy2(os.path.join(dirname, 'yacc.py'), os.path.join(dirname, 'yacc.py.bak'))
+    with open(os.path.join(dirname, 'yacc.py'), 'r') as f:
+        lines = f.readlines()
+
+    parse_start, parse_end = get_source_range(lines, 'parsedebug')
+    parseopt_start, parseopt_end = get_source_range(lines, 'parseopt')
+    parseopt_notrack_start, parseopt_notrack_end = get_source_range(lines, 'parseopt-notrack')
+
+    # Get the original source
+    orig_lines = lines[parse_start:parse_end]
+
+    # Filter the DEBUG sections out
+    parseopt_lines = filter_section(orig_lines, 'DEBUG')
+
+    # Filter the TRACKING sections out
+    parseopt_notrack_lines = filter_section(parseopt_lines, 'TRACKING')
+
+    # Replace the parser source sections with updated versions
+    lines[parseopt_notrack_start:parseopt_notrack_end] = parseopt_notrack_lines
+    lines[parseopt_start:parseopt_end] = parseopt_lines
+
+    lines = [line.rstrip()+'\n' for line in lines]
+    with open(os.path.join(dirname, 'yacc.py'), 'w') as f:
+        f.writelines(lines)
+
+    print('Updated yacc.py')
+
+if __name__ == '__main__':
+    main()
+
+
+
+
+