from sys import argv
import re
keyword = ['else', 'if', 'int', 'while', 'return', 'void', 'float']
oper = ['+', '-', '*', '/', '=', '<', '>', '<=', '>=', '==', '!=']
delim = ['\t','\n',',',';','(',')','{','}','[',']', ' ']
num = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
p = re.compile(r'(\d+(\.\d+)?([E][+|-]?\d+)?)')
scripty, filename = argv
#file = open(filename)
comment_count = 0
line_comment = 0
is_comment = False
i = 0
iden = "" #null string for identifiers to be built up
print_list = []
end_comment = False #This is a bool value for a block comment
float_str = ""
def is_keyword(kw):
if kw in keyword:
return True
return False
def is_delim(char):
if char in delim:
return True
return False
def which_delim(char):
if char in delim:
if char != '\t' and char != '\n' and char != ' ':
print char
def is_digit(char):
if char in num:
return True
return False
def is_char(char):
c = 0
c = ord(char)
if c >= 97 and c <= 122:
return True
return False
def is_oper(char):
if char in oper:
return True
return False
def is_num(str):
try:
int(str)
return True
except:
return False
def is_float(str):
m = p.match(str)
length = len(str)
if m and length == len(m.group(0)):
print "FLOAT: %s" %m.group(0)
return True
else:
return False
for line in open(filename):
if line != '\n':
print "Input: %s" % (line),
while line[i] != '\n': #i and enumerate allows to iterate through line
if line[i] is '/':
if line[i + 1] is '/' and comment_count is 0: # it's a line comment print it out
line_comment += 1
elif line[i + 1] is '*':
i += 1
comment_count += 1
elif (line[i] is '*') and (line[i+1] is '/') and comment_count > 0:
comment_count -= 1
i += 1
if comment_count == 0:
end_comment = True
if comment_count is 0 and line_comment is 0 and end_comment == False:
if is_digit(line[i]): #check for float
j = i
while not is_delim(line[j]):
float_str += line[j]
j += 1
if is_float(float_str):
if(j < len(line)):
i = j
iden = ''
float_str = '' #reset string at end use
if is_char(line[i]) or is_digit(line[i]) and not is_oper(line[i]):
iden += line[i]
if is_delim(line[i]) and iden == '': #for delims w/ blank space
which_delim(line[i])
if is_oper(line[i]) and iden is '':
temp = line[i] + line[i + 1]
if(is_oper(temp)):
print temp
i += 1
else:
print line[i]
if not is_char(line[i]) and not is_digit(line[i]) and not is_oper(line[i]) and iden is not '' and not is_delim(line[i]):
if is_keyword(iden):
print "keyword: %s" % iden
print "ERROR: %s" % line[i]
elif is_oper(iden):
print iden
print "Error: %s" % line[i]
elif is_num(iden):
print "NUM: %s" % iden
print "Error: %s" % line[i]
else:
print "ID: %s" % iden
print "Error: %s" % line[i]
iden = ''
elif not is_char(line[i]) and not is_digit(line[i]) and not is_oper(line[i]) and not is_delim(line[i]):
print "Error: %s" % line[i]
if (is_delim(line[i]) or is_oper(line[i])) and iden != '':
if is_keyword(iden):
print "keyword: %s" % iden
elif is_oper(line[i]):
temp = line[i] + line[i + 1]
if is_oper(temp):
if is_keyword(iden):
print "keyword: %s" % iden
print temp
i += 1
else:
print "ID: %s" % iden
print line[i]
elif is_num(iden):
print "NUM: %s" % iden
elif is_oper(iden):
temp = iden + line[i + 1]
if is_oper(temp):
print temp
i += 1
else:
print iden
else:
print "ID: %s" % iden
which_delim(line[i])
iden = ''
i += 1 #increment i
end_comment = False
if line[i] == '\n' and iden != '':
if is_keyword(iden):
print "keyword: %s" % iden
elif is_oper(iden):
print iden
else:
print "ID: %s" % iden
iden = ''
line_comment = 0 # reset line commment number
i = 0 #reset i
Lexer for C- in Python
from sys import argv
import re
keyword = ['else', 'if', 'int', 'while', 'return', 'void', 'float']
oper = ['+', '-', '*', '/', '=', '<', '>', '<=', '>=', '==', '!=']
delim = ['\t','\n',',',';','(',')','{','}','[',']', ' ']
num = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
p = re.compile(r'(\d+(\.\d+)?([E][+|-]?\d+)?)')
scripty, filename = argv
#file = open(filename)
comment_count = 0
line_comment = 0
is_comment = False
i = 0
iden = "" #null string for identifiers to be built up
print_list = []
end_comment = False #This is a bool value for a block comment
float_str = ""
def is_keyword(kw):
if kw in keyword:
return True
return False
def is_delim(char):
if char in delim:
return True
return False
def which_delim(char):
if char in delim:
if char != '\t' and char != '\n' and char != ' ':
print char
def is_digit(char):
if char in num:
return True
return False
def is_char(char):
c = 0
c = ord(char)
if c >= 97 and c <= 122:
return True
return False
def is_oper(char):
if char in oper:
return True
return False
def is_num(str):
try:
int(str)
return True
except:
return False
def is_float(str):
m = p.match(str)
length = len(str)
if m and length == len(m.group(0)):
print "FLOAT: %s" %m.group(0)
return True
else:
return False
for line in open(filename):
if line != '\n':
print "Input: %s" % (line),
while line[i] != '\n': #i and enumerate allows to iterate through line
if line[i] is '/':
if line[i + 1] is '/' and comment_count is 0: # it's a line comment print it out
line_comment += 1
elif line[i + 1] is '*':
i += 1
comment_count += 1
elif (line[i] is '*') and (line[i+1] is '/') and comment_count > 0:
comment_count -= 1
i += 1
if comment_count == 0:
end_comment = True
if comment_count is 0 and line_comment is 0 and end_comment == False:
if is_digit(line[i]): #check for float
j = i
while not is_delim(line[j]):
float_str += line[j]
j += 1
if is_float(float_str):
if(j < len(line)):
i = j
iden = ''
float_str = '' #reset string at end use
if is_char(line[i]) or is_digit(line[i]) and not is_oper(line[i]):
iden += line[i]
if is_delim(line[i]) and iden == '': #for delims w/ blank space
which_delim(line[i])
if is_oper(line[i]) and iden is '':
temp = line[i] + line[i + 1]
if(is_oper(temp)):
print temp
i += 1
else:
print line[i]
if not is_char(line[i]) and not is_digit(line[i]) and not is_oper(line[i]) and iden is not '' and not is_delim(line[i]):
if is_keyword(iden):
print "keyword: %s" % iden
print "ERROR: %s" % line[i]
elif is_oper(iden):
print iden
print "Error: %s" % line[i]
elif is_num(iden):
print "NUM: %s" % iden
print "Error: %s" % line[i]
else:
print "ID: %s" % iden
print "Error: %s" % line[i]
iden = ''
elif not is_char(line[i]) and not is_digit(line[i]) and not is_oper(line[i]) and not is_delim(line[i]):
print "Error: %s" % line[i]
if (is_delim(line[i]) or is_oper(line[i])) and iden != '':
if is_keyword(iden):
print "keyword: %s" % iden
elif is_oper(line[i]):
temp = line[i] + line[i + 1]
if is_oper(temp):
if is_keyword(iden):
print "keyword: %s" % iden
print temp
i += 1
else:
print "ID: %s" % iden
print line[i]
elif is_num(iden):
print "NUM: %s" % iden
elif is_oper(iden):
temp = iden + line[i + 1]
if is_oper(temp):
print temp
i += 1
else:
print iden
else:
print "ID: %s" % iden
which_delim(line[i])
iden = ''
i += 1 #increment i
end_comment = False
if line[i] == '\n' and iden != '':
if is_keyword(iden):
print "keyword: %s" % iden
elif is_oper(iden):
print iden
else:
print "ID: %s" % iden
iden = ''
line_comment = 0 # reset line commment number
i = 0 #reset i
from sys import argv
import re
keyword = ['else', 'if', 'int', 'while', 'return', 'void', 'float']
oper = ['+', '-', '*', '/', '=', '<', '>', '<=', '>=', '==', '!=']
delim = ['\t','\n',',',';','(',')','{','}','[',']', ' ']
num = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
p = re.compile(r'(\d+(\.\d+)?([E][+|-]?\d+)?)')
scripty, filename = argv
#file = open(filename)
comment_count = 0
line_comment = 0
is_comment = False
i = 0
iden = "" #null string for identifiers to be built up
print_list = []
end_comment = False #This is a bool value for a block comment
float_str = ""
def is_keyword(kw):
if kw in keyword:
return True
return False
def is_delim(char):
if char in delim:
return True
return False
def which_delim(char):
if char in delim:
if char != '\t' and char != '\n' and char != ' ':
print char
def is_digit(char):
if char in num:
return True
return False
def is_char(char):
c = 0
c = ord(char)
if c >= 97 and c <= 122:
return True
return False
def is_oper(char):
if char in oper:
return True
return False
def is_num(str):
try:
int(str)
return True
except:
return False
def is_float(str):
m = p.match(str)
length = len(str)
if m and length == len(m.group(0)):
print "FLOAT: %s" %m.group(0)
return True
else:
return False
for line in open(filename):
if line != '\n':
print "Input: %s" % (line),
while line[i] != '\n': #i and enumerate allows to iterate through line
if line[i] is '/':
if line[i + 1] is '/' and comment_count is 0: # it's a line comment print it out
line_comment += 1
elif line[i + 1] is '*':
i += 1
comment_count += 1
elif (line[i] is '*') and (line[i+1] is '/') and comment_count > 0:
comment_count -= 1
i += 1
if comment_count == 0:
end_comment = True
if comment_count is 0 and line_comment is 0 and end_comment == False:
if is_digit(line[i]): #check for float
j = i
while not is_delim(line[j]):
float_str += line[j]
j += 1
if is_float(float_str):
if(j < len(line)):
i = j
iden = ''
float_str = '' #reset string at end use
if is_char(line[i]) or is_digit(line[i]) and not is_oper(line[i]):
iden += line[i]
if is_delim(line[i]) and iden == '': #for delims w/ blank space
which_delim(line[i])
if is_oper(line[i]) and iden is '':
temp = line[i] + line[i + 1]
if(is_oper(temp)):
print temp
i += 1
else:
print line[i]
if not is_char(line[i]) and not is_digit(line[i]) and not is_oper(line[i]) and iden is not '' and not is_delim(line[i]):
if is_keyword(iden):
print "keyword: %s" % iden
print "ERROR: %s" % line[i]
elif is_oper(iden):
print iden
print "Error: %s" % line[i]
elif is_num(iden):
print "NUM: %s" % iden
print "Error: %s" % line[i]
else:
print "ID: %s" % iden
print "Error: %s" % line[i]
iden = ''
elif not is_char(line[i]) and not is_digit(line[i]) and not is_oper(line[i]) and not is_delim(line[i]):
print "Error: %s" % line[i]
if (is_delim(line[i]) or is_oper(line[i])) and iden != '':
if is_keyword(iden):
print "keyword: %s" % iden
elif is_oper(line[i]):
temp = line[i] + line[i + 1]
if is_oper(temp):
if is_keyword(iden):
print "keyword: %s" % iden
print temp
i += 1
else:
print "ID: %s" % iden
print line[i]
elif is_num(iden):
print "NUM: %s" % iden
elif is_oper(iden):
temp = iden + line[i + 1]
if is_oper(temp):
print temp
i += 1
else:
print iden
else:
print "ID: %s" % iden
which_delim(line[i])
iden = ''
i += 1 #increment i
end_comment = False
if line[i] == '\n' and iden != '':
if is_keyword(iden):
print "keyword: %s" % iden
elif is_oper(iden):
print iden
else:
print "ID: %s" % iden
iden = ''
line_comment = 0 # reset line commment number
i = 0 #reset i
Lexer for C- in python improvement suggestions.Python
I am currently taking a compilers course where we are designing a compiler for C- (which is a subset of C). Our first step was the lexer and I have written that but I believe that it is not very "pythonic" and I was hoping someone could help me make it more "pythonic" as I believe doing this would make the future parts of this assignment far simpler and more manageable.
1.The accepted keywords are as follows:
else if int return void while float
2.The special symbols are:
+ - * / < <= > >= == != = ; , ( ) [ ] { } /* */ //
Other tokens are ID, NUM (for ints) or FLOAT defined by the following regular expressions:
FLOAT = (\d+(.\d+)?([E][+|-]?\d+)?) ID = letter letter* NUM = digit digit* letter = a|...|z|A|...|Z digit = 0|...|9
The accepted keywords are as follows:
else if int return void while floatThe special symbols are:
+ - * / < <= > >= == != = ; , ( ) [ ] { } /* */ //Other tokens are ID, NUM (for ints) or FLOAT defined by the following regular expressions:
FLOAT = (\d+(\.\d+)?([E][+|-]?\d+)?) ID = letter letter* NUM = digit digit* letter = a|...|z|A|...|Z digit = 0|...|9
lowercaseLowercase and uppercase are distinct.
- White spaceWhitespace consists of blanks, newlines, and tabs. White space is ignore except that it must separate IDs
IDs, NUMsNUMs, FLOATsFLOATs, and keywords. - Comments are surrounded by the C notations /* ... */ //
/* ... */ //and CAN (don't know why) be nested.
The program will read in a C- file and output the line followed by every ID, keyword, NUM, and FLOAT, keyword, NUM, and FLOAT` in order that they appear as well as outputting every special symbol. (Comments are ignored and so is white space. Anything that is invalid is to be displayed as an error and the program resume as normal.) The program does not determine if the program is valid it is simply breaking it up.
/**/ /*/* */ */
/*/*/****This**********/*/ */
/**************/
/*************************
i = 333; ******************/ */
iiii = 3@33;
int g 4 cd (int u, int v) {
/**/ /*/* */ */ /*/*/****This**********/*/ */ /**************/ /************************* i = 333; ******************/ */ iiii = 3@33; int g 4 cd (int u, int v) {
INPUT: /**/ /*/* */ */
INPUT: /*/*/****This**********/*/ */
INPUT: /**************/
INPUT: /*************************
INPUT: i = 333; ******************/ */
*
/
INPUT: iiii = 3@33;
ID: iiii
=
NUM: 3
Error: @33
;
INPUT: int g 4 cd (int u, int v) {
keyword: int
ID: g
NUM: 4
ID: cd
(
keyword: int
ID: u
,
keyword: int
ID: v
)
{
INPUT: /**/ /*/* */ */ INPUT: /*/*/****This**********/*/ */ INPUT: /**************/ INPUT: /************************* INPUT: i = 333; ******************/ */ * / INPUT: iiii = 3@33; ID: iiii = NUM: 3 Error: @33 ; INPUT: int g 4 cd (int u, int v) { keyword: int ID: g NUM: 4 ID: cd ( keyword: int ID: u , keyword: int ID: v ) {
Currently I am currently running through line by line-by-line and then character by character and building up the tokens but I feel like there is a much more straight forward way of doing it.
Here is my code:
Lexer for C- in python improvement suggestions.
I am currently taking a compilers course where we are designing a compiler for C- (which is a subset of C) Our first step was the lexer and I have written that but I believe that it is not very "pythonic" and I was hoping someone could help me make it more "pythonic" as I believe doing this would make the future parts of this assignment far simpler and more manageable.
1.The accepted keywords are as follows:
else if int return void while float
2.The special symbols are:
+ - * / < <= > >= == != = ; , ( ) [ ] { } /* */ //
Other tokens are ID, NUM (for ints) or FLOAT defined by the following regular expressions:
FLOAT = (\d+(.\d+)?([E][+|-]?\d+)?) ID = letter letter* NUM = digit digit* letter = a|...|z|A|...|Z digit = 0|...|9
lowercase and uppercase are distinct
- White space consists of blanks, newlines, and tabs. White space is ignore except that it must separate IDs, NUMs, FLOATs, and keywords.
- Comments are surrounded by the C notations /* ... */ // and CAN (don't know why) be nested
The program will read in a C- file and output the line followed by every ID, keyword, NUM, and FLOAT in order that they appear as well as outputting every special symbol. (Comments are ignored and so is white space. Anything that is invalid is to be displayed as an error and the program resume as normal.) The program does not determine if the program is valid it is simply breaking it up.
/**/ /*/* */ */
/*/*/****This**********/*/ */
/**************/
/*************************
i = 333; ******************/ */
iiii = 3@33;
int g 4 cd (int u, int v) {
INPUT: /**/ /*/* */ */
INPUT: /*/*/****This**********/*/ */
INPUT: /**************/
INPUT: /*************************
INPUT: i = 333; ******************/ */
*
/
INPUT: iiii = 3@33;
ID: iiii
=
NUM: 3
Error: @33
;
INPUT: int g 4 cd (int u, int v) {
keyword: int
ID: g
NUM: 4
ID: cd
(
keyword: int
ID: u
,
keyword: int
ID: v
)
{
Currently I am running through line by line and then character by character and building up the tokens but I feel like there is a much more straight forward way of doing it.
Here is my code:
Lexer for C in Python
I am currently taking a compilers course where we are designing a compiler for C- (which is a subset of C). Our first step was the lexer and I have written that but I believe that it is not very "pythonic" and I was hoping someone could help me make it more "pythonic" as I believe doing this would make the future parts of this assignment far simpler and more manageable.
The accepted keywords are as follows:
else if int return void while floatThe special symbols are:
+ - * / < <= > >= == != = ; , ( ) [ ] { } /* */ //Other tokens are ID, NUM (for ints) or FLOAT defined by the following regular expressions:
FLOAT = (\d+(\.\d+)?([E][+|-]?\d+)?) ID = letter letter* NUM = digit digit* letter = a|...|z|A|...|Z digit = 0|...|9
Lowercase and uppercase are distinct.
- Whitespace consists of blanks, newlines, and tabs. White space is ignore except that it must separate
IDs,NUMs,FLOATs, and keywords. - Comments are surrounded by the C notations
/* ... */ //and CAN (don't know why) be nested.
The program will read in a C- file and output the line followed by every ID, keyword, NUM, and FLOAT` in order that they appear as well as outputting every special symbol. (Comments are ignored and so is white space. Anything that is invalid is to be displayed as an error and the program resume as normal.) The program does not determine if the program is valid it is simply breaking it up.
/**/ /*/* */ */ /*/*/****This**********/*/ */ /**************/ /************************* i = 333; ******************/ */ iiii = 3@33; int g 4 cd (int u, int v) {
INPUT: /**/ /*/* */ */ INPUT: /*/*/****This**********/*/ */ INPUT: /**************/ INPUT: /************************* INPUT: i = 333; ******************/ */ * / INPUT: iiii = 3@33; ID: iiii = NUM: 3 Error: @33 ; INPUT: int g 4 cd (int u, int v) { keyword: int ID: g NUM: 4 ID: cd ( keyword: int ID: u , keyword: int ID: v ) {
I am currently running through line-by-line and then character by character and building up the tokens but I feel like there is a much more straight forward way of doing it.
default