import re, sys
class CompactTokenizer():
def __init__(self,file):
self.pat = re.compile('([\[@\]])')
self.whitespacepat = re.compile('\n? *$')
self.file = file
self.tokens = []
def nextToken(self):
while True:
if len(self.tokens)==0:
self.line = self.file.readline().strip()
if self.line:
self.tokens = [tok for tok in
self.pat.split(self.line) if tok!='']
else:
self.token = None
break
self.token = self.tokens.pop(0)
if not self.whitespacepat.match(self.token):
return self.token
def getToken(self):
return self.token
def skip(self,sym):
if self.token==sym:
return self.nextToken()
else:
raise ValueError('skip:%s expected but %s found'%(sym,self.token))