import re, sys class CompactTokenizer(): def __init__(self,file): self.pat = re.compile('([\[@\]])') self.whitespacepat = re.compile('\n? *$') self.file = file self.tokens = [] def nextToken(self): while True: if len(self.tokens)==0: self.line = self.file.readline().strip() if self.line: self.tokens = [tok for tok in self.pat.split(self.line) if tok!=''] else: self.token = None break self.token = self.tokens.pop(0) if not self.whitespacepat.match(self.token): return self.token def getToken(self): return self.token def skip(self,sym): if self.token==sym: return self.nextToken() else: raise ValueError('skip:%s expected but %s found'%(sym,self.token))