import re, sys

class CompactTokenizer():
    
    def __init__(self,file):
        self.pat = re.compile('([\[@\]])')
        self.whitespacepat = re.compile('\n? *$')
        self.file = file
        self.tokens = []

    def nextToken(self):
        while True:
            if len(self.tokens)==0:
                self.line = self.file.readline().strip()
                if self.line:
                    self.tokens = [tok for tok in 
                                   self.pat.split(self.line) if tok!='']
                else:
                    self.token = None
                    break
            self.token = self.tokens.pop(0)
            if not self.whitespacepat.match(self.token):
                return self.token
            
    def getToken(self):
        return self.token

    def skip(self,sym):
        if self.token==sym:
            return self.nextToken()
        else:
            raise ValueError('skip:%s expected but %s found'%(sym,self.token))