require 'strscan' class CompactTokenizer attr_reader :token def initialize(file) @re = /\[|\]|\n|@|[^\[\]\n@]+/ # handle different token types @scanner = StringScanner.new(file.read) end # get next non-whitespace token and return it def nextToken loop do @token = @scanner.scan(@re) break if @token !~ /^\n? *$/ #skip tokens with only whitespace end @token end # if the current token is "sym" then get next one # otherwise, output error message def skip(sym) if (@token==sym) nextToken else # output error message giving the position and the current line str = @scanner.string pos = @scanner.pos raise "skip:#{sym} expected but #{@token} found at position #{pos}:"+ str[str.rindex("\n",pos)||0 ... str.index("\n",pos)||str.length] end end end