Lexer Unit Tests

The lexer Tokens are objects of the token type. Each token (e.g., "whitespace") has a new Token subclass created for it: "TWhitespace". In fact, there are numbers allocated to the tokens, but there's no good way to specify those numbers as a simple identifier - Tok.Whitespace. So using strings, and the name of the class, seems like a win.

package lexer;

import java.io.*;
import org.junit.*;
import static org.junit.Assert.*;

public class LexerTest {
	public Lexer init_lexer( String input ) {
		Lexer lex = new Lexer( new PushbackReader( new StringReader( input ) ) );
		return lex;
	}
	
	public String next_token_class( Lexer lex ) throws Exception {
		String tclass = lex.next().getClass().getName();
		tclass = tclass.substring( tclass.lastIndexOf( '.' ) + 1 );
		return tclass;
	}
	
	public void test_tokens( String input, String ... tokens ) throws Exception {
		Lexer lex = init_lexer( input );
		
		for( String tok : tokens ) {
			String tclass = next_token_class( lex );
			assertEquals( "Token types should match", tok, tclass );
		}
	}

	
	@Test
	public void binaryIntegers( ) throws Exception {
		test_tokens( "0b0", "TLitNumber" );
		test_tokens( "0b011101", "TLitNumber" );
		test_tokens( "0b_11", "TLitNumber" );
	}
	
	@Test
	public void decimalIntegers( ) throws Exception {
		test_tokens( "0", "TLitNumber" );
		test_tokens( "2", "TLitNumber" );
		test_tokens( "10", "TLitNumber" );
		test_tokens( "1 2", "TLitNumber", "TWhitespace", "TLitNumber" );
	}
	
	@Test
	public void octalIntegers( ) throws Exception {
		test_tokens( "0o0", "TLitNumber" );
		test_tokens( "0o___4", "TLitNumber" ); 
		test_tokens( "0o377", "TLitNumber" );
	}

	@Test
	public void whitespace( ) throws Exception {
		test_tokens( " ", "TWhitespace" ); 
		test_tokens( "\t", "TWhitespace" );
		test_tokens( "\004", "TWhitespace" );
		test_tokens( "\n\n", "TWhitespace" );
		test_tokens( "\r\n", "TWhitespace" );
	}
}