fun input_file is =
    let fun getstring s=
         case input (is, 1) of
               "" => s
              |c  =>getstring(s^c)
    in getstring "" end;
     
fun read_in_file fname=
	let val is=open_in (fname)
            val str=input_file (is)
            val tmp=close_in(is)
	in 
	str
	end;

fun isAlphaNumeric s =
	s >="A" andalso s<="Z" orelse
	s >="a" andalso s<="z" orelse
	s >="0" andalso s<="9" orelse 
	s ="_" orelse s=":" orelse s="-";

fun tokenize nil "" generated = generated|
    tokenize nil cur generated = (generated @ [cur])|
    tokenize (hd::tl) cur generated = if (isAlphaNumeric(hd)) then (tokenize tl (cur^hd) generated) 
						      else (if cur="" then (tokenize tl ("") (generated@[hd]))
								     else (tokenize tl ("") (generated@[cur]@[hd])));

fun outputtokens nil =nil|
    outputtokens (hd::tl)= (output(std_out,hd^"_\t"); outputtokens tl);


fun get_tokens fname= let val strlist=(explode (read_in_file fname))
			  val tokenlist=(tokenize strlist ("") nil)
			 
		      in
        	   	  tokenlist 
		      end;