Packages

gustav

0.1.4
0.1.8 0.1.7 0.1.6 0.1.5 0.1.4 0.1.3 0.1.2 0.1.1 0.1.0

Small helper library

Current section

Files

Jump to
Readme
9 Versions
0 Dependencies
Dependants
Files
Activity
gustav src scheme tokenizer.erl
Raw

src/scheme/tokenizer.erl 

-module(tokenizer).



-export([tokenize/1]).



-ifdef(EUNIT).

-include_lib("eunit/include/eunit.hrl").

-compile(export_all).

-endif.



%%====================================================================

%% API functions

%%====================================================================



tokenize(Text) ->

    clean(split(grow_open(grow_close(Text)))).



%%====================================================================

%% Internal functions

%%====================================================================



grow_open(Text) ->

    replace(Text, "(", " ( ").



grow_close(Text) ->

    replace(Text, ")", " ) ").



replace(Text, From, To) ->

    lists:append(string:replace(Text, From, To, all)).



split(Text) ->

    string:split(Text, " ", all).



clean(Tokens) ->

    [T || T <- Tokens, length(T) > 0].



%%====================================================================

%% Tests

%%====================================================================



-ifdef(EUNIT).



tokenize_minimal_test() ->

    Text = "(define (f x y) (+ x y))",

    Tokens = tokenize(Text),

    ?assert(Tokens =:= ["(", "define",

			"(", "f", "x", "y", ")",

			"(", "+", "x", "y", ")", ")"]).



-endif.