//Title: Tokeniser with arbitrary, single-character divisions between tokens
//Credit to: Jp
//Contributed by: Jp

/*
Breaks up a string into bits of string around
arbitrary characters. 'text' is the string you're
tokenising, and the 'tokenbreaker' list should
contain a string consisting of characters that the
tokens should be split up around. Dividers aren't
included in the tokens
*/

proc
    Tokenise(text, list/tokenbreaker)
        var
            list/tokens=list()
            temp
            currtoken
            
        for(var/a=1, a<=lentext(text), a++)
            temp=copytext(text,a,a+1)
            if(!(temp in tokenbreaker)) currtoken+=temp
            else
                if(currtoken) tokens+=currtoken
                currtoken=""
        if(currtoken) tokens+=currtoken
        return tokens
    
    Tokenize(t, list/tb)
        return Tokenise(t,tb) //Silly a-meri-cans


//Testing code:
mob/verb/test(t as message)
    var/list/o=Tokenise(t, list(" ", ",", ".", ascii2text(10), ascii2text(13))
    src << "Tokenise:"
    for(var/k in o) src << k