The gettoken( ) function

The gettoken( ) function is used much like strtok( ). On the first call, pass a pointer to

the string to be tokenized. On subsequent calls, pass a null pointer. It returns a pointer to

the next token in the string. It returns a null pointer when there are no more tokens. To

tokenize a new string, simply start the process over by passing a pointer to the new string.

The simple gettoken( ) function, along with a main( ) function to demonstrate its use, is

shown here:

// Demonstrate a custom gettoken() function that can

// return the tokens that comprise very simple expressions.

#include <iostream>

#include <cstring>

#include <cctype>

using namespace std;

const char *gettoken(const char *str);

int main() {

char sampleA[] = "max=12+3/89; count27 = 19*(min+floor);";

char sampleB[] = "while(i < max) i = counter * 2;";

const char *tok;

// Tokenize the first string.

tok = gettoken(sampleA);

cout << "Here are the tokens in: " << sampleA << endl;

while(tok) {

cout << tok << endl;

tok = gettoken(NULL);

}

cout << "\n\n";

// Restart gettoken() by passing the second string.

tok = gettoken(sampleB);

cout << "Here are the tokens in: " << sampleB << endl;

while(tok) {

cout << tok << endl;

tok = gettoken(NULL);

}

return 0;

}

// pointer if there are no more tokens.

#define MAX_TOK_SIZE 128

const char *gettoken(const char *str) {

static char token[MAX_TOK_SIZE+1];

static const char *ptr;

int count; // holds the current character count

char *tokptr;

if(str) {

ptr = str;

}

tokptr = token;

count = 0;

while(isspace(*ptr)) ptr++;

if(isalpha(*ptr)) {

while(isalpha(*ptr) || isdigit(*ptr)) {

*tokptr++ = *ptr++;

++count;

if(count == MAX_TOK_SIZE) break;

}

} else if(isdigit(*ptr)) {

while(isdigit(*ptr)) {

*tokptr++ = *ptr++;

++count;

if(count == MAX_TOK_SIZE) break;

}

} else if(ispunct(*ptr)) {

*tokptr++ = *ptr++;

} else return NULL;

// Null-terminate the token.

*tokptr = '\0';

return token;

}