A Class for Get Tokens, Parse Strings, Count Words, Search Tokens

Problem/Question/Abstract:

A general way for get tokens, parse strings, count words and search for a specific token on a string.

Answer:

There are many functions and ways for get a token on a string. I have written a general class for handle tokens and parse strings.

The class is named as TToken. I want to describe it with some examples.

var
  xTk: TToken;
  i: Integer;
  s: string;
  b: Boolean;
begin
  xTk := TToken.Create;

  {//////////
  the class has some variables:
  TEXT contains the string to handle
  SEPS contains the set of characters that separate tokens
  }

  xTk.Text := 'Here is my example. Contains commas, dots and spaces.';
  xTk.Seps := [' ', ',', '.'];

  {//////////
  with the method COUNT I can count the number of tokens.
  I can use it on two ways, I can call the method and the variable NUMTOKENS save the number of tokens or I can assign the method to a memory variable. Here is the example of the two ways.
  }

  i := xTk.Count;
  ShowMessage(IntToStr(i));
  ShowMessage(IntToStr(xTk.NumTokens));

  {//////////
  When I want to search all tokens on a sequential way Im going to use the methods FIRT and NEXT. Im going to use two Variables: MORETOKENS and LASTTOKEN. MORETOKENS is a boolean variabale that indicates that after I execute the First or Next method I have a token that is saved on the LASTTOKEN variable
  }

  xTk.First;
  while xTk.MoreTokens do
  begin
    ShowMessage(xTk.LastToken);
    xTk.Next;
  end;

  {//////////
  I can assign the Firt and Next method to a memory variable and I can use the variable NOTOKEN that contains the negative value of MORETOKENS
  }

  s := xTk.First;
  while not xTk.NoToken do
  begin
    ShowMessage(s);
    s := xTk.Next;
  end;

  {//////////
  I can search for a specific token with the SEARCH method
  }

  b := xTk.Search('IS');
  if b then
    ShowMessage('Found it')
  else
    ShowMessage('Not found it');

  b := xTk.Search('YOUR');
  if b then
    ShowMessage('Found it')
  else
    ShowMessage('Not found it');

  xTk.Free;

end;

The class is:

unit UToken;

{
Class for handle Tokens
Author: Alejandro Castro
}

interface

uses SysUtils;

type
  TToken = class(Tobject)
  private
    xCharText: string;
    function xGetToken(xText: string): string;
  public
    Seps: set of char; // Separators
    Text: string; // string to handle
    LastToken: string; // token readed with first or next method
    NoToken: Boolean; // flag that indicate that there ARENT more tokens
    MoreTokens: Boolean; // flag that indicate that there ARE more tokens
    NumTokens: Integer; // no of tokens on Text

    constructor Create;
    function Count: Integer; // count the number of tokens
    function First: string; // Find the First Token
    function Next: string; // Find the Next Token
    function Search(TokSearch: string): Boolean; // Search for a specific token

  end;

implementation

constructor TToken.Create;
begin
  Seps := [];
  Text := '';
  xCharText := '';
  NoToken := True;
  MoreTokens := False;
  LastToken := '';
end;

function TToken.Count: Integer;
var
  i, xLen: Integer;
  xFlag: Boolean;
begin
  NumTokens := 0;
  xLen := length(Text);
  i := 1;
  xFlag := False;
  while (i <= xLen) and (xLen <> 0) do
  begin
    if (Text[i] in Seps) then
      xFlag := False
    else
    begin
      if (not xFlag) then
      begin
        xFlag := True;
        inc(NumTokens);
      end;
    end;
    inc(i);
  end;
  Result := NumTokens;
end;

function TToken.Next: string;
begin
  Result := xGetToken(xCharText);
  LastToken := Result;
  if Result = '' then
    NoToken := True
  else
    NoToken := False;
  MoreTokens := not NoToken;
end;

function TToken.First: string;
begin

  Result := xGetToken(Text);
  LastToken := Result;
  if Result = '' then
    NoToken := True
  else
    NoToken := False;
  MoreTokens := not NoToken;

end;

function TToken.xGetToken(xText: string): string;
var
  i, xLen: Integer;
  xFlag, xAgain: Boolean;
begin
  Result := '';
  xLen := length(xText);
  i := 1;
  xFlag := False;
  xAgain := True;
  while (i <= xLen) and (xLen <> 0) and (xAgain) do
  begin
    if (xText[i] in Seps) then
    begin
      xAgain := (xAgain and (not xFlag));
      xFlag := False
    end
    else
    begin
      if (not xFlag) then
      begin
        xFlag := True;
        xAgain := true;
      end;
      Result := Result + xText[i];
    end;
    inc(i);
  end;
  xCharText := copy(xText, i, xLen);
end;

function TToken.Search(TokSearch: string): Boolean;
var
  xAgain: Boolean;
begin
  Result := False;
  xAgain := True;
  First;
  while (not noToken) and (xAgain) do
  begin
    if UpperCase(LastToken) = UpperCase(TokSearch) then
    begin
      Result := true;
      xAgain := False;
    end;
    Next;
  end;
end;
end.


Component Download: http://www.baltsoft.com/files/dkb/attachment/UToken.zip