Downloading a URL’s HTML

Problem/Question/Abstract:

Downloading a URL’s HTML

Answer:

The objects I present in this article allow you to download data from any URL using the GET method, using only the standard socket components included with Delphi 4+. The object (TabHTTPRequest) is capable of connecting directly to a web server and then requesting a file, the object can also pass a query string; as of this writing it can only get a file using the GET method and using a query string. If there is sufficient interest I can expand the object to also handle POST and cookies, as well as interpreting the result so the return header can be used, so let me know what you guys think!

TInternetURI – This object takes a URI (uniform resource indicator) and splits it into it’s various components to allow the GET object to accept a URL such as http://www.borland.com/delphi/ as a parameter. You do not need to use this object directly.  This object however, follows the complete RFC standard for HTTP addresses and can be used to interpret any URL into its various components.

TabHTTPRequest – This object is designed to connect to a web server and download the HTML, which can then be used in your application.

A couple examples:

URL:

http://www.borland.com/delphi/

CODE:

with TabHTTPRequest.Create do
begin
Get('http://www.borland.com/delphi/');
// Work with result (ex. mmURL.Text := ResultData.DataString);
Free;
end; // with

URL:

http://www.borland.com/rad/delandcppletter.html

CODE:

with TabHTTPRequest.Create do
begin
Get('http://www.borland.com/rad/delandcppletter.html’);
// Work with result (ex. mmURL.Text := ResultData.DataString);
Free;
end; // with

URL: (This is an actual search on yahoo)

http://google.yahoo.com/bin/query?p=delphi+3000&hc=0&hs=0

CODE:

with TabHTTPRequest.Create do
begin
Get('http://google.yahoo.com/bin/query?p=delphi+3000&hc=0&hs=0');
// Work with result (ex. mmURL.Text := ResultData.DataString);
Free;
end; // with

Once get has been called you can access the HTML through the ResultData property:

mmHTML.Lines.Text := URLObject.ResultData.DataString;

I hope you found this article and function to be useful; I’d love to hear your comments, suggestions, etc.

The following is the source code for the functions described above, feel free to use the code in your own programs, but please leave my name and address intact!

I also have a complete test program available by request via e-mail.

// ---------------------------ooo------------------------------ \\
// ©2000 David Lederman
// dlederman@ssccompany.com
// ---------------------------ooo------------------------------ \\
unit abHTTPGet;

interface

uses
Classes, Sysutils, ScktComp;

// ---------------------------ooo------------------------------ \\
// This type will crack a Uniform Resource Indicator
// ---------------------------ooo------------------------------ \\
type
TInternetURI = class(TObject)
private
function CrackScheme(var URIData: string): string;
function CrackLocation(var URIData: string): string;
function CrackQuery(var URIData: string): string;
function CrackParams(var URIData: string): string;
public
Scheme: string;
NetLocation: string;
Path: string;
Query: string;
Fragment: string;
Params: string;
constructor Create(URIData: string);
destructor Destroy; override;
end;

type
TabHTTPRequest = class
private
iBuffer: string;
Socket: TClientSocket;
public
ResultData: TStringStream;
HostToConnect: string;
PortToConnect: Integer;
FileToGet: string;
TimeOut: Integer;
function Get: Boolean; overload;
function Get(URL: string): Boolean; overload;
constructor Create;
destructor Destroy; override;
end;

// ---------------------------ooo------------------------------ \\
// Global HTTP Routines
// ---------------------------ooo------------------------------ \\
function TrimToToken(Token: Char; var DataToParse: string; CopyToken: Boolean = True;
MaxCount: Integer = 1): string;
function TrimPastToken(Token: Char; var DataToParse: string; CopyToken: Boolean =
True; MaxCount: Integer = 1): string;

implementation

{ TabHTTPRequest }

constructor TabHTTPRequest.Create;
begin
// Simply Set Defaults
HostToConnect := 'www.InternetToolsCorp.com';
PortToConnect := 80;
FileToGet := '/';
TimeOut := 5000;
// Create the socket object
Socket := TClientSocket.Create(nil);
Socket.ClientType := ctBlocking;
// Create the result stream
ResultData := TStringStream.Create('');
end;

destructor TabHTTPRequest.Destroy;
begin
// Free the helper objects
Socket.Free;
ResultData.Free;
inherited;
end;

function TabHTTPRequest.Get: Boolean;
var
Waiter: TWinSocketStream;
BufferData: array[0..4028] of char;
DataRead: Integer;
BufferString: string;
begin
// Setup the Request
Waiter := nil;
iBuffer := '';
Socket.Host := HostToConnect;
Socket.Port := PortToConnect;
// Reset the data stream
ResultData.Size := 0;
try
// Do the request
// Open the connection
//  Socket.Open;
Socket.Open;
// Create the waiter
Waiter := TWinSocketStream.Create(Socket.Socket, TimeOut);
// Prepare the request
BufferString := 'GET ' + FileToGet + ' HTTP/1.1' + #13#10 + 'Host: ' +
HostToConnect + #13#10 + #13#10;
// Write the Request
Waiter.Write(BufferString[1], Length(BufferString));
Waiter.Free;
Waiter := nil;
// Now process the result of the request
while Socket.Socket.Connected do
begin
try
// Create the waiter
Waiter := TWinSocketStream.Create(Socket.Socket, TimeOut);
// Wait for data
if Waiter.WaitForData(TimeOut) then
begin
// Try to read a chunck of data
DataRead := Waiter.Read(BufferData, SizeOf(BufferData));
// Check if we got data
if DataRead = 0 then
begin
// Get out
Socket.Close;
end
else
begin
// Save the data to the stream
ResultData.Write(BufferData, DataRead);
end;
end
else
begin
Socket.Close;
end;
finally
Waiter.Free;
Waiter := nil;
end;
end;
// close the socket
if Socket.Active then
Socket.Close;
Result := True;
// Clean up
if Waiter <> nil then
Waiter.Free;
except
// Free the waiter object
if Waiter <> nil then
Waiter.Free;
// Close the socket if it's open
if Socket.Active then
Socket.Close;
// reraise the exception
raise;
end;
end;

function TabHTTPRequest.Get(URL: string): Boolean;
begin
// Crack the URL
try
// Make sure than a scheme is in place
if Pos('://', URL) = 0 then
begin
// Simply Prepend the HTTP
URL := 'http://' + URL;
end;
// Make sure that a / is in the URL
if Pos('/', Copy(URL, 8, Length(URL))) = 0 then
begin
// Simply Append the trailing /
URL := URL + '/';
end;

with TInternetURI.Create(URL) do
begin
// Check if there is a port in the net location
if Pos(':', NetLocation) <> 0 then
begin
// Copy the host name
HostToConnect := Copy(NetLocation, 1, Pos(':', NetLocation) - 1);
// Copy the port
PortToConnect := StrToInt(Copy(NetLocation, Pos(':', NetLocation) + 1,
Length(NetLocation)));
end
else
begin
HostToConnect := NetLocation;
PortToConnect := 80;
end;
FileToGet := '';
// Set the File to get
if Query <> '' then
FileToGet := Path + '?' + Query;
if FileToGet = '' then
FileToGet := '/';
Free
end; // with
// Now simply call get
Result := Get;
except
raise;
end;
end;

{ TInternetURI }

function TInternetURI.CrackLocation(var URIData: string): string;
var
StartPos, EndPos: Integer;
begin
// Step 1. - See if the network ID is here
StartPos := Pos('//', URIData);
// If the starting // is not found then there is no network location
if StartPos = 0 then
Exit;
// Delete the first //
Delete(URIData, StartPos, 2);
// Now look for the trailing slash
EndPos := Pos('/', URIData);
if (EndPos = 0) or (EndPos = 1) then
Exit;
// Now Copy the String Upto the /
Result := Copy(URIData, 1, EndPos - 1);
// Now Delete the network location
Delete(URIData, 1, EndPos - 1);
end;

function TInternetURI.CrackParams(var URIData: string): string;
var
StartPos: Integer;
begin
// Step 1. - See if the query is here
StartPos := Pos(';', URIData);
// If the starting ; is not found then there are no params
if StartPos = 0 then
Exit;
// Copy the Params String
Result := Copy(URIData, StartPos + 1, Length(URIData));
Delete(URIData, StartPos, Length(URIData));
end;

function TInternetURI.CrackQuery(var URIData: string): string;
var
StartPos: Integer;
begin
// Step 1. - See if the query is here
StartPos := Pos('?', URIData);
// If the starting ? is not found then there is no query
if StartPos = 0 then
Exit;
// Copy the Query String
Result := Copy(URIData, StartPos + 1, Length(URIData));
Delete(URIData, StartPos, Length(URIData));
end;

function TInternetURI.CrackScheme(var URIData: string): string;
const
AllowedChars = ['A'..'Z', 'a'..'z', '0'..'9', '+', '-', '.'];
var
tString, WorkData: string;
i: Integer;
StringLength: Integer;
InValidScheme: Boolean;
begin
// Step 1. - Get To The First
WorkData := TrimToToken(':', URIData, False);
if WorkData = '' then
begin
Result := '';
Exit;
end;
// Get The String Length
StringLength := Length(WorkData);
// See if any invalid characters are in the system
InValidScheme := False;
for i := 1 to StringLength do
begin
// Check if the char is valid
InValidScheme := (WorkData[i] in AllowedChars) = False;
if InValidScheme then
Break;
end;
if InValidScheme then
begin
// we need to return the data back to the string
URIData := WorkData + ':' + URIData;
end
else
begin
Result := WorkData;
end;
end;

constructor TInternetURI.Create(URIData: string);
begin
// Step 1. - Copy The Fragment
Fragment := TrimPastToken('#', URIData, False);
// Step 2. - Crack the Scheme
Scheme := CrackScheme(URIData);
// Step 3. - Crack the Network Location
NetLocation := CrackLocation(URIData);
// Step 4. - Crack the Query
Query := CrackQuery(URIData);
// Step 5. - Crack the Parameters
Params := CrackParams(URIData);
// Finally !! Copy the Path (which should be all that is remaining)
Path := URIData;
end;

destructor TInternetURI.Destroy;
begin
inherited;

end;

// ---------------------------ooo------------------------------ \\
// Global routines for HTTP Processing
// ---------------------------ooo------------------------------ \\
// ---------------------------ooo------------------------------ \\
// This function will take the DataToParse and create a string
// list seperating the data using the user-defined tokens.
// ---------------------------ooo------------------------------ \\

function TokenizeString(Tokens: TSysCharSet; DataToParse: string): TStringList;
var
StringLength: Integer;
i, CurPos, StartPos: Integer;
tempString: string;
begin
try
// Create the result set
Result := TStringList.Create;
// Get The String Length
StringLength := Length(DataToParse);
// Setup the search
CurPos := 1;
StartPos := 1;
// Look for the tokens
for i := 1 to StringLength do
begin
// Increment the current position
Inc(CurPos);
// See if the char is in the token list
if DataToParse[i] in Tokens then
begin
// copy the string to current
tempString := Copy(DataToParse, StartPos, (CurPos - 1) - StartPos);
Result.Add(tempstring);
StartPos := i + 1;
end;
end;
// Copy the final string (if neccesary)
if (StartPos - 1) <> StringLength then
begin
tempString := Copy(DataToParse, StartPos, StringLength);
Result.Add(tempString);
end;
except
Result.Free;
Result := nil;
end;
end;

// ---------------------------ooo------------------------------ \\
// This function will use the TokenizeString routine to get
// all the occurences of Token then return all the string to
// the right of MaxCount occurences.
// ---------------------------ooo------------------------------ \\

function TrimToToken(Token: Char; var DataToParse: string; CopyToken: Boolean = True;
MaxCount: Integer = 1): string;
var
i: Integer;
begin
// First Tokenize the string
with TokenizeString([Token], DataToParse) do
begin
// Check if there were any occurences of Token
if Count = 0 then
begin
// Return blank then free and exit
Result := '';
Free;
Exit;
end;
// reset the final string
DataToParse := '';
for i := 0 to (MaxCount - 1) do
begin
// concat the string
if CopyToken then
Result := Result + Strings[i] + Token
else
Result := Result + Strings[i];
end;
// Copy and remaining data
for i := (MaxCount) to Pred(Count) do
begin
DataToParse := DataToParse + Strings[i];
end;
Free;
end;
end;

// ---------------------------ooo------------------------------ \\
// This function will use the TokenizeString routine to get
// all the occurences of Token then return all the string to
// the left of MaxCount occurences.
// ---------------------------ooo------------------------------ \\

function TrimPastToken(Token: Char; var DataToParse: string; CopyToken: Boolean =
True; MaxCount: Integer = 1): string;
var
i: Integer;
begin
// First Tokenize the string
with TokenizeString([Token], DataToParse) do
begin
// Check if there were any occurences of Token
if Count = 0 then
begin
// Return blank then free and exit
Result := '';
Free;
Exit;
end;
// reset the final string
DataToParse := '';
for i := 0 to (MaxCount - 1) do
begin
// concat the string
DataToParse := DataToParse + Strings[i];
end;
// Copy and remaining data
for i := (MaxCount) to Pred(Count) do
begin
if CopyToken then
Result := Result + Token + Strings[i]
else
Result := Result + Strings[i];
end;
Free;
end;
end;

end.