How to copy multiple files into one (2)

Problem/Question/Abstract:

What is the quickest way of merging loads of files together, and being able to pull them out when needed in the application all files have unique names, I need to merge the files as the application could create 10000+ and all them being in one dirctory, well lets say windows does not handle it very well specially the fact that they are all small file with the odd occasion of a 15mb file, so I need a better way off managing it not interested in compression I want something that is as quick or quicker than access an individual file.

Answer:

Solve 1:

If you do not need random access to the files in the larger file (in which case you need an index, a kind of directory) you can simply concatenate the source files, storing the file name and size for each file in front of the files data.

procedure ConCatFiles(const targetname: string; const Sourcenames: TStrings);
var
  i: Integer;
  target, source: TFileStream;
  fsize: Longint;
begin
  target := TFileStream.Create(targetname, fmCreate);
  try
    for i := 0 to Sourcenames.Count - 1 do
    begin
      source := TFileStream.Create(Sourcenames[i], fmOpenread or fmShareDenyNone);
      try
        fsize := Length(Sourcenames[i]);
        target.Write(fsize, Sizeof(fsize));
        target.Write(Sourcenames[i][1], fsize);
        fsize := source.size;
        target.Write(fsize, Sizeof(fsize));
        target.Copyfrom(source, 0);
      finally
        source.free;
      end;
    end;
  finally
    target.Free;
  end;
end;

procedure UnmergeFiles(const sourcename: string);
var
  i: Integer;
  target, source: TFileStream;
  fsize, sourcesize: Longint;
  fname: string;
begin
  source := TFileStream.Create(sourcename, fmOpenread or fmShareDenyNone);
  try
    sourcesize := source.size;
    while source.position < sourcesize do
    begin
      source.Read(fsize, Sizeof(fsize));
      SetLength(fname, fsize);
      source.Read(fname[1], fsize);
      target := TFileStream.Create(fname, fmCreate);
      try
        source.Read(fsize, Sizeof(fsize));
        target.Copyfrom(source, fsize);
      finally
        target.free;
      end;
    end;
  finally
    source.Free;
  end;
end;

Untested! And of course you should think about how to handle pathes in this context.


Solve 2:

I've written a little example that doesn't consume too much memory. It concatenates and compresses files into one destination file (CompressFiles) and can restore then in a given location (DecompressFiles).

{ ... }
implementation

{$R *.dfm}

uses
  zLib;

procedure CompressFiles(Files: TStrings; const Filename: string);
var
  infile, outfile, tmpFile: TFileStream;
  compr: TCompressionStream;
  i, l: Integer;
  s: string;
begin
  if Files.Count > 0 then
  begin
    outFile := TFileStream.Create(Filename, fmCreate);
    try
      {the number of files}
      l := Files.Count;
      outfile.Write(l, SizeOf(l));
      for i := 0 to Files.Count - 1 do
      begin
        infile := TFileStream.Create(Files[i], fmOpenRead);
        try
          {the original filename}
          s := ExtractFilename(Files[i]);
          l := Length(s);
          outfile.Write(l, SizeOf(l));
          outfile.Write(s[1], l);
          {the original filesize}
          l := infile.Size;
          outfile.Write(l, SizeOf(l));
          {compress and store the file temporary}
          tmpFile := TFileStream.Create('tmp', fmCreate);
          compr := TCompressionStream.Create(clMax, tmpfile);
          try
            compr.CopyFrom(infile, l);
          finally
            compr.Free;
            tmpFile.Free;
          end;
          {append the compressed file to the destination file}
          tmpFile := TFileStream.Create('tmp', fmOpenRead);
          try
            outfile.CopyFrom(tmpFile, 0);
          finally
            tmpFile.Free;
          end;
        finally
          infile.Free;
        end;
      end;
    finally
      outfile.Free;
    end;
    DeleteFile('tmp');
  end;
end;

procedure DecompressFiles(const Filename, DestDirectory: string);
var
  dest, s: string;
  decompr: TDecompressionStream;
  infile, outfile: TFilestream;
  i, l, c: Integer;
begin
  dest := IncludeTrailingPathDelimiter(DestDirectory);
  infile := TFileStream.Create(Filename, fmOpenRead);
  try
    {number of files}
    infile.Read(c, SizeOf(c));
    for i := 1 to c do
    begin
      {read filename}
      infile.Read(l, SizeOf(l));
      SetLength(s, l);
      infile.Read(s[1], l);
      {read filesize}
      infile.Read(l, SizeOf(l));
      {decompress the files and store it}
      s := dest + s; {include the path}
      outfile := TFileStream.Create(s, fmCreate);
      decompr := TDecompressionStream.Create(infile);
      try
        outfile.CopyFrom(decompr, l);
      finally
        outfile.Free;
        decompr.Free;
      end;
    end;
  finally
    infile.Free;
  end;
end;