// PCompress Library
// Copyright (C) 1999 Britt Yenne
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
// 
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
// 
// You should have received a copy of the GNU Library General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
// 
// To contact the author, visit:  http://www.jump.net/~syenne/pilot/

#define PCOMP_INTERNAL
#include "pcomp.h"

#ifdef UNIX
# include <fcntl.h>
# include <stdio.h>
#else
# pragma pack(2)
# include <Common.h>
# include <SysAll.h>
#endif

//
// This algorithm is a variant of the one used by gzip.  The main
// modifications are to support a tighter memory space and a very slow
// processor.  The goal is still to match occurrences of repeating
// strings (LZ77) and then Huffman-encode the result.  One benefit of
// palmtops is that the entire input database is always in memory,
// obviating the need for fixed blocking or input buffering.
//

//
// For performance analysis.
//
#ifdef UNIX
# define PERF
# define Perf(w)	w
# define Assert(c)	((c) || (printf("assert line %d\n", __LINE__), exit(1)))
static s32bit hashMiss = 0, hashHits = 0;
static s32bit tableInserts = 0;
#else
# define Perf(w)
# define Assert(c)
#endif

//
// Output buffer management.  The compression essentially occurs in two
// passes, where the first pass emits a stream of tokens which are then
// Huffman-encoded in the second pass.  Adaptive Huffman (or similar
// algorithms) are not used because they take up memory and affect
// performance.  See the FAQ.
//
// Since there are more than 256 tokens and I don't want to waste memory
// storing 16-bit tokens, I encode the high tokens using a special Marker
// character.  A Marker followed by another Marker is the literal Marker
// character.  A Marker followed by any other character is the token
// (0x100 | character).
//
#define Marker		((u16bit)0xd9)
#define OutBufSize	1024
#define OutBufHiwat	(OutBufSize-10)
#define OutputByte(c)	{ \
  if ((outBuf[outLen] = (c)) == Marker)	\
    outBuf[++outLen] = Marker;		\
  ctl->hist[outBuf[outLen]]++;		\
  outLen++;				\
}
#define OutputTok(c)	outBuf[outLen++] = Marker, \
			outBuf[outLen++] = (u8bit)(c), \
			ctl->hist[c]++
#define OutputArg(c)	ctl->hist[outBuf[outLen++] = (c)]++

#ifdef PCOMP_SHLIB
//
// This must be the first function in the file.  This sets up the
// dispatch table for the shared library.  Many thanks to the ISAAC
// group for this function:
//    http://www.isaac.cs.berkeley.edu/pilot/shlib.html
//
ULong install_dispatcher(UInt refnum, SysLibTblEntryPtr entryP);
Ptr *gettable(void);

ULong
start(UInt refnum, SysLibTblEntryPtr entryP)
{
  ULong ret;

  asm("
  movel %%fp@(10),%%sp@-
  movew %%fp@(8),%%sp@-
  jsr install_dispatcher(%%pc)
  movel %%d0,%0
  jmp out(%%pc)
gettable:
  lea jmptable(%%pc),%%a0
  rts
jmptable:
  dc.w 74
  dc.w 26
  dc.w 30
  dc.w 34
  dc.w 38
  dc.w 42
  dc.w 46
  dc.w 50
  dc.w 54
  dc.w 58
  dc.w 62
  dc.w 66
  dc.w 70
  jmp PCompLibOpen(%%pc)
  jmp PCompLibClose(%%pc)
  jmp PCompLibSleep(%%pc)
  jmp PCompLibWake(%%pc)
  jmp PCompOpen(%%pc)
  jmp PCompClose(%%pc)
  jmp PCompGet(%%pc)
  jmp PCompSet(%%pc)
  jmp PCompDeflate(%%pc)
  jmp PCompBuildTree(%%pc)
  jmp PCompEncode(%%pc)
  jmp PCompDecode(%%pc)
  .asciz \"PCompress Library\"
.even
out:
  " : "=r" (ret) :);
  return ret;
}

ULong
install_dispatcher(UInt refnum, SysLibTblEntryPtr entryP)
{
  Ptr *table = gettable();
  entryP->dispatchTblP = table;
  entryP->globalsP = NULL;
  return 0;
}

Err
PCompLibOpen(UInt refNum)
{
  return 0;
}

Err
PCompLibClose(UInt refNum, UIntPtr numappsP)
{
  return 0;
}

Err
PCompLibSleep(UInt refNum)
{
  return 0;
}

Err
PCompLibWake(UInt refNum)
{
  return 0;
}
#endif	// PCOMP_SHLIB

//
// A la gzip, the following are used to limit the match algorithm when
// the user wants faster, sloppier compression.
//
static const struct {
  u16bit tableSize;
  u8bit goodLength;
  u8bit bestLength;
  u8bit maxChain;
} levelParms[10] = {
  {    1,   0,   0,   0},  // 0 - use run and Huffman encodings only
  { 3000,   4,   8,   4},  // 1
  { 4000,   4,  16,   8},  // 2
  { 5000,   4,  16,  16},  // 3
  { 6000,   4,  32,  32},  // 4
  { 7000,   8,  32,  32},  // 5
  { 8000,  16, 128,  64},  // 6
  { 8000, 255, 255, 255},  // 7
  {12000,   8,  32,  64},  // 8
  {14000, 255, 255, 255}   // 9 - best compression
};

//
// Allocate and return a new PCompCtl control instance.
//
PCompCtl
PCompOpen(UInt refNum, void*(*Alloc)(u32bit), void (*Free)(void*))
{
  PCompCtl ctl;
  int i;

  if ((ctl = (PCompCtl)Alloc(sizeof(struct pCompCtl))) == 0)
    return 0;
  for (i = 0; i < sizeof(struct pCompCtl); i++)
    ((char*)ctl)[i] = 0;
  ctl->Alloc = Alloc;
  ctl->Free = Free;
  ctl->level = 2;
  return ctl;
}

//
// Dispose of a PCompCtl control instance.
//
void
PCompClose(UInt refNum, PCompCtl ctl)
{
  if (ctl->hist)
    ctl->Free(ctl->hist);
  if (ctl->tree)
    ctl->Free(ctl->tree);
  if (ctl->tokBits)
    ctl->Free(ctl->tokBits);
  if (ctl->tokLen)
    ctl->Free(ctl->tokLen);
  ctl->Free(ctl);
}

//
// Set a parameter of a PCompCtl control instance.
//
int
PCompSet(UInt refNum, PCompCtl ctl, u8bit type, void *val)
{
  switch (type) {
    case pCompLevel:
      if ((s32bit)val < 0 || (s32bit)val > 9)
	return rcFail;
      ctl->level = (s32bit)val;
      break;
    case pCompProgress:
      ctl->Progress = (int(*)(PCompCtl, s32bit))val;
      break;
    case pCompFlush:
      ctl->Flush = (s32bit(*)(PCompCtl, u8bit*, u16bit, Vector**))val;
      break;
    case pCompNewRecord:
      ctl->NewRecord = (int(*)(PCompCtl))val;
      break;
    case pCompNewSection:
      ctl->NewSection = (int(*)(PCompCtl))val;
      break;
    case pCompGetChunk:
      ctl->GetChunk = (s32bit(*)(PCompCtl, u8bit**))val;
      break;
    case pCompUserData:
      ctl->userData = val;
      break;
    default:
      return rcFail;
  }
  return 0;
}

//
// Get a parameter of a PCompCtl control instance.
//
void *
PCompGet(UInt refNum, PCompCtl ctl, u8bit type)
{
  switch (type) {
    case pCompLevel:
      return (void*)((u32bit)ctl->level);
    case pCompProgress:
      return (void*)ctl->Progress;
    case pCompFlush:
      return (void*)ctl->Flush;
    case pCompNewRecord:
      return (void*)ctl->NewRecord;
    case pCompNewSection:
      return (void*)ctl->NewSection;
    case pCompGetChunk:
      return (void*)ctl->GetChunk;
    case pCompUserData:
      return ctl->userData;
  }
  return 0;
}

//
// This routine performs the string-matching.
// It takes an array of vectors (ie. records) which combine to form a
// complete input database.  To keep things simple, strings can be
// matched between different vectors but cannot span over vector
// boundaries.
//
// Note that this function MUST be able to get called multiple times
// for the same database if the record count is too high to fit into
// one vector array.  The "eof" parameter indicates whether this is
// the last call to Deflate().
//
// Calls:
//   Flush() to write pass 1 output for Encode() to read later
//
s32bit
PCompDeflate(UInt refNum, PCompCtl ctl, Vector *vect, s32bit count,
  u8bit eof)
{
  register u8bit *s1, *s2, *cmpEnd;
  u8bit *ptr, *saveS1, *outBuf = 0;
  s32bit extent, i, p, mlen, high, pos, nextp = 0, v;
  s32bit slen, ii, iv, rc;
  u16bit word, *head = 0, chain, endChain, outLen = 0, runLen;
  u16bit *table = 0, t, tableCur, tableSize;

  //
  // Allocate needed memory.
  //
  if (!ctl->hist) {
    if ((ctl->hist = (u32bit*)ctl->Alloc((TokLast+1) * sizeof(u32bit))) == 0) {
      rc = rcNoMem;
      goto cleanup;
    }
    for (i = 0; i <= TokLast; i++)
      ctl->hist[i] = 0;
  }
  tableSize = levelParms[ctl->level].tableSize;
  if ((table = (u16bit*)ctl->Alloc(tableSize * sizeof(u16bit))) == 0
   || (head = (u16bit*)ctl->Alloc(HashSize * sizeof(u16bit))) == 0
   || (outBuf = (u8bit*)ctl->Alloc(OutBufSize)) == 0) {
    rc = rcNoMem;
    goto cleanup;
  }
  for (i = 0; i < HashSize; i++)
    head[i] = TableNil;

  //
  // Process each input vector.
  //
  i = 0;
  for (v = 0; v < count; v++) {
    if (v > 0) {
      OutputTok(TokEOR);
    }
    ptr = vect[v].ptr;
    if ((extent = vect[v].extent) - i < MinMatch) {
      //
      // Vector has fewer than MinMatch bytes -- output them as literals.
      //
      while (i < extent)
	OutputByte(ptr[i++]);
      if (outLen >= OutBufHiwat) {
	if ((rc = ctl->Flush(ctl, outBuf, outLen, 0)) < 0)
	  goto cleanup;
	outLen = 0;
      }
    }
    else {
      tableCur = i % tableSize;
      runLen = 0;
      word = 0;
      nextp = i;
      for (p = 0; p < MinMatch-1; p++)
	word = (word << HashShift) ^ ptr[i++];
      for (; i < extent; i++) {
	//
	// Check output buffer and report progress periodically.
	//
	if (outLen >= OutBufHiwat) {
	  if ((rc = ctl->Flush(ctl, outBuf, outLen, 0)) < 0)
	    goto cleanup;
	  outLen = 0;
	  if (ctl->Progress && (rc = ctl->Progress(ctl, i)) < 0)
	    goto cleanup;
	}

	//
	// Calculate the hash word.
	//
	word = ((word << HashShift) ^ ptr[i]) & HashMask;
	p = i - (MinMatch - 1);	// start of the string hashed in 'word'

#ifdef PERFDEBUG
      printf("%06x: %02x%02x%02x%02x word=%04x i=%d nextp=%x\n", p, (int)ptr[p], (int)ptr[p+1], (int)ptr[p+2], (int)ptr[p+3], word, i, nextp);
#endif

	Assert(runLen == 0);
	if (p == nextp) {
	  //
	  // Search for the longest previous occurrence of this string.
	  // If we're sitting at a long run of the same character, then
	  // consider the length of the run against the length of any
	  // previous string.  (See the FAQ or pcomp_int.h for more
	  // info about RUN_HACK)
	  //
	  chain = 0;
	  endChain = levelParms[ctl->level].maxChain;

#ifdef RUN_HACK
	  s2 = (s1 = ptr + p) + 1;
	  cmpEnd = ptr + extent;
	  while (s2 < cmpEnd && *s2 == *s1 && ((++s2) - s1) < MaxMatch+1)
	    continue;
	  if ((high = ((s2 - s1) - 1)) >= MinMatch) {
	    //
	    // This is a long run of the same character.
	    //
	    pos = p;
	    runLen = s2 - s1;
	  }
#else
	  high = 0;
#endif

	  iv = v;
	  for (t = head[word]; t != TableNil; t = table[t]) {
	    //
	    // Calculate where the string resides based on its offset.
	    //
	    if (t < tableCur)
	      ii = p - (tableCur - t);
	    else
	      ii = p - tableSize + (t - tableCur);
	    for (; iv > 0 && ii < vect[iv-1].extent; iv--)
	      continue;
	    s2 = ptr + p;
	    saveS1 = s1 = vect[iv].ptr + ii;

	    //
	    // Check to see if this really is a matching string.  For best
	    // results, we should compare MinMatch bytes.  Note that the
	    // last compare should leave s1 and s2 pointing to the last byte
	    // compared.
	    //
	    if (*(s1++) == *(s2++)
	     && *(s1++) == *(s2++)
	     && *(s1++) == *(s2++)
	     && *s1 == *s2) {
	      chain++;
	      Perf(hashHits++);
	      //
	      // Calculate how many chars we can compare.
	      //
	      if ((slen = vect[iv].extent - ii) > MaxMatch)
		slen = MaxMatch;
	      if (extent - p < slen)
		slen = extent - p;
	      cmpEnd = saveS1 + slen;

	      //
	      // Find matching string length.
	      //
	      while (s1 + 8 < cmpEnd)
		if (*(++s1) != *(++s2) || *(++s1) != *(++s2)
		 || *(++s1) != *(++s2) || *(++s1) != *(++s2)
		 || *(++s1) != *(++s2) || *(++s1) != *(++s2)
		 || *(++s1) != *(++s2) || *(++s1) != *(++s2))
		   break;
	      while (s1 < cmpEnd && *s1 == *s2)
		s1++, s2++;

	      if ((mlen = s1 - saveS1) > high && s1 <= cmpEnd) {
		if ((high = mlen) > MaxMatch)
		  high = MaxMatch;
		pos = saveS1 - vect[iv].ptr;
		if (mlen >= levelParms[ctl->level].goodLength) {
		  if (mlen >= levelParms[ctl->level].bestLength)
		    break;
		  endChain = levelParms[ctl->level].maxChain >> 1;
		}
	      }
	    }
	    else {
	      Perf(if (s1 - saveS1 >= 3) hashMiss++);
#ifdef PERFDEBUG
	      printf("  %02x%02x%02x%02x%02x FALSE POSITIVE at %06x\n",
		(s32bit)ptr[p], (s32bit)ptr[p+1], (s32bit)ptr[p+2],
		(s32bit)ptr[p+3], (s32bit)ptr[p+4],
		(s32bit)(saveS1 - vect[iv].ptr));
#endif
	    }
	    if (table[t] == t
	    || (u16bit)(t-table[t]) > (u16bit)(t-tableCur)
	    || (chain >= endChain))
	      break;
	  }

	  //
	  // If a match was found, output it.  Otherwise, output a single
	  // literal byte.
	  //
	  if (high >= MinMatch) {
	    if ((pos = p - pos) == 0 && runLen > 0) {
	      //
	      // The run is a longer length -- use it instead
	      //
	      pos = 1;
	      OutputByte(ptr[p++]);
#ifdef DEBUG
	      printf("%06x: byte %02x\n", p-1, (int)ptr[p-1]);
#endif
	    }
	    Assert(pos > 0 && pos < 65536);
	    if (pos <= 256) {
	      OutputTok(TokMatch1);
	      OutputArg((pos-1) & 0xff);
	    }
	    else {
	      OutputTok(TokMatch2);
	      OutputArg((pos >> 8) & 0xff);
	      OutputArg(pos & 0xff);
	    }
	    OutputArg(high - 3);
	    nextp = p + high;
#ifdef DEBUG
	    printf("%06x: %02x%02x%02x%02x match %d bytes at %06x\n",
	      p,
	      (s32bit)ptr[p], (s32bit)ptr[p+1], (s32bit)ptr[p+2],
	      (s32bit)ptr[p+3],
	      high, p - pos);
#endif
	  }
	  else {
#ifdef DEBUG
	    printf("%06x: byte %02x\n", p, (int)ptr[p]);
#endif
	    OutputByte(ptr[p]);
	    nextp++;
	  }
	}

	//
	// Update the table to reflect this string.
	//
	if (ctl->level > 0) {
	  table[tableCur] = head[word];
	  head[word] = tableCur;
	}

	//
	// Skip past long runs -- we don't want them to affect the
	// string table.  We know the hash word is filled with the
	// first MinMatch bytes of the run, which should all be the
	// same byte.  Hence, we can just set 'i' forward without
	// recalculating the hash word.
	//
	if (runLen) {
	  i += runLen - MinMatch;
	  tableCur += runLen - MinMatch;
	  runLen = 0;
	}
	if (++tableCur >= tableSize)
	  tableCur -= tableSize;
      }
      for (; nextp < extent; nextp++) {
#ifdef DEBUG
	printf("%06x: byte %02x\n", nextp, (int)ptr[nextp]);
#endif
	OutputByte(ptr[nextp]);
      }
    }
  }
  OutputTok(eof ? TokEOF : TokEOS);
  rc = ctl->Flush(ctl, outBuf, outLen, 0);

cleanup:
  //
  // Free memory.
  //
  if (table)
    ctl->Free(table);
  if (head)
    ctl->Free(head);
  if (outBuf)
    ctl->Free(outBuf);
  if (rc < 0 && ctl->hist)
    ctl->Free(ctl->hist), ctl->hist = 0;

  return rc;
}

//
// For debugging
//
#ifdef UNIX
static void
Traverse(Tree *tree, int node, char *buf, int level)
{
  if (tree[node].left == TreeNil) {
    buf[level] = '\0';
    printf("%03d: %s\n", tree[node].token, buf);
  }
  else {
    buf[level] = '0';
    Traverse(tree, tree[node].left, buf, level+1);
    buf[level] = '1';
    Traverse(tree, tree[node].right, buf, level+1);
  }
}
#endif

//
// Construct the Huffman tree from statistical data generated during a
// previous call to Deflate().  Performance here is really not so much of
// an issue, so I didn't bother myself too much about it.
//
// Calls:
//   nothing
//
int
PCompBuildTree(UInt refNum, PCompCtl ctl)
{
  int i, ins = 0, ind[2], head, rc;
  u32bit bits;
  u8bit len;
  u32bit *weight = 0;	// weight of node
  u16bit *next = 0;	// next node in linked list

  //
  // Allocate needed memory.
  //
  if ((!ctl->tree && (ctl->tree = (Tree*)ctl->Alloc(TreeMax * sizeof(Tree))) == 0)
   || (!ctl->tokBits && (ctl->tokBits = (u32bit*)ctl->Alloc((TokLast+1) * sizeof(u32bit))) == 0)
   || (!ctl->tokLen && (ctl->tokLen = (u8bit*)ctl->Alloc((TokLast+1) * sizeof(u8bit))) == 0)
   || (weight = (u32bit*)ctl->Alloc(TreeMax * sizeof(u32bit))) == 0
   || (next = (u16bit*)ctl->Alloc(TreeMax * sizeof(u16bit))) == 0) {
    rc = rcNoMem;
    goto cleanup;
  }
  for (i = 0; i <= TokLast; i++)
    ctl->tokLen[i] = 0;

  //
  // Start with a list of leaf nodes where the weight is the
  // occurence count for each token.
  //
  head = TreeNil;
  for (i = 0; i <= TokLast; i++)
    if (ctl->hist[i] > 0) {
      weight[ins] = ctl->hist[i];
      ctl->tree[ins].token = i;
      next[ins] = head;
      ctl->tree[ins].left = TreeNil;
      ctl->tree[ins].right = TreeNil;
      head = ins++;
    }

  if (head != TreeNil) {
    //
    // Construct the Huffman tree.
    //
    while (next[head] != TreeNil) {
      //
      // Find the two nodes with the lowest weight.
      //
      ind[0] = ind[1] = -1;
      for (i = head; i != TreeNil; i = next[i]) {
	if (ind[0] < 0 || weight[i] < weight[ind[0]]) {
	  ind[1] = ind[0];
	  ind[0] = i;
	}
	else if (ind[1] < 0 || weight[i] < weight[ind[1]])
	  ind[1] = i;
      }

      //
      // Remove the nodes.
      //
      while (head == ind[0] || head == ind[1])
	head = next[head];
      for (i = head; i != TreeNil; i = next[i])
	while (next[i] == ind[0] || next[i] == ind[1])
	  next[i] = next[next[i]];

      //
      // Combine the nodes.
      //
      weight[ins] = weight[ind[0]] + weight[ind[1]];
      ctl->tree[ins].left = ind[0];
      ctl->tree[ins].right = ind[1];
      next[ins] = head;
      next[ind[0]] = next[ind[1]] = ins;
      head = ins++;
    }

    //
    // Traverse the tree and build the bitfields.
    //
    if (ctl->tree[head].left == TreeNil) {
      //
      // Handle the degenerate case of one node.
      //
      ctl->tokBits[ctl->tree[head].token] = 0;
      ctl->tokLen[ctl->tree[head].token] = 1;
    }
    else {
      i = head;
      bits = 0;
      len = 0;
      while (i != TreeNil) {
	if (ctl->tree[i].left != TreeNil) {
	  i = ctl->tree[i].left;
	  bits <<= 1;
	  len++;
	}
	else {
	  //
	  // Handle this node.
	  //
	  ctl->tokBits[ctl->tree[i].token] = bits;
	  ctl->tokLen[ctl->tree[i].token] = len;

	  //
	  // Find the next right-most node.
	  //
	  while (ctl->tree[next[i]].right == i) {
	    bits >>= 1;
	    len--;
	    if ((i = next[i]) == head) {
	      i = TreeNil;
	      break;
	    }
	  }
	  if (i != TreeNil) {
	    bits |= 1;
	    i = ctl->tree[next[i]].right;
	  }
	}
      }
    }
  }
  ctl->treeHead = head;
  rc = 0;

cleanup:
  //
  // Free memory.
  //
  if (ctl->hist)
    ctl->Free(ctl->hist), ctl->hist = 0;
  if (weight)
    ctl->Free(weight);
  if (next)
    ctl->Free(next);
  if (rc < 0) {
    if (ctl->tree)
      ctl->Free(ctl->tree), ctl->tree = 0;
    if (ctl->tokBits)
      ctl->Free(ctl->tokBits), ctl->tokBits = 0;
    if (ctl->tokLen)
      ctl->Free(ctl->tokLen), ctl->tokLen = 0;
  }
#if 0
  if (ctl->tree) { char buf[32]; Traverse(ctl->tree, ctl->treeHead, buf, 0); }
#endif
  return rc;
}

//
// Handle little-endian architectures.
//
#ifndef linux
# define FixBuffer(buf,len)
#else
static void
FixBuffer(u32bit *buf, int len)
{
  int i;
  u8bit *s;
  u32bit val;
  
  s = (u8bit*)buf;
  for (i = 0; i < len; i++) {
    val = buf[i];
    *(s++) = (val >> 24);
    *(s++) = (val >> 16);
    *(s++) = (val >> 8);
    *(s++) = val;
  }
}
#endif

//
// This routine reads the output of a previous Deflate() pass and
// encodes it with the Huffman bitfields generated by BuildTree().
//
// Calls:
//   GetChunk() to read the output from Deflate()
//   Flush() to write encoded output
//
s32bit
PCompEncode(UInt refNum, PCompCtl ctl)
{
  int i;
  u8bit *s, mark, args, *end, len, remain;
  u16bit tok, outLen, *parent = 0;
  u32bit bits, *outBuf = 0;
  s32bit rc;

  //
  // Allocate needed memory.
  //
  if ((outBuf = (u32bit*)ctl->Alloc(OutBufSize)) == 0
   || (parent = (u16bit*)ctl->Alloc(TreeMax * sizeof(u16bit))) == 0) {
    rc = rcNoMem;
    goto cleanup;
  }
  outLen = 0;

  //
  // The first two bytes are the magic number and the version number.
  //
  outBuf[0] = (((u32bit)PCompMagic) << 24) | (((u32bit)PCompVersion) << 16);
  remain = 16;

  //
  // Encode the Huffman tree.
  //
  i = ctl->treeHead;
  parent[i] = TreeNil;
  while (i != TreeNil) {
    if (ctl->tree[i].left == TreeNil) {
      //
      // Leaf node -- output a token.
      //
      if (ctl->tree[i].token > 0xff || ctl->tree[i].token == Marker) {
	bits = (((u32bit)1)<<16) | (Marker<<8) | (ctl->tree[i].token & 0xff);
	len = 17;
      }
      else {
	bits = (1<<8) | ctl->tree[i].token;
	len = 9;
      }
      while (i != ctl->treeHead && ctl->tree[parent[i]].right == i)
	i = parent[i];
      i = (i == ctl->treeHead) ? TreeNil : ctl->tree[parent[i]].right;
    }
    else {
      //
      // Non-leaf node -- traverse branches
      //
      bits = 0;
      len = 1;
      parent[ctl->tree[i].left] = i;
      parent[ctl->tree[i].right] = i;
      i = ctl->tree[i].left;
    }

    if (len > remain) {
      outBuf[outLen] |= (bits >> (len - remain));
      len -= remain;
      if (++outLen >= (OutBufSize/4)) {
	FixBuffer(outBuf, outLen);
	if ((rc = ctl->Flush(ctl, (u8bit*)outBuf, OutBufSize, 0)) < 0)
	  goto cleanup;
	outLen = 0;
      }
      outBuf[outLen] = 0;
      remain = 32;
    }
    outBuf[outLen] |= (bits << (remain - len));
    remain -= len;
  }

  //
  // Encode each input chunk.
  //
  mark = args = 0;
  while ((rc = ctl->GetChunk(ctl, &s)) > 0)
    for (end = s + rc; s < end; s++) {
      tok = *s;
      if (mark) {
	if (tok != Marker) {
	  tok |= 0x100;
	  if (tok == TokMatch1)
	    args = 2;
	  else if (tok == TokMatch2)
	    args = 3;
	}
	mark = 0;
      }
      else if (args > 0) {
	args--;
      }
      else if (tok == Marker) {
	mark = 1;
	continue;
      }

#ifdef PERF
      if (tok > TokLast || ctl->tokLen[tok] <= 0 || ctl->tokLen[tok] > 32) {
	printf("ASSERT - token=%03x len=%d\n", tok, ctl->tokLen[tok]);
	exit(1);
      }
#endif

      len = ctl->tokLen[tok];
      bits = ctl->tokBits[tok];
      if (len > remain) {
	outBuf[outLen] |= (bits >> (len - remain));
	len -= remain;
	if (++outLen >= (OutBufSize/4)) {
	  FixBuffer(outBuf, outLen);
	  if ((rc = ctl->Flush(ctl, (u8bit*)outBuf, OutBufSize, 0)) < 0)
	    goto cleanup;
	  outLen = 0;
	}
	outBuf[outLen] = 0;
	remain = 32;
      }
      outBuf[outLen] |= (bits << (remain - len));
      remain -= len;
    }
  if (rc >= 0) {
    if (remain < 32)
      outLen++;
    FixBuffer(outBuf, outLen);
    rc = ctl->Flush(ctl, (u8bit*)outBuf, outLen * 4, 0);
  }

cleanup:
  //
  // Free memory.
  //
  if (outBuf)
    ctl->Free(outBuf);
  if (parent)
    ctl->Free(parent);
  if (ctl->tree)
    ctl->Free(ctl->tree), ctl->tree = 0;
  if (ctl->tokBits)
    ctl->Free(ctl->tokBits), ctl->tokBits = 0;
  if (ctl->tokLen)
    ctl->Free(ctl->tokLen), ctl->tokLen = 0;
  return rc;
}

//
// This macro defines the getChunk operation used by Decode().
//
#define GETCHUNK { \
  if ((rc = ctl->GetChunk(ctl, &s)) <= 0) {	\
    if (rc == 0)				\
      rc = rcBadData;				\
    goto cleanup;				\
  }						\
  end = s + rc;					\
}

//
// This macro defines the flush operation which happens in too many dang
// places in Decode().  It's important that we update byteCount and
// outLen properly and consistently, hence the macro.
//
#define FLUSHBUF { \
  Assert(outLen <= OutBufSize + MaxMatch);				\
  if ((rc = ctl->Flush(ctl, outBuf, outLen, &vect)) < 0)		\
    goto cleanup;							\
  byteCount += outLen;							\
  outLen = 0;								\
}

//
// This routine reads the output of a previous Encode() pass and
// decodes it into the original text.
//
// Calls:
//   GetChunk() to read compressed data from Encode()
//   Flush() to write uncompressed output
//   NewRecord() at record (vector) boundaries
//   NewSection() at section boundaries
//
s32bit
PCompDecode(UInt refNum, PCompCtl ctl)
{
  int i, ins;
  u8bit *s, mark, *end, remain, *outBuf = 0, args, arg[4], *copyPos, eof;
  u16bit outLen, *parent = 0, tok, copyLen, treeHead;
  s32bit rc, byteCount, count, offset, v;
  Vector *vect;
  Tree *tree = 0;

  //
  // Allocate needed memory.
  //
  if ((outBuf = (u8bit*)ctl->Alloc(OutBufSize + MaxMatch)) == 0
   || (parent = (u16bit*)ctl->Alloc(TreeMax * sizeof(u16bit))) == 0
   || (tree = (Tree*)ctl->Alloc(TreeMax * sizeof(Tree))) == 0) {
    rc = rcNoMem;
    goto cleanup;
  }

  //
  // Check the magic and version numbers.  For now there's only one
  // version number, so we just bail if it doesn't match.
  //
  GETCHUNK;
  if (*s != PCompMagic) {
    rc = rcBadData;
    goto cleanup;
  }
  if (++s >= end) {
    GETCHUNK;
  }
  if (*s != PCompVersion) {
    rc = rcBadVersion;
    goto cleanup;
  }
  remain = 0;

  //
  // Decode the Huffman tree.
  //
  i = treeHead = 0;
  ins = 1;
  args = 0;
  while (i != TreeNil) {
    if (remain <= 0) {
      if (++s >= end) {
	GETCHUNK;
      }
      remain = 8;
    }
    remain--;
    if (*s & (1<<remain)) {
      //
      // Leaf node -- read a token.
      //
      tree[i].left = tree[i].right = TreeNil;
      for (mark = 0;; mark = 1) {
	tree[i].token = (u8bit)(*s << (8 - remain));
	if (++s >= end) {
	  GETCHUNK;
	}
	tree[i].token |= (*s >> remain);
	if (mark) {
	  if (tree[i].token != Marker)
	    tree[i].token |= 0x100;
	  break;
	}
	else if (tree[i].token != Marker)
	  break;
      }

      while (i != treeHead && tree[parent[i]].right == i)
	i = parent[i];
      i = (i == treeHead) ? TreeNil : tree[parent[i]].right;
    }
    else {
      //
      // Non-leaf node -- create branches
      //
      if (ins > TreeMax-2) {
	rc = rcBadData;
	goto cleanup;
      }
      parent[tree[i].left = ins++] = i;
      parent[tree[i].right = ins++] = i;
      i = tree[i].left;
    }
  }
  ctl->Free(parent), parent = 0;
#if 0
  { char buf[32]; Traverse(tree, treeHead, buf, 0); }
#endif

  //
  // If the tree has only one node, then the file is zero-length and
  // the tree contains only the TokEOF token.
  //
  if (tree[treeHead].left == TreeNil) {
    rc = ctl->Flush(ctl, outBuf, 0, 0);
    goto cleanup;
  }

  //
  // Decode each input chunk.
  //
  eof = 0;
  count = byteCount = 0;
  outLen = 0;
  i = treeHead;
  mark = (remain > 0) ? (1<<(remain-1)) : 0;
  for (;;) {
    for (; s < end; s++) {
      for (; mark; mark >>= 1) {
	i = (*s & mark) ? tree[i].right : tree[i].left;
	if (tree[i].left == TreeNil) {
	  //
	  // Leaf node -- we have ourselves a token.
	  //
	  if (args > 0) {
	    arg[--args] = tree[i].token;
	    if (args == 0) {
	      //
	      // Repeat a previous string.
	      //
	      if (tok == TokMatch1)
		offset = ((u16bit)arg[1]) + 1;
	      else
		offset = (u16bit)((((u16bit)arg[2])<<8) | arg[1]);
	      if ((offset = byteCount + outLen - offset) < 0) {
		rc = rcBadData;
		goto cleanup;
	      }
	      copyLen = arg[0] + 3;
	      if (count == 0 || offset >= vect[count-1].extent) {
		//
		// The string to repeat is in the current record.  It
		// may be already flushed to the output database, still
		// in the output buffer, or spread between both.
		//
		if (offset >= byteCount) {
		  //
		  // String is completely within the output buffer.
		  //
		  copyPos = outBuf + (offset - byteCount);
		}
		else {
		  //
		  // String has been at least partially flushed.
		  //
		  copyPos = vect[count].ptr + offset;
		  if (offset + copyLen > byteCount) {
		    //
		    // String is partially flushed and partially in the
		    // output buffer.  Copy what has been flushed and
		    // leave the rest for below.
		    //
		    copyLen = byteCount - offset;
		    for (; copyLen > 0; copyLen--)
		      outBuf[outLen++] = *(copyPos++);
		    copyPos = outBuf;
		    copyLen = (offset + arg[0] + 3) - byteCount;
		  }
		}
	      }
	      else {
		//
		// The string to repeat is in a previous record.
		//
		for (v = count-1; v > 0 && offset < vect[v-1].extent; v--)
		  continue;
		copyPos = vect[v].ptr + offset;
		if (offset + copyLen > vect[v].extent) {
		  rc = rcBadData;
		  goto cleanup;
		}
	      }
	      for (; copyLen > 0; copyLen--)
		outBuf[outLen++] = *(copyPos++);
	      if (outLen >= OutBufHiwat-8) {
		FLUSHBUF;
	      }
	    }
	  }
	  else {
	    switch (tree[i].token) {
	      case TokMatch1:
		tok = TokMatch1;
		args = 2;
		break;

	      case TokMatch2:
		tok = TokMatch2;
		args = 3;
		break;

	      case TokEOR:
		FLUSHBUF;
		if ((rc = ctl->NewRecord(ctl)) < 0)
		  goto cleanup;
		count++;
		break;

	      case TokEOS:
		FLUSHBUF;
		if ((rc = ctl->NewSection(ctl)) < 0)
		  goto cleanup;
		byteCount = 0;
		count = 0;
		break;

	      case TokEOF:
		mark = 0;
		end = s;
		eof = 1;
		break;

	      default:
		outBuf[outLen++] = tree[i].token;
	    }
	  }
	  i = treeHead;
	}
      }
      if (outLen >= OutBufHiwat-8) {
	FLUSHBUF;
      }
      mark = 0x80;
    }
    if ((rc = ctl->GetChunk(ctl, &s)) <= 0)
      break;
    end = s + rc;
  }
  if (rc >= 0)
    rc = (eof == 0) ? rcBadData : ctl->Flush(ctl, outBuf, outLen, &vect);

cleanup:
  //
  // Free memory.
  //
  if (outBuf)
    ctl->Free(outBuf);
  if (parent)
    ctl->Free(parent);
  if (tree)
    ctl->Free(tree), tree = 0;
  return rc;
}

#ifdef UNIX
static s32bit totalMem = 0, maxMem = 0;

static void *
Alloc(u32bit size)
{
  s32bit *out;

  if ((out = (s32bit*)malloc(size + sizeof(s32bit))) == 0) {
    fprintf(stderr, "malloc(%d) failed!\n", size);
    exit(0);
  }
  *out = size;
  totalMem += size;
  if (totalMem > maxMem)
    maxMem = totalMem;

#ifdef DEBUG
  printf("Alloc(%d) -- total %d\n", size, totalMem);
#endif

  return (void*)(&out[1]);
}

static void
Free(void *ptr)
{
  s32bit *sizep;

  sizep = (s32bit*)((char*)ptr - sizeof(s32bit));
  totalMem -= *sizep;

#ifdef DEBUG
  printf("Free(%d) -- total %d\n", *sizep, totalMem);
#endif

  free(sizep);
}

static int
Progress(PCompCtl ctl, s32bit i)
{
  static s32bit foo, last = 50000;

  if (i < last)
    foo = 50000;
  last = i;
  if (i >= foo) {
    printf("...%d...\n", i);
    foo += 50000;
  }
  return 0;
}

static char *outBuf = 0, *chunkBuf;
static int outLen = 0, outMax = 0;
static int chunkPos, chunkLen;
static Vector outVect;

static s32bit
FlushBuffer(PCompCtl ctl, u8bit *buf, u16bit len, Vector **vectp)
{
  while (outLen + len >= outMax) {
    outMax += 4096;
    if (outBuf)
      outBuf = (u8bit*)realloc(outBuf, outMax);
    else
      outBuf = (u8bit*)malloc(outMax);
    if (outBuf == 0)
      return -1;
  }
  memcpy(outBuf + outLen, buf, len);
  outLen += len;

  if (vectp) {
    outVect.ptr = outBuf;
    outVect.extent = outLen;
    *vectp = &outVect;
  }

  return outLen;
}

static s32bit
NewRecord(PCompCtl ctl)
{
  return -1;	// notimp for unix
}

static s32bit
NewSection(PCompCtl ctl)
{
  return -1;	// notimp for unix
}

static s32bit
GetChunk(PCompCtl ctl, u8bit **bufPtr)
{
  s32bit rc;

  *bufPtr = chunkBuf + chunkPos;
  if ((rc = chunkLen - chunkPos) > 4096)
    rc = 4096;
  chunkPos += rc;
  return rc;
}

int
main(s32bit argc, char **argv)
{
  s32bit fd, len, rc, i, nvect = 1, level = 2;
  static u8bit in[3000000];
  char *fname = "data";
  Vector vect[3];
  PCompCtl ctl;

  if (argc >= 2)
    fname = argv[1];
  if (argc >= 3)
    level = atoi(argv[2]);
  if ((len = atoi(fname)) > 0) {
    for (i = 0; i < len; i++)
      in[i] = i & 0x0f;
  }
  else {
    if ((fd = open(fname, O_RDONLY)) < 0) {
      perror(fname);
      exit(1);
    }
    len = read(fd, in, sizeof(in));
    close(fd);
  }
  printf("Compressing %d bytes...\n", len);

  if ((ctl = PCompOpen(0, Alloc, Free)) == 0) {
    printf("PCompOpen failed\n");
    exit(1);
  }
  PCompSet(0, ctl, pCompLevel, (void*)level);
  PCompSet(0, ctl, pCompProgress, Progress);
  PCompSet(0, ctl, pCompFlush, FlushBuffer);
  PCompSet(0, ctl, pCompGetChunk, GetChunk);
  PCompSet(0, ctl, pCompNewRecord, NewRecord);
  PCompSet(0, ctl, pCompNewSection, NewSection);

  vect[0].ptr = in;
  vect[0].extent = len;
  vect[1].ptr = vect[0].ptr - len;
  vect[1].extent = vect[0].extent + len;
  vect[2].ptr = vect[1].ptr - len;
  vect[2].extent = vect[1].extent + len;
  if ((rc = PCompDeflate(0, ctl, vect, nvect, 1)) < 0) {
    fprintf(stderr, "Deflate failed!\n");
    exit(1);
  }
  printf("Deflate: %d bytes, outBuf=%08x outLen=%d outMax=%d mem=%d\n",
    rc, outBuf, outLen, outMax, totalMem);

#ifdef PERF
  printf("  Hash hits=%d misses=%d\n", hashHits, hashMiss);
  printf("  Table inserts=%d\n", tableInserts);
#endif

  if ((fd = open("pass1", O_WRONLY | O_CREAT | O_TRUNC, 0666)) >= 0) {
    write(fd, outBuf, outLen);
    close(fd);
  }

  if ((rc = PCompBuildTree(0, ctl)) < 0) {
    fprintf(stderr, "BuildTree failed!\n");
    exit(1);
  }
  printf("BuildTree: %d, mem=%d\n", rc, totalMem);

#if 0
  for (i = 0; i <= TokLast; i++)
    if (ctl->tokLen[i] > 0) {
      printf("%03x: ", i);
      for (len = 1 << (ctl->tokLen[i]-1); len; len >>= 1)
	putchar((ctl->tokBits[i] & len) ? '1' : '0');
      putchar('\n');
    }
#endif

  chunkBuf = outBuf;
  chunkLen = outLen;
  chunkPos = 0;
  outBuf = 0;
  outLen = outMax = 0;
  if ((rc = PCompEncode(0, ctl)) < 0) {
    fprintf(stderr, "Encode failed!\n");
    exit(1);
  }
  printf("Encode: %d bytes mem=%d\n", rc, totalMem);

#ifdef PERF
  printf("  Max memory usage: %d\n", maxMem);
#endif

  if ((fd = open("pass2", O_WRONLY | O_CREAT | O_TRUNC, 0666)) >= 0) {
    write(fd, outBuf, outLen);
    close(fd);
  }

  if (chunkBuf)
    free(chunkBuf);
  chunkBuf = outBuf;
  chunkLen = outLen;
  chunkPos = 0;
  outBuf = 0;
  outLen = outMax = 0;
  if ((rc = PCompDecode(0, ctl)) < 0) {
    fprintf(stderr, "Decode failed! rc=%d\n", rc);
    exit(1);
  }
  printf("Decode: %d bytes mem=%d\n", rc, totalMem);

  if ((fd = open("pass3", O_WRONLY | O_CREAT | O_TRUNC, 0666)) >= 0) {
    write(fd, outBuf, outLen);
    close(fd);
  }

  return 0;
}
#endif
