Logo Search packages:      
Sourcecode: scamper version File versions  Download package

scamper_file_warts.c

/*
 * scamper_file_warts.c
 *
 * the Waikato ARTS file format replacement
 *
 * $Id: scamper_file_warts.c,v 1.104.2.11 2008/04/04 23:49:04 mjl Exp $
 *
 * Copyright (C) 2004-2008 The University of Waikato
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, version 2.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 */

#include <sys/types.h>
#include <sys/time.h>
#include <sys/socket.h>

#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/ip_icmp.h>
#include <netinet/ip6.h>
#include <netinet/icmp6.h>

#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>

#if defined(__APPLE__)
#include <stdint.h>
#endif

#include <assert.h>

#if defined(DMALLOC)
#include <dmalloc.h>
#endif

#include "scamper_addr.h"
#include "scamper_list.h"
#include "scamper_tlv.h"
#include "scamper_trace.h"
#include "scamper_ping.h"
#include "scamper_file.h"
#include "scamper_file_warts.h"

#include "mjl_splaytree.h"
#include "utils.h"

#define WARTS_MAGIC 0x1205

/*
 * trace attributes: 2 bytes each.
 * the first 4 bits are the type, the second 12 bits are the length
 */
#define WARTS_TRACE_ATTR_HDR(type, len) ((type << 12) | len)
#define WARTS_TRACE_ATTR_HDR_TYPE(hdr)  ((hdr >> 12) & 0xf)
#define WARTS_TRACE_ATTR_HDR_LEN(hdr)    (hdr & 0x0fff)
#define WARTS_TRACE_ATTR_EOF       0x0000
#define WARTS_TRACE_ATTR_PMTUD     0x1
#define WARTS_TRACE_ATTR_LASTDITCH 0x2

/* how many entries to grow the table by each time */
#define WARTS_ADDR_TABLEGROW  1000
#define WARTS_LIST_TABLEGROW  1
#define WARTS_CYCLE_TABLEGROW 1

/*
 * warts_addr / warts_list / warts_cycle
 *
 * these structures associate a scamper structure with an id number used
 * to represent the structure on disk.
 */
typedef struct warts_addr
{
  scamper_addr_t *addr;
  uint32_t id;
} warts_addr_t;
typedef struct warts_list
{
  scamper_list_t *list;
  uint32_t id;
} warts_list_t;
typedef struct warts_cycle
{
  scamper_cycle_t *cycle;
  uint32_t id;
} warts_cycle_t;

/*
 * warts_hdr
 *
 * this object is written at the start of every object.
 * the magic field is a special integer value that signifies a new warts
 * record.
 * the type field says what type of record follows.
 * the length field reports the length of the following record.
 */
typedef struct warts_hdr
{
  uint16_t magic;
  uint16_t type;
  uint32_t len;
} warts_hdr_t;

/*
 * warts_state
 *
 * warts keeps state of lists, cycles, and addresses declared in a warts
 * file.  each resource is stored either in a tree (for fast searching) or
 * a table (for fast indexing).  when a file is open for writing, the tree
 * is used.  when a file is open for reading, the table is used.  each null
 * entry is used for the first ([0]) entry in the corresponding table.
 */
typedef struct warts_state
{
  int               ispipe;

  /* list state */
  uint32_t          list_count;
  splaytree_t      *list_tree;
  warts_list_t    **list_table;
  warts_list_t      list_null;

  /* cycle state */
  uint32_t          cycle_count;
  splaytree_t      *cycle_tree;
  warts_cycle_t   **cycle_table;
  warts_cycle_t     cycle_null;

  /* address state */
  uint32_t          addr_count;
  splaytree_t      *addr_tree;
  warts_addr_t    **addr_table;
  warts_addr_t      addr_null;

} warts_state_t;

/*
 * warts_var
 *
 * warts often stores optional items of data with each object.  it does
 * this by declaring an array of bits that declare which optional bits of
 * data will be stored.  the warts_var structure is a convenient way of
 * encouraging the code for each object to be consistent.
 *
 * the id field corresponds to a bit
 * the size field records how large the field is stored on disk; -1 is variable
 * the tlv_id field records the id for a scamper_tlv_t if the data item is
 * stored optionally in the data structure itself.
 */
typedef struct warts_var
{
  int     id;
  ssize_t size;
  int     tlv_id;
} warts_var_t;
#define WARTS_VAR_COUNT(array) (sizeof(array)/sizeof(warts_var_t))
#define WARTS_VAR_MFB(array) ((WARTS_VAR_COUNT(array) / 7) + \
                        (WARTS_VAR_COUNT(array) % 7 == 0 ? 0 : 1))

typedef int (*wpr_t)(const uint8_t *,uint32_t *,const uint32_t,void *, void *);
typedef void (*wpw_t)(uint8_t *,uint32_t *,const uint32_t,const void *,void *);

typedef struct warts_param_reader
{
  void       *data;
  wpr_t       read;
  void       *param;
} warts_param_reader_t;

typedef struct warts_param_writer
{
  const void *data;
  wpw_t       write;
  void       *param;
} warts_param_writer_t;

/*
 * the optional bits of a list structure
 */
#define WARTS_LIST_DESCR      1              /* description of list */
#define WARTS_LIST_MONITOR    2              /* canonical name of monitor */
static const warts_var_t list_vars[] =
{
  {WARTS_LIST_DESCR,   -1, -1},
  {WARTS_LIST_MONITOR, -1, -1},
};
static const unsigned int list_vars_mfb = WARTS_VAR_MFB(list_vars);

/*
 * the optional bits of a cycle start structure
 */
#define WARTS_CYCLE_STOP_TIME 1              /* time at which cycle ended */
#define WARTS_CYCLE_HOSTNAME  2              /* hostname at cycle point */
static const warts_var_t cycle_vars[] =
{
  {WARTS_CYCLE_STOP_TIME,  4, -1},
  {WARTS_CYCLE_HOSTNAME,  -1, -1},
};
static const unsigned int cycle_vars_mfb = WARTS_VAR_MFB(cycle_vars);

/*
 * the optional bits of a trace structure
 */
#define WARTS_TRACE_LIST_ID    1              /* list id assigned by warts */
#define WARTS_TRACE_CYCLE_ID   2              /* cycle id assigned by warts */
#define WARTS_TRACE_ADDR_SRC   3              /* source address key */
#define WARTS_TRACE_ADDR_DST   4              /* destination address key */
#define WARTS_TRACE_START      5              /* start timestamp */
#define WARTS_TRACE_STOP_R     6              /* stop reason */
#define WARTS_TRACE_STOP_D     7              /* stop data */
#define WARTS_TRACE_FLAGS      8              /* flags */
#define WARTS_TRACE_ATTEMPTS   9              /* attempts */
#define WARTS_TRACE_HOPLIMIT   10             /* hoplimit */
#define WARTS_TRACE_TYPE       11             /* type */
#define WARTS_TRACE_PROBE_S    12             /* probe size */
#define WARTS_TRACE_PORT_SRC   13             /* source port */
#define WARTS_TRACE_PORT_DST   14             /* destination port */
#define WARTS_TRACE_FIRSTHOP   15             /* first hop */
#define WARTS_TRACE_TOS        16             /* type of service bits */
#define WARTS_TRACE_WAIT       17             /* how long to wait per probe */
#define WARTS_TRACE_LOOPS      18             /* max loops before stopping */
#define WARTS_TRACE_HOPCOUNT   19             /* hop_count */
#define WARTS_TRACE_GAPLIMIT   20             /* gap limit */
#define WARTS_TRACE_GAPACTION  21             /* gap action */
#define WARTS_TRACE_LOOPACTION 22            /* loop action */
static const warts_var_t trace_vars[] =
{
  {WARTS_TRACE_LIST_ID,    4, -1},
  {WARTS_TRACE_CYCLE_ID,   4, -1},
  {WARTS_TRACE_ADDR_SRC,   4, -1},
  {WARTS_TRACE_ADDR_DST,   4, -1},
  {WARTS_TRACE_START,      8, -1},
  {WARTS_TRACE_STOP_R,     1, -1},
  {WARTS_TRACE_STOP_D,     1, -1},
  {WARTS_TRACE_FLAGS,      1, -1},
  {WARTS_TRACE_ATTEMPTS,   1, -1},
  {WARTS_TRACE_HOPLIMIT,   1, -1},
  {WARTS_TRACE_TYPE,       1, -1},
  {WARTS_TRACE_PROBE_S,    2, -1},
  {WARTS_TRACE_PORT_SRC,   2, -1},
  {WARTS_TRACE_PORT_DST,   2, -1},
  {WARTS_TRACE_FIRSTHOP,   1, -1},
  {WARTS_TRACE_TOS,        1, -1},
  {WARTS_TRACE_WAIT,       1, -1},
  {WARTS_TRACE_LOOPS,      1, -1},
  {WARTS_TRACE_HOPCOUNT,   2, -1},
  {WARTS_TRACE_GAPLIMIT,   1, -1},
  {WARTS_TRACE_GAPACTION,  1, -1},
  {WARTS_TRACE_LOOPACTION, 1, -1},
};
static const unsigned int trace_vars_mfb = WARTS_VAR_MFB(trace_vars);

/*
 * the optional bits of a trace pmtud structure
 */
#define WARTS_TRACE_PMTUD_IFMTU  1           /* interface mtu */
#define WARTS_TRACE_PMTUD_PMTU   2           /* path mtu */
#define WARTS_TRACE_PMTUD_OUTMTU 3           /* mtu to gateway */
static const warts_var_t pmtud_vars[] =
{
  {WARTS_TRACE_PMTUD_IFMTU,  2, -1},
  {WARTS_TRACE_PMTUD_PMTU,   2, -1},
  {WARTS_TRACE_PMTUD_OUTMTU, 2, SCAMPER_TRACE_PMTUD_TLV_OUTMTU},
};
static const unsigned int pmtud_vars_mfb = WARTS_VAR_MFB(pmtud_vars);

/*
 * the optional bits of a trace hop structure
 */
#define WARTS_TRACE_HOP_ADDR         1       /* address id */
#define WARTS_TRACE_HOP_PROBE_TTL    2       /* probe ttl */
#define WARTS_TRACE_HOP_REPLY_TTL    3       /* reply ttl */
#define WARTS_TRACE_HOP_FLAGS        4       /* flags */
#define WARTS_TRACE_HOP_PROBE_ID     5       /* probe id */
#define WARTS_TRACE_HOP_RTT          6       /* round trip time */
#define WARTS_TRACE_HOP_ICMP_TC      7       /* icmp type / code */
#define WARTS_TRACE_HOP_PROBE_SIZE   8       /* probe size */
#define WARTS_TRACE_HOP_REPLY_SIZE   9       /* reply size */
#define WARTS_TRACE_HOP_REPLY_IPID   10      /* ipid of reply packet */
#define WARTS_TRACE_HOP_REPLY_IPTOS  11      /* tos bits of reply packet */
#define WARTS_TRACE_HOP_NHMTU        12      /* next hop mtu in ptb message */
#define WARTS_TRACE_HOP_INNER_IPLEN  13      /* ip->len from inside icmp */
#define WARTS_TRACE_HOP_INNER_IPTTL  14      /* ip->ttl from inside icmp */
#define WARTS_TRACE_HOP_TCP_FLAGS    15      /* tcp->flags of reply packet */
#define WARTS_TRACE_HOP_INNER_IPTOS  16      /* ip->tos byte inside icmp */
#define WARTS_TRACE_HOP_ICMPEXT      17      /* RFC 4884 icmp extension data */
static const warts_var_t hop_vars[] =
{
  {WARTS_TRACE_HOP_ADDR,         4, -1},
  {WARTS_TRACE_HOP_PROBE_TTL,    1, -1},
  {WARTS_TRACE_HOP_REPLY_TTL,    1, -1},
  {WARTS_TRACE_HOP_FLAGS,        1, -1},
  {WARTS_TRACE_HOP_PROBE_ID,     1, -1},
  {WARTS_TRACE_HOP_RTT,          4, -1},
  {WARTS_TRACE_HOP_ICMP_TC,      2, -1},
  {WARTS_TRACE_HOP_PROBE_SIZE,   2, -1},
  {WARTS_TRACE_HOP_REPLY_SIZE,   2, -1},
  {WARTS_TRACE_HOP_REPLY_IPID,   2, SCAMPER_TRACE_HOP_TLV_REPLY_IPID},
  {WARTS_TRACE_HOP_REPLY_IPTOS,  1, SCAMPER_TRACE_HOP_TLV_REPLY_IPTOS},
  {WARTS_TRACE_HOP_NHMTU,        2, SCAMPER_TRACE_HOP_TLV_NHMTU},
  {WARTS_TRACE_HOP_INNER_IPLEN,  2, SCAMPER_TRACE_HOP_TLV_INNER_IPLEN},
  {WARTS_TRACE_HOP_INNER_IPTTL,  1, SCAMPER_TRACE_HOP_TLV_INNER_IPTTL},
  {WARTS_TRACE_HOP_TCP_FLAGS,    1, -1},
  {WARTS_TRACE_HOP_INNER_IPTOS,  1, SCAMPER_TRACE_HOP_TLV_INNER_IPTOS},
  {WARTS_TRACE_HOP_ICMPEXT      -1, -1},
};
static const unsigned int hop_vars_mfb = WARTS_VAR_MFB(hop_vars);

/*
 * the optional bits of a ping structure
 */
#define WARTS_PING_LIST_ID         1
#define WARTS_PING_CYCLE_ID        2
#define WARTS_PING_ADDR_SRC        3
#define WARTS_PING_ADDR_DST        4
#define WARTS_PING_START           5
#define WARTS_PING_STOP_R          6
#define WARTS_PING_STOP_D          7
#define WARTS_PING_PATTERN_LEN     8
#define WARTS_PING_PATTERN_BYTES   9
#define WARTS_PING_PROBE_COUNT    10
#define WARTS_PING_PROBE_SIZE     11
#define WARTS_PING_PROBE_WAIT     12
#define WARTS_PING_PROBE_TTL      13
#define WARTS_PING_REPLY_COUNT    14
#define WARTS_PING_PING_SENT      15
static const warts_var_t ping_vars[] =
{
  {WARTS_PING_LIST_ID,        4, -1},
  {WARTS_PING_CYCLE_ID,       4, -1},
  {WARTS_PING_ADDR_SRC,       4, -1},
  {WARTS_PING_ADDR_DST,       4, -1},
  {WARTS_PING_START,          8, -1},
  {WARTS_PING_STOP_R,         1, -1},
  {WARTS_PING_STOP_D,         1, -1},
  {WARTS_PING_PATTERN_LEN,    2, -1},
  {WARTS_PING_PATTERN_BYTES, -1, -1},
  {WARTS_PING_PROBE_COUNT,    2, -1},
  {WARTS_PING_PROBE_SIZE,     2, -1},
  {WARTS_PING_PROBE_WAIT,     1, -1},
  {WARTS_PING_PROBE_TTL,      1, -1},
  {WARTS_PING_REPLY_COUNT,    2, -1},
  {WARTS_PING_PING_SENT,      2, -1},
};
static const unsigned int ping_vars_mfb = WARTS_VAR_MFB(ping_vars);

#define WARTS_PING_REPLY_ADDR        1
#define WARTS_PING_REPLY_FLAGS       2
#define WARTS_PING_REPLY_REPLY_TTL   3
#define WARTS_PING_REPLY_REPLY_SIZE  4
#define WARTS_PING_REPLY_ICMP_TC     5
#define WARTS_PING_REPLY_RTT         6
#define WARTS_PING_REPLY_PROBE_ID    7
static const warts_var_t ping_reply_vars[] =
{
  {WARTS_PING_REPLY_ADDR,       4, -1},
  {WARTS_PING_REPLY_FLAGS,      1, -1},
  {WARTS_PING_REPLY_REPLY_TTL,  1, -1},
  {WARTS_PING_REPLY_REPLY_SIZE, 2, -1},
  {WARTS_PING_REPLY_ICMP_TC,    2, -1},
  {WARTS_PING_REPLY_RTT,        4, -1},
  {WARTS_PING_REPLY_PROBE_ID,   2, -1},
};
static const unsigned int ping_reply_vars_mfb = WARTS_VAR_MFB(ping_reply_vars);

typedef struct warts_hop_state
{
  scamper_trace_hop_t *hop;
  uint32_t             addr;
  uint8_t              flags[WARTS_VAR_MFB(hop_vars)];
  uint16_t             flags_len;
  uint16_t             params_len;
} warts_hop_state_t;

typedef struct warts_ping_reply_state
{
  scamper_ping_reply_t *reply;
  uint32_t              addr;
  uint8_t               flags[WARTS_VAR_MFB(ping_reply_vars)];
  uint16_t              flags_len;
  uint16_t              params_len;
} warts_ping_reply_state_t;

/*
 * set_flag
 *
 * small routine to set a flag bit.  this exists because the 8th bit of
 * each byte used for flags is used to indicate when another set of flags
 * follows the byte.
 */
static void set_flag(uint8_t *flags, const int id, int *max_id)
{
  int i, j;

  assert(id > 0);

  if(id % 7 == 0)
    {
      i = (id / 7) - 1;
      j = 7;
    }
  else
    {
      i = id / 7;
      j = id % 7;
    }

  flags[i] |= (0x1 << (j-1));

  if(max_id != NULL && *max_id < id) *max_id = id;

  return;
}

/*
 * fold_flags
 *
 * go through and set each link bit in the flag set, as appropriate.
 * conveniently return the count of the number of bytes required to store
 * the flags.
 */
static uint16_t fold_flags(uint8_t *flags, const int max_id)
{
  uint16_t i, j;

  /* if no flags are set, it is still a requirement to include a zero byte */
  if(max_id == 0)
    {
      return 1;
    }

  /* figure out how many bytes have been used */
  j = max_id / 7;
  if((max_id % 7) != 0) j++;

  /*
   * j has to be greater than zero by the above logic.  however, the for
   * loop below will go bananas if it is not
   */
  assert(j > 0);

  /* skip through and set the 'more flags' bit for all flag bytes necessary */
  for(i=0; i<j-1; i++)
    {
      flags[i] |= 0x80;
    }

  return j;
}

static void insert_uint16(uint8_t *buf, uint32_t *off, const uint32_t len,
                    const uint16_t *in, void *param)
{
  uint16_t tmp = htons(*in);

  assert(len - *off >= 2);

  memcpy(&buf[*off], &tmp, 2);
  *off += 2;
  return;
}

static void insert_uint16_tlv(uint8_t *buf, uint32_t *off, const uint32_t len,
                        const scamper_tlv_t *in, uint8_t *type)
{
  const scamper_tlv_t *tlv = scamper_tlv_get(in, *type);
  assert(tlv != NULL);
  assert(tlv->tlv_len == 2);
  insert_uint16(buf, off, len, &tlv->tlv_val_16, NULL);
  return;
}

static void insert_uint32(uint8_t *buf, uint32_t *off, const uint32_t len,
                    const uint32_t *in, void *param)
{
  uint32_t tmp = htonl(*in);

  assert(len - *off >= 4);

  memcpy(&buf[*off], &tmp, 4);
  *off += 4;
  return;
}

static void insert_byte(uint8_t *buf, uint32_t *off, const uint32_t len,
                  const uint8_t *in, void *param)
{
  assert(len - *off >= 1);
  buf[(*off)++] = *in;
  return;
}

static void insert_byte_tlv(uint8_t *buf, uint32_t *off, const uint32_t len,
                      const scamper_tlv_t *in, uint8_t *type)
{
  const scamper_tlv_t *tlv = scamper_tlv_get(in, *type);
  assert(tlv != NULL);
  assert(tlv->tlv_len == 1);
  insert_byte(buf, off, len, &tlv->tlv_val_8, NULL);
  return;
}

static void insert_bytes_uint16(uint8_t *buf,uint32_t *off,const uint32_t len,
                        const void *vin, uint16_t *count)
{
  assert(len - *off >= *count);
  memcpy(buf + *off, vin, *count);
  *off += *count;
  return;
}

static void insert_string(uint8_t *buf, uint32_t *off, const uint32_t len,
                    const char *in, void *param)
{
  uint8_t c;
  int i = 0;

  do
    {
      assert(len - *off > 0);
      buf[(*off)++] = c = in[i++];
    }
  while(c != '\0');

  return;
}

/*
 * insert_timeval
 *
 * this function may cause trouble in the future with timeval struct members
 * changing types and so on.
 */
static void insert_timeval(uint8_t *buf, uint32_t *off, const uint32_t len,
                     const struct timeval *in, void *param)
{
  uint32_t t32;

  assert(len - *off >= 8);

  t32 = htonl(in->tv_sec);
  memcpy(buf + *off, &t32, 4); *off += 4;
  
  t32 = htonl(in->tv_usec);
  memcpy(buf + *off, &t32, 4); *off += 4;

  return;
}

static void insert_rtt(uint8_t *buf, uint32_t *off, const uint32_t len,
                   const struct timeval *tv, void *param)
{
  uint32_t t32 = (tv->tv_sec * 1000000) + tv->tv_usec;
  insert_uint32(buf, off, len, &t32, NULL);
  return;
}

static int extract_string(const uint8_t *buf, uint32_t *off,
                    const uint32_t len, char **out, void *param)
{
  uint32_t i;

  for(i=*off; i<len; i++)
    {
      /* scan for the null terminator */
      if(buf[i] == '\0')
      {
        if((*out = memdup(buf+*off, (size_t)(i-*off+1))) == NULL)
          {
            return -1;
          }

        *off = i+1;
        return 0;
      }
    }

  return -1;
}

static int extract_uint32(const uint8_t *buf, uint32_t *off,
                    const uint32_t len, uint32_t *out, void *param)
{
  if(len - *off < 4)
    {
      return -1;
    }

  memcpy(out, buf + *off, 4); *off += 4;
  *out = ntohl(*out);
  return 0;
}

static int extract_uint16(const uint8_t *buf, uint32_t *off,
                    const uint32_t len, uint16_t *out, void *param)
{
  if(len - *off < 2)
    {
      return -1;
    }

  memcpy(out, buf + *off, 2); *off += 2;
  *out = ntohs(*out);
  return 0;
}

static int extract_uint16_tlv(const uint8_t *buf, uint32_t *off,
                        const uint32_t len, scamper_tlv_t **tlvs,
                        uint8_t *type)
{
  uint16_t t16;

  if(extract_uint16(buf, off, len, &t16, NULL) != 0)
    {
      return -1;
    }

  if(scamper_tlv_set(tlvs, *type, 2, &t16) == NULL)
    {
      return -1;
    }

  return 0;
}

static int extract_byte(const uint8_t *buf, uint32_t *off,
                  const uint32_t len, uint8_t *out, void *param)
{
  if(len - *off < 1)
    {
      return -1;
    }

  *out = buf[(*off)++];
  return 0;  
}

static int extract_bytes_uint16(const uint8_t *buf, uint32_t *off,
                        const uint32_t len, uint8_t **out,
                        uint16_t *req)
{
  if(len - *off < *req)
    {
      return -1;
    }

  if(*req == 0)
    {
      *out = NULL;
    }
  else
    {
      if((*out = malloc(*req)) == NULL)
      {
        return -1;
      }

      memcpy(*out, buf + *off, *req);
      *off += *req;
    }

  return 0;
}

static int extract_byte_tlv(const uint8_t *buf, uint32_t *off,
                      const uint32_t len,
                      scamper_tlv_t **tlvs, uint8_t *type)
{
  uint8_t  t8;

  if(extract_byte(buf, off, len, &t8, NULL) != 0)
    {
      return -1;
    }

  if(scamper_tlv_set(tlvs, *type, 1, &t8) == NULL)
    {
      return -1;
    }

  return 0;
}

static int extract_addr(const uint8_t *buf, uint32_t *off,
                  const uint32_t len,
                  scamper_addr_t **addr, warts_state_t *state)
{
  uint32_t id;

  if(extract_uint32(buf, off, len, &id, NULL) != 0)
    {
      return -1;
    }

  if(id >= state->addr_count)
    {
      return -1;
    }

  *addr = scamper_addr_use(state->addr_table[id]->addr);
  return 0;
}

static int extract_list(const uint8_t *buf, uint32_t *off,
                  const uint32_t len,
                  scamper_list_t **list, warts_state_t *state)
{
  uint32_t id;

  if(extract_uint32(buf, off, len, &id, NULL) != 0)
    {
      return -1;
    }

  if(id >= state->list_count)
    {
      return -1;
    }

  *list = scamper_list_use(state->list_table[id]->list);
  return 0;
}

static int extract_cycle(const uint8_t *buf, uint32_t *off,
                   const uint32_t len,
                   scamper_cycle_t **cycle, warts_state_t *state)
{
  uint32_t id;

  if(extract_uint32(buf, off, len, &id, NULL) != 0)
    {
      return -1;
    }

  if(id >= state->cycle_count)
    {
      return -1;
    }

  *cycle = scamper_cycle_use(state->cycle_table[id]->cycle);
  return 0;
}

static int extract_timeval(const uint8_t *buf, uint32_t *off,
                     const uint32_t len, struct timeval *tv, void *param)
{
  uint32_t t32;

  if(extract_uint32(buf, off, len, &t32, NULL) != 0)
    {
      return -1;
    }
  tv->tv_sec = t32;

  if(extract_uint32(buf, off, len, &t32, NULL) != 0)
    {
      return -1;
    }
  tv->tv_usec = t32;

  return 0;
}

static int extract_rtt(const uint8_t *buf, uint32_t *off, const uint32_t len,
                   struct timeval *tv, void *param)
{
  uint32_t t32;

  if(extract_uint32(buf, off, len, &t32, NULL) != 0)
    {
      return -1;
    }

  tv->tv_sec  = t32 / 1000000;
  tv->tv_usec = t32 % 1000000;
  return 0;
}

static int warts_params_read(const uint8_t *buf, uint32_t *off, uint32_t len,
                       warts_param_reader_t *handlers, int handler_cnt)
{
  warts_param_reader_t *handler;
  const uint8_t *flags = &buf[*off];
  uint16_t flags_len, params_len;
  uint32_t final_off;
  int      i, j, id;

  /* if there are no flags set at all, then there's nothing left to do */
  if(flags[0] == 0)
    {
      (*off)++;
      return 0;
    }

  /* figure out how long the flags block is */
  flags_len = 0;
  while((buf[*off] & 0x80) != 0 && *off < len)
    {
      (*off)++; flags_len++;
    }
  flags_len++; (*off)++;
  if(*off > len) goto err;

  /* the length field */
  if(extract_uint16(buf, off, len, &params_len, NULL) != 0)
    {
      goto err;
    }

  /*
   * this calculation is required so we handle the case where we have
   * new parameters that we don't know how to handle (i.e. so we can skip
   * over them)
   */
  final_off = *off + params_len;

  /* read all flag bytes */
  for(i=0; i<flags_len; i++)
    {
      /* if no flags are set in this byte, then skip over it */
      if((flags[i] & 0x7f) == 0)
      {
        continue;
      }

      /* try each bit in this byte */
      for(j=0; j<7; j++)
      {
        /* if this flag is unset, then skip the rest of the loop */
        if((flags[i] & (0x1 << j)) == 0)
          {
            continue;
          }

        /*
         * if the id is greater than we have handlers for, then we've
         * got to the end of what we can parse.
         */
        if((id = (i*7)+j) >= handler_cnt)
          {
            goto done;
          }

        handler = &handlers[id];
        if(handler->read(buf, off, len, handler->data, handler->param) == -1)
          {
            goto err;
          }
      }
    }

 done:
  *off = final_off;
  return 0;

 err:
  return -1;  
}

static void warts_params_write(uint8_t *buf, uint32_t *off,
                         const uint32_t len,
                         const uint8_t *flags,
                         const uint16_t flags_len,
                         const uint16_t params_len,
                         const warts_param_writer_t *handlers,
                         const int handler_cnt)
{
  int i, j, id;
  uint16_t tmp;

  /* write the flag bytes out */
  tmp = flags_len;
  insert_bytes_uint16(buf, off, len, flags, &tmp);

  /*
   * if there are flags specified, then write the parameter length out.
   * otherwise, there are no parameters to write, so we are done.
   */
  if(flags[0] != 0)
    {
      insert_uint16(buf, off, len, &params_len, NULL);
    }
  else
    {
      assert(params_len == 0);
      return;
    }

  /* handle writing the parameter for each flight out */
  for(i=0; i<flags_len; i++)
    {
      /* skip flag bytes where no flags are set */
      if((flags[i] & 0x7f) == 0)
      {
        continue;
      }

      /* try each flag bit in the byte */
      for(j=0; j<7; j++)
      {
        /* skip over unset flags */
        if((flags[i] & (0x1 << j)) == 0)
          {
            continue;
          }

        /* this is the parameter id for the flag */
        id = (i*7)+j;

        /*
         * if the id is greater than we have handlers for, then either there
         * is some code missing, or there is a bug.
         */
        assert(id < handler_cnt);

        /* actually write the data out */
        handlers[id].write(buf,off,len,handlers[id].data,handlers[id].param);
      }
    }

  return;
}

/*
 * warts_hdr_read
 *
 */
static int warts_hdr_read(const scamper_file_t *sf, warts_hdr_t *hdr)
{
  const uint32_t len = 8;
  int      fd = scamper_file_getfd(sf);
  uint8_t  buf[len];
  size_t   rc;
  int      ret;
  uint32_t off = 0;

  if((ret = read_wrap(fd, buf, &rc, len)) != 0)
    {
      /* have we hit the eof? */
      if(rc == 0 && ret == -2)
      {
        return 0;
      }

      fprintf(stderr, "warts_hdr_read: read %d of %d bytes\n", (int)rc, len);
      return -1;
    }

  /* these three statements are guaranteed not to fail... */
  extract_uint16(buf, &off, len, &hdr->magic, NULL);
  extract_uint16(buf, &off, len, &hdr->type, NULL);
  extract_uint32(buf, &off, len, &hdr->len, NULL);

  assert(off == len);

  return 1;
}

/*
 * warts_hdr_write
 *
 */
static int warts_hdr_write(const scamper_file_t *sf,
                     const uint16_t type, const uint32_t len)
{
  const uint16_t hdr_magic = WARTS_MAGIC;
  const uint16_t hdr_len = 8;
  warts_state_t *state = scamper_file_getstate(sf);
  int      fd = scamper_file_getfd(sf);
  uint32_t off = 0;
  off_t    pos = 0;
  uint8_t  buf[hdr_len];
  size_t   wc;

  insert_uint16(buf, &off, hdr_len, &hdr_magic, NULL);
  insert_uint16(buf, &off, hdr_len, &type, NULL);
  insert_uint32(buf, &off, hdr_len, &len, NULL);

  assert(off == hdr_len);

  if(state->ispipe == 0 && (pos = lseek(fd, 0, SEEK_CUR)) == (off_t)-1)
    {
      if(errno != ESPIPE)
      {
        return -1;
      }
      state->ispipe = 1;
    }

  if(write_wrap(fd, buf, &wc, hdr_len) != 0)
    {
      /* truncate if a partial header was written */
      if(wc != 0 && state->ispipe == 0)
      {
        ftruncate(fd, pos);
      }
      return -1;
    }

  return 0;
}

static uint8_t *warts_read(const scamper_file_t *sf, const warts_hdr_t *hdr)
{
  uint8_t *buf;
  int      fd;

  if((fd = scamper_file_getfd(sf)) == -1)
    {
      return NULL;
    }

  if((buf = (void *)malloc(hdr->len)) == NULL)
    {
      return NULL;
    }

  if(read_wrap(fd, buf, NULL, hdr->len) != 0)
    {
      free(buf);
      return NULL;
    }

  return buf;
}

static int warts_skip(const scamper_file_t *sf, uint32_t bytes)
{
  warts_state_t *state = scamper_file_getstate(sf);
  int fd = scamper_file_getfd(sf);
  uint8_t buf[512];
  size_t len;

  if(state->ispipe == 0)
    {
      if(lseek(fd, bytes, SEEK_CUR) != -1)
      {
        return 0;
      }

      if(errno != ESPIPE)
      {
        return -1;
      }

      state->ispipe = 1;
    }

  while(bytes != 0)
    {
      len = (sizeof(buf) < bytes) ? sizeof(buf) : bytes;
      if(read_wrap(fd, buf, NULL, len) != 0)
      {
        return -1;
      }
      bytes -= len;
    }

  return 0;
}

/*
 * warts_write
 *
 * this function will write a record to disk, appending a warts_header
 * on the way out to the disk.  if the write fails for whatever reason
 * (as in the disk is full and only a partial recrd can be written), then
 * the write will be retracted in its entirety.
 */
static int warts_write(const scamper_file_t *sf, const uint8_t type,
                   const void *buf, const size_t len)
{
  warts_state_t *state = scamper_file_getstate(sf);
  off_t off = 0;
  int fd = scamper_file_getfd(sf);

  if(state->ispipe == 0 && (off = lseek(fd, 0, SEEK_CUR)) == (off_t)-1)
    {
      if(errno != ESPIPE)
      {
        return -1;
      }
      state->ispipe = 1;
    }

  if(warts_hdr_write(sf, type, len) == -1)
    {
      return -1;
    }

  if(write_wrap(fd, buf, NULL, len) != 0)
    {
      /*
       * if we could not write the buf out, then truncate the warts file
       * at the hdr we just wrote out above.
       */
      if(state->ispipe == 0)
      {
        ftruncate(fd, off);
      }
      return -1;
    }

  return 0;
}

static int warts_addr_cmp(const void *a, const void *b)
{
  const warts_addr_t *wa = (const warts_addr_t *)a;
  const warts_addr_t *wb = (const warts_addr_t *)b;
  return scamper_addr_cmp(wa->addr, wb->addr);
}

static warts_addr_t *warts_addr_alloc(scamper_addr_t *addr, uint32_t id)
{
  warts_addr_t *wa;
  if((wa = malloc_zero(sizeof(warts_addr_t))) != NULL)
    {
      wa->addr = scamper_addr_use(addr);
      wa->id = id;
    }
  return wa;
}

static void warts_addr_free(warts_addr_t *wa)
{
  if(wa->addr != NULL) scamper_addr_free(wa->addr);
  free(wa);
  return;
}

/*
 * warts_addr_read
 *
 * read an address structure out of the file and record it in the splay
 * tree of addresses.
 *
 * each address record consists of
 *   - an id assigned to the address, modulo 255
 *   - the address family the address belongs to
 *   - the address [length determined by record length]
 */
static scamper_addr_t *warts_addr_read(const scamper_file_t *sf,
                               const warts_hdr_t *hdr)
{
  warts_state_t  *state = scamper_file_getstate(sf);
  scamper_addr_t *addr = NULL;
  warts_addr_t   *wa = NULL, **table;
  uint8_t        *buf = NULL;
  size_t          size;

  /* the data has to be at least 3 bytes long to be valid */
  assert(hdr->len > 2);

  if((state->addr_count % WARTS_ADDR_TABLEGROW) == 0)
    {
      size = sizeof(warts_addr_t*)*(state->addr_count + WARTS_ADDR_TABLEGROW);
      if((table = realloc(state->addr_table, size)) == NULL)
      {
        goto err;
      }
      state->addr_table = table;
    }

  /* read the address record from the file */
  if((buf = warts_read(sf, hdr)) == NULL)
    {
      goto err;
    }

  /*
   * sanity check that the warts id recorded in the file matches what we
   * think it should be.
   */
  if(state->addr_count % 255 != buf[0])
    {
      goto err;
    }

  /* allocate a scamper address using the record read from disk */
  if((addr = scamper_addr_alloc(buf[1], buf+2)) == NULL)
    {
      goto err;
    }

  /* finally, allocate the warts address to wrap the scamper address */
  if((wa = warts_addr_alloc(addr, state->addr_count)) == NULL)
    {
      goto err;
    }

  state->addr_table[state->addr_count++] = wa;
  scamper_addr_free(addr);
  free(buf);

  return addr;

 err:
  if(addr != NULL) scamper_addr_free(addr);
  if(wa != NULL) warts_addr_free(wa);
  if(buf != NULL) free(buf);
  return NULL;
}

/*
 * warts_addr_write
 *
 * the address passed in does not have an id associated with it yet, so
 * allocate one and write it to disk.
 */
static int warts_addr_write(const scamper_file_t *sf,
                      scamper_addr_t *addr, uint32_t *id)
{
  warts_state_t *state = scamper_file_getstate(sf);
  warts_addr_t  *wa = NULL;
  uint8_t        tmp[18], *buf = NULL;
  size_t         size;

  /* create the warts addr struct to associate an id with an address */
  if((wa = warts_addr_alloc(addr, state->addr_count)) == NULL)
    {
      goto err;
    }

  /*
   * create the structure to write to disk.
   *
   * where possible, use the tmp buffer on the stack, rather than allocating
   * a buffer with malloc.
   */
  if((size = scamper_addr_size(addr) + 2) <= sizeof(tmp))
    {
      buf = tmp;
    }
  else if((buf = malloc(size)) == NULL)
    {
      goto err;
    }
  buf[0] = wa->id % 255;
  buf[1] = addr->type;
  memcpy(buf+2, addr->addr, scamper_addr_size(addr));

  /* insert the warts_addr into a tree so it can be found quickly */
  if(splaytree_insert(state->addr_tree, wa) == NULL)
    {
      goto err;
    }

  /* write the warts address record to disk */
  if(warts_write(sf, SCAMPER_FILE_OBJ_ADDR, buf, size) == -1)
    {
      goto err;
    }
  if(buf != NULL && buf != tmp) free(buf);

  state->addr_count++;
  *id = wa->id;
  return 0;

 err:
  if(wa != NULL)
    {
      splaytree_remove_item(state->addr_tree, wa);
      warts_addr_free(wa);
    }
  if(buf != NULL && buf != tmp) free(buf);
  return -1;
}

/*
 * warts_addr_getid
 *
 */
static int warts_addr_getid(const scamper_file_t *sf,
                      scamper_addr_t *sa, uint32_t *id)
{
  warts_state_t *state = scamper_file_getstate(sf);
  warts_addr_t   findme, *wa;

  if(sa == NULL)
    {
      *id = 0;
      return 0;
    }

  findme.addr = sa;
  findme.id = 0;
  if((wa = splaytree_find(state->addr_tree, &findme)) != NULL)
    {
      *id = wa->id;
      return 0;
    }

  if(warts_addr_write(sf, sa, id) == 0)
    {
      return 0;
    }

  return -1;
}

static int warts_list_cmp(const void *va, const void *vb)
{
  const warts_list_t *wa = (const warts_list_t *)va;
  const warts_list_t *wb = (const warts_list_t *)vb;
  return scamper_list_cmp(wa->list, wb->list);
}

static warts_list_t *warts_list_alloc(scamper_list_t *list, uint32_t id)
{
  warts_list_t *wl;
  if((wl = malloc_zero(sizeof(warts_list_t))) != NULL)
    {
      wl->list = scamper_list_use(list);
      wl->id = id;
    }
  return wl;
}

static void warts_list_free(warts_list_t *wl)
{
  if(wl->list != NULL) scamper_list_free(wl->list);
  free(wl);
  return;
}

/*
 * warts_list_params
 *
 * put together an outline of the optional bits for a list structure,
 * including the flags structure that sits at the front, and the size (in
 * bytes) of the various parameters that will be optionally included in the
 * file.
 */
static void warts_list_params(const scamper_list_t *list, uint8_t *flags,
                        uint16_t *flags_len, uint16_t *params_len)
{
  int max_id = 0;

  /* unset all the flags */
  memset(flags, 0, list_vars_mfb);
  *params_len = 0;

  if(list->descr != NULL)
    {
      set_flag(flags, WARTS_LIST_DESCR,   &max_id);
      *params_len += strlen(list->descr) + 1;
    }

  if(list->monitor != NULL)
    {
      set_flag(flags, WARTS_LIST_MONITOR, &max_id);
      *params_len += strlen(list->monitor) + 1;
    }

  *flags_len = fold_flags(flags, max_id);

  return;
}

/*
 * warts_list_params_read
 *
 */
static int warts_list_params_read(scamper_list_t *list,
                          uint8_t *buf, uint32_t *off, uint32_t len)
{
  warts_param_reader_t handlers[] = {
    {&list->descr,   (wpr_t)extract_string, NULL}, /* WARTS_LIST_DESCR   */
    {&list->monitor, (wpr_t)extract_string, NULL}, /* WARTS_LIST_MONITOR */
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_reader_t);

  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

static void warts_list_params_write(const scamper_list_t *list,
                            uint8_t *buf, uint32_t *off,
                            const uint32_t len,
                            const uint8_t *flags,
                            const uint16_t flags_len,
                            const uint16_t params_len)
{
  warts_param_writer_t handlers[] = {
    {list->descr,   (wpw_t)insert_string, NULL}, /* WARTS_LIST_DESCR */
    {list->monitor, (wpw_t)insert_string, NULL}, /* WARTS_LIST_MONITOR */
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_writer_t);

  warts_params_write(buf, off, len, flags, flags_len, params_len, handlers,
                 handler_cnt);
  return;
}

/*
 * warts_list_read
 *
 * each list record consists of
 *   - a 4 byte id assigned to the list by warts
 *   - a 4 byte list id assigned by a human
 *   - the name of the list
 *   - optional parameters (e.g. list description, monitor)
 */
static scamper_list_t *warts_list_read(const scamper_file_t *sf,
                               const warts_hdr_t *hdr)
{
  warts_state_t *state = scamper_file_getstate(sf);
  scamper_list_t *list = NULL;
  warts_list_t *wl = NULL, **table;
  uint8_t  *buf = NULL;
  size_t    size;
  uint32_t  i = 0;
  uint32_t  id;

  /*
   * must at least include the warts list id, the human-assigned list-id,
   * a name, and some amount of flags + parameters
   */
  if(hdr->len < 4 + 4 + 2 + 1)
    {
      goto err;
    }

  if((state->list_count % WARTS_LIST_TABLEGROW) == 0)
    {
      size = sizeof(warts_list_t *)*(state->list_count + WARTS_LIST_TABLEGROW);
      if((table = realloc(state->list_table, size)) == NULL)
      {
        goto err;
      }
      state->list_table = table;
    }

  /* read the list record from the file */
  if((buf = warts_read(sf, hdr)) == NULL)
    {
      goto err;
    }

  /* preallocate an empty list structure */
  if((list = malloc_zero(sizeof(scamper_list_t))) == NULL)
    {
      goto err;
    }
  list->refcnt = 1;

  /*
   * sanity check that the warts id recorded in the file matches what we
   * think it should be.
   */
  if(extract_uint32(buf, &i, hdr->len, &id, NULL) != 0 ||
     id != state->list_count)
    {
      goto err;
    }

  /* get the list id (assigned by a human) and name */
  if(extract_uint32(buf, &i, hdr->len, &list->id, NULL) != 0 ||
     extract_string(buf, &i, hdr->len, &list->name, NULL) != 0)
    {
      goto err;
    }

  if(warts_list_params_read(list, buf, &i, hdr->len) != 0)
    {
      goto err;
    }

  if((wl = warts_list_alloc(list, state->list_count)) == NULL)
    {
      goto err;
    }

  state->list_table[state->list_count++] = wl;
  scamper_list_free(list);
  free(buf);

  return list;

 err:
  if(list != NULL) scamper_list_free(list);
  if(wl != NULL)   warts_list_free(wl);
  if(buf != NULL)  free(buf);
  return NULL;
}

/*
 * warts_list_write
 *
 * take a list structure and write it to disk.  update the state held, too
 */
static int warts_list_write(const scamper_file_t *sf, scamper_list_t *list,
                      uint32_t *id)
{
  warts_state_t *state = scamper_file_getstate(sf);
  warts_list_t *wl = NULL;
  uint8_t  *buf = NULL;
  uint8_t   flags[list_vars_mfb];
  uint32_t  off = 0, len;
  uint16_t  name_len, flags_len, params_len;

  /* we require a list name */
  if(list->name == NULL)
    {
      goto err;
    }

  /* allocate a warts wrapping structure for the list */
  if((wl = warts_list_alloc(list, state->list_count)) == NULL)
    {
      goto err;
    }

  /* figure out how large the record will be */
  name_len = strlen(list->name) + 1;
  warts_list_params(list, flags, &flags_len, &params_len);
  len = 4 + 4 + name_len + flags_len + params_len;
  if(params_len != 0) len += 2;

  /* allocate the record */
  if((buf = malloc(len)) == NULL)
    {
      goto err;
    }

  /* list id assigned by warts */
  insert_uint32(buf, &off, len, &wl->id, NULL);

  /* list id assigned by a person */
  insert_uint32(buf, &off, len, &list->id, NULL);

  /* list name */
  insert_bytes_uint16(buf, &off, len, list->name, &name_len);

  /* copy in the flags for any parameters */
  warts_list_params_write(list, buf, &off, len, flags, flags_len, params_len);

  assert(off == len);

  if(splaytree_insert(state->list_tree, wl) == NULL)
    {
      goto err;
    }

  /* write the list record to disk */
  if(warts_write(sf, SCAMPER_FILE_OBJ_LIST, buf, len) == -1)
    {
      goto err;
    }

  state->list_count++;
  *id = wl->id;
  free(buf);
  return 0;

 err:
  if(wl != NULL)
    {
      splaytree_remove_item(state->list_tree, wl);
      warts_list_free(wl);
    }
  if(buf != NULL) free(buf);
  return -1;
}

/*
 * warts_list_getid
 *
 * given a scamper_list structure, return the id to use internally to
 * uniquely identify it.  allocate the id if necessary.
 */
static int warts_list_getid(const scamper_file_t *sf, scamper_list_t *list,
                      uint32_t *id)
{
  warts_state_t *state = scamper_file_getstate(sf);
  warts_list_t findme, *wl;

  if(list == NULL)
    {
      *id = 0;
      return 0;
    }

  /* see if there is a tree entry for this list */
  findme.list = list;
  if((wl = splaytree_find(state->list_tree, &findme)) != NULL)
    {
      *id = wl->id;
      return 0;
    }

  /* no tree entry, so write it to a file and return the assigned id */
  if(warts_list_write(sf, list, id) == 0)
    {
      return 0;
    }

  return -1;
}

static int warts_cycle_cmp(const void *va, const void *vb)
{
  const warts_cycle_t *a = (const warts_cycle_t *)va;
  const warts_cycle_t *b = (const warts_cycle_t *)vb;
  return scamper_cycle_cmp(a->cycle, b->cycle);
}

static warts_cycle_t *warts_cycle_alloc(scamper_cycle_t *cycle, uint32_t id)
{
  warts_cycle_t *wc;
  if((wc = malloc_zero(sizeof(warts_cycle_t))) != NULL)
    {
      wc->cycle = scamper_cycle_use(cycle);
      wc->id = id;
    }
  return wc;
}

static void warts_cycle_free(warts_cycle_t *cycle)
{
  if(cycle->cycle != NULL) scamper_cycle_free(cycle->cycle);
  free(cycle);
  return;
}

static void warts_cycle_params(const scamper_cycle_t *cycle, uint8_t *flags,
                         uint16_t *flags_len, uint16_t *params_len)
{
  int max_id;

  /* unset all the flags, reset max_id */
  memset(flags, 0, cycle_vars_mfb);
  max_id = 0;

  *params_len = 0;

  if(cycle->hostname != NULL)
    {
      set_flag(flags, WARTS_CYCLE_HOSTNAME, &max_id);
      *params_len += strlen(cycle->hostname) + 1;
    }

  if(cycle->stop_time != 0)
    {
      set_flag(flags, WARTS_CYCLE_STOP_TIME, &max_id);
      *params_len += 4;
    }

  /* figure out how many bytes the flags will require */
  *flags_len = fold_flags(flags, max_id);

  return;
}

static void warts_cycle_params_write(const scamper_cycle_t *cycle,
                             uint8_t *buf, uint32_t *off,
                             const uint32_t len,
                             const uint8_t *flags,
                             const uint16_t flags_len,
                             const uint16_t params_len)
{
  warts_param_writer_t handlers[] = {
    {&cycle->stop_time, (wpw_t)insert_uint32, NULL},
    {cycle->hostname,   (wpw_t)insert_string, NULL},
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_writer_t);
  warts_params_write(buf, off, len, flags, flags_len, params_len, handlers,
                 handler_cnt);
  return;
}

static int warts_cycle_params_read(scamper_cycle_t *cycle,
                           uint8_t *buf, uint32_t *off, uint32_t len)
{
  warts_param_reader_t handlers[] = {
    {&cycle->stop_time, (wpr_t)extract_uint32, NULL},
    {&cycle->hostname,  (wpr_t)extract_string, NULL},
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_reader_t);
  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

/*
 * warts_cycle_read
 *
 * 4 byte cycle id (assigned by warts from counter)
 * 4 byte list id (assigned by warts)
 * 4 byte cycle id (assigned by human)
 * 4 byte time since the epoch, representing start time of the cycle
 * 1 byte flags (followed by optional data items)
 */
static scamper_cycle_t *warts_cycle_read(const scamper_file_t *sf,
                               const warts_hdr_t *hdr)
{
  warts_state_t *state = scamper_file_getstate(sf);
  scamper_cycle_t *cycle = NULL;
  warts_cycle_t *wc = NULL, **table;
  size_t   size;
  uint8_t *buf = NULL;
  uint32_t id;
  uint32_t off = 0;

  /* ensure the cycle_start object is large enough to be valid */
  if(hdr->len < 4 + 4 + 4 + 4 + 1)
    {
      goto err;
    }

  if((state->cycle_count % WARTS_CYCLE_TABLEGROW) == 0)
    {
      size = sizeof(warts_list_t *)*(state->cycle_count+WARTS_CYCLE_TABLEGROW);
      if((table = realloc(state->cycle_table, size)) == NULL)
      {
        goto err;
      }
      state->cycle_table = table;
    }

  /* read the cycle_start structure out of the file */
  if((buf = warts_read(sf, hdr)) == NULL)
    {
      goto err;
    }

  /*
   * sanity check that the warts id recorded in the file matches what we
   * think it should be.
   */
  if(extract_uint32(buf, &off, hdr->len, &id, NULL) != 0 ||
     id != state->cycle_count)
    {
      goto err;
    }

  /* the _warts_ list id for the cycle */
  if(extract_uint32(buf, &off, hdr->len, &id, NULL) != 0 ||
     id >= state->list_count)
    {
      goto err;
    }

  if((cycle = scamper_cycle_alloc(state->list_table[id]->list)) == NULL)
    {
      goto err;
    }

  /*
   * the second 4 bytes is the actual cycle id assigned by a human.
   * the third 4 bytes is seconds since the epoch.
   */
  if(extract_uint32(buf, &off, hdr->len, &cycle->id, NULL) != 0 ||
     extract_uint32(buf, &off, hdr->len, &cycle->start_time, NULL) != 0)
    {
      goto err;
    }

  if(warts_cycle_params_read(cycle, buf, &off, hdr->len) != 0)
    {
      goto err;
    }

  if((wc = warts_cycle_alloc(cycle, state->cycle_count)) == NULL)
    {
      goto err;
    }

  state->cycle_table[state->cycle_count++] = wc;
  scamper_cycle_free(cycle);
  free(buf);

  return cycle;

 err:
  if(cycle != NULL)
    {
      if(cycle->list != NULL) scamper_list_free(cycle->list);
      free(cycle);
    }
  if(buf != NULL) free(buf);
  return NULL;
}

/*
 * warts_cycle_write
 *
 * write out a cycle record.  depending on whether the type is a start point,
 * or a cycle definition, some
 *
 * 4 byte cycle id (assigned by warts from counter)
 * 4 byte list id (assigned by warts)
 * 4 byte cycle id (assigned by human)
 * 4 byte time since the epoch, representing start time of the cycle
 * 1 byte flags (followed by optional data items)
 */
static int warts_cycle_write(const scamper_file_t *sf, scamper_cycle_t *cycle,
                       const int type, uint32_t *id)
{
  warts_state_t *state = scamper_file_getstate(sf);
  warts_cycle_t *wc = NULL;
  uint32_t warts_list_id;
  uint8_t *buf = NULL;
  uint8_t  flags[cycle_vars_mfb];
  uint16_t flags_len, params_len;
  uint32_t off = 0, len;

  /* find the list associated w/ the cycle, as we require the warts list id */
  if(warts_list_getid(sf, cycle->list, &warts_list_id) == -1)
    {
      goto err;
    }

  /* allocate warts_cycle wrapping struct to associate a warts-assigned id */
  if((wc = warts_cycle_alloc(cycle, state->cycle_count)) == NULL)
    {
      goto err;
    }

  /* figure out the shape the optional parameters will take */
  warts_cycle_params(cycle, flags, &flags_len, &params_len);

  /* allocate a temporary buf for recording the cycle */
  len = 4 + 4 + 4 + 4 + flags_len + params_len;
  if(params_len != 0) len += 2;
  if((buf = malloc(len)) == NULL)
    {
      goto err;
    }

  /* cycle and list ids, assigned by warts from counters */
  insert_uint32(buf, &off, len, &wc->id, NULL);
  insert_uint32(buf, &off, len, &warts_list_id, NULL);

  /* human cycle id, timestamp */
  insert_uint32(buf, &off, len, &cycle->id, NULL);
  insert_uint32(buf, &off, len, &cycle->start_time, NULL);

  /* copy in the optionally-included parameters */
  warts_cycle_params_write(cycle, buf,&off,len, flags, flags_len, params_len);

  assert(off == len);

  if(splaytree_insert(state->cycle_tree, wc) == NULL)
    {
      goto err;
    }

  if(warts_write(sf, type, buf, len) == -1)
    {
      goto err;
    }

  if(id != NULL) *id = wc->id;
  state->cycle_count++;
  free(buf);

  return 0;

 err:
  if(wc != NULL)
    {
      splaytree_remove_item(state->cycle_tree, wc);
      warts_cycle_free(wc);
    }
  if(buf != NULL) free(buf);
  return -1;
}

/*
 * warts_cycle_stop_read
 *
 * a cycle_stop record consists of the cycle id (assigned by warts from a
 * counter), a timestamp, and some optional parameters.
 *
 * the want parameter specifies if the cycle record is required by the caller.
 */
static scamper_cycle_t *warts_cycle_stop_read(const scamper_file_t *sf,
                                    const warts_hdr_t *hdr,
                                    const int want)
{
  warts_state_t *state = scamper_file_getstate(sf);
  scamper_cycle_t *cycle;
  uint32_t  off = 0;
  uint32_t  id;
  uint8_t  *buf = NULL;

  if(hdr->len < 4 + 4 + 1)
    {
      goto err;
    }

  if((buf = warts_read(sf, hdr)) == NULL)
    {
      goto err;
    }

  /*
   * get an index into the stored cycles.
   *
   * if the id does not make sense (is larger than any cycle currently
   * defined, or is the null cycle entry, or there is no current cycle
   * for this id) then we have a problem...
   */
  if(extract_uint32(buf, &off, hdr->len, &id, NULL) != 0 || 
     id >= state->cycle_count || id == 0 || state->cycle_table[id] == NULL)
    {
      goto err;
    }

  /* embed the stop timestamp with the cycle object */
  cycle = state->cycle_table[id]->cycle;
  if(extract_uint32(buf, &off, hdr->len, &cycle->stop_time, NULL) != 0)
    {
      goto err;
    }

  /*
   * don't need the cycle in the array any longer; get a reference to the
   * cycle, since the caller may want the structure.  free the warts_cycle
   * wrapping structure, though
   */
  if(want != 0) scamper_cycle_use(cycle);
  warts_cycle_free(state->cycle_table[id]);
  state->cycle_table[id] = NULL;

  free(buf);

  return cycle;

 err:
  if(buf != NULL) free(buf);
  return NULL;
}

static int warts_cycle_getid(const scamper_file_t *sf, scamper_cycle_t *cycle,
                       uint32_t *id)
{
  warts_state_t *state = scamper_file_getstate(sf);
  warts_cycle_t findme, *wc;

  /* if no cycle is specified, we use the special value zero */
  if(cycle == NULL)
    {
      *id = 0;
      return 0;
    }

  /* see if there is an entry for this cycle */
  findme.cycle = cycle;
  if((wc = splaytree_find(state->cycle_tree, &findme)) != NULL)
    {
      *id = wc->id;
      return 0;
    }

  if(warts_cycle_write(sf, cycle, SCAMPER_FILE_OBJ_CYCLE_DEF, id) == 0)
    {
      return 0;
    }

  return -1;
}

/*
 * warts_cycle_stop_write
 *
 * this function writes a record denoting the end of the cycle pointed to
 * by the cycle parameter.
 * it writes
 *  the 4 byte cycle id assigned by warts
 *  the 4 byte stop time
 *  where applicable, additional parameters
 */
static int warts_cycle_stop_write(const scamper_file_t *sf,
                          scamper_cycle_t *cycle)
{
  uint32_t wc_id;
  uint8_t *buf = NULL;
  uint32_t off = 0, len;
  uint8_t  flag = 0;

  assert(cycle != NULL);

  if(warts_cycle_getid(sf, cycle, &wc_id) != 0)
    {
      goto err;
    }

  len = 4 + 4 + 1;
  if((buf = malloc(len)) == NULL)
    {
      goto err;
    }

  insert_uint32(buf, &off, len, &wc_id, NULL);
  insert_uint32(buf, &off, len, &cycle->stop_time, NULL);
  insert_byte(buf, &off, len, &flag, NULL);

  assert(off == len);

  if(warts_write(sf, SCAMPER_FILE_OBJ_CYCLE_STOP, buf, len) == -1)
    {
      goto err;
    }

  free(buf);
  return 0;

 err:
  if(buf != NULL) free(buf);
  return -1;
}

static void warts_trace_params(const scamper_trace_t *trace, uint8_t *flags,
                         uint16_t *flags_len, uint16_t *params_len)
{
  int max_id = 0;

  /* unset all the flags possible */
  memset(flags, 0, trace_vars_mfb);
  *params_len = 0;

  /* for now, we include the base data items */
  set_flag(flags, WARTS_TRACE_LIST_ID,    &max_id); *params_len += 4;
  set_flag(flags, WARTS_TRACE_CYCLE_ID,   &max_id); *params_len += 4;
  set_flag(flags, WARTS_TRACE_ADDR_SRC,   &max_id); *params_len += 4;
  set_flag(flags, WARTS_TRACE_ADDR_DST,   &max_id); *params_len += 4;
  set_flag(flags, WARTS_TRACE_START,      &max_id); *params_len += 8;
  set_flag(flags, WARTS_TRACE_STOP_R,     &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_STOP_D,     &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_FLAGS,      &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_ATTEMPTS,   &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_HOPLIMIT,   &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_TYPE,       &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_PROBE_S,    &max_id); *params_len += 2;
  set_flag(flags, WARTS_TRACE_PORT_SRC,   &max_id); *params_len += 2;
  set_flag(flags, WARTS_TRACE_PORT_DST,   &max_id); *params_len += 2;
  set_flag(flags, WARTS_TRACE_FIRSTHOP,   &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_TOS,        &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_WAIT,       &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_LOOPS,      &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_HOPCOUNT,   &max_id); *params_len += 2;
  set_flag(flags, WARTS_TRACE_GAPLIMIT,   &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_GAPACTION,  &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_LOOPACTION, &max_id); *params_len += 1;

  *flags_len = fold_flags(flags, max_id);

  return;
}

static int warts_trace_params_read(scamper_trace_t *trace,warts_state_t *state,
                           uint8_t *buf, uint32_t *off, uint32_t len)
{
  warts_param_reader_t handlers[] = {
    {&trace->list,        (wpr_t)extract_list,    state},
    {&trace->cycle,       (wpr_t)extract_cycle,   state},
    {&trace->src,         (wpr_t)extract_addr,    state},
    {&trace->dst,         (wpr_t)extract_addr,    state},
    {&trace->start,       (wpr_t)extract_timeval, NULL},
    {&trace->stop_reason, (wpr_t)extract_byte,    NULL},
    {&trace->stop_data,   (wpr_t)extract_byte,    NULL},
    {&trace->flags,       (wpr_t)extract_byte,    NULL},
    {&trace->attempts,    (wpr_t)extract_byte,    NULL},
    {&trace->hoplimit,    (wpr_t)extract_byte,    NULL},
    {&trace->type,        (wpr_t)extract_byte,    NULL},
    {&trace->probe_size,  (wpr_t)extract_uint16,  NULL},
    {&trace->sport,       (wpr_t)extract_uint16,  NULL},
    {&trace->dport,       (wpr_t)extract_uint16,  NULL},
    {&trace->firsthop,    (wpr_t)extract_byte,    NULL},
    {&trace->tos,         (wpr_t)extract_byte,    NULL},
    {&trace->wait,        (wpr_t)extract_byte,    NULL},
    {&trace->loops,       (wpr_t)extract_byte,    NULL},
    {&trace->hop_count,   (wpr_t)extract_uint16,  NULL},
    {&trace->gaplimit,    (wpr_t)extract_byte,    NULL},
    {&trace->gapaction,   (wpr_t)extract_byte,    NULL},
    {&trace->loopaction,  (wpr_t)extract_byte,    NULL},
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_reader_t);

  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

static int warts_trace_params_write(const scamper_trace_t *trace,
                            const scamper_file_t *sf,
                            uint8_t *buf, uint32_t *off,
                            const uint32_t len,
                            const uint8_t *flags,
                            const uint16_t flags_len,
                            const uint16_t params_len)
{
  uint32_t list_id, cycle_id, src_id, dst_id;
  warts_param_writer_t handlers[] = {
    {&list_id,            (wpw_t)insert_uint32,  NULL},
    {&cycle_id,           (wpw_t)insert_uint32,  NULL},
    {&src_id,             (wpw_t)insert_uint32,  NULL},
    {&dst_id,             (wpw_t)insert_uint32,  NULL},
    {&trace->start,       (wpw_t)insert_timeval, NULL},
    {&trace->stop_reason, (wpw_t)insert_byte,    NULL},
    {&trace->stop_data,   (wpw_t)insert_byte,    NULL},
    {&trace->flags,       (wpw_t)insert_byte,    NULL},
    {&trace->attempts,    (wpw_t)insert_byte,    NULL},
    {&trace->hoplimit,    (wpw_t)insert_byte,    NULL},
    {&trace->type,        (wpw_t)insert_byte,    NULL},
    {&trace->probe_size,  (wpw_t)insert_uint16,  NULL},
    {&trace->sport,       (wpw_t)insert_uint16,  NULL},
    {&trace->dport,       (wpw_t)insert_uint16,  NULL},
    {&trace->firsthop,    (wpw_t)insert_byte,    NULL},
    {&trace->tos,         (wpw_t)insert_byte,    NULL},
    {&trace->wait,        (wpw_t)insert_byte,    NULL},
    {&trace->loops,       (wpw_t)insert_byte,    NULL},
    {&trace->hop_count,   (wpw_t)insert_uint16,  NULL},
    {&trace->gaplimit,    (wpw_t)insert_byte,    NULL},
    {&trace->gapaction,   (wpw_t)insert_byte,    NULL},
    {&trace->loopaction,  (wpw_t)insert_byte,    NULL},
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_writer_t);

  if(warts_list_getid(sf,  trace->list,  &list_id)  == -1) return -1;
  if(warts_cycle_getid(sf, trace->cycle, &cycle_id) == -1) return -1;
  if(warts_addr_getid(sf,  trace->src,   &src_id)   == -1) return -1;
  if(warts_addr_getid(sf,  trace->dst,   &dst_id)   == -1) return -1;

  warts_params_write(buf, off, len, flags, flags_len, params_len, handlers,
                 handler_cnt);
  return 0;
}

static int warts_hop_read_icmp_tc(const uint8_t *buf, uint32_t *off,
                          uint32_t len, scamper_trace_hop_t *hop,
                          void *param)
{
  if(len - *off < 2)
    {
      return -1;
    }
  hop->hop_icmp_type = buf[(*off)++];
  hop->hop_icmp_code = buf[(*off)++];
  return 0;
}

static void warts_hop_write_icmp_tc(uint8_t *buf, uint32_t *off,
                            const uint32_t len,
                            const scamper_trace_hop_t *hop,
                            void *param)
{
  assert(len - *off >= 2);
  buf[(*off)++] = hop->hop_icmp_type;
  buf[(*off)++] = hop->hop_icmp_code;
  return;
}

static int warts_hop_read_probe_id(const uint8_t *buf, uint32_t *off,
                           uint32_t len, uint8_t *out, void *param)
{
  if(len - *off < 1)
    {
      return -1;
    }
  *out = buf[(*off)++] + 1;
  return 0;
}

static void warts_hop_write_probe_id(uint8_t *buf, uint32_t *off,
                             const uint32_t len, const uint8_t *in,
                             void *param)
{
  assert(len - *off >= 1);
  buf[(*off)++] = *in - 1;
  return;
}

static int warts_hop_read_icmpext(const uint8_t *buf, uint32_t *off,
                          uint32_t len, scamper_trace_hop_t *hop,
                          void *param)
{
  uint16_t tmp;
  uint16_t u16;

  /* make sure there's enough left for the length field */
  if(len - *off < 2)
    {
      return -1;
    }

  /* extract the length field that says how much data is left past it */
  memcpy(&tmp, &buf[*off], 2);
  tmp = ntohs(tmp);

  *off += 2;

  assert(tmp > 0);

  /* make sure there's enough left for the extension data */
  if(len - *off < tmp)
    {
      return -1;
    }

  while(tmp >= 4)
    {
      memcpy(&u16, &buf[*off], 2); u16 = ntohs(u16);
      if(len - *off < 2 + 1 + 1 + u16)
      {
        return -1;
      }

      if(scamper_trace_hop_icmpext_add(hop, buf[*off+2], buf[*off+3], u16,
                               &buf[*off+4]) != 0)
      {
        return -1;
      }

      *off += (2 + 1 + 1 + u16);
      tmp  -= (2 + 1 + 1 + u16);
    }

  assert(tmp == 0);
  return 0;
}

static void warts_hop_write_icmpext(uint8_t *buf, uint32_t *off,
                            const uint32_t len,
                            const scamper_trace_hop_t *hop,
                            void *param)
{
  const scamper_trace_hop_icmpext_t *ie;
  uint16_t tmp = 0;
  uint16_t u16;

  for(ie=hop->hop_icmpext; ie != NULL; ie = ie->ie_next)
    {
      assert(*off + tmp + 1 + 1 + 2 + ie->ie_dl <= len);

      /* convert the data length field to network byte order and write */
      u16 = htons(ie->ie_dl);
      memcpy(&buf[*off + 2 + tmp], &u16, 2); tmp += 2;

      /* write the class num/type fields */
      buf[*off + 2 + tmp] = ie->ie_cn; tmp++;
      buf[*off + 2 + tmp] = ie->ie_ct; tmp++;

      /* write any data */
      if(ie->ie_dl != 0)
      {
        memcpy(&buf[*off + 2 + tmp], ie->ie_data, ie->ie_dl);
        tmp += ie->ie_dl;
      }
    }

  /* write, at the start of the data, the length of the icmp extension data */
  u16 = htons(tmp);
  memcpy(&buf[*off], &u16, 2);
  *off = *off + 2 + tmp;

  return;
}

static void warts_hop_params(const scamper_trace_hop_t *hop, uint8_t *flags,
                       uint16_t *flags_len, uint16_t *params_len)
{
  static const int tlv_idx[] = {
    WARTS_TRACE_HOP_REPLY_IPID,  /* SCAMPER_TRACE_HOP_TLV_REPLY_IPID */
    WARTS_TRACE_HOP_REPLY_IPTOS, /* SCAMPER_TRACE_HOP_TLV_REPLY_IPTOS */
    WARTS_TRACE_HOP_NHMTU,       /* SCAMPER_TRACE_HOP_TLV_NHMTU */
    WARTS_TRACE_HOP_INNER_IPLEN, /* SCAMPER_TRACE_HOP_TLV_INNER_IPLEN */
    WARTS_TRACE_HOP_INNER_IPTTL, /* SCAMPER_TRACE_HOP_TLV_INNER_IPTTL */
    WARTS_TRACE_HOP_INNER_IPTOS, /* SCAMPER_TRACE_HOP_TLV_INNER_IPTOS */
  };
  scamper_trace_hop_icmpext_t *ie;
  scamper_tlv_t *tlv;
  int max_id = 0;

  /* unset all the flags possible */
  memset(flags, 0, hop_vars_mfb);
  *params_len = 0;

  /* for now, we include all the base data items */
  set_flag(flags, WARTS_TRACE_HOP_ADDR,       &max_id); *params_len += 4;
  set_flag(flags, WARTS_TRACE_HOP_PROBE_TTL,  &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_HOP_REPLY_TTL,  &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_HOP_FLAGS,      &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_HOP_PROBE_ID,   &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_HOP_RTT,        &max_id); *params_len += 4;
  set_flag(flags, WARTS_TRACE_HOP_PROBE_SIZE, &max_id); *params_len += 2;
  set_flag(flags, WARTS_TRACE_HOP_REPLY_SIZE, &max_id); *params_len += 2;

  if((hop->hop_flags & SCAMPER_TRACE_HOP_FLAG_TCP) == 0)
    {
      set_flag(flags, WARTS_TRACE_HOP_ICMP_TC,    &max_id); *params_len += 2;
    }
  else
    {
      set_flag(flags, WARTS_TRACE_HOP_TCP_FLAGS,  &max_id); *params_len += 1;
    }

  /* go through the TLVs and decide which flags to set */
  for(tlv = hop->hop_tlvs; tlv != NULL; tlv = tlv->tlv_next)
    {
      assert(tlv->tlv_type-1 < (int)(sizeof(tlv_idx)/sizeof(int)));
      set_flag(flags, tlv_idx[tlv->tlv_type-1], &max_id);
      *params_len += tlv->tlv_len;
    }

  if(hop->hop_icmpext != NULL)
    {
      set_flag(flags, WARTS_TRACE_HOP_ICMPEXT, &max_id);
      *params_len += 2;
      for(ie = hop->hop_icmpext; ie != NULL; ie = ie->ie_next)
      {
        *params_len += (2 + 1 + 1 + ie->ie_dl);
      }
    }

  *flags_len = fold_flags(flags, max_id);

  return;
}

static int warts_hop_read(scamper_trace_hop_t *hop, warts_state_t *state,
                    const uint8_t *buf, uint32_t *off, uint32_t len)
{
  uint8_t types[] = {
    SCAMPER_TRACE_HOP_TLV_REPLY_IPID,
    SCAMPER_TRACE_HOP_TLV_REPLY_IPTOS,
    SCAMPER_TRACE_HOP_TLV_NHMTU,
    SCAMPER_TRACE_HOP_TLV_INNER_IPLEN,
    SCAMPER_TRACE_HOP_TLV_INNER_IPTTL,
    SCAMPER_TRACE_HOP_TLV_INNER_IPTOS,
  };

  warts_param_reader_t handlers[] = {
    {&hop->hop_addr,       (wpr_t)extract_addr,            state},
    {&hop->hop_probe_ttl,  (wpr_t)extract_byte,            NULL},
    {&hop->hop_reply_ttl,  (wpr_t)extract_byte,            NULL},
    {&hop->hop_flags,      (wpr_t)extract_byte,            NULL},
    {&hop->hop_probe_id,   (wpr_t)warts_hop_read_probe_id, NULL},
    {&hop->hop_rtt,        (wpr_t)extract_rtt,             NULL},
    {hop,                  (wpr_t)warts_hop_read_icmp_tc,  NULL},
    {&hop->hop_probe_size, (wpr_t)extract_uint16,          NULL},
    {&hop->hop_reply_size, (wpr_t)extract_uint16,          NULL},
    {&hop->hop_tlvs,       (wpr_t)extract_uint16_tlv,      &types[0]},
    {&hop->hop_tlvs,       (wpr_t)extract_byte_tlv,        &types[1]},
    {&hop->hop_tlvs,       (wpr_t)extract_uint16_tlv,      &types[2]},
    {&hop->hop_tlvs,       (wpr_t)extract_uint16_tlv,      &types[3]},
    {&hop->hop_tlvs,       (wpr_t)extract_byte_tlv,        &types[4]},
    {&hop->hop_tcp_flags,  (wpr_t)extract_byte,            NULL},
    {&hop->hop_tlvs,       (wpr_t)extract_byte_tlv,        &types[5]},
    {hop,                  (wpr_t)warts_hop_read_icmpext,  NULL},
  };

  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_reader_t);

  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

static void warts_hop_write(const warts_hop_state_t *state,
                      uint8_t *buf, uint32_t *off, uint32_t len)
{
  scamper_trace_hop_t *hop = state->hop;

  uint8_t types[] = {
    SCAMPER_TRACE_HOP_TLV_REPLY_IPID,
    SCAMPER_TRACE_HOP_TLV_REPLY_IPTOS,
    SCAMPER_TRACE_HOP_TLV_NHMTU,
    SCAMPER_TRACE_HOP_TLV_INNER_IPLEN,
    SCAMPER_TRACE_HOP_TLV_INNER_IPTTL,
    SCAMPER_TRACE_HOP_TLV_INNER_IPTOS,
  };

  warts_param_writer_t handlers[] = {
    {&state->addr,         (wpw_t)insert_uint32,            NULL},
    {&hop->hop_probe_ttl,  (wpw_t)insert_byte,              NULL},
    {&hop->hop_reply_ttl,  (wpw_t)insert_byte,              NULL},
    {&hop->hop_flags,      (wpw_t)insert_byte,              NULL},
    {&hop->hop_probe_id,   (wpw_t)warts_hop_write_probe_id, NULL},
    {&hop->hop_rtt,        (wpw_t)insert_rtt,               NULL},
    {hop,                  (wpw_t)warts_hop_write_icmp_tc,  NULL},
    {&hop->hop_probe_size, (wpw_t)insert_uint16,            NULL},
    {&hop->hop_reply_size, (wpw_t)insert_uint16,            NULL},
    {hop->hop_tlvs,        (wpw_t)insert_uint16_tlv,        &types[0]},
    {hop->hop_tlvs,        (wpw_t)insert_byte_tlv,          &types[1]},
    {hop->hop_tlvs,        (wpw_t)insert_uint16_tlv,        &types[2]},
    {hop->hop_tlvs,        (wpw_t)insert_uint16_tlv,        &types[3]},
    {hop->hop_tlvs,        (wpw_t)insert_byte_tlv,          &types[4]},
    {&hop->hop_tcp_flags,  (wpw_t)insert_byte,              NULL},
    {hop->hop_tlvs,        (wpw_t)insert_byte_tlv,          &types[5]},
    {hop,                  (wpw_t)warts_hop_write_icmpext,  NULL},
  };

  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_writer_t);

  warts_params_write(buf, off, len, state->flags, state->flags_len,
                 state->params_len, handlers, handler_cnt);
  return;
}

static int warts_hops_read(scamper_trace_hop_t **hops, warts_state_t *state,
                     const uint8_t *buf, uint32_t *off, uint32_t len,
                     uint16_t count)
{
  scamper_trace_hop_t *head = NULL, *hop = NULL;
  int i;

  for(i=0; i<count; i++)
    {
      /*
       * the hop list is stored in a linked list; add each new hop to the
       * end of the list
       */
      if(hop != NULL)
      {
        hop->hop_next = scamper_trace_hop_alloc();
        hop = hop->hop_next;
      }
      else
      {
        head = hop = scamper_trace_hop_alloc();
      }

      /* could not allocate an empty hop structure ... */
      if(hop == NULL)
      {
        goto err;
      }

      if(warts_hop_read(hop, state, buf, off, len) != 0)
      {
        goto err;
      }
    }

  *hops = head;

  return 0;

 err:
  while(head != NULL)
    {
      hop = head;
      head = head->hop_next;
      scamper_trace_hop_free(hop);
    }
  return -1;
}

static void warts_trace_pmtud_params(const scamper_trace_t *trace,
                             uint8_t *flags, uint16_t *flags_len,
                             uint16_t *params_len)
{
  static const int tlv_idx[] = {
    WARTS_TRACE_PMTUD_OUTMTU, /* SCAMPER_TRACE_PMTUD_TLV_OUTMTU */
  };
  scamper_tlv_t *tlv;
  int max_id = 0;

  /* unset all the flags possible */
  memset(flags, 0, pmtud_vars_mfb);
  *params_len = 0;

  /* for now, we include the base data items */
  set_flag(flags, WARTS_TRACE_PMTUD_IFMTU, &max_id); *params_len += 2;
  set_flag(flags, WARTS_TRACE_PMTUD_PMTU, &max_id);  *params_len += 2;

  /* go through the TLVs and decide which flags to set */
  for(tlv = trace->pmtud->tlvs; tlv != NULL; tlv = tlv->tlv_next)
    {
      set_flag(flags, tlv_idx[tlv->tlv_type-1], &max_id);
      *params_len += tlv->tlv_len;
    }

  *flags_len = fold_flags(flags, max_id);

  return;
}

static int warts_trace_pmtud_read(scamper_trace_t *trace, warts_state_t *state,
                          const uint8_t *buf, uint32_t *off,
                          uint32_t len)
{
  uint8_t outmtu = SCAMPER_TRACE_PMTUD_TLV_OUTMTU;
  warts_param_reader_t handlers[] = {
    {&trace->pmtud->ifmtu, (wpr_t)extract_uint16,     NULL},
    {&trace->pmtud->pmtu,  (wpr_t)extract_uint16,     NULL},
    {&trace->pmtud->tlvs,  (wpr_t)extract_uint16_tlv, &outmtu},
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_reader_t);
  scamper_trace_hop_t *hops;
  uint16_t count;

  if((trace->pmtud = malloc_zero(sizeof(scamper_trace_pmtud_t))) == NULL ||
     warts_params_read(buf, off, len, handlers, handler_cnt) != 0 ||
     extract_uint16(buf, off, len, &count, NULL) != 0)
    {
      goto err;
    }

  if(count != 0)
    {
      if(warts_hops_read(&hops, state, buf, off, len, count) != 0)
      {
        goto err;
      }

      trace->pmtud->hops = hops;
    }

  return 0;

 err:
  return -1;
}

static void warts_trace_pmtud_write(const scamper_trace_t *trace, uint8_t *buf,
                            uint32_t *off, uint32_t len,
                            uint8_t *flags, uint16_t flags_len,
                            uint16_t params_len)
{
  uint16_t outmtu;
  warts_param_writer_t handlers[] = {
    {&trace->pmtud->ifmtu, (wpw_t)insert_uint16, NULL},
    {&trace->pmtud->pmtu,  (wpw_t)insert_uint16, NULL},
    {&outmtu,              (wpw_t)insert_uint16, NULL},
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_writer_t);

  SCAMPER_TRACE_PMTUD_GET_OUTMTU(trace->pmtud, outmtu);

  warts_params_write(buf, off, len, flags, flags_len, params_len, handlers,
                 handler_cnt);
  return;
}

static int warts_trace_lastditch_read(scamper_trace_t *trace,
                              warts_state_t *state, const uint8_t *buf,
                              uint32_t *off, uint32_t len)
{
  scamper_trace_hop_t *hops;
  uint16_t count;

  if(warts_params_read(buf, off, len, NULL, 0) != 0 ||
     extract_uint16(buf, off, len, &count, NULL) != 0)
    {
      goto err;
    }

  if(count != 0)
    {
      if(warts_hops_read(&hops, state, buf, off, len, count) != 0)
      {
        goto err;
      }

      trace->lastditch = hops;
    }

  return 0;

 err:
  return -1;
}

/*
 * warts_trace_read
 *
 */
static scamper_trace_t *warts_trace_read(const scamper_file_t *sf,
                               const warts_hdr_t *hdr)
{
  warts_state_t       *state = scamper_file_getstate(sf);
  scamper_trace_t     *trace = NULL;
  uint8_t             *buf = NULL;
  uint32_t             i, off = 0;
  scamper_trace_hop_t *hops = NULL;
  scamper_trace_hop_t *hop;
  uint16_t             count;
  int                  max_ttl;
  uint8_t              type;
  uint16_t             len;
  uint16_t             junk16;

  if((buf = warts_read(sf, hdr)) == NULL)
    {
      goto err;
    }

  if((trace = scamper_trace_alloc()) == NULL)
    {
      goto err;
    }

  /* read the trace's parameters */
  if(warts_trace_params_read(trace, state, buf, &off, hdr->len) != 0)
    {
      goto err;
    }

  /*
   * the next two bytes tell us how many scamper_hops to read out of trace
   * if we did not get any responses, we are done.
   */
  if(extract_uint16(buf, &off, hdr->len, &count, NULL) != 0)
    {
      goto err;
    }

  /* read all the hop records */
  if(warts_hops_read(&hops, state, buf, &off, hdr->len, count) != 0)
    {
      goto err;
    }

  /* work out the maximum ttl probed with that got a response */
  max_ttl = 0;
  for(i=0, hop = hops; i < count; i++)
    {
      if(hop->hop_probe_ttl > max_ttl)
      {
        max_ttl = hop->hop_probe_ttl;
      }
      hop = hop->hop_next;
    }

  /*
   * if the hop_count field was provided in the file, then
   * make sure it makes sense based on the hop data we've just scanned
   */
  if(trace->hop_count != 0)
    {
      if(trace->hop_count < max_ttl)
      {
        goto err;
      }
    }
  else
    {
      trace->hop_count = max_ttl;
    }

  /* allocate enough hops to string the trace together */
  if(scamper_trace_hops_alloc(trace, trace->hop_count) == -1)
    {
      goto err;
    }

  if(hops == NULL)
    {
      assert(count == 0);
      goto done;
    }

  /*
   * now loop through the hops array stored in this procedure
   * and assemble the responses into trace->hops.
   */
  trace->hops[hops->hop_probe_ttl-1] = hop = hops;
  while(hop->hop_next != NULL) 
    {
      if(hop->hop_probe_ttl != hop->hop_next->hop_probe_ttl)
      {
        i = hop->hop_next->hop_probe_ttl-1;
        trace->hops[i] = hop->hop_next;
        hop->hop_next = NULL;
        hop = trace->hops[i];
      }
      else hop = hop->hop_next;
    }
  hops = NULL;

  for(;;)
    {
      if(extract_uint16(buf, &off, hdr->len, &junk16, NULL) != 0)
      {
        goto err;
      }
      if(junk16 == WARTS_TRACE_ATTR_EOF)
      {
        break;
      }

      type = WARTS_TRACE_ATTR_HDR_TYPE(junk16);
      len  = WARTS_TRACE_ATTR_HDR_LEN(junk16);

      if(type == WARTS_TRACE_ATTR_PMTUD)
      {
        i = off;
        if(warts_trace_pmtud_read(trace, state, buf, &i, hdr->len) != 0)
          {
            goto err;
          }
      }
      else if(type == WARTS_TRACE_ATTR_LASTDITCH)
      {
        i = off;
        if(warts_trace_lastditch_read(trace, state, buf, &i, hdr->len) != 0)
          {
            goto err;
          }
      }

      off += len;
    }

  assert(off == hdr->len);

 done:
  free(buf);
  return trace;

 err:
  if(hops != NULL) free(hops);
  if(buf != NULL) free(buf);
  if(trace != NULL) scamper_trace_free(trace);
  return NULL;
}

static int warts_hop_state(const scamper_file_t *sf, scamper_trace_hop_t *hop,
                     warts_hop_state_t *state, uint32_t *len)
{
  /* for each hop, remember the address id */
  if(warts_addr_getid(sf, hop->hop_addr, &state->addr) == -1)
    {
      return -1;
    }

  /* for each hop, figure out how much space it will take up */
  warts_hop_params(hop, state->flags, &state->flags_len, &state->params_len);

  /* store the actual hop record with the state structure too */
  state->hop = hop;

  /* increase length required for the trace record */
  *len += state->flags_len + 2 + state->params_len;

  return 0;
}

static int warts_trace_write(const scamper_file_t *sf,
                       const scamper_trace_t *trace)
{
  scamper_trace_hop_t *hop;
  uint8_t             *buf = NULL;
  uint8_t              trace_flags[trace_vars_mfb];
  uint16_t             trace_flags_len, trace_params_len;
  warts_hop_state_t   *hop_state = NULL;
  uint16_t             hop_recs;
  uint8_t              pmtud_flags[pmtud_vars_mfb];
  uint16_t             pmtud_flags_len = 0, pmtud_params_len = 0;
  warts_hop_state_t   *pmtud_state = NULL;
  uint16_t             pmtud_recs = 0;
  uint32_t             pmtud_len = 0;
  warts_hop_state_t   *lastditch_state = NULL;
  uint16_t             lastditch_recs = 0;
  uint32_t             lastditch_len = 0;
  uint16_t             junk16;
  uint8_t              junk8;
  uint32_t             off = 0, len, len2;
  size_t               size;
  int                  i, j;

  assert(trace != NULL);

  /* figure out which trace data items we'll store in this record */
  warts_trace_params(trace, trace_flags, &trace_flags_len, &trace_params_len);

  /*
   * this represents the length of the trace's flags and parameters, and the
   * 2-byte field that records the number of hop records that follow
   */
  len = trace_flags_len + 2 + trace_params_len + 2;

  /* for each hop, figure out what is going to be stored in this record */
  if((hop_recs = scamper_trace_hop_count(trace)) > 0)
    {
      size = hop_recs * sizeof(warts_hop_state_t);
      if((hop_state = (warts_hop_state_t *)malloc(size)) == NULL)
      {
        goto err;
      }

      for(i=0, j=0; i<trace->hop_count; i++)
      {
        for(hop = trace->hops[i]; hop != NULL; hop = hop->hop_next)
          {
            /* record basic hop state */
            len2 = len;
            if(warts_hop_state(sf, hop, &hop_state[j++], &len2) == -1)
            {
              goto err;
            }
            if(len2 < len)
            {
              printf("len wrapped\n");
              goto err;
            }
            len = len2;
          }
      }
    }

  /* figure out how much space we need for PMTUD data, if we have it */
  if(trace->pmtud != NULL)
    {
      /* figure out what the structure of the pmtud header looks like */
      warts_trace_pmtud_params(trace, pmtud_flags, &pmtud_flags_len,
                         &pmtud_params_len);

      /* count the number of hop records */
      pmtud_recs = scamper_trace_pmtud_hop_count(trace);

      /* allocate an array of address indexes for the pmtud hop addresses */
      size = pmtud_recs * sizeof(warts_hop_state_t);
      if((pmtud_state = (warts_hop_state_t *)malloc(size)) == NULL)
      {
        goto err;
      }

      /* flags + params + number of hop records for pmtud structure */
      pmtud_len = pmtud_flags_len + 2 + pmtud_params_len + 2;

      /* record hop state for each pmtud hop */
      for(hop = trace->pmtud->hops, j=0; hop != NULL; hop = hop->hop_next)
      {
        if(warts_hop_state(sf, hop, &pmtud_state[j++], &pmtud_len) == -1)
          {
            goto err;
          }
      }

      len += (2 + pmtud_len); /* 2 = size of attribute header */
    }

  if(trace->lastditch != NULL)
    {
      /* count the number of last-ditch hop records */
      lastditch_recs = scamper_trace_lastditch_hop_count(trace);

      /* allocate an array of hop state structs for the lastditch hops */
      size = lastditch_recs * sizeof(warts_hop_state_t);
      if((lastditch_state = (warts_hop_state_t *)malloc(size)) == NULL)
      {
        goto err;
      }

      /* need to record count of lastditch hops and a single zero flags byte */
      lastditch_len = 3;

      /* record hop state for each lastditch reply */
      for(hop = trace->lastditch, j=0; hop != NULL; hop = hop->hop_next)
      {
        if(warts_hop_state(sf, hop,
                       &lastditch_state[j++], &lastditch_len) == -1)
          {
            goto err;
          }
      }

      len += (2 + lastditch_len); /* 2 = size of attribute header */
    }

  len += 2; /* EOF */

  if((buf = malloc(len)) == NULL)
    {
      goto err;
    }

  /* write trace parameters */
  if(warts_trace_params_write(trace, sf, buf, &off, len, trace_flags,
                        trace_flags_len, trace_params_len) == -1)
    {
      goto err;
    }

  /* hop record count */
  insert_uint16(buf, &off, len, &hop_recs, NULL);

  /* write each traceroute hop record */
  for(i=0; i<hop_recs; i++)
    {
      warts_hop_write(&hop_state[i], buf, &off, len);
    }
  if(hop_state != NULL)
    {
      free(hop_state);
      hop_state = NULL;
    }

  /* write the PMTUD data */
  if(trace->pmtud != NULL)
    {
      junk16 = WARTS_TRACE_ATTR_HDR(WARTS_TRACE_ATTR_PMTUD, pmtud_len);
      insert_uint16(buf, &off, len, &junk16, NULL);
                
      /* write details of the pmtud measurement */
      warts_trace_pmtud_write(trace, buf, &off, len,
                        pmtud_flags, pmtud_flags_len, pmtud_params_len);

      /* write the number of hop records */
      insert_uint16(buf, &off, len, &pmtud_recs, NULL);

      for(i=0; i<pmtud_recs; i++)
      {
        warts_hop_write(&pmtud_state[i], buf, &off, len);
      }
      if(pmtud_state != NULL)
      {
        free(pmtud_state);
        pmtud_state = NULL;
      }
    }

  /* write the last-ditch data */
  if(trace->lastditch != NULL)
    {
      /* write the attribute header */
      junk16 = WARTS_TRACE_ATTR_HDR(WARTS_TRACE_ATTR_LASTDITCH, lastditch_len);
      insert_uint16(buf, &off, len, &junk16, NULL);

      /* write the last-ditch flags: currently zero */
      junk8 = 0;
      insert_byte(buf, &off, len, &junk8, NULL);

      /* write the number of hop records */
      insert_uint16(buf, &off, len, &lastditch_recs, NULL);

      for(i=0; i<lastditch_recs; i++)
      {
        warts_hop_write(&lastditch_state[i], buf, &off, len);
      }
      free(lastditch_state);
      lastditch_state = NULL;
    }

  /* write the end of trace attributes header */
  junk16 = WARTS_TRACE_ATTR_EOF;
  insert_uint16(buf, &off, len, &junk16, NULL);

  assert(off == len);

  if(warts_write(sf, SCAMPER_FILE_OBJ_TRACE, buf, len) == -1)
    {
      goto err;
    }

  free(buf);
  return 0;

 err:
  if(buf != NULL) free(buf);
  if(hop_state != NULL) free(hop_state);
  if(pmtud_state != NULL) free(pmtud_state);
  if(lastditch_state != NULL) free(lastditch_state);
  return -1;
}

static void warts_ping_reply_params(const scamper_ping_reply_t *reply,
                            uint8_t *flags, uint16_t *flags_len,
                            uint16_t *params_len)
{
  int max_id = 0;

  /* unset all the flags possible */
  memset(flags, 0, ping_reply_vars_mfb);
  *params_len = 0;

  set_flag(flags, WARTS_PING_REPLY_ADDR,       &max_id); *params_len += 4;
  set_flag(flags, WARTS_PING_REPLY_FLAGS,      &max_id); *params_len += 1;
  set_flag(flags, WARTS_PING_REPLY_REPLY_TTL,  &max_id); *params_len += 1;
  set_flag(flags, WARTS_PING_REPLY_REPLY_SIZE, &max_id); *params_len += 2;
  set_flag(flags, WARTS_PING_REPLY_ICMP_TC,    &max_id); *params_len += 2;
  set_flag(flags, WARTS_PING_REPLY_RTT,        &max_id); *params_len += 4;
  set_flag(flags, WARTS_PING_REPLY_PROBE_ID,   &max_id); *params_len += 2;

  *flags_len = fold_flags(flags, max_id);

  return;
}

static int warts_ping_reply_state(const scamper_file_t *sf,
                          scamper_ping_reply_t *reply,
                          warts_ping_reply_state_t *state,
                          uint32_t *len)
{
  if(warts_addr_getid(sf, reply->addr, &state->addr) == -1)
    {
      return -1;
    }

  warts_ping_reply_params(reply, state->flags,
                    &state->flags_len,&state->params_len);

  state->reply = reply;

  *len += state->flags_len + 2 + state->params_len;

  return 0;
}

static int warts_ping_reply_read_icmptc(const uint8_t *buf, uint32_t *off,
                              uint32_t len,
                              scamper_ping_reply_t *reply,
                              void *param)
{
  if(len - *off < 2)
    {
      return -1;
    }
  reply->icmp_type = buf[(*off)++];
  reply->icmp_code = buf[(*off)++];
  return 0;
}

static void warts_ping_reply_write_icmptc(uint8_t *buf, uint32_t *off,
                                const uint32_t len,
                                const scamper_ping_reply_t *reply,
                                void *param)
{
  assert(len - *off >= 2);

  buf[(*off)++] = reply->icmp_type;
  buf[(*off)++] = reply->icmp_code;

  return;
}

static int warts_ping_reply_read(scamper_ping_reply_t *reply,
                         warts_state_t *state, const uint8_t *buf,
                         uint32_t *off, uint32_t len)
{
  warts_param_reader_t handlers[] = {
    {&reply->addr,       (wpr_t)extract_addr,                 state},
    {&reply->flags,      (wpr_t)extract_byte,                 NULL},
    {&reply->reply_ttl,  (wpr_t)extract_byte,                 NULL},
    {&reply->reply_size, (wpr_t)extract_uint16,               NULL},
    {reply,              (wpr_t)warts_ping_reply_read_icmptc, NULL},
    {&reply->rtt,        (wpr_t)extract_rtt,                  NULL},
    {&reply->probe_id,   (wpr_t)extract_uint16,               NULL},
  };

  const int handler_cnt = sizeof(handlers) / sizeof(warts_param_reader_t);

  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

static void warts_ping_reply_write(const warts_ping_reply_state_t *state,
                           uint8_t *buf, uint32_t *off, uint32_t len)
{
  scamper_ping_reply_t *reply = state->reply;

  warts_param_writer_t handlers[] = {
    {&state->addr,       (wpw_t)insert_uint32,                 NULL},
    {&reply->flags,      (wpw_t)insert_byte,                   NULL},
    {&reply->reply_ttl,  (wpw_t)insert_byte,                   NULL},
    {&reply->reply_size, (wpw_t)insert_uint16,                 NULL},
    {reply,              (wpw_t)warts_ping_reply_write_icmptc, NULL},
    {&reply->rtt,        (wpw_t)insert_rtt,                    NULL},
    {&reply->probe_id,   (wpw_t)insert_uint16,                 NULL},
  };

  const int handler_cnt = sizeof(handlers) / sizeof(warts_param_writer_t);

  warts_params_write(buf, off, len, state->flags, state->flags_len,
                 state->params_len, handlers, handler_cnt);
  return;
}

static void warts_ping_params(const scamper_ping_t *ping, uint8_t *flags,
                        uint16_t *flags_len, uint16_t *params_len)
{
  uint16_t pad_len = ping->pattern_len;
  int max_id = 0;

  /* unset all the flags possible */
  memset(flags, 0, ping_vars_mfb);
  *params_len = 0;

  set_flag(flags, WARTS_PING_LIST_ID,       &max_id); *params_len += 4;
  set_flag(flags, WARTS_PING_CYCLE_ID,      &max_id); *params_len += 4;
  set_flag(flags, WARTS_PING_ADDR_SRC,      &max_id); *params_len += 4;
  set_flag(flags, WARTS_PING_ADDR_DST,      &max_id); *params_len += 4;
  set_flag(flags, WARTS_PING_START,         &max_id); *params_len += 8;
  set_flag(flags, WARTS_PING_STOP_R,        &max_id); *params_len += 1;
  set_flag(flags, WARTS_PING_STOP_D,        &max_id); *params_len += 1;
  set_flag(flags, WARTS_PING_PATTERN_LEN,   &max_id); *params_len += 2;
  set_flag(flags, WARTS_PING_PATTERN_BYTES, &max_id); *params_len += pad_len;
  set_flag(flags, WARTS_PING_PROBE_COUNT,   &max_id); *params_len += 2;
  set_flag(flags, WARTS_PING_PROBE_SIZE,    &max_id); *params_len += 2;
  set_flag(flags, WARTS_PING_PROBE_WAIT,    &max_id); *params_len += 1;
  set_flag(flags, WARTS_PING_PROBE_TTL,     &max_id); *params_len += 1;
  set_flag(flags, WARTS_PING_REPLY_COUNT,   &max_id); *params_len += 2;
  set_flag(flags, WARTS_PING_PING_SENT,     &max_id); *params_len += 2;

  *flags_len = fold_flags(flags, max_id);

  return;
}

static int warts_ping_params_read(scamper_ping_t *ping, warts_state_t *state,
                          uint8_t *buf, uint32_t *off, uint32_t len)
{
  warts_param_reader_t handlers[] = {
    {&ping->list,          (wpr_t)extract_list,         state},
    {&ping->cycle,         (wpr_t)extract_cycle,        state},
    {&ping->src,           (wpr_t)extract_addr,         state},
    {&ping->dst,           (wpr_t)extract_addr,         state},
    {&ping->start,         (wpr_t)extract_timeval,      NULL},
    {&ping->stop_reason,   (wpr_t)extract_byte,         NULL},
    {&ping->stop_data,     (wpr_t)extract_byte,         NULL},
    {&ping->pattern_len,   (wpr_t)extract_uint16,       NULL},
    {&ping->pattern_bytes, (wpr_t)extract_bytes_uint16, &ping->pattern_len},
    {&ping->probe_count,   (wpr_t)extract_uint16,       NULL},
    {&ping->probe_size,    (wpr_t)extract_uint16,       NULL},
    {&ping->probe_wait,    (wpr_t)extract_byte,         NULL},
    {&ping->probe_ttl,     (wpr_t)extract_byte,         NULL},
    {&ping->reply_count,   (wpr_t)extract_uint16,       NULL},
    {&ping->ping_sent,     (wpr_t)extract_uint16,       NULL},
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_reader_t);

  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

static int warts_ping_params_write(const scamper_ping_t *ping,
                           const scamper_file_t *sf,
                           uint8_t *buf, uint32_t *off,
                           const uint32_t len,
                           const uint8_t *flags,
                           const uint16_t flags_len,
                           const uint16_t params_len)
{
  uint32_t list_id, cycle_id, src_id, dst_id;
  uint16_t pad_len = ping->pattern_len;
  warts_param_writer_t handlers[] = {
    {&list_id,             (wpw_t)insert_uint32,       NULL},
    {&cycle_id,            (wpw_t)insert_uint32,       NULL},
    {&src_id,              (wpw_t)insert_uint32,       NULL},
    {&dst_id,              (wpw_t)insert_uint32,       NULL},
    {&ping->start,         (wpw_t)insert_timeval,      NULL},
    {&ping->stop_reason,   (wpw_t)insert_byte,         NULL},
    {&ping->stop_data,     (wpw_t)insert_byte,         NULL},
    {&ping->pattern_len,   (wpw_t)insert_uint16,       NULL},
    {&ping->pattern_bytes, (wpw_t)insert_bytes_uint16, &pad_len},
    {&ping->probe_count,   (wpw_t)insert_uint16,       NULL},
    {&ping->probe_size,    (wpw_t)insert_uint16,       NULL},
    {&ping->probe_wait,    (wpw_t)insert_byte,         NULL},
    {&ping->probe_ttl,     (wpw_t)insert_byte,         NULL},
    {&ping->reply_count,   (wpw_t)insert_uint16,       NULL},
    {&ping->ping_sent,     (wpw_t)insert_uint16,       NULL},
  };

  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_writer_t);

  if(warts_list_getid(sf,  ping->list,  &list_id)  == -1) return -1;
  if(warts_cycle_getid(sf, ping->cycle, &cycle_id) == -1) return -1;
  if(warts_addr_getid(sf,  ping->src,   &src_id)   == -1) return -1;
  if(warts_addr_getid(sf,  ping->dst,   &dst_id)   == -1) return -1;

  warts_params_write(buf, off, len, flags, flags_len, params_len, handlers,
                 handler_cnt);
  return 0;
}

static scamper_ping_t *warts_ping_read(const scamper_file_t *sf,
                               const warts_hdr_t *hdr)
{
  warts_state_t *state = scamper_file_getstate(sf);
  scamper_ping_t *ping = NULL;
  uint8_t *buf = NULL;
  uint32_t off = 0;
  uint16_t i;
  scamper_ping_reply_t *reply;
  uint16_t reply_count;

  if((buf = warts_read(sf, hdr)) == NULL)
    {
      goto err;
    }

  if((ping = scamper_ping_alloc()) == NULL)
    {
      goto err;
    }

  if(warts_ping_params_read(ping, state, buf, &off, hdr->len) != 0)
    {
      goto err;
    }

  /*
   * determine how many replies to read.
   * if there are no replies, then we are done.
   */
  if(extract_uint16(buf, &off, hdr->len, &reply_count, NULL) != 0)
    {
      goto err;
    }
  if(reply_count == 0) goto done;

  /* allocate the ping_replies array */
  if(scamper_ping_replies_alloc(ping, ping->ping_sent) != 0)
    {
      goto err;
    }

  /* for each reply, read it and insert it into the ping structure */
  for(i=0; i<reply_count; i++)
    {
      if((reply = scamper_ping_reply_alloc()) == NULL)
      {
        goto err;
      }

      if(warts_ping_reply_read(reply, state, buf, &off, hdr->len) != 0)
      {
        goto err;
      }

      if(scamper_ping_reply_append(ping, reply) != 0)
      {
        goto err;
      }
    }

  assert(off == hdr->len);

 done:
  free(buf);
  return ping;

 err:
  if(buf != NULL) free(buf);
  if(ping != NULL) scamper_ping_free(ping);
  return NULL;
}

static int warts_ping_write(const scamper_file_t *sf,
                      const scamper_ping_t *ping)
{
  warts_ping_reply_state_t *reply_state = NULL;
  scamper_ping_reply_t *reply;
  uint8_t *buf = NULL;
  uint8_t  flags[ping_vars_mfb];
  uint16_t flags_len, params_len;
  uint32_t len, off = 0;
  uint16_t reply_count;
  size_t   size;
  int      i, j;

  assert(ping != NULL);

  /* figure out which ping data items we'll store in this record */
  warts_ping_params(ping, flags, &flags_len, &params_len);

  /* length of the ping's flags, parameters, and number of reply records */
  len = flags_len + 2 + params_len + 2;

  if((reply_count = scamper_ping_reply_count(ping)) > 0)
    {
      size = reply_count * sizeof(warts_ping_reply_state_t);
      if((reply_state = (warts_ping_reply_state_t *)malloc(size)) == NULL)
      {
        goto err;
      }

      for(i=0, j=0; i<ping->ping_sent; i++)
      {
        for(reply=ping->ping_replies[i]; reply != NULL; reply = reply->next)
          {
            if(warts_ping_reply_state(sf,reply,&reply_state[j++],&len) == -1)
            {
              goto err;
            }
          }
      }
    }

  if((buf = malloc(len)) == NULL)
    {
      goto err;
    }

  if(warts_ping_params_write(ping, sf, buf, &off, len, flags, flags_len,
                       params_len) == -1)
    {
      goto err;
    }

  /* reply record count */
  insert_uint16(buf, &off, len, &reply_count, NULL);

  /* write each ping reply record */
  for(i=0; i<reply_count; i++)
    {
      warts_ping_reply_write(&reply_state[i], buf, &off, len);
    }
  if(reply_state != NULL)
    {
      free(reply_state);
      reply_state = NULL;
    }

  assert(off == len);

  if(warts_write(sf, SCAMPER_FILE_OBJ_PING, buf, len) == -1)
    {
      goto err;
    }

  free(buf);
  return 0;

 err:
  if(buf != NULL) free(buf);
  return -1;
}

/*
 * scamper_file_warts_read
 *
 */
int scamper_file_warts_read(scamper_file_t *sf, scamper_file_filter_t *filter,
                      uint16_t *type, void **data)
{
  scamper_list_t  *list;
  scamper_cycle_t *cycle;
  scamper_addr_t  *addr;
  warts_hdr_t      hdr;
  int              fd;
  int              isfilter;
  int              tmp;

  fd = scamper_file_getfd(sf);

  for(;;)
    {
      /* read the header for the next record from the file */
      if((tmp = warts_hdr_read(sf, &hdr)) == 0)
      {
        /* EOF */
        *data = NULL;
        break;
      }
      else if(tmp == -1)
      {
        /* partial record */
        return -1;
      }

      /* if the header does not pass a basic sanity check, then give up */
      if(hdr.magic != WARTS_MAGIC)
      {
        return -1;
      }

      /* does the caller wants to know about this type? */
      if((isfilter = scamper_file_filter_isset(filter, hdr.type)) == 1)
      {
        *type = hdr.type;
      }

      if(hdr.type == SCAMPER_FILE_OBJ_ADDR)
      {
        if((addr = warts_addr_read(sf, &hdr)) == NULL) return -1;
        if(isfilter == 1) {*data = scamper_addr_use(addr); return 0; }
      }
      else if(hdr.type == SCAMPER_FILE_OBJ_LIST)
      {
        if((list = warts_list_read(sf, &hdr)) == NULL) return -1;
        if(isfilter == 1) {*data = scamper_list_use(list); return 0; }
      }
      else if(hdr.type == SCAMPER_FILE_OBJ_CYCLE_DEF ||
            hdr.type == SCAMPER_FILE_OBJ_CYCLE_START)
      {
        if((cycle = warts_cycle_read(sf, &hdr)) == NULL) return -1;
        if(isfilter == 1) {*data = scamper_cycle_use(cycle); return 0; }
      }
      else if(hdr.type == SCAMPER_FILE_OBJ_CYCLE_STOP)
      {
        if((cycle = warts_cycle_stop_read(sf, &hdr, isfilter)) == NULL)
          {
            return -1;
          }
        if(isfilter == 1) {*data = cycle; return 0; }
      }
      else if(isfilter == 0)
      {
        /* reader doesn't care what the data is, and neither do we */
        if(warts_skip(sf, hdr.len) != 0) return -1;
      }
      else if(hdr.type == SCAMPER_FILE_OBJ_TRACE)
      {
        if((*data = warts_trace_read(sf, &hdr)) == NULL) return -1;
        return 0;
      }
      else if(hdr.type == SCAMPER_FILE_OBJ_PING)
      {
        if((*data = warts_ping_read(sf, &hdr)) == NULL) return -1;
        return 0;
      }
      else
      {
        /* we don't know about this object */
        return -1;
      }
    }

  return 0;
}

int scamper_file_warts_write_ping(const scamper_file_t *sf,
                          const scamper_ping_t *ping)
{
  return warts_ping_write(sf, ping);
}

int scamper_file_warts_write_trace(const scamper_file_t *sf,
                           const scamper_trace_t *trace)
{
  return warts_trace_write(sf, trace);
}

int scamper_file_warts_write_cycle_start(const scamper_file_t *sf,
                               scamper_cycle_t *c)
{
  return warts_cycle_write(sf, c, SCAMPER_FILE_OBJ_CYCLE_START, NULL);
}

int scamper_file_warts_write_cycle_stop(const scamper_file_t *sf,
                              scamper_cycle_t *c)
{
  return warts_cycle_stop_write(sf, c);
}

/*
 * scamper_file_warts_init_read
 *
 * initialise the scamper_file_t's state structure so that it is all set
 * for reading.  the first entry of the list and cycle tables is pre-set
 * to be null for data objects that don't have associated list/cycle
 * objects.
 */
int scamper_file_warts_init_read(scamper_file_t *sf)
{
  warts_state_t *state;
  size_t size;

  if((state = (warts_state_t *)malloc_zero(sizeof(warts_state_t))) == NULL)
    {
      goto err;
    }

  size = sizeof(warts_addr_t *) * WARTS_ADDR_TABLEGROW;
  if((state->addr_table = malloc(size)) == NULL)
    {
      goto err;
    }
  state->addr_table[0] = &state->addr_null;
  state->addr_count = 1;

  size = sizeof(warts_list_t *) * WARTS_LIST_TABLEGROW;
  if((state->list_table = malloc(size)) == NULL)
    {
      goto err;
    }
  state->list_table[0] = &state->list_null;
  state->list_count = 1;

  size = sizeof(warts_cycle_t *) * WARTS_CYCLE_TABLEGROW;
  if((state->cycle_table = malloc(size)) == NULL)
    {
      goto err;
    }
  state->cycle_table[0] = &state->cycle_null;
  state->cycle_count = 1;

  scamper_file_setstate(sf, state);
  return 0;

 err:
  if(state != NULL)
    {
      if(state->addr_table != NULL) free(state->addr_table);
      if(state->list_table != NULL) free(state->list_table);
      if(state->cycle_table != NULL) free(state->cycle_table);
      free(state);
    }
  return -1;
}

/*
 * scamper_file_warts_init_write
 *
 * get the scamper_file_t object ready to write warts objects and keep state
 */
int scamper_file_warts_init_write(scamper_file_t *sf)
{
  warts_state_t *state = NULL;

  if((state = (warts_state_t *)malloc_zero(sizeof(warts_state_t))) == NULL)
    {
      goto err;
    }

  if((state->addr_tree = splaytree_alloc(warts_addr_cmp)) == NULL)
    {
      goto err;
    }
  state->addr_count = 1;

  if((state->list_tree = splaytree_alloc(warts_list_cmp)) == NULL)
    {
      goto err;
    }
  state->list_count = 1;

  if((state->cycle_tree = splaytree_alloc(warts_cycle_cmp)) == NULL)
    {
      goto err;
    }
  state->cycle_count = 1;

  scamper_file_setstate(sf, state);

  return 0;

 err:
  if(state != NULL)
    {
      if(state->addr_tree != NULL)  splaytree_free(state->addr_tree, NULL);
      if(state->list_tree != NULL)  splaytree_free(state->list_tree, NULL);
      if(state->cycle_tree != NULL) splaytree_free(state->cycle_tree, NULL);
      free(state);
    }
  return -1;
}

/*
 * scamper_file_warts_init_append
 *
 * go through the file and form the address, list, and cycle dictionaries
 */
int scamper_file_warts_init_append(scamper_file_t *sf)
{
  warts_state_t *state;
  warts_hdr_t    hdr;
  int            i, fd;
  uint32_t       j;

  /* init the warts structures as if we were reading the file */
  if(scamper_file_warts_init_read(sf) == -1)
    {
      return -1;
    }

  fd = scamper_file_getfd(sf);

  for(;;)
    {
      if((i = warts_hdr_read(sf, &hdr)) == -1)
      {
        return -1;
      }
      else if(i == 0) break;

      if(hdr.magic != WARTS_MAGIC || hdr.type == 0)
      {
        return -1;
      }

      switch(hdr.type)
      {
      case SCAMPER_FILE_OBJ_ADDR:
        if(warts_addr_read(sf, &hdr) == NULL) return -1;
        break;

      case SCAMPER_FILE_OBJ_LIST:
        if(warts_list_read(sf, &hdr) == NULL) return -1;
        break;

      case SCAMPER_FILE_OBJ_CYCLE_START:
      case SCAMPER_FILE_OBJ_CYCLE_DEF:
        if(warts_cycle_read(sf, &hdr) == NULL) return -1;
        break;

      case SCAMPER_FILE_OBJ_CYCLE_STOP:
        if(warts_cycle_stop_read(sf, &hdr, 0) == NULL) return -1;
        break;

      default:
        if(lseek(fd, hdr.len, SEEK_CUR) == -1)
          {
            return -1;
          }
        break;
      }      
    }

  /* get the state structure created in init_read */
  state = scamper_file_getstate(sf);

  /*
   * all the addresses are in a table.  put them into a splay tree so we can
   * find them quickly, and then trash the address table
   */
  if((state->addr_tree = splaytree_alloc(warts_addr_cmp)) == NULL)
    {
      return -1;
    }
  for(j=1; j<state->addr_count; j++)
    {
      if(splaytree_insert(state->addr_tree, state->addr_table[j]) == NULL)
      {
        return -1;
      }
    }
  free(state->addr_table); state->addr_table = NULL;

  if((state->list_tree = splaytree_alloc(warts_list_cmp)) == NULL)
    {
      return -1;
    }
  for(j=1; j<state->list_count; j++)
    {
      if(splaytree_insert(state->list_tree, state->list_table[j]) == NULL)
      {
        return -1;
      }
    }
  free(state->list_table); state->list_table = NULL;

  if((state->cycle_tree = splaytree_alloc(warts_cycle_cmp)) == NULL)
    {
      return -1;
    }
  for(j=1; j<state->cycle_count; j++)
    {
      /* don't install finished cycles into the splaytree */
      if(state->cycle_table[j] == NULL)
      {
        continue;
      }

      if(splaytree_insert(state->cycle_tree, state->cycle_table[j]) == NULL)
      {
        return -1;
      }
    }
  free(state->cycle_table); state->cycle_table = NULL;

  return 0;
}

int scamper_file_warts_is(const scamper_file_t *sf)
{
  uint16_t magic16;
  int fd = scamper_file_getfd(sf);

  if(lseek(fd, 0, SEEK_SET) == -1)
    {
      return 0;
    }

  if(read_wrap(fd, &magic16, NULL, sizeof(magic16)) != 0)
    {
      return 0;
    }

  if(ntohs(magic16) == WARTS_MAGIC)
    {
      if(lseek(fd, 0, SEEK_SET) == -1)
      {
        return 0;
      }
      return 1;
    }

  return 0;
}

static void warts_free_state(splaytree_t *tree, void **table,
                       unsigned int count, splaytree_free_t free_cb)
{
  unsigned int i;

  if(table != NULL)
    {
      for(i=1; i<count; i++)
      {
        if(table[i] != NULL)
          {
            free_cb(table[i]);
          }
      }
      free(table);
    }
  if(tree != NULL)
    {
      splaytree_free(tree, free_cb);
    }

  return;
}

void scamper_file_warts_free_state(scamper_file_t *sf)
{
  warts_state_t *state;

  /* there may not actually be state allocated with the file ... */
  if((state = scamper_file_getstate(sf)) == NULL)
    {
      return;
    }

  warts_free_state(state->list_tree,
               (void **)state->list_table, state->list_count,
               (splaytree_free_t)warts_list_free);

  warts_free_state(state->cycle_tree,
               (void **)state->cycle_table, state->cycle_count,
               (splaytree_free_t)warts_cycle_free);

  warts_free_state(state->addr_tree,
               (void **)state->addr_table, state->addr_count,
               (splaytree_free_t)warts_addr_free);

  free(state);

  return;
}

Generated by  Doxygen 1.6.0   Back to index