source: trunk/src/os2ahci/os2ahci.c@ 75

/******************************************************************************
 * os2ahci.c - main file for os2ahci driver
 *
 * Copyright (c) 2010 Christian Mueller. Parts copied from/inspired by the
 * Linux AHCI driver; those parts are (c) Linux AHCI/ATA maintainers
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include "os2ahci.h"
#include "bldday.h"

/* -------------------------- macros and constants ------------------------- */

/* parse integer command line parameter */
#define drv_parm_int(s, value, type, radix)               \
  {                                                       \
    char _far *_ep;                                       \
    if ((s)[1] != ':') {                                  \
      cprintf("missing colon (:) after /%c\n", *(s));     \
      goto init_fail;                                     \
    }                                                     \
    value = (type) strtol((s) + 2,                        \
                          (const char _far* _far*) &_ep,  \
                          radix);                         \
    s = _ep;                                              \
  }

/* ------------------------ typedefs and structures ------------------------ */

/* -------------------------- function prototypes -------------------------- */

void     _cdecl small_code_          (void);

/* ------------------------ global/static variables ------------------------ */

int             debug = 0;         /* if > 0, print debug messages to COM1 */
int             thorough_scan;     /* if != 0, perform thorough PCI scan */
int             init_reset;        /* if != 0, reset ports during init */

PFN             Device_Help = 0;   /* pointer to device helper entry point */
ULONG           RMFlags = 0;       /* required by resource manager library */
PFN             RM_Help0 = NULL;   /* required by resource manager library */
PFN             RM_Help3 = NULL;   /* required by resource manager library */
HDRIVER         rm_drvh;           /* resource manager driver handle */
char            rm_drvname[80];    /* driver name as returned by RM */
USHORT          add_handle;        /* driver handle (RegisterDeviceClass) */
UCHAR           timer_pool[TIMER_POOL_SIZE]; /* timer pool */

/* resource manager driver information structure */
DRIVERSTRUCT rm_drvinfo = {
  "OS2AHCI",                       /* driver name */
  "AHCI SATA Driver",              /* driver description */
  "GNU",                           /* vendor name */
  CMVERSION_MAJOR,                 /* RM interface version major */
  CMVERSION_MINOR,                 /* RM interface version minor */
  BLD_YEAR, BLD_MONTH, BLD_DAY,    /* date */
  0,                               /* driver flags */
  DRT_ADDDM,                       /* driver type */
  DRS_ADD,                         /* driver sub type */
  NULL                             /* driver callback */
};

ULONG           drv_lock;          /* driver-level spinlock */
IORB_QUEUE      driver_queue;      /* driver-level IORB queue */
AD_INFO         ad_infos[MAX_AD];  /* adapter information list */
int             ad_info_cnt;       /* number of entries in ad_infos[] */
int             init_complete;     /* if != 0, initialization has completed */

/* apapter/port-specific options saved when parsing the command line */
u8              link_speed[MAX_AD][AHCI_MAX_PORTS];
u8              disable_ncq[MAX_AD][AHCI_MAX_PORTS];

static char     init_msg[] = "OS2AHCI driver version %d.%02d\n";
static char     exit_msg[] = "OS2AHCI driver *not* installed\n";
static char     eval_msg[] = ANSI_CLR_RED ANSI_CLR_BRIGHT "Evaluation version "
                             "- not licensed for production use.\n" ANSI_RESET;


/* ----------------------------- start of code ----------------------------- */

/******************************************************************************
 * OS/2 device driver main strategy function. This function is only used
 * for initialization purposes; all other calls go directly to the adapter
 * device driver's strategy function.
 */
USHORT _cdecl c_strat(RPH _far *req)
{
  u16 rc;

  switch (req->Cmd) {

  case CMDInitBase:
    rc = init_drv((RPINITIN _far *) req);
    break;

  case CMDShutdown:
    rc = exit_drv(((RPSAVERESTORE _far *) req)->FuncCode);
    break;

  default:
    rc = STDON | STATUS_ERR_UNKCMD;
    break;
  }

  return(rc);
}

/******************************************************************************
 * Intialize the os2ahci driver. This includes command line parsing, scanning
 * the PCI bus for supported AHCI adapters, etc.
 */
USHORT init_drv(RPINITIN _far *req)
{
  RPINITOUT _far *rsp = (RPINITOUT _far *) req;
  DDD_PARM_LIST _far *ddd_pl = (DDD_PARM_LIST _far *) req->InitArgs;
  APIRET rmrc;
  char _far *cmd_line;
  char _far *s;
  int adapter_index = -1;
  int port_index = -1;
  u16 vendor;
  u16 device;

  /* set device helper entry point */
  Device_Help = req->DevHlpEP;

  /* create driver-level spinlock */
  DevHelp_CreateSpinLock(&drv_lock);

  if (debug) {
    /* initialize debug interface (COM1) */
    init_com1();
  }

  /* print initialization message */
  cprintf(init_msg, VERSION / 100, VERSION % 100);
  cprintf(eval_msg);

  /* register driver with resource manager */
  if ((rmrc = RMCreateDriver(&rm_drvinfo, &rm_drvh)) != RMRC_SUCCESS) {
    cprintf("failed to register driver with resource manager (rc = %d)\n", rmrc);
    goto init_fail;
  }

  /* parse command line parameters */
  cmd_line = (char _far *) ((u32) ddd_pl & 0xffff0000l) + ddd_pl->cmd_line_args;

  for (s = cmd_line; *s != 0; s++) {
    if (*s == '/' && s[1] != '\0') {
      s++;
      switch(tolower(*s)) {

      case 'c':
        /* set COM port base address for debug messages */
        drv_parm_int(s, com_base, u16, 16);
        break;

      case 'd':
        /* increase debug level */
        debug++;
        break;

      case 'i':
        /* add specfied PCI ID as a supported generic AHCI adapter  */
        drv_parm_int(s, vendor, u16, 16);
        drv_parm_int(s, device, u16, 16);
        if (add_pci_id(vendor, device)) {
          cprintf("failed to add PCI ID %04x:%04x\n", vendor, device);
          goto init_fail;
        }
        thorough_scan = 1;
        break;

      case 't':
        /* perform thorough PCI scan (i.e. look for individual supported PCI IDs) */
        thorough_scan = 1;
        break;

      case 'r':
        /* reset ports during initialization */
        init_reset = 1;
        break;

      case 'a':
        /* set adapter index for adapter and port-related options */
        drv_parm_int(s, adapter_index, int, 10);
        if (adapter_index < 0 || adapter_index >= MAX_AD) {
          cprintf("invalid adapter index (%d)\n", adapter_index);
          goto init_fail;
        }
        break;

      case 'p':
        /* set port index for port-related options */
        drv_parm_int(s, port_index, int, 10);
        if (port_index < 0 || port_index >= AHCI_MAX_PORTS) {
          cprintf("invalid port index (%d)\n", port_index);
          goto init_fail;
        }
        break;

      case 's':
        /* set link speed of current port on current adapter */
        drv_parm_int(s, link_speed[adapter_index][port_index], u8, 10);
        init_reset = 1;
        break;

      case 'n':
        /* disable NCQ */
        if (adapter_index == -1) {
          /* disable NCQ on all adapters and ports */
          memset(disable_ncq, 0x01, sizeof(disable_ncq));
        } else if (port_index == -1) {
          /* disable NCQ on all ports of this adapter */
          memset(disable_ncq[adapter_index], 0x01, sizeof(*disable_ncq));
        } else {
          /* disable NCQ for this adapter and port */
          disable_ncq[adapter_index][port_index] = 1;
        }
        break;

      default:
        cprintf("invalid option: /%c\n", *s);
        goto init_fail;
      }
    }
  }

  /* scan PCI bus for supported devices */
  scan_pci_bus();

  if (ad_info_cnt > 0) {
    /* initialization succeeded and we found at least one AHCI adapter */
    ADD_InitTimer(timer_pool, sizeof(timer_pool));
    mdelay_cal();

    if (DevHelp_RegisterDeviceClass("OS2AHCI", (PFN) add_entry, 0, 1,
                                    &add_handle)) {
      cprintf("error: couldn't register device class\n");
      goto init_fail;
    }

    /* allocate context hooks */
    if (DevHelp_AllocateCtxHook(mk_NPFN(restart_hook), &restart_ctxhook_h) != 0 ||
        DevHelp_AllocateCtxHook(mk_NPFN(reset_hook), &reset_ctxhook_h) != 0 ||
        DevHelp_AllocateCtxHook(mk_NPFN(engine_hook), &engine_ctxhook_h)) {
      cprintf("failed to allocate task-time context hooks\n");
      goto init_fail;
    }

    rsp->CodeEnd = (u16) end_of_code;
    rsp->DataEnd = (u16) &end_of_data;
    return(STDON);

  } else {
    /* no adapters found */
    cprintf(" No adapters found.\n");
  }

init_fail:
  /* initialization failed; set segment sizes to 0 and return error */
  rsp->CodeEnd = 0;
  rsp->DataEnd = 0;

  /* free context hooks */
  if (engine_ctxhook_h != 0)  DevHelp_FreeCtxHook(engine_ctxhook_h);
  if (reset_ctxhook_h != 0)   DevHelp_FreeCtxHook(reset_ctxhook_h);
  if (restart_ctxhook_h != 0) DevHelp_FreeCtxHook(restart_ctxhook_h);

  if (rm_drvh != 0) {
    /* remove driver from resource manager */
    RMDestroyDriver(rm_drvh);
  }

  cprintf(exit_msg);
  return(STDON | ERROR_I24_QUIET_INIT_FAIL);
}

/******************************************************************************
 * Device driver exit handler. This handler is called when OS/2 shuts down and
 * flushes the write caches of all attached devices.
 *
 * NOTE: Errors are ignored because there's no way we could stop the shutdown
 *       or do something about the error, unless retrying endlessly is
 *       considered an option.
 */
USHORT exit_drv(int func)
{
  int a;
  int p;
  int d;

  dprintf("exit_drv(%d) called\n", func);

  if (func == 0) {
    /* we're only interested in the second phase of the shutdown */
    return(STDON);
  }

  for (a = 0; a < ad_info_cnt; a++) {
    AD_INFO *ai = ad_infos + a;

    for (p = 0; p <= ai->port_max; p++) {
      for (d = 0; d <= ai->ports[p].dev_max; d++) {
        if (ai->ports[p].devs[d].present) {
          ahci_flush_cache(ai, p, d);
        }
      }
    }
  }

  return(STDON);
}

/******************************************************************************
 * ADD entry point. This is the main entry point for all ADD requests. Due to
 * the asynchronous nature of ADD drivers, this function primarily queues the
 * IORB(s) to the corresponding adapter or port queues, then triggers the
 * state machine to initiate processing queued IORBs.
 *
 * NOTE: In order to prevent race conditions or engine stalls, certain rules
 *       around locking, unlocking and IORB handling in general have been
 *       established. Refer to the comments in "trigger_engine()" for
 *       details.
 */
void _cdecl _far _loadds add_entry(IORBH _far *first_iorb)
{
  IORBH _far *iorb;
  IORBH _far *next = NULL;

  spin_lock(drv_lock);

  for (iorb = first_iorb; iorb != NULL; iorb = next) {
    /* Queue this IORB. Queues primarily exist on port level but there are
     * some requests which affect the whole driver, most notably
     * IOCC_CONFIGURATION. In either case, adding the IORB to the driver or
     * port queue will change the links, thus we need to save the original
     * link in 'next'.
     */
    next = (iorb->RequestControl | IORB_CHAIN) ? iorb->pNxtIORB : 0;

    iorb->Status = 0;
    iorb->ErrorCode = 0;
    memset(&iorb->ADDWorkSpace, 0x00, sizeof(ADD_WORKSPACE));

    if (iorb_driver_level(iorb)) {
      /* adapter-level IORB */
      iorb->UnitHandle = 0;
      iorb_queue_add(&driver_queue, iorb);

    } else {
      /* port-level IORB */
      int a = iorb_unit_adapter(iorb);
      int p = iorb_unit_port(iorb);
      int d = iorb_unit_device(iorb);

      if (a >= ad_info_cnt ||
          p > ad_infos[a].port_max ||
          d > ad_infos[a].ports[p].dev_max ||
          (ad_infos[a].port_map & (1UL << p)) == 0) {

        /* unit handle outside of the allowed range */
        dprintf("warning: IORB for %d.%d.%d out of range\n", a, p, d);
        iorb->Status = IORB_ERROR;
        iorb->ErrorCode = IOERR_CMD_SYNTAX;
        complete_iorb(iorb);
        continue;
      }

      iorb_queue_add(&ad_infos[a].ports[p].iorb_queue, iorb);
    }
  }

  /* trigger state machine */
  trigger_engine();

  spin_unlock(drv_lock);
}

/******************************************************************************
 * Trigger IORB queue engine. This is a wrapper function for trigger_engine_1()
 * which will try to get all IORBs sent on their way a couple of times. If
 * there are still IORBs ready for processing after this, this function will
 * hand off to a context hook which will continue to trigger the engine until
 * all IORBs have been sent.
 */
void trigger_engine(void)
{
  int i;

  for (i = 0; i < 3; i++) {
    if (trigger_engine_1() == 0) {
      /* done -- all IORBs have been sent on their way */
      return;
    }
  }

  /* Something keeps bouncing; hand off to the engine context hook which will
   * keep trying in the background.
   */
  DevHelp_ArmCtxHook(0, engine_ctxhook_h);
}

/******************************************************************************
 * Trigger IORB queue engine in order to send commands in the driver/port IORB
 * queues to the AHCI hardware. This function will return the number of IORBs
 * sent. Keep in mind that IORBs might "bounce" if the adapter/port is not in
 * a state to accept the command, thus it might take quite a few calls to get
 * all IORBs on their way. This is why there's a wrapper function which tries
 * it a few times, then hands off to a context hook which will keep trying in
 * the background.
 *
 * IORBs might complete before send_iorb() has returned, at any time during
 * interrupt processing or on another CPU on SMP systems. IORB completion
 * means modifications to the corresponding IORB queue (the completed IORB
 * is removed from the queue) thus we need to protect the IORB queues from
 * race conditions. The safest approach short of keeping the driver-level
 * spinlock aquired permanently is to keep it throughout this function and
 * release it temporarily in send_iorb().
 *
 * This implies that the handler functions are fully responsible for aquiring
 * the driver-level spinlock when they need it, and for releasing it again.
 *
 * As a rule of thumb, get the driver-level spinlock whenever accessing
 * volatile variables (IORB queues, values in ad_info[], ...).
 *
 * Additional Notes:
 *
 * - This function is expected to be called with the spinlock aquired
 *
 * - Adapters can be flagged as 'busy' which means no new IORBs are sent (they
 *   just remain in the queue). This can be used to release the driver-level
 *   spinlock while making sure no new IORBs are going to hit the hardware.
 *   In order to prevent engine stalls, all handlers using this functionality
 *   need to invoke trigger_engine() after resetting the busy flag.
 *
 * - Driver-level IORBs are not synchronized by adapter-level 'busy' flags.
 *   However, the driver-level queue is worked "one entry at a time" which
 *   means that no new IORBs will be queued on the driver-level queue until
 *   the head element has completed processing. This means that driver-
 *   level IORB handlers don't need to protect against each other. But they
 *   they do need to keep in mind interference with port-level IORBs:
 *
 *   - Driver-level IORB handlers must obtain the spinlock and/or flag all
 *     adapters as 'busy' which are affected by the driver-level IORB
 *
 *   - Driver-level IORB handlers must not access the hardware of a
 *     particular adapter if it's flagged as 'busy'
 */
int trigger_engine_1(void)
{
  IORBH _far *iorb;
  IORBH _far *next;
  int iorbs_sent = 0;
  int a;
  int p;

  iorbs_sent = 0;

  /* process driver-level IORBs */
  if ((iorb = driver_queue.root) != NULL && !add_workspace(iorb)->processing) {
    send_iorb(iorb);
    iorbs_sent++;
  }

  /* process port-level IORBs */
  for (a = 0; a < ad_info_cnt; a++) {
    AD_INFO *ai = ad_infos + a;
    if (ai->busy) {
      /* adapter is busy; don't process any IORBs */
      continue;
    }
    for (p = 0; p <= ai->port_max; p++) {
      /* send all queued IORBs on this port */
      next = NULL;
      for (iorb = ai->ports[p].iorb_queue.root; iorb != NULL; iorb = next) {
        next = iorb->pNxtIORB;
        if (!add_workspace(iorb)->processing) {
          send_iorb(iorb);
          iorbs_sent++;
        }
      }
    }
  }

  return(iorbs_sent);
}

/******************************************************************************
 * Send a single IORB to the corresponding AHCI adapter/port. This is just a
 * switch board for calling the corresponding iocc_*() handler function.
 *
 * NOTE: This function is expected to be called with the driver-level spinlock
 *       aquired. It will release it before calling any of the handler
 *       functions and re-aquire it when done.
 */
void send_iorb(IORBH _far *iorb)
{
  /* Mark IORB as "processing" before doing anything else. Once the IORB is
   * marked as "processing", we can release the spinlock because subsequent
   * invocations of trigger_engine() (e.g. at interrupt time) will ignore this
   * IORB.
   */
  add_workspace(iorb)->processing = 1;
  spin_unlock(drv_lock);

  switch (iorb->CommandCode) {

  case IOCC_CONFIGURATION:
    iocc_configuration(iorb);
    break;

  case IOCC_DEVICE_CONTROL:
    iocc_device_control(iorb);
    break;

  case IOCC_UNIT_CONTROL:
    iocc_unit_control(iorb);
    break;

  case IOCC_GEOMETRY:
    iocc_geometry(iorb);
    break;

  case IOCC_EXECUTE_IO:
    iocc_execute_io(iorb);
    break;

  case IOCC_UNIT_STATUS:
    iocc_unit_status(iorb);
    break;

  case IOCC_ADAPTER_PASSTHRU:
    iocc_adapter_passthru(iorb);
    break;

  default:
    /* unsupported call */
    iorb_seterr(iorb, IOERR_CMD_NOT_SUPPORTED);
    iorb_done(iorb);
    break;
  }

  /* re-aquire spinlock before returning to trigger_engine() */
  spin_lock(drv_lock);
}

/******************************************************************************
 * Handle IOCC_CONFIGURATION requests.
 */
void iocc_configuration(IORBH _far *iorb)
{
  int a;

  switch (iorb->CommandModifier) {

  case IOCM_COMPLETE_INIT:
    /* Complete initialization. From now on, we won't have to restore the BIOS
     * configuration after each command and we're fully operational (i.e. will
     * use interrupts, timers and context hooks instead of polling).
     */
    if (!init_complete) {
      dprintf("leaving initialization mode\n");
      spin_lock(drv_lock);
      for (a = 0; a < ad_info_cnt; a++) {
        ahci_complete_init(ad_infos + a);
      }
      init_complete = 1;
      spin_unlock(drv_lock);
    }
    iorb_done(iorb);
    break;

  case IOCM_GET_DEVICE_TABLE:
    /* construct a device table */
    iocm_device_table(iorb);
    break;

  default:
    iorb_seterr(iorb, IOERR_CMD_NOT_SUPPORTED);
    iorb_done(iorb);
    break;
  }
}

/******************************************************************************
 * Handle IOCC_DEVICE_CONTROL requests.
 */
void iocc_device_control(IORBH _far *iorb)
{
  AD_INFO *ai = ad_infos + iorb_unit_adapter(iorb);
  IORBH _far *ptr;
  IORBH _far *next = NULL;
  int p = iorb_unit_port(iorb);
  int d = iorb_unit_device(iorb);

  switch (iorb->CommandModifier) {

  case IOCM_ABORT:
    /* abort all pending commands on specified port and device */
    spin_lock(drv_lock);
    for (ptr = ai->ports[p].iorb_queue.root; ptr != NULL; ptr = next) {
      next = ptr->pNxtIORB;
      /* move all matching IORBs to the abort queue */
      if (ptr != iorb && iorb_unit_device(ptr) == d) {
        iorb_queue_del(&ai->ports[p].iorb_queue, ptr);
        iorb_queue_add(&abort_queue, ptr);
        ptr->ErrorCode = IOERR_CMD_ABORTED;
      }
    }
    spin_unlock(drv_lock);

    /* trigger reset context hook which will finish the abort processing */
    DevHelp_ArmCtxHook(0, reset_ctxhook_h);
    break;

  case IOCM_SUSPEND:
  case IOCM_RESUME:
  case IOCM_GET_QUEUE_STATUS:
    /* Suspend/resume operations allow access to the hardware for other
     * entities such as IBMIDECD.FLT. Since os2ahci implements both ATA
     * and ATAPI in the same driver, this won't be required.
     */
    iorb_seterr(iorb, IOERR_CMD_NOT_SUPPORTED);
    break;

  case IOCM_LOCK_MEDIA:
  case IOCM_UNLOCK_MEDIA:
  case IOCM_EJECT_MEDIA:
    /* unit control commands to lock, unlock and eject media */
    /* will be supported later... */
    iorb_seterr(iorb, IOERR_CMD_NOT_SUPPORTED);
    break;

  default:
    iorb_seterr(iorb, IOERR_CMD_NOT_SUPPORTED);
    break;
  }

  iorb_done(iorb);
}

/******************************************************************************
 * Handle IOCC_UNIT_CONTROL requests.
 */
void iocc_unit_control(IORBH _far *iorb)
{
  IORB_UNIT_CONTROL _far *iorb_uc = (IORB_UNIT_CONTROL _far *) iorb;
  int a = iorb_unit_adapter(iorb);
  int p = iorb_unit_port(iorb);
  int d = iorb_unit_device(iorb);

  spin_lock(drv_lock);
  switch (iorb->CommandModifier) {

  case IOCM_ALLOCATE_UNIT:
    /* allocate unit for exclusive access */
    if (ad_infos[a].ports[p].devs[d].allocated) {
      iorb_seterr(iorb, IOERR_UNIT_ALLOCATED);
    } else {
      ad_infos[a].ports[p].devs[d].allocated = 1;
    }
    break;

  case IOCM_DEALLOCATE_UNIT:
    /* deallocate exclusive access to unit */
    if (!ad_infos[a].ports[p].devs[d].allocated) {
      iorb_seterr(iorb, IOERR_UNIT_NOT_ALLOCATED);
    } else {
      ad_infos[a].ports[p].devs[d].allocated = 0;
    }
    break;

  case IOCM_CHANGE_UNITINFO:
    /* Change unit (device) information. One reason for this IOCM is the
     * interface for filter device drivers: a filter device driver can
     * either change existing UNITINFOs or permanently allocate units
     * and fabricate new [logical] units; the former is the reason why we
     * must store the pointer to the updated UNITNIFO for subsequent
     * IOCC_CONFIGURATION/IOCM_GET_DEVICE_TABLE calls.
     */
    if (!ad_infos[a].ports[p].devs[d].allocated) {
      iorb_seterr(iorb, IOERR_UNIT_NOT_ALLOCATED);
      break;
    }
    ad_infos[a].ports[p].devs[d].unit_info = iorb_uc->pUnitInfo;
    break;

  default:
    iorb_seterr(iorb, IOERR_CMD_NOT_SUPPORTED);
    break;
  }

  spin_unlock(drv_lock);
  iorb_done(iorb);
}

/******************************************************************************
 * Scan all ports for AHCI devices and construct a DASD device table.
 *
 * NOTE: This function may be called multiple times. Only the first invocation
 *       will actually scan for devices; all subsequent calls will merely
 *       return the results of the initial scan, potentially augmented by
 *       modified unit infos after IOCC_CONFIGURATION/IOCM_CHANGE_UNITINFO
 *       requests.
 */
void iocm_device_table(IORBH _far *iorb)
{
  IORB_CONFIGURATION _far *iorb_conf;
  DEVICETABLE _far *dt;
  char _far *pos;
  int rc;
  int a;
  int p;
  int d;

  iorb_conf = (IORB_CONFIGURATION _far *) iorb;
  dt = iorb_conf->pDeviceTable;

  spin_lock(drv_lock);

  /* initialize device table header */
  dt->ADDLevelMajor = ADD_LEVEL_MAJOR;
  dt->ADDLevelMinor = ADD_LEVEL_MINOR;
  dt->ADDHandle     = add_handle;
  dt->TotalAdapters = ad_info_cnt;

  /* Initial position of dynamic portion of device table (i.e. behind the
   * array of ADAPTERINFO pointers, pAdapter, in the device table)
   */
  pos = (char _far *) (dt->pAdapter + ad_info_cnt);

  for (a = 0; a < ad_info_cnt; a++) {
    ADAPTERINFO _far *ptr = (ADAPTERINFO _far *) pos;
    AD_INFO *ad_info = ad_infos + a;
    int units = 0;

    /* sanity check for sufficient space in device table */
    if ((u32) (ptr + 1) - (u32) dt > iorb_conf->DeviceTableLen) {
      dprintf("error: device table provided by DASD too small\n");
      iorb_seterr(iorb, IOERR_CMD_SW_RESOURCE);
      goto iocm_device_table_done;
    }

    /* set ADAPTERINFO offset in device table */
    dt->pAdapter[a] = (ADAPTERINFO _near *) ((u32) ptr & 0xffff);

    /* fill in adapter information structure in device table */
    memset(ptr, 0x00, sizeof(*ptr));
    sprintf(ptr->AdapterName, "AHCI_%d", a);
    ptr->AdapterDevBus   = AI_DEVBUS_ST506 | AI_DEVBUS_32BIT;
    ptr->AdapterIOAccess = AI_IOACCESS_BUS_MASTER;
    ptr->AdapterHostBus  = AI_HOSTBUS_OTHER | AI_BUSWIDTH_32BIT;
    ptr->AdapterFlags    = AF_16M | AF_HW_SCATGAT;

    /* AHCI limits S/G elements to 22 bits, thus we'll report only half of
     * our S/G list buffers to reduce complexity. The command preparation code
     * will always try to map as many S/G elements as possible so the physical
     * S/G list capacity is not really wasted except in rare conditions where
     * we need to split commands with long S/G lists without any suitable split
     * points except those at the reported MaxHWSGList.
     */
    ptr->MaxHWSGList     = AHCI_MAX_SG / 2;

    if (!ad_info->port_scan_done) {
      /* First call; need to scan AHCI hardware for devices. Since this might
       * be a lengthy operation, especially when init_reset is set, we'll mark
       * the adapter as busy (new IORBs will only be queued but not executed)
       * and release the spinlock while scanning the ports so interrupts will
       * be processed.
       */
      if (ad_info->busy) {
        dprintf("error: port scan requested while adapter was busy\n");
        iorb_seterr(iorb, IOERR_CMD_SW_RESOURCE);
        goto iocm_device_table_done;
      }
      ad_info->busy = 1;
      spin_unlock(drv_lock);
      rc = ahci_scan_ports(ad_info);
      spin_lock(drv_lock);
      ad_info->busy = 0;

      if (rc != 0) {
        dprintf("error: port scan failed on adapter #%d\n", a);
        iorb_seterr(iorb, IOERR_CMD_SW_RESOURCE);
        goto iocm_device_table_done;
      }
      ad_info->port_scan_done = 1;
    }

    /* insert devices (units) into the device table */
    for (p = 0; p <= ad_info->port_max; p++) {
      for (d = 0; d <= ad_info->ports[p].dev_max; d++) {
        if (ad_info->ports[p].devs[d].present) {
          UNITINFO _far *ui = ptr->UnitInfo + units;

          /* sanity check for sufficient space in device table */
          if ((u32) (ui + 1) - (u32) dt > iorb_conf->DeviceTableLen) {
            dprintf("error: device table provided by DASD too small\n");
            iorb_seterr(iorb, IOERR_CMD_SW_RESOURCE);
            goto iocm_device_table_done;
          }

          if (ad_info->ports[p].devs[d].unit_info == NULL) {
            /* provide initial information about this device (unit) */
            memset(ui, 0x00, sizeof(*ui));
            ui->AdapterIndex = a;
            ui->UnitIndex    = units;
            ui->UnitHandle   = iorb_unit(a, p, d);
            ui->UnitType     = ad_info->ports[p].devs[d].dev_type;
            ui->QueuingCount = ad_info->ports[p].devs[d].ncq_max;;
            if (ad_info->ports[p].devs[d].removable) {
              ui->UnitFlags |= UF_REMOVABLE;
            }
          } else {
            /* copy updated device (unit) information (IOCM_CHANGE_UNITINFO) */
            memcpy(ui, ad_info->ports[p].devs[d].unit_info, sizeof(*ui));
          }
          units++;
        }
      }
    }

    /* set total device (unit) count for this adapter */
    ptr->AdapterUnits = units;

    /* calculate offset for next adapter */
    pos = (char _far *) (ptr->UnitInfo + units);
  }

iocm_device_table_done:
  spin_unlock(drv_lock);
  iorb_done(iorb);
}

/******************************************************************************
 * Handle IOCC_GEOMETRY requests.
 */
void iocc_geometry(IORBH _far *iorb)
{
  switch (iorb->CommandModifier) {

  case IOCM_GET_MEDIA_GEOMETRY:
  case IOCM_GET_DEVICE_GEOMETRY:
    add_workspace(iorb)->idempotent = 1;
    ahci_get_geometry(iorb);
    break;

  default:
    iorb_seterr(iorb, IOERR_CMD_NOT_SUPPORTED);
    iorb_done(iorb);
  }
}

/******************************************************************************
 * Handle IOCC_EXECUTE_IO requests.
 */
void iocc_execute_io(IORBH _far *iorb)
{
  switch (iorb->CommandModifier) {

  case IOCM_READ:
    add_workspace(iorb)->idempotent = 1;
    ahci_read(iorb);
    break;

  case IOCM_READ_VERIFY:
    add_workspace(iorb)->idempotent = 1;
    ahci_verify(iorb);
    break;

  case IOCM_WRITE:
    add_workspace(iorb)->idempotent = 1;
    ahci_write(iorb);
    break;

  case IOCM_WRITE_VERIFY:
    add_workspace(iorb)->idempotent = 1;
    ahci_write(iorb);
    break;

  default:
    iorb_seterr(iorb, IOERR_CMD_NOT_SUPPORTED);
    iorb_done(iorb);
  }
}

/******************************************************************************
 * Handle IOCC_UNIT_STATUS requests.
 */
void iocc_unit_status(IORBH _far *iorb)
{
  switch (iorb->CommandModifier) {

  case IOCM_GET_UNIT_STATUS:
    add_workspace(iorb)->idempotent = 1;
    ahci_unit_ready(iorb);
    break;

  default:
    iorb_seterr(iorb, IOERR_CMD_NOT_SUPPORTED);
    iorb_done(iorb);
  }
}

/******************************************************************************
 * Handle IOCC_ADAPTER_PASSTHROUGH requests.
 */
void iocc_adapter_passthru(IORBH _far *iorb)
{
  switch (iorb->CommandModifier) {

  case IOCM_EXECUTE_CDB:
    add_workspace(iorb)->idempotent = 0;
    ahci_execute_cdb(iorb);
    break;

  case IOCM_EXECUTE_ATA:
    add_workspace(iorb)->idempotent = 0;
    ahci_execute_ata(iorb);
    break;

  default:
    iorb_seterr(iorb, IOERR_CMD_NOT_SUPPORTED);
    iorb_done(iorb);
  }
}

/******************************************************************************
 * Add an IORB to the specified queue.
 */
void iorb_queue_add(IORB_QUEUE _far *queue, IORBH _far *iorb)
{
  if (iorb_priority(iorb) {
    /* priority IORB; insert at first position */
    iorb->pNxtIORB = queue->root;
    queue->root = iorb;

  } else {
    /* append IORB to end of queue */
    iorb->pNxtIORB = NULL;

    if (queue->root == NULL) {
      queue->root = iorb;
    } else {
      queue->tail->pNxtIORB = iorb;
    }
    queue->tail = iorb;
  }

  if (debug) {
    /* determine queue type (local, driver, abort or port) and minimum debug
     * level; otherwise, queue debug prints can become really confusing.
     */
    char *queue_type;
    int min_debug = 1;

    if ((u32) queue >> 16 == (u32) (void _far *) &queue >> 16) {
      /* this queue is on the stack */
      queue_type = "local";
      min_debug = 2;

    } else if (queue == &driver_queue) {
      queue_type = "driver";

    } else if (queue == &abort_queue) {
      queue_type = "abort";
      min_debug = 2;

    } else {
      queue_type = "port";
    }

    if (debug >= min_debug) {
      printf("IORB %Fp queued (cmd = %d/%d, queue = %Fp [%s], timeout = %ld)\n",
             iorb, iorb->CommandCode, iorb->CommandModifier, queue, queue_type,
             iorb->Timeout);
    }
  }
}

/******************************************************************************
 * Remove an IORB from the specified queue.
 */
int iorb_queue_del(IORB_QUEUE _far *queue, IORBH _far *iorb)
{
  IORBH _far *_iorb;
  IORBH _far *_prev = NULL;
  int found = 0;

  for (_iorb = queue->root; _iorb != NULL; _iorb = _iorb->pNxtIORB) {
    if (_iorb == iorb) {
      /* found the IORB to be removed */
      if (_prev != NULL) {
        _prev->pNxtIORB = _iorb->pNxtIORB;
      } else {
        queue->root = _iorb->pNxtIORB;
      }
      if (_iorb == queue->tail) {
        queue->tail = _prev;
      }
      found = 1;
      break;
    }
    _prev = _iorb;
  }

  if (found) {
    ddprintf("IORB %Fp removed (queue = %Fp)\n", iorb, queue);
  } else {
    dprintf("IORB %Fp not found in queue %Fp\n", iorb, queue);
  }

  return(!found);
}

/******************************************************************************
 * Set the error code in the specified IORB
 *
 * NOTE: This function does *not* call iorb_done(). It merely sets the IORB
 *       status to the specified error code.
 */
void iorb_seterr(IORBH _far *iorb, USHORT error_code)
{
  iorb->ErrorCode = error_code;
  iorb->Status = IORB_ERROR;
}

/******************************************************************************
 * Mark the specified IORB as done and notify the asynchronous post function,
 * if any. The IORB is also removed from the corresponding IORB queue.
 *
 * NOTES: This function does not clear the Status field; it merely adds the
 *        IORB_DONE flag.
 *
 *        This function is expected to be called *without* the corresponding
 *        driver-level drv_lock aquired. It will aquire the spinlock before
 *        updating the IORB queue and release it before notifying the upstream
 *        code in order to prevent deadlocks.
 *
 *        Due to this logic, this function is only good for simple task-time
 *        completions. Functions working on lists of IORBs (such as interrupt
 *        handlers or context hooks) should implement their own logic. See
 *        abort_ctxhook() for an example.
 */
void iorb_done(IORBH _far *iorb)
{
  int a = iorb_unit_adapter(iorb);
  int p = iorb_unit_port(iorb);

  /* remove IORB from corresponding queue */
  spin_lock(drv_lock);
  if (iorb_driver_level(iorb)) {
    iorb_queue_del(&driver_queue, iorb);
  } else {
    iorb_queue_del(&ad_infos[a].ports[p].iorb_queue, iorb);
  }
  aws_free(add_workspace(iorb));
  spin_unlock(drv_lock);

  complete_iorb(iorb);
}

/******************************************************************************
 * Requeue the specified IORB such that it will be sent downstream for
 * processing again. This includes freeing all resources currently allocated
 * (timer, buffer, ...) and resetting the flags to 0.
 *
 * The following flags are preserved:
 *  - no_ncq
 */
void iorb_requeue(IORBH _far *iorb)
{
  ADD_WORKSPACE _far *aws = add_workspace(iorb);
  u16 no_ncq = aws->no_ncq;

  aws_free(aws);
  memset(aws, 0x00, sizeof(*aws));
  aws->no_ncq = no_ncq;
}

/******************************************************************************
 * Timeout handler for I/O commands. Since timeout handling can involve
 * lengthy operations like port resets, the main code is located in a
 * separate function which is invoked via a context hook.
 */
void _cdecl _far timeout_callback(ULONG timer_handle, ULONG p1,
                                  ULONG p2)
{
  IORBH _far *iorb = (IORBH _far *) p1;
  int a = iorb_unit_adapter(iorb);
  int p = iorb_unit_port(iorb);

  ADD_CancelTimer(timer_handle);
  dprintf("timeout for IORB %Fp\n", iorb);

  /* Move the timed-out IORB to the abort queue. Since it's possible that the
   * IORB has completed after the timeout has expired but before we got to
   * this line of code, we'll check the return code of iorb_queue_del(): If it
   * returns an error, the IORB must have completed a few microseconds ago and
   * there is no timeout.
   */
  spin_lock(drv_lock);
  if (iorb_queue_del(&ad_infos[a].ports[p].iorb_queue, iorb) == 0) {
    iorb_queue_add(&abort_queue, iorb);
    iorb->ErrorCode = IOERR_ADAPTER_TIMEOUT;
  }
  spin_unlock(drv_lock);

  /* Trigger abort processing function. We don't really care whether this
   * succeeds because the only reason why it would fail should be multiple
   * calls to DevHelp_ArmCtxHook() before the context hook had a chance to
   * start executing, which leaves two scenarios:
   *
   *  - We succeded in arming the context hook. Fine.
   *
   *  - We armed the context hook a second time before it had a chance to
   *    start executing. In this case, the already scheduled context hook
   *    will process our IORB as well.
   */
  DevHelp_ArmCtxHook(0, reset_ctxhook_h);
}

/******************************************************************************
 * small_code_ - this dummy func resolves the undefined reference linker
 * error that occurrs when linking WATCOM objects with DDK's link.exe
 */
void _cdecl small_code_(void)
{
}