Changeset 2841

Timestamp:

Aug 26, 2016, 10:04:50 PM (9 years ago)

Author:

bird

Message:

work in progress

File:

• trunk/src/kmk/kmkbuiltin/submit.c (modified) (10 diffs)

trunk/src/kmk/kmkbuiltin/submit.c

@@ -38 +38 @@
 #include <string.h>
 #include <errno.h>
+#include <assert.h>
 #ifdef HAVE_ALLOCA_H
 # include <alloca.h>
@@ -50 +51 @@
 #endif
 
+#include "kbuild.h"
 #include "kmkbuiltin.h"
 #include "err.h"
 
-#ifdef __OS2__
-# define INCL_BASE
-# include <os2.h>
-# ifndef LIBPATHSTRICT
-#  define LIBPATHSTRICT 3
-# endif
-#endif
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** Hashes a pid. */
+#define KWORKER_PID_HASH(a_pid) ((size_t)(a_pid) % 61)
+
 
 /*********************************************************************************************************************************
@@ -71 +73 @@
     /** Pointer to the previous worker instance. */
     PWORKERINSTANCE         pPrev;
+    /** Pointer to the next worker with the same pid hash slot. */
+    PWORKERINSTANCE         pNextPidHash;
     /** 32 or 64. */
     unsigned                cBits;
+    /** The process ID of the kWorker process. */
+    pid_t                   pid;
+#ifdef KBUILD_OS_WINDOWS
     /** The process handle. */
     HANDLE                  hProcess;
-
+    /** The bi-directional pipe we use to talk to the kWorker process. */
+    HANDLE                  hPipe;
+    /** For overlapped read (have valid event semaphore). */
+    OVERLAPPED              OverlappedRead;
+    /** The 32-bit exit code read bufffer. */
+    uint32_t                u32ReadResult;
+#else
+    /** The socket descriptor we use to talk to the kWorker process. */
+    int                     fdSocket;
+#endif
+
+    /** What it's busy with.  NULL if idle. */
+    struct child           *pBusyWith;
 } WORKERINSTANCE;
+
+
+typedef struct WORKERLIST
+{
+    /** The head of the list.  NULL if empty. */
+    PWORKERINSTANCE         pHead;
+    /** The tail of the list.  NULL if empty. */
+    PWORKERINSTANCE         pTail;
+    /** Number of list entries. */
+    size_t                  cEntries;
+} WORKERLIST;
+typedef WORKERLIST *PWORKERLIST;
 
 
@@ -82 +113 @@
 *   Global Variables                                                                                                             *
 *********************************************************************************************************************************/
-static PWORKERINSTANCE g_pIdleHead;
-static PWORKERINSTANCE g_pIdleTail;
-
+/** List of idle worker.*/
+static WORKERLIST           g_IdleList;
+/** List of busy workers. */
+static WORKERLIST           g_BusyList;
+/** PID hash table for the workers.
+ * @sa KWORKER_PID_HASH() */
+static PWORKERINSTANCE      g_apPidHash[61];
+
+#ifdef KBUILD_OS_WINDOWS
+/** For naming the pipes.
+ * Also indicates how many worker instances we've spawned. */
+static unsigned             g_uWorkerSeqNo = 0;
+#endif
+
+/** @var g_cArchBits
+ * The bit count of the architecture this binary is compiled for. */
+/** @var g_szArch
+ * The name of the architecture this binary is compiled for. */
+/** @var g_cArchBits
+ * The bit count of the alternative architecture. */
+/** @var g_szAltArch
+ * The name of the alternative architecture. */
+#if defined(KBUILD_ARCH_AMD64)
+static unsigned             g_cArchBits    = 64;
+static char const           g_szArch[]     = "amd64";
+static unsigned             g_cAltArchBits = 32;
+static char const           g_szAltArch[]  = "x86";
+#elif defined(KBUILD_ARCH_X86)
+static unsigned             g_cArchBits    = 32;
+static char const           g_szArch[]     = "x86";
+static unsigned             g_cAltArchBits = 64;
+static char const           g_szAltArch[]  = "amd64";
+#else
+# error "Port me!"
+#endif
+
+
+/**
+ * Unlinks a worker instance from a list.
+ *
+ * @param   pList               The list.
+ * @param   pWorker             The worker.
+ */
+static void kSubmitListUnlink(PWORKERLIST pList, PWORKERINSTANCE pWorker)
+{
+    PWORKERINSTANCE pNext = pWorker->pNext;
+    PWORKERINSTANCE pPrev = pWorker->pPrev;
+
+    if (pNext)
+    {
+        assert(pNext->pPrev == pWorker);
+        pNext->pPrev = pPrev;
+    }
+    else
+    {
+        assert(pList->pHead == pWorker);
+        pList->pHead = pPrev;
+    }
+
+    if (pPrev)
+    {
+        assert(pPrev->pNext == pWorker);
+        pPrev->pNext = pNext;
+    }
+    else
+    {
+        assert(pList->pTail == pWorker);
+        pList->pTail = pNext;
+    }
+
+    assert(pList->cEntries > 0);
+    pList->cEntries--;
+
+    pWorker->pNext = NULL;
+    pWorker->pPrev = NULL;
+}
+
+
+/**
+ * Appends a worker instance to the tail of a list.
+ *
+ * @param   pList               The list.
+ * @param   pWorker             The worker.
+ */
+static void kSubmitListAppend(PWORKERLIST pList, PWORKERINSTANCE pWorker)
+{
+    PWORKERINSTANCE pTail = pList->pTail;
+
+    assert(pTail != pWorker);
+    assert(pList->pHead != pWorker);
+
+    pWorker->pNext = NULL;
+    pWorker->pPrev = pTail;
+    if (pTail != NULL)
+    {
+        assert(pTail->pNext == NULL);
+        pTail->pNext = pWorker;
+    }
+    else
+    {
+        assert(pList->pHead == NULL);
+        pList->pHead = pWorker;
+        pList->pTail = pWorker;
+    }
+
+    pList->cEntries++;
+}
+
+
+/**
+ * Looks up a worker by its process ID.
+ *
+ * @returns Pointer to the worker instance if found. NULL if not.
+ * @param   pid                 The process ID of the worker.
+ */
+static PWORKERINSTANCE kSubmitFindWorkerByPid(pid_t pid)
+{
+    PWORKERINSTANCE pWorker = g_apPidHash[KWORKER_PID_HASH(pid)];
+    while (pWorker && pWorker->pid != pid)
+        pWorker = pWorker->pNextPidHash;
+    return pWorker;
+}
+
+
+/**
+ * Creates a new worker process.
+ *
+ * @returns 0 on success, non-zero value on failure.
+ * @param   pWorker             The worker structure.  Caller does the linking
+ *                              (as we might be reusing an existing worker
+ *                              instance because a worker shut itself down due
+ *                              to high resource leak level).
+ * @param   cVerbosity          The verbosity level.
+ */
+static int kSubmitSpawnWorker(PWORKERINSTANCE pWorker, int cVerbosity)
+{
+#if defined(KBUILD_OS_WINDOWS) || defined(KBUILD_OS_OS2)
+    static const char s_szWorkerName[] = "kWorker.exe";
+#else
+    static const char s_szWorkerName[] = "kWorker";
+#endif
+    const char     *pszBinPath = get_kbuild_bin_path();
+    size_t const    cchBinPath = strlen(pszBinPath);
+    size_t          cchExectuable;
+    size_t const    cbExecutableBuf = GET_PATH_MAX;
+    PATH_VAR(szExecutable);
+
+    /*
+     * Construct the executable path.
+     */
+    if (   pWorker->cBits == g_cArchBits
+        ?  cchBinPath + 1 + sizeof(s_szWorkerName) <= cbExecutableBuf
+        :  cchBinPath + 1 - sizeof(g_szArch) + sizeof(g_szAltArch) + sizeof(s_szWorkerName) <= cbExecutableBuf )
+    {
+#ifdef KBUILD_OS_WINDOWS
+        static DWORD        s_fDenyRemoteClients = ~(DWORD)0;
+        wchar_t             wszPipeName[64];
+        HANDLE              hWorkerPipe;
+        SECURITY_ATTRIBUTES SecAttrs = { /*nLength:*/ sizeof(SecAttrs), /*pAttrs:*/ NULL, /*bInheritHandle:*/ TRUE };
+#else
+        int                 aiPair[2] = { -1, -1 };
+#endif
+
+        memcpy(szExecutable, pszBinPath, cchBinPath);
+        cchExectuable = cchBinPath;
+
+        /* Replace the arch bin directory extension with the alternative one if requested. */
+        if (pWorker->cBits != g_cArchBits)
+        {
+            if (   cchBinPath < sizeof(g_szArch)
+                || memcmp(&szExecutable[cchBinPath - sizeof(g_szArch) + 1], g_szArch, sizeof(g_szArch) - 1) != 0)
+                return errx(1, "KBUILD_BIN_PATH does not end with main architecture (%s) as expected: %s", pszBinPath, g_szArch);
+            cchExectuable -= sizeof(g_szArch) - 1;
+            memcpy(&szExecutable[cchExectuable], g_szAltArch, sizeof(g_szAltArch) - 1);
+            cchExectuable += sizeof(g_szAltArch) - 1;
+        }
+
+        /* Append a slash and the worker name. */
+        szExecutable[cchExectuable++] = '/';
+        memcpy(&szExecutable[cchExectuable], s_szWorkerName, sizeof(s_szWorkerName));
+
+#ifdef KBUILD_OS_WINDOWS
+        /*
+         * Create the bi-directional pipe.  Worker end is marked inheritable, our end is not.
+         */
+        if (s_fDenyRemoteClients == ~(DWORD)0)
+            s_fDenyRemoteClients = GetVersion() >= 0x60000 ? PIPE_REJECT_REMOTE_CLIENTS : 0;
+        _snwprintf(wszPipeName, sizeof(wszPipeName), L"\\\\.\\pipe\\kmk-%u-kWorker-%u", getpid(), g_uWorkerSeqNo++);
+        hWorkerPipe = CreateNamedPipeW(wszPipeName,
+                                       PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED | FILE_FLAG_FIRST_PIPE_INSTANCE /* win2k sp2+ */,
+                                       PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT | s_fDenyRemoteClients,
+                                       1 /* cMaxInstances */,
+                                       64 /*cbOutBuffer*/,
+                                       65536 /*cbInBuffer*/,
+                                       0 /*cMsDefaultTimeout -> 50ms*/,
+                                       &SecAttrs /* inherit */);
+        if (hWorkerPipe != INVALID_HANDLE_VALUE)
+        {
+            pWorker->hPipe = CreateFileW(wszPipeName,
+                                         GENERIC_READ | GENERIC_WRITE,
+                                         0 /* dwShareMode - no sharing */,
+                                         NULL /*pSecAttr - no inherit */,
+                                         OPEN_EXISTING,
+                                         FILE_FLAG_OVERLAPPED,
+                                         NULL /*hTemplate*/);
+            if (pWorker->hPipe != INVALID_HANDLE_VALUE)
+            {
+                pWorker->OverlappedRead.hEvent = CreateEventW(NULL /*pSecAttrs - no inherit*/, TRUE /*bManualReset*/,
+                                                              TRUE /*bInitialState*/, NULL /*pwszName*/);
+                if (pWorker->OverlappedRead.hEvent != NULL)
+                {
+                    char        szHandleArg[16];
+                    const char *apszArgs[4] = { szExecutable, "--pipe", szHandleArg, NULL };
+                    _snprintf(szHandleArg, sizeof(szHandleArg), "%p", hWorkerPipe);
+
+                    /*
+                     * Create the worker process.
+                     */
+                    pWorker->hProcess = (HANDLE) _spawnve(_P_NOWAIT, szExecutable, apszArgs, environ);
+                    if ((intptr_t)pWorker->hProcess != -1)
+                    {
+                        CloseHandle(hWorkerPipe);
+                        pWorker->pid = GetProcessId(pWorker->hProcess);
+                        if (cVerbosity > 0)
+                            fprintf(stderr, "kSubmit: created %d bit worker %d\n", pWorker->cBits, pWorker->pid);
+                        return 0;
+                    }
+                    err(1, "_spawnve(,%s,,)", szExecutable);
+                    CloseHandle(pWorker->OverlappedRead.hEvent);
+                    pWorker->OverlappedRead.hEvent = INVALID_HANDLE_VALUE;
+                }
+                else
+                    errx(1, "CreateEventW failed: %u", GetLastError());
+                CloseHandle(pWorker->hPipe);
+                pWorker->hPipe = INVALID_HANDLE_VALUE;
+            }
+            else
+                errx(1, "Opening named pipe failed: %u", GetLastError());
+            CloseHandle(hWorkerPipe);
+        }
+        else
+            errx(1, "CreateNamedPipeW failed: %u", GetLastError());
+
+#else
+        /*
+         * Create a socket pair.
+         */
+        if (socketpair(AF_LOCAL, SOCK_STREAM, 0, aiPair) == 0)
+        {
+            pWorker->fdSocket = aiPair[1];
+        }
+        else
+            err(1, "socketpair");
+#endif
+    }
+    else
+        errx(1, "KBUILD_BIN_PATH is too long");
+    return -1;
+}
+
+
+/**
+ * Selects an idle worker or spawns a new one.
+ *
+ * @returns Pointer to the selected worker instance.  NULL on error.
+ * @param   pWorker             The idle worker instance to respawn.
+ *                              On failure this will be freed!
+ * @param   cBitsWorker         The worker bitness - 64 or 32.
+ */
+static int kSubmitRespawnWorker(PWORKERINSTANCE pWorker, int cVerbosity)
+{
+    size_t idxHash;
+
+    /*
+     * Clean up after the old worker.
+     */
+#ifdef KBUILD_OS_WINDOWS
+    DWORD   rcWait;
+
+    /* Close the pipe handle first, breaking the pipe in case it's not already
+       busted up.  Close the event semaphore too before waiting for the process. */
+    if (!CloseHandle(pWorker->hPipe))
+        warnx("CloseHandle(pWorker->hPipe): %u", GetLastError());
+    pWorker->hPipe = INVALID_HANDLE_VALUE;
+
+    if (!CloseHandle(pWorker->OverlappedRead.hEvent))
+        warnx("CloseHandle(pWorker->OverlappedRead.hEvent): %u", GetLastError());
+    pWorker->OverlappedRead.hEvent = INVALID_HANDLE_VALUE;
+
+    /* It's probably shutdown already, if not give it 10 milliseconds before
+       we terminate it forcefully. */
+    rcWait = WaitForSingleObject(pWorker->hProcess, 10);
+    if (rcWait != WAIT_OBJECT_0)
+    {
+        BOOL fRc = TerminateProcess(pWorker->hProcess, 127);
+        rcWait = WaitForSingleObject(pWorker->hProcess, 100);
+        if (rcWait != WAIT_OBJECT_0)
+            warnx("WaitForSingleObject returns %u (and TerminateProcess %d)", rcWait, fRc);
+    }
+
+    if (!CloseHandle(pWorker->hProcess))
+        warnx("CloseHandle(pWorker->hProcess): %u", GetLastError());
+    pWorker->hProcess = INVALID_HANDLE_VALUE;
+
+#else
+    pid_t   pidWait;
+    int     rc;
+
+    if (close(pWorker->fdSocket) != 0)
+        warn("close(pWorker->fdSocket)");
+    pWorker->fdSocket = -1;
+
+    kill(pWorker->pid, SIGTERM);
+    pidWait = waitpid(pWorker->pid, &rc, 0);
+    if (pidWait != pWorker->pid)
+        warn("waitpid(pWorker->pid,,0)");
+#endif
+
+    /*
+     * Unlink it from the hash table.
+     */
+    idxHash = KWORKER_PID_HASH(pWorker->pid);
+    if (g_apPidHash[idxHash] == pWorker)
+        g_apPidHash[idxHash] = pWorker->pNext;
+    else
+    {
+        PWORKERINSTANCE pPrev = g_apPidHash[idxHash];
+        while (pPrev && pPrev->pNext != pWorker)
+            pPrev = pPrev->pNext;
+        assert(pPrev != NULL);
+        if (pPrev)
+            pPrev->pNext = pWorker->pNext;
+    }
+    pWorker->pid = -1;
+
+    /*
+     * Respawn it.
+     */
+    if (kSubmitSpawnWorker(pWorker, cVerbosity) == 0)
+    {
+        /*
+         * Insert it into the process ID hash table and idle list.
+         */
+        size_t idxHash = KWORKER_PID_HASH(pWorker->pid);
+        pWorker->pNextPidHash = g_apPidHash[idxHash];
+        g_apPidHash[idxHash] = pWorker;
+        return 0;
+    }
+
+    kSubmitListUnlink(&g_IdleList, pWorker);
+    free(pWorker);
+    return -1;
+}
+
+
+/**
+ * Selects an idle worker or spawns a new one.
+ *
+ * @returns Pointer to the selected worker instance.  NULL on error.
+ * @param   cBitsWorker         The worker bitness - 64 or 32.
+ */
+static PWORKERINSTANCE kSubmitSelectWorkSpawnNewIfNecessary(unsigned cBitsWorker, int cVerbosity)
+{
+    /*
+     * Lookup up an idle worker.
+     */
+    PWORKERINSTANCE pWorker = g_IdleList.pHead;
+    while (pWorker)
+    {
+        if (pWorker->cBits == cBitsWorker)
+            return pWorker;
+        pWorker = pWorker->pNext;
+    }
+
+    /*
+     * Create a new worker instance.
+     */
+    pWorker = (PWORKERINSTANCE)xcalloc(sizeof(*pWorker));
+    pWorker->cBits = cBitsWorker;
+    if (kSubmitSpawnWorker(pWorker, cVerbosity) == 0)
+    {
+        /*
+         * Insert it into the process ID hash table and idle list.
+         */
+        size_t idxHash = KWORKER_PID_HASH(pWorker->pid);
+        pWorker->pNextPidHash = g_apPidHash[idxHash];
+        g_apPidHash[idxHash] = pWorker;
+
+        kSubmitListAppend(&g_IdleList, pWorker);
+        return pWorker;
+    }
+
+    free(pWorker);
+    return NULL;
+}
+
+
+/**
+ * Composes a JOB mesage for a worker.
+ *
+ * @returns Pointer to the message.
+ * @param   pszExecutable   The executable to run.
+ * @param   papszArgs       The argument vector.
+ * @param   papszEnvVars    The environment vector.
+ * @param   pszCwd          The current directory.
+ * @param   pcbMsg          Where to return the message length.
+ */
+static void *kSubmitComposeJobMessage(const char *pszExecutable, char **papszArgs, char **papszEnvVars,
+                                      const char *pszCwd, uint32_t *pcbMsg)
+{
+    size_t   i;
+    size_t   cbTmp;
+    uint32_t cbMsg;
+    uint8_t *pbMsg;
+    uint8_t *pbCursor;
+
+    /*
+     * Calculate the message length first.
+     */
+    cbMsg  = sizeof(cbMsg);
+    cbMsg += sizeof("JOB");
+    cbMsg += strlen(pszExecutable) + 1;
+
+    for (i = 0; papszArgs[i] != NULL; i++)
+        cbMsg += strlen(papszArgs[i]) + 1;
+    cbMsg += 1;
+
+    for (i = 0; papszEnvVars[i] != NULL; i++)
+        cbMsg += strlen(papszEnvVars[i]) + 1;
+    cbMsg += 1;
+
+    cbMsg += strlen(pszCwd) + 1;
+
+    /*
+     * Compose the message.
+     */
+    pbMsg = pbCursor = xmalloc(cbMsg);
+
+    memcpy(pbCursor, &cbMsg, sizeof(cbMsg));
+    pbCursor += sizeof(cbMsg);
+    memcpy(pbCursor, "JOB", sizeof("JOB"));
+    pbCursor += sizeof("JOB");
+
+    cbTmp = strlen(pszExecutable) + 1;
+    memcpy(pbCursor, pszExecutable, cbTmp);
+    pbCursor += cbTmp;
+
+    for (i = 0; papszArgs[i] != NULL; i++)
+    {
+        cbTmp = strlen(papszArgs[i]) + 1;
+        memcpy(pbCursor, papszArgs[i], cbTmp);
+        pbCursor += cbTmp;
+    }
+    *pbCursor++ = '\0';
+
+    for (i = 0; papszEnvVars[i] != NULL; i++)
+    {
+        cbTmp = strlen(papszEnvVars[i]) + 1;
+        memcpy(pbCursor, papszEnvVars[i], cbTmp);
+        pbCursor += cbTmp;
+    }
+    *pbCursor++ = '\0';
+
+    cbTmp = strlen(pszCwd) + 1;
+    memcpy(pbCursor, pszCwd, cbTmp);
+    pbCursor += cbTmp;
+
+    assert(pbCursor - pbMsg == (size_t)cbMsg);
+
+    /* done */
+    *pcbMsg = cbMsg;
+    return pbMsg;
+}
+
+
+/**
+ * Sends the job message to the given worker, respawning the worker if
+ * necessary.
+ *
+ * @returns 0 on success, non-zero on failure.
+ *
+ * @param   pWorker             The work to send the request to.  The worker is
+ *                              on the idle list.
+ * @param   pvMsg               The message to send.
+ * @param   cbMsg               The size of the message.
+ * @param   cVerbosity          The verbosity level.
+ */
+static int kSubmitSendJobMessage(PWORKERINSTANCE pWorker, void const *pvMsg, uint32_t cbMsg, int cVerbosity)
+{
+    int cRetries = 1;
+    for (;; cRetries--)
+    {
+        /*
+         * Try write the message.
+         */
+        uint32_t        cbLeft = cbMsg;
+        uint8_t const  *pbLeft = (uint8_t const  *)pvMsg;
+#ifdef KBUILD_OS_WINDOWS
+        DWORD           dwErr;
+        DWORD           cbWritten;
+        while (WriteFile(pWorker->hPipe, pbLeft, cbLeft, &cbWritten, NULL /*pOverlapped*/))
+        {
+            assert(cbWritten <= cbLeft);
+            cbLeft -= cbWritten;
+            if (!cbLeft)
+                return 0;
+
+            /* This scenario shouldn't really ever happen. But just in case... */
+            pbLeft += cbWritten;
+        }
+        dwErr = GetLastError();
+        if (   (   dwErr != ERROR_BROKEN_PIPE
+                && dwErr != ERROR_NO_DATA)
+            || cRetries <= 0)
+            return errx(1, "Error writing to worker: %u", dwErr);
+#else
+        ssize_t cbWritten
+        while ((cbWritten = write(pWorker->fdSocket, pbLeft, cbLeft)) >= 0)
+        {
+            assert(cbWritten <= cbLeft);
+            cbLeft -= cbWritten;
+            if (!cbLeft)
+                return 0;
+
+            pbLeft += cbWritten;
+        }
+        if (  (   errno != EPIPE
+               && errno != ENOTCONN
+               && errno != ECONNRESET))
+            || cRetries <= 0)
+            return err(1, "Error writing to worker");
+# error "later"
+#endif
+
+        /*
+         * Broken connection. Try respawn the worker.
+         */
+        if (kSubmitRespawnWorker(pWorker, cVerbosity) != 0)
+            return 2;
+    }
+}
+
+
+/**
+ * Handles the --set var=value option.
+ *
+ * @returns 0 on success, non-zero exit code on error.
+ * @param   papszEnv            The environment vector.
+ * @param   pcEnvVars           Pointer to the variable holding the number of
+ *                              environment variables held by @a papszEnv.
+ * @param   pcAllocatedEnvVars  Pointer to the variable holding max size of the
+ *                              environment vector.
+ * @param   cVerbosity          The verbosity level.
+ * @param   pszValue            The var=value string to apply.
+ */
+static int kSubmitOptEnvSet(char **papszEnv, unsigned *pcEnvVars, unsigned *pcAllocatedEnvVars,
+                            int cVerbosity, const char *pszValue)
+{
+    const char *pszEqual = strchr(pszValue, '=');
+    if (pszEqual)
+    {
+        unsigned iEnvVar;
+        unsigned cEnvVars = *pcEnvVars;
+        size_t const cchVar = pszValue - pszEqual;
+        for (iEnvVar = 0; iEnvVar < cEnvVars; iEnvVar++)
+            if (   strncmp(papszEnv[iEnvVar], pszValue, cchVar) == 0
+                && papszEnv[iEnvVar][cchVar] == '=')
+            {
+                if (cVerbosity > 0)
+                    fprintf(stderr, "kSubmit: replacing '%s' with '%s'\n", papszEnv[iEnvVar], pszValue);
+                free(papszEnv[iEnvVar]);
+                papszEnv[iEnvVar] = xstrdup(pszValue);
+                break;
+            }
+        if (iEnvVar == cEnvVars)
+        {
+            /* Append new variable. We probably need to resize the vector. */
+            if ((cEnvVars + 2) > *pcAllocatedEnvVars)
+            {
+                *pcAllocatedEnvVars = (cEnvVars + 2 + 0xf) & ~(unsigned)0xf;
+                papszEnv = (char **)xrealloc(papszEnv, *pcAllocatedEnvVars * sizeof(papszEnv[0]));
+            }
+            papszEnv[cEnvVars++] = xstrdup(pszValue);
+            papszEnv[cEnvVars]   = NULL;
+            *pcEnvVars = cEnvVars;
+            if (cVerbosity > 0)
+                fprintf(stderr, "kSubmit: added '%s'\n", papszEnv[iEnvVar]);
+        }
+        else
+        {
+            /* Check for duplicates. */
+            for (iEnvVar++; iEnvVar < cEnvVars; iEnvVar++)
+                if (   strncmp(papszEnv[iEnvVar], pszValue, cchVar) == 0
+                    && papszEnv[iEnvVar][cchVar] == '=')
+                {
+                    if (cVerbosity > 0)
+                        fprintf(stderr, "kSubmit: removing duplicate '%s'\n", papszEnv[iEnvVar]);
+                    free(papszEnv[iEnvVar]);
+                    cEnvVars--;
+                    if (iEnvVar != cEnvVars)
+                        papszEnv[iEnvVar] = papszEnv[cEnvVars];
+                    papszEnv[cEnvVars] = NULL;
+                    iEnvVar--;
+                }
+        }
+    }
+    else
+        return errx(1, "Missing '=': -E %s", pszValue);
+
+    return 0;
+}
+
+
+/**
+ * Handles the --unset var option.
+ *
+ * @returns 0 on success, non-zero exit code on error.
+ * @param   papszEnv            The environment vector.
+ * @param   pcEnvVars           Pointer to the variable holding the number of
+ *                              environment variables held by @a papszEnv.
+ * @param   cVerbosity          The verbosity level.
+ * @param   pszVarToRemove      The name of the variable to remove.
+ */
+static int kSubmitOptEnvUnset(char **papszEnv, unsigned *pcEnvVars, int cVerbosity, const char *pszVarToRemove)
+{
+    if (strchr(pszVarToRemove, '=') == NULL)
+    {
+        unsigned     cRemoved = 0;
+        size_t const cchVar   = strlen(pszVarToRemove);
+        unsigned     cEnvVars = *pcEnvVars;
+        unsigned     iEnvVar;
+
+        for (iEnvVar = 0; iEnvVar < cEnvVars; iEnvVar++)
+            if (   strncmp(papszEnv[iEnvVar], pszVarToRemove, cchVar) == 0
+                && papszEnv[iEnvVar][cchVar] == '=')
+            {
+                if (cVerbosity > 0)
+                    fprintf(stderr, !cRemoved ? "kSubmit: removing '%s'\n"
+                            : "kSubmit: removing duplicate '%s'\n", papszEnv[iEnvVar]);
+                free(papszEnv[iEnvVar]);
+                cEnvVars--;
+                if (iEnvVar != cEnvVars)
+                    papszEnv[iEnvVar] = papszEnv[cEnvVars];
+                papszEnv[cEnvVars] = NULL;
+                cRemoved++;
+                iEnvVar--;
+            }
+        *pcEnvVars = cEnvVars;
+
+        if (cVerbosity > 0 && !cRemoved)
+            fprintf(stderr, "kSubmit: not found '%s'\n", pszVarToRemove);
+    }
+    else
+        return errx(1, "Found invalid variable name character '=' in: -U %s", pszVarToRemove);
+    return 0;
+}
+
+
+
+/**
+ * Handles the --chdir dir option.
+ *
+ * @returns 0 on success, non-zero exit code on error.
+ * @param   pszCwd              The CWD buffer.  Contains current CWD on input,
+ *                              modified by @a pszValue on output.
+ * @param   cbCwdBuf            The size of the CWD buffer.
+ * @param   pszValue            The --chdir value to apply.
+ */
+static int kSubmitOptChDir(char *pszCwd, size_t cbCwdBuf, const char *pszValue)
+{
+    size_t cchNewCwd = strlen(pszValue);
+    size_t offDst;
+    if (cchNewCwd)
+    {
+#ifdef HAVE_DOS_PATHS
+        if (*pszValue == '/' || *pszValue == '\\')
+        {
+            if (pszValue[1] == '/' || pszValue[1] == '\\')
+                offDst = 0; /* UNC */
+            else if (pszCwd[1] == ':' && isalpha(pszCwd[0]))
+                offDst = 2; /* Take drive letter from CWD. */
+            else
+                return errx(1, "UNC relative CWD not implemented: cur='%s' new='%s'", pszCwd, pszValue);
+        }
+        else if (   pszValue[1] == ':'
+                 && isalpha(pszValue[0]))
+        {
+            if (pszValue[2] == '/'|| pszValue[2] == '\\')
+                offDst = 0; /* DOS style absolute path. */
+            else if (   pszCwd[1] == ':'
+                     && tolower(pszCwd[0]) == tolower(pszValue[0]) )
+            {
+                pszValue += 2; /* Same drive as CWD, append drive relative path from value. */
+                cchNewCwd -= 2;
+                offDst = strlen(pszCwd);
+            }
+            else
+            {
+                /* Get current CWD on the specified drive and append value. */
+                int iDrive = tolower(pszValue[0]) - 'a' + 1;
+                if (!_getdcwd(iDrive, pszCwd, cbCwdBuf))
+                    return err(1, "_getdcwd(%d,,) failed", iDrive);
+                pszValue += 2;
+                cchNewCwd -= 2;
+            }
+        }
+#else
+        if (*pszValue == '/')
+            offDst = 0;
+#endif
+        else
+            offDst = strlen(pszCwd); /* Relative path, append to the existing CWD value. */
+
+        /* Do the copying. */
+#ifdef HAVE_DOS_PATHS
+        if (offDst > 0 && pszCwd[offDst - 1] != '/' && pszCwd[offDst - 1] != '\\')
+#else
+        if (offDst > 0 && pszCwd[offDst - 1] != '/')
+#endif
+             pszCwd[offDst++] = '/';
+        if (offDst + cchNewCwd >= cbCwdBuf)
+            return errx(1, "Too long CWD: %*.*s%s", offDst, offDst, pszCwd, pszValue);
+        memcpy(&pszCwd[offDst], pszValue, cchNewCwd + 1);
+    }
+    /* else: relative, no change - quitely ignore. */
+    return 0;
+}
 
 
@@ -133 +888 @@
     unsigned        iEnvVar;
     unsigned        cEnvVars;
-    char          **papszEnv = NULL;
-    const char     *pszCwd = NULL;
-    unsigned        cBitsWorker = 0;
-    int             fWatcomBrainDamage = 0;
-    int             cVerbosity = 0;
-    size_t const    cbCwdBuf = GET_PATH_MAX;
+    char          **papszEnv            = NULL;
+    const char     *pszExecutable       = NULL;
+    const char     *pszCwd              = NULL;
+    unsigned        cBitsWorker         = g_cArchBits;
+    int             fWatcomBrainDamage  = 0;
+    int             cVerbosity          = 0;
+    size_t const    cbCwdBuf            = GET_PATH_MAX;
     PATH_VAR(szCwd);
 
@@ -215 +971 @@
                 else if (strcmp(pszArg, "verbose") == 0)
                     chOpt = 'v';
+                else if (strcmp(pszArg, "executable") == 0)
+                    chOpt = 'e';
                 else
                 {
@@ -232 +990 @@
                     case 'U':
                     case 'C':
+                    case 'e':
                         if (*pszArg != '\0')
                             pszValue = pszArg + (*pszArg == ':' || *pszArg == '=');
@@ -255 +1014 @@
 
                     case 'E':
-                    {
-                        const char *pszEqual = strchr(pszValue, '=');
-                        if (pszEqual)
-                        {
-                            size_t const cchVar = pszValue - pszEqual;
-                            for (iEnvVar = 0; iEnvVar < cEnvVars; iEnvVar++)
-                                if (   strncmp(papszEnv[iEnvVar], pszValue, cchVar) == 0
-                                    && papszEnv[iEnvVar][cchVar] == '=')
-                                {
-                                    if (cVerbosity > 0)
-                                        fprintf(stderr, "kSubmit: replacing '%s' with '%s'\n", papszEnv[iEnvVar], pszValue);
-                                    free(papszEnv[iEnvVar]);
-                                    papszEnv[iEnvVar] = xstrdup(pszValue);
-                                    break;
-                                }
-                            if (iEnvVar == cEnvVars)
-                            {
-                                /* Append new variable. We probably need to resize the vector. */
-                                if ((cEnvVars + 2) > cAllocatedEnvVars)
-                                {
-                                    cAllocatedEnvVars = (cEnvVars + 2 + 0xf) & ~(unsigned)0xf;
-                                    pChild->environment = papszEnv = (char **)xrealloc(papszEnv,
-                                                                                       cAllocatedEnvVars * sizeof(papszEnv[0]));
-                                }
-                                papszEnv[cEnvVars++] = xstrdup(pszValue);
-                                papszEnv[cEnvVars]   = NULL;
-                                if (cVerbosity > 0)
-                                    fprintf(stderr, "kSubmit: added '%s'\n", papszEnv[iEnvVar]);
-                            }
-                            else
-                            {
-                                /* Check for duplicates. */
-                                for (iEnvVar++; iEnvVar < cEnvVars; iEnvVar++)
-                                    if (   strncmp(papszEnv[iEnvVar], pszValue, cchVar) == 0
-                                        && papszEnv[iEnvVar][cchVar] == '=')
-                                    {
-                                        if (cVerbosity > 0)
-                                            fprintf(stderr, "kSubmit: removing duplicate '%s'\n", papszEnv[iEnvVar]);
-                                        free(papszEnv[iEnvVar]);
-                                        cEnvVars--;
-                                        if (iEnvVar != cEnvVars)
-                                            papszEnv[iEnvVar] = papszEnv[cEnvVars];
-                                        papszEnv[cEnvVars] = NULL;
-                                        iEnvVar--;
-                                    }
-                            }
-                        }
-                        else
-                            return errx(1, "Missing '=': -E %s", pszValue);
-                        break;
-                    }
+                        rcExit = kSubmitOptEnvSet(papszEnv, &cEnvVars, &cAllocatedEnvVars, cVerbosity, pszValue);
+                        pChild->environment = papszEnv;
+                        if (rcExit == 0)
+                            break;
+                        return rcExit;
 
                     case 'U':
-                    {
-                        if (strchr(pszValue, '=') == NULL)
-                        {
-                            unsigned     cRemoved = 0;
-                            size_t const cchVar = strlen(pszValue);
-                            for (iEnvVar = 0; iEnvVar < cEnvVars; iEnvVar++)
-                                if (   strncmp(papszEnv[iEnvVar], pszValue, cchVar) == 0
-                                    && papszEnv[iEnvVar][cchVar] == '=')
-                                {
-                                    if (cVerbosity > 0)
-                                        fprintf(stderr, !cRemoved ? "kSubmit: removing '%s'\n"
-                                                : "kSubmit: removing duplicate '%s'\n", papszEnv[iEnvVar]);
-                                    free(papszEnv[iEnvVar]);
-                                    cEnvVars--;
-                                    if (iEnvVar != cEnvVars)
-                                        papszEnv[iEnvVar] = papszEnv[cEnvVars];
-                                    papszEnv[cEnvVars] = NULL;
-                                    cRemoved++;
-                                    iEnvVar--;
-                                }
-                            if (cVerbosity > 0 && !cRemoved)
-                                fprintf(stderr, "kSubmit: not found '%s'\n", pszValue);
-                        }
-                        else
-                            return errx(1, "Found invalid variable name character '=' in: -U %s", pszValue);
-                        break;
-                    }
+                        rcExit = kSubmitOptEnvUnset(papszEnv, &cEnvVars, cVerbosity, pszValue);
+                        if (rcExit == 0)
+                            break;
+                        return rcExit;
 
                     case 'C':
-                    {
-                        size_t cchNewCwd = strlen(pszValue);
-                        size_t offDst;
-                        if (cchNewCwd)
-                        {
-#ifdef HAVE_DOS_PATHS
-                            if (*pszValue == '/' || *pszValue == '\\')
-                            {
-                                if (pszValue[1] == '/' || pszValue[1] == '\\')
-                                    offDst = 0; /* UNC */
-                                else if (szCwd[1] == ':' && isalpha(szCwd[0]))
-                                    offDst = 2; /* Take drive letter from CWD. */
-                                else
-                                    return errx(1, "UNC relative CWD not implemented: cur='%s' new='%s'", szCwd, pszValue);
-                            }
-                            else if (   pszValue[1] == ':'
-                                     && isalpha(pszValue[0]))
-                            {
-                                if (pszValue[2] == '/'|| pszValue[2] == '\\')
-                                    offDst = 0; /* DOS style absolute path. */
-                                else if (   szCwd[1] == ':'
-                                         && tolower(szCwd[0]) == tolower(pszValue[0]) )
-                                {
-                                    pszValue += 2; /* Same drive as CWD, append drive relative path from value. */
-                                    cchNewCwd -= 2;
-                                    offDst = strlen(szCwd);
-                                }
-                                else
-                                {
-                                    /* Get current CWD on the specified drive and append value. */
-                                    int iDrive = tolower(pszValue[0]) - 'a' + 1;
-                                    if (!_getdcwd(iDrive, szCwd, cbCwdBuf))
-                                        return err(1, "_getdcwd(%d,,) failed", iDrive);
-                                    pszValue += 2;
-                                    cchNewCwd -= 2;
-                                }
-                            }
-#else
-                            if (*pszValue == '/')
-                                offDst = 0;
-#endif
-                            else
-                                offDst = strlen(szCwd); /* Relative path, append to the existing CWD value. */
-
-                            /* Do the copying. */
-#ifdef HAVE_DOS_PATHS
-                            if (offDst > 0 && szCwd[offDst - 1] != '/' && szCwd[offDst - 1] != '\\')
-#else
-                            if (offDst > 0 && szCwd[offDst - 1] != '/')
-#endif
-                                 szCwd[offDst++] = '/';
-                            if (offDst + cchNewCwd >= cbCwdBuf)
-                                return errx(1, "Too long CWD: %*.*s%s", offDst, offDst, szCwd, pszValue);
-                            memcpy(&szCwd[offDst], pszValue, cchNewCwd + 1);
-                        }
-                        /* else: relative, no change - quitely ignore. */
-                        break;
-                    }
+                        rcExit = kSubmitOptChDir(szCwd, cbCwdBuf, pszValue);
+                        if (rcExit == 0)
+                            break;
+                        return rcExit;
 
                     case '3':
@@ -402 +1038 @@
                     case '6':
                         cBitsWorker = 64;
+                        break;
+
+                    case 'e':
+                        pszExecutable = pszValue;
                         break;
 
@@ -433 +1073 @@
     if (iArg < argc)
     {
-
+        uint32_t        cbMsg;
+        void           *pvMsg   = kSubmitComposeJobMessage(pszExecutable, &argv[iArg], papszEnv, szCwd, &cbMsg);
+        PWORKERINSTANCE pWorker = kSubmitSelectWorkSpawnNewIfNecessary(cBitsWorker, cVerbosity);
+        if (pWorker)
+        {
+            rcExit = kSubmitSendJobMessage(pWorker, pvMsg, cbMsg, cVerbosity);
+            if (rcExit == 0)
+            {
+                pWorker->pBusyWith = pChild;
+                /** @todo integrate with sub_proc.c / whatever. */
+            }
+        }
+        else
+            rcExit = 1;
+        free(pvMsg);
     }
     else