Changeset 3361 for trunk/src

Timestamp:

Jun 8, 2020, 9:27:14 PM (5 years ago)

Author:

bird

Message:

kWorker: Adjustments for VC++ 14.2. Fixed bug in GetFileAttributesExW/A where we'd forget to set the return value on success. Reduced the size of the Tls DLLs, but having to add more as the re-load-same-dll hack doesn't work anymore.

Location:

trunk/src/kWorker

Files:

• Makefile.kmk (modified) (3 diffs)
• kWorker.c (modified) (104 diffs)
• kWorkerTlsXxxK.c (modified) (2 diffs)

trunk/src/kWorker/Makefile.kmk

@@ -34 +34 @@
 kWorker_DEFS.debug = K_STRICT
 kWorker_DEFS.release = NASSERT
+ifeq ($(USERNAME),bird)
+kWorker_CFLAGS = -W4 -wd4127 -wd4100 -wd4214 -wd4201 -wd4204
+endif
 kWorker_SOURCES = \
         kWorker.c \
@@ -50 +53 @@
         $(PATH_SDK_WINDDK71_LIB_WNET)/psapi.lib
 kWorker_LDFLAGS.win = \
-        /BASE:0x10000 /DYNAMICBASE:NO /FIXED
+        /DYNAMICBASE:NO /FIXED
+kWorker_LDFLAGS.win.x86   = /BASE:0x00010000
+kWorker_LDFLAGS.win.amd64 = /BASE:0x0000000420000000
+
 #kWorker_LDFLAGS.win.x86 = \
 #       /SAFESEH:NO - doesn't help anyone.
@@ -159 +165 @@
 # A couple of dummy DLLs we use for grabbing LDR TLS entries.
 #
-DLLS += kWorkerTls1K kWorkerTls64K kWorkerTls512K
+DLLS += kWorkerTls1K kWorkerTls1K01 kWorkerTls1K02 kWorkerTls1K03 kWorkerTls1K04 kWorkerTls1K05 kWorkerTls1K06 kWorkerTls1K07 \
+        kWorkerTls1K08 kWorkerTls1K09 kWorkerTls1K10 kWorkerTls1K11 kWorkerTls1K12 kWorkerTls1K13 kWorkerTls1K14 kWorkerTls1K15
 kWorkerTls1K_TEMPLATE   = BIN-STATIC-THREADED
-kWorkerTls1K_DEFS       = KWORKER_BASE=0x10000 TLS_SIZE=1024
+kWorkerTls1K_DEFS       = TLS_SIZE=1024
 kWorkerTls1K_SOURCES    = kWorkerTlsXxxK.c
 kWorkerTls1K_LDFLAGS    = /Entry:DummyDllEntry
 
+kWorkerTls1K01_EXTENDS = kWorkerTls1K
+kWorkerTls1K02_EXTENDS = kWorkerTls1K
+kWorkerTls1K03_EXTENDS = kWorkerTls1K
+kWorkerTls1K04_EXTENDS = kWorkerTls1K
+kWorkerTls1K05_EXTENDS = kWorkerTls1K
+kWorkerTls1K06_EXTENDS = kWorkerTls1K
+kWorkerTls1K07_EXTENDS = kWorkerTls1K
+kWorkerTls1K08_EXTENDS = kWorkerTls1K
+kWorkerTls1K09_EXTENDS = kWorkerTls1K
+kWorkerTls1K10_EXTENDS = kWorkerTls1K
+kWorkerTls1K11_EXTENDS = kWorkerTls1K
+kWorkerTls1K12_EXTENDS = kWorkerTls1K
+kWorkerTls1K13_EXTENDS = kWorkerTls1K
+kWorkerTls1K14_EXTENDS = kWorkerTls1K
+kWorkerTls1K15_EXTENDS = kWorkerTls1K
+
+DLLS += kWorkerTls64K kWorkerTls512K
+
 kWorkerTls64K_TEMPLATE  = BIN-STATIC-THREADED
-kWorkerTls64K_DEFS      = KWORKER_BASE=0x10000 TLS_SIZE=65536
+kWorkerTls64K_DEFS      = TLS_SIZE=65536
 kWorkerTls64K_SOURCES   = kWorkerTlsXxxK.c
 kWorkerTls64K_LDFLAGS   = /Entry:DummyDllEntry
 
 kWorkerTls512K_TEMPLATE = BIN-STATIC-THREADED
-kWorkerTls512K_DEFS     = KWORKER_BASE=0x10000 TLS_SIZE=524288
+kWorkerTls512K_DEFS     = TLS_SIZE=524288
 kWorkerTls512K_SOURCES  = kWorkerTlsXxxK.c
 kWorkerTls512K_LDFLAGS  = /Entry:DummyDllEntry

trunk/src/kWorker/kWorker.c

@@ -320 +320 @@
     /** The windows module handle. */
     HMODULE             hOurMod;
+    /** Parent (real) module if this is a virtual API module (api-ms-*.dll or
+     * ext-ms-*.dll). Referenced. */
+    PKWMODULE           pVirtualApiMod;
     /** The of the loaded image bits. */
     KSIZE               cbImage;
@@ -348 +351 @@
             /** The state. */
             KWMODSTATE          enmState;
+            /** The re-init state. */
+            KWMODSTATE          enmReInitState;
 #if defined(KBUILD_OS_WINDOWS) && defined(KBUILD_ARCH_AMD64)
             /** The number of entries in the table. */
@@ -443 +448 @@
 
 
+/** One TLS DLL. */
+typedef struct KWTLSDLL
+{
+    const wchar_t  *pwszName;   /**< The DLL name. */
+    KBOOL           fUsed;      /**< Set if used, clear if not. */
+} KWTLSDLL;
+typedef KWTLSDLL *PKWTLSDLL;
+
+/**
+ * TLS DLL tracker.
+ */
+typedef struct KWTLSDLLENTRY
+{
+    KU32            cbTls;      /**< Max TLS size. */
+    KU32            cDlls;      /**< Number of DLLs we ship (paDlls). */
+    PKWTLSDLL       paDlls;     /**< Array of DLLs we ship. */
+} KWTLSDLLENTRY;
+typedef KWTLSDLLENTRY *PKWTLSDLLENTRY;
+
+
 /**
  * A cached file.
@@ -794 +819 @@
 typedef struct KWSANDBOX
 {
+    /** Jump buffer (first for alignment reasons). */
+    jmp_buf     JmpBuf;
     /** The tool currently running in the sandbox. */
     PKWTOOL     pTool;
-    /** Jump buffer. */
-    jmp_buf     JmpBuf;
     /** The thread ID of the main thread (owner of JmpBuf). */
     DWORD       idMainThread;
@@ -866 +891 @@
     PKWFSTEMPFILE   pTempFileHead;
 
+    /** Critical section protecting pVirtualAllocHead. */
+    CRITICAL_SECTION VirtualAllocLock;
     /** Head of the virtual alloc allocations. */
     PKWVIRTALLOC    pVirtualAllocHead;
@@ -948 +975 @@
     KU32        iSlot;
 
-    /** The CRT module data.    */
+    /** The CRT module data. */
     PKWMODULE   pModule;
-    /** Pointer to the malloc function.   */
+    /** Pointer to the malloc function. */
     void * (__cdecl *pfnMalloc)(size_t);
+    /** Pointer to the beginthreadex function. */
+    uintptr_t (__cdecl *pfnBeginThreadEx)(void *, unsigned, unsigned (__stdcall *)(void *), void *, unsigned, unsigned *);
 
 } KWCRTSLOT;
@@ -1064 +1093 @@
 static PKWMODULE    g_pModPendingTlsAlloc = NULL;
 
+/** The 1KB TLS DLLs. */
+static KWTLSDLL     g_aTls1KDlls[] =
+{
+    { L"kWorkerTls1K.dll",   K_FALSE },
+    { L"kWorkerTls1K01.dll", K_FALSE },
+    { L"kWorkerTls1K02.dll", K_FALSE },
+    { L"kWorkerTls1K03.dll", K_FALSE },
+    { L"kWorkerTls1K04.dll", K_FALSE },
+    { L"kWorkerTls1K05.dll", K_FALSE },
+    { L"kWorkerTls1K06.dll", K_FALSE },
+    { L"kWorkerTls1K07.dll", K_FALSE },
+    { L"kWorkerTls1K08.dll", K_FALSE },
+    { L"kWorkerTls1K09.dll", K_FALSE },
+    { L"kWorkerTls1K10.dll", K_FALSE },
+    { L"kWorkerTls1K11.dll", K_FALSE },
+    { L"kWorkerTls1K12.dll", K_FALSE },
+    { L"kWorkerTls1K13.dll", K_FALSE },
+    { L"kWorkerTls1K14.dll", K_FALSE },
+    { L"kWorkerTls1K15.dll", K_FALSE },
+};
+
+/** The 64KB TLS DLLs. */
+static KWTLSDLL     g_aTls64KDlls[] =
+{
+    { L"kWorkerTls64K.dll", K_FALSE },
+};
+
+/** The 512KB TLS DLLs. */
+static KWTLSDLL     g_aTls512KDlls[] =
+{
+    { L"kWorkerTls512K.dll", K_FALSE },
+};
+
+/** The TLS DLLs grouped by size. */
+static KWTLSDLLENTRY const g_aTlsDlls[] =
+{
+    {     1024, K_ELEMENTS(g_aTls1KDlls),   g_aTls1KDlls },
+    {  64*1024, K_ELEMENTS(g_aTls64KDlls),  g_aTls64KDlls },
+    { 512*1024, K_ELEMENTS(g_aTls512KDlls), g_aTls512KDlls },
+};
+
 /** CRT slots.
  * @note The number of entires here must match CRT_SLOT_FUNCTION_WRAPPER. */
 static KWCRTSLOT    g_aCrtSlots[32];
+
+/** windbg .reload statements. vs   */
+char                g_szReloads[4096];
+/** Current offset into g_szReloads. */
+KU32 volatile       g_cchReloads;
 
 /** The file system cache. */
@@ -1206 +1281 @@
                                        const char *pszSearchPath, PKWMODULE *ppMod);
 static PKWMODULE kwLdrModuleForLoadedNative(const char *pszName, KBOOL fEnsureCrtSlot, KBOOL fAlwaysPresent);
+static PKWMODULE kwLdrModuleForLoadedNativeByHandle(HMODULE hModule, KBOOL fEnsureCrtSlot, const char *pszLogName);
+static int kwLdrModuleCreateCrtSlot(PKWMODULE pModule);
+static PKWMODULE kwToolLocateModuleByHandle(PKWTOOL pTool, HMODULE hmod);
 static char *kwSandboxDoGetEnvA(PKWSANDBOX pSandbox, const char *pchVar, KSIZE cchVar);
 static KBOOL kwSandboxHandleTableEnter(PKWSANDBOX pSandbox, PKWHANDLE pHandle, HANDLE hHandle);
@@ -1771 +1849 @@
 {
     kHlpAssert(pMod->cRefs > 0);
-    kHlpAssert(pMod->cRefs < 64);
+    kHlpAssert(pMod->cRefs < 64 || pMod->fNative /* kernelbase.dll and VC++ 14.2 */);
     pMod->cRefs++;
     return pMod;
@@ -1789 +1867 @@
         if (!pMod->fExe)
         {
-            PKWMODULE pPrev = NULL;
             unsigned  idx   = pMod->uHashPath % K_ELEMENTS(g_apModules);
             if (g_apModules[idx] == pMod)
@@ -1864 +1941 @@
     }
     else
-        kHlpAssert(pMod->cRefs < 64);
+        kHlpAssert(pMod->cRefs < 64 || pMod->fNative /* kernelbase.dll and VC++ 14.2 */);
 }
 
@@ -1968 +2045 @@
                 {
                     IMAGE_IMPORT_BY_NAME const *pName     = (IMAGE_IMPORT_BY_NAME const *)&pbImage[off];
-                    KSIZE const                 cchSymbol = kHlpStrLen(pName->Name);
+                    KSIZE const                 cchSymbol = kHlpStrLen((const char *)&pName->Name[0]);
                     KU32                        i         = g_cSandboxNativeReplacements;
                     while (i-- > 0)
@@ -1977 +2054 @@
                                 || kHlpStrICompAscii(g_aSandboxNativeReplacements[i].pszModule, pszImport) == 0)
                             {
-                                KW_LOG(("%s: replacing %s!%s\n", pMod->pLdrMod->pszName, pszImport, pName->Name));
+                                KWLDR_LOG(("%s: replacing %s!%s\n", pMod->pLdrMod->pszName, pszImport, pName->Name));
 
                                 /* The .rdata section is normally read-only, so we need to make it writable first. */
@@ -2003 +2080 @@
                                             case PAGE_EXECUTE_WRITECOPY:
                                                 /* Already writable, nothing to do. */
+                                                fRc = TRUE;
                                                 break;
 
@@ -2081 +2159 @@
  */
 static PKWMODULE kwLdrModuleCreateForNativekLdrModule(PKLDRMOD pLdrMod, const char *pszPath, KSIZE cbPath, KU32 uHashPath,
-                                                      KBOOL fDoReplacements)
+                                                      KBOOL fDoReplacements, PKWMODULE pVirtualApiMod)
 {
     /*
@@ -2089 +2167 @@
     if (pMod)
     {
-        pMod->pwszPath      = (wchar_t *)(pMod + 1);
+        pMod->pwszPath          = (wchar_t *)(pMod + 1);
         kwStrToUtf16(pszPath, (wchar_t *)pMod->pwszPath, cbPath * 2);
-        pMod->pszPath       = (char *)kHlpMemCopy((char *)&pMod->pwszPath[cbPath * 2], pszPath, cbPath);
-        pMod->uHashPath     = uHashPath;
-        pMod->cRefs         = 1;
-        pMod->offFilename   = (KU16)(kHlpGetFilename(pszPath) - pszPath);
-        pMod->offFilenameW  = (KU16)(kwPathGetFilenameW(pMod->pwszPath) - pMod->pwszPath);
-        pMod->fExe          = K_FALSE;
-        pMod->fNative       = K_TRUE;
-        pMod->pLdrMod       = pLdrMod;
-        pMod->hOurMod       = (HMODULE)(KUPTR)pLdrMod->aSegments[0].MapAddress;
-        pMod->cbImage       = (KSIZE)kLdrModSize(pLdrMod);
-        pMod->iCrtSlot      = KU8_MAX;
-        pMod->fNeedReInit   = K_FALSE;
+        pMod->pszPath           = (char *)kHlpMemCopy((char *)&pMod->pwszPath[cbPath * 2], pszPath, cbPath);
+        pMod->uHashPath         = uHashPath;
+        pMod->cRefs             = 1;
+        pMod->offFilename       = (KU16)(kHlpGetFilename(pszPath) - pszPath);
+        pMod->offFilenameW      = (KU16)(kwPathGetFilenameW(pMod->pwszPath) - pMod->pwszPath);
+        pMod->fExe              = K_FALSE;
+        pMod->fNative           = K_TRUE;
+        pMod->pLdrMod           = pLdrMod;
+        pMod->hOurMod           = (HMODULE)(KUPTR)pLdrMod->aSegments[0].MapAddress;
+        pMod->cbImage           = (KSIZE)kLdrModSize(pLdrMod);
+        pMod->iCrtSlot          = KU8_MAX;
+        pMod->fNeedReInit       = K_FALSE;
         pMod->pszMsPdbSrvEndpoint = NULL;
         pMod->fReInitOnMsPdbSrvEndpointChange = kHlpStrNICompAscii(&pMod->pszPath[pMod->offFilename], TUPLE("mspdb")) == 0;
+        pMod->pVirtualApiMod    = pVirtualApiMod;
+        if (pVirtualApiMod)
+            kwLdrModuleRetain(pVirtualApiMod);
 
         if (fDoReplacements)
@@ -2113 +2194 @@
         }
 
-        KW_LOG(("New module: %p LB %#010x %s (native)\n",
-                (KUPTR)pMod->pLdrMod->aSegments[0].MapAddress, kLdrModSize(pMod->pLdrMod), pMod->pszPath));
+        KWLDR_LOG(("New module: %p LB %#010x %s (native%s%s)\n",
+                   (KUPTR)pMod->pLdrMod->aSegments[0].MapAddress, kLdrModSize(pMod->pLdrMod), pMod->pszPath,
+                   pVirtualApiMod ? ", virtual api => " : "", pVirtualApiMod ? pVirtualApiMod->pszPath : ""));
         g_cModules++;
         return kwLdrModuleLink(pMod);
@@ -2134 +2216 @@
 static PKWMODULE kwLdrModuleCreateNative(const char *pszPath, KU32 uHashPath, KBOOL fDoReplacements)
 {
+    PKLDRMOD pLdrMod;
+    int rc;
+
     /*
-     * Open the module and check the type.
+     * HACK ALERT! Make sure the application path is searched when looking for
+     * imports in the module we're loading.
      */
-    PKLDRMOD pLdrMod;
-    int rc = kLdrModOpenNative(pszPath, KLDRMOD_OPEN_FLAGS_NATIVE_ALLOW_INIT_TERM, &pLdrMod);
+    /** @todo improve on this hack!  */
+    PKWMODULE pExe = g_Sandbox.pTool ? g_Sandbox.pTool->u.Sandboxed.pExe : NULL;
+    if (pExe)
+    {
+        /* HACK ALERT! */
+        wchar_t *pwzFilename = (wchar_t *)&pExe->pwszPath[pExe->offFilenameW];
+        wchar_t wcSaved = pExe->pwszPath[pExe->offFilenameW];
+        *pwzFilename = '\0';
+        if (!SetDllDirectoryW(pExe->pwszPath))
+            kwErrPrintf("SetDllDirectoryW failed: %u\n", GetLastError());
+        *pwzFilename = wcSaved;
+    }
+
+    /*
+     * Load the library and create a module structure for it.
+     */
+    rc = kLdrModOpenNative(pszPath, KLDRMOD_OPEN_FLAGS_NATIVE_ALLOW_INIT_TERM, &pLdrMod);
     if (rc == 0)
     {
-        PKWMODULE pMod = kwLdrModuleCreateForNativekLdrModule(pLdrMod, pszPath, kHlpStrLen(pszPath) + 1,
-                                                              uHashPath, fDoReplacements);
+        KSIZE     cchPath = kHlpStrLen(pszPath);
+        PKWMODULE pMod = kwLdrModuleCreateForNativekLdrModule(pLdrMod, pszPath, cchPath + 1, uHashPath,
+                                                              fDoReplacements, NULL /*pVirtualApiMod*/);
         if (pMod)
             return pMod;
@@ -2148 +2250 @@
     }
     return NULL;
+}
+
+
+/**
+ * Checks if the given name could be a virtual API module or not.
+ */
+static KBOOL kwLdrIsVirtualApiModule(const char *pszName, KSIZE cchName)
+{
+    if (cchName <= 7)
+        return K_FALSE;
+    switch (*pszName)
+    {
+        default:
+            return K_FALSE;
+        case 'a':
+        case 'A':
+            if (pszName[1] != 'p' && pszName[1] != 'P')
+                return K_FALSE;
+            if (pszName[2] != 'i' && pszName[2] != 'I')
+                return K_FALSE;
+            break;
+        case 'e':
+        case 'E':
+            if (pszName[1] != 'x' && pszName[1] != 'X')
+                return K_FALSE;
+            if (pszName[2] != 't' && pszName[2] != 'T')
+                return K_FALSE;
+            break;
+    }
+    if (pszName[3] != '-')
+        return K_FALSE;
+    if (pszName[4] != 'm' && pszName[4] != 'M')
+        return K_FALSE;
+    if (pszName[5] != 's' && pszName[5] != 'S')
+        return K_FALSE;
+    if (pszName[6] != '-')
+        return K_FALSE;
+    return K_TRUE;
+}
+
+
+/**
+ * Try load what seems to be a virtual API DLL.
+ *
+ * This is a worker for kwLdrModuleResolveAndLookup and
+ * kwSandbox_Kernel32_LoadLibraryExA_VirtualApiModule.
+ *
+ * @returns Pointer to module on success, NULL on failure.
+ * @param   pszName             The name of the module.  This must be
+ *                              normalized already!
+ * @param   cchName             The length of the name.
+ */
+static PKWMODULE kwLdrModuleTryLoadVirtualDll(const char *pszName, KSIZE cchName)
+{
+    HMODULE     hModule;
+
+    /*
+     * Look it up in the hash table.
+     */
+    KU32 const  uHashPath = kwStrHash(pszName);
+    unsigned    idxHash   = uHashPath % K_ELEMENTS(g_apModules);
+    PKWMODULE   pMod      = g_apModules[idxHash];
+    if (pMod)
+    {
+        do
+        {
+            if (   pMod->uHashPath == uHashPath
+                && kHlpStrComp(pMod->pszPath, pszName) == 0)
+                return kwLdrModuleRetain(pMod);
+            pMod = pMod->pNextHash;
+        } while (pMod);
+    }
+
+    /*
+     * Not found. Try load it.
+     */
+    hModule = LoadLibraryA(pszName);
+    if (!hModule)
+    {
+        KWLDR_LOG(("kwLdrModuleTryLoadVirtualDll: %s failed (%u)\n", pszName, GetLastError()));
+        return NULL;
+    }
+
+    /*
+     * Loaded successfully.  Create a module for the real module.
+     */
+    pMod = kwLdrModuleForLoadedNativeByHandle(hModule, K_FALSE /*fEnsureCrtSlot*/, pszName);
+    if (pMod)
+    {
+        /* Create a module for the virtual API name too, unless it is actually a real DLL. */
+        if (stricmp(&pMod->pszPath[pMod->offFilename], pszName) != 0)
+        {
+            PKLDRMOD pLdrMod;
+            int rc = kLdrModOpenNativeByHandle((KUPTR)hModule, KLDRMOD_OPEN_FLAGS_NATIVE_ALLOW_INIT_TERM, &pLdrMod);
+            if (rc == 0)
+            {
+                PKWMODULE pVirtMod = kwLdrModuleCreateForNativekLdrModule(pLdrMod, pszName, cchName + 1, kwStrHash(pszName),
+                                                                          K_FALSE /*fDoReplacements*/, pMod /*pVirtualApiMod*/);
+                if (pVirtMod)
+                {
+                    kwLdrModuleRelease(pMod);
+                    pMod = pVirtMod;
+                }
+                else
+                {
+                    kLdrModClose(pLdrMod);
+                    kwErrPrintf("out of memory\n");
+                }
+            }
+            else
+                kwErrPrintf("kLdrModOpenNativeByHandle failed for %p / '%s': %d\n", hModule, pszName, rc);
+        }
+        else
+            KWLDR_LOG(("kwLdrModuleTryLoadVirtualDll: %s -> %s - A real DLL!\n", pszName, pMod->pszPath));
+    }
+
+    return pMod;
 }
 
@@ -2214 +2433 @@
                             pMod->u.Manual.aQuickCopyChunks[iChunk].pbSrc    = &pMod->u.Manual.pbCopy[(KSIZE)paSegs[iSeg].RVA];
                             pMod->u.Manual.aQuickCopyChunks[iChunk].cbToCopy = paSegs[iSeg].cbMapped - cbTrailingZeros;
-                            pMod->u.Manual.cQuickCopyChunks = iChunk + 1;
+                            pMod->u.Manual.cQuickCopyChunks = (KU8)(iChunk + 1);
                             KWLDR_LOG(("aQuickCopyChunks[%u]: %#p LB %#" KSIZE_PRI " <- %p (%*.*s)\n", iChunk,
                                        pMod->u.Manual.aQuickCopyChunks[iChunk].pbDst,
@@ -2231 +2450 @@
                                                                             + pMod->u.Manual.aQuickCopyChunks[iChunk].cbToCopy;
                             pMod->u.Manual.aQuickZeroChunks[iZero].cbToZero = cbTrailingZeros;
-                            pMod->u.Manual.cQuickZeroChunks = iZero + 1;
+                            pMod->u.Manual.cQuickZeroChunks = (KU8)(iZero + 1);
                             KWLDR_LOG(("aQuickZeroChunks[%u]: %#p LB %#" KSIZE_PRI " <- zero (%*.*s)\n", iZero,
                                        pMod->u.Manual.aQuickZeroChunks[iZero].pbDst,
@@ -2286 +2505 @@
          * Try sabotage the DLL name so we can load this module again.
          */
+/** @todo this doesn't work W10 18363   */
         pHead = &pPeb->Ldr->InMemoryOrderModuleList;
         for (pCur = pHead->Blink; pCur != pHead; pCur = pCur->Blink)
@@ -2330 +2550 @@
         KU32 const                  cEntries  = pTlsDir->Size / sizeof(IMAGE_TLS_DIRECTORY);
         KU32                        iEntry;
+        KU32                        iTlsDll;
+        KU32                        iTlsDllSub;
         KUPTR                       offIndex;
         KUPTR                       offCallbacks;
@@ -2384 +2606 @@
         /*
          * Make the allocation by loading a new instance of one of the TLS dlls.
-         * The DLL will make a call to
+         * The DLL will make a call to kwLdrTlsAllocationHook.
          */
         offIndex     = (KUPTR)paEntries[0].AddressOfIndex     - (KUPTR)pMod->u.Manual.pbLoad;
@@ -2390 +2612 @@
         puCallbacks  = (KUPTR const *)&pbImg[offCallbacks];
         cbData = paEntries[0].SizeOfZeroFill + (paEntries[0].EndAddressOfRawData - paEntries[0].StartAddressOfRawData);
-        if (cbData <= 1024)
-            pwszTlsDll = L"kWorkerTls1K.dll";
-        else if (cbData <= 65536)
-            pwszTlsDll = L"kWorkerTls64K.dll";
-        else if (cbData <= 524288)
-            pwszTlsDll = L"kWorkerTls512K.dll";
-        else
-        {
-            kwErrPrintf("TLS data size in %s is too big: %u (%#p), max 512KB\n", pMod->pszPath, (unsigned)cbData, cbData);
-            return -1;
-        }
+
+        /** @todo find better strategy here. Like temporary copy or whatever when
+         *        there is more than a single user. */
+        for (iTlsDll = 0; cbData > g_aTlsDlls[iTlsDll].cbTls;)
+            if (++iTlsDll >= K_ELEMENTS(g_aTlsDlls))
+            {
+                kwErrPrintf("TLS data size in %s is too big: %u (%#p), max 512KB\n", pMod->pszPath, (unsigned)cbData, cbData);
+                return -1;
+            }
+        for (iTlsDllSub = 0; g_aTlsDlls[iTlsDll].paDlls[iTlsDllSub].fUsed;)
+            if (++iTlsDllSub >= g_aTlsDlls[iTlsDll].cDlls)
+            {
+                kwErrPrintf("No unused TLS DLLs for %s of size %u!\n", pMod->pszPath, (unsigned)cbData);
+                return -1;
+            }
+
+        g_aTlsDlls[iTlsDll].paDlls[iTlsDllSub].fUsed = K_TRUE;
+        pwszTlsDll = g_aTlsDlls[iTlsDll].paDlls[iTlsDllSub].pwszName;
 
         pMod->u.Manual.idxTls         = KU32_MAX;
@@ -2542 +2771 @@
                         {
                             KI32 iImp;
+                            KU32 cchReloads;
 
                             /*
@@ -2548 +2778 @@
                             pMod->hOurMod = (HMODULE)pMod->u.Manual.pbLoad;
                             kwLdrModuleLink(pMod);
-                            KW_LOG(("New module: %p LB %#010x %s (kLdr)\n",
-                                    pMod->u.Manual.pbLoad, pMod->cbImage, pMod->pszPath));
-                            KW_LOG(("TODO: .reload /f %s=%p\n", pMod->pszPath, pMod->u.Manual.pbLoad));
+                            KWLDR_LOG(("New module: %p LB %#010x %s (kLdr)\n",
+                                       pMod->u.Manual.pbLoad, pMod->cbImage, pMod->pszPath));
+                            KWLDR_LOG(("TODO: .reload /f %s=%p\n", pMod->pszPath, pMod->u.Manual.pbLoad));
                             kwDebuggerPrintf("TODO: .reload /f %s=%p\n", pMod->pszPath, pMod->u.Manual.pbLoad);
+                            cchReloads = g_cchReloads;
+                            if (cchReloads + 80 < sizeof(g_szReloads))
+                            {
+                                cchReloads += _snprintf(&g_szReloads[cchReloads], sizeof(g_szReloads) - cchReloads,
+                                                        "%s.reload /f %s=%p\n", cchReloads ? "; " : "",
+                                                        pMod->pszPath, pMod->u.Manual.pbLoad);
+                                g_cchReloads = cchReloads;
+                            }
 
                             for (iImp = 0; iImp < cImports; iImp++)
@@ -2611 +2849 @@
                                         pMod->u.Manual.pvBits = pMod->u.Manual.pbCopy;
                                         pMod->u.Manual.enmState = KWMODSTATE_NEEDS_BITS;
+                                        pMod->u.Manual.enmReInitState = KWMODSTATE_NEEDS_BITS;
+                                        if (   g_Sandbox.pTool
+                                            && (   g_Sandbox.pTool->u.Sandboxed.enmHint == KWTOOLHINT_VISUAL_CPP_CL
+                                                || g_Sandbox.pTool->u.Sandboxed.enmHint == KWTOOLHINT_VISUAL_CPP_LINK)
+                                            && !pMod->fExe)
+                                            pMod->u.Manual.enmReInitState = KWMODSTATE_READY;
                                         g_cModules++;
                                         g_cNonNativeModules++;
@@ -2675 +2919 @@
                         || !g_aSandboxReplacements[i].fOnlyExe)
                     {
-                        KW_LOG(("replacing %s!%s\n",&pImpMod->pszPath[pImpMod->offFilename], g_aSandboxReplacements[i].pszFunction));
-                        *puValue = g_aSandboxReplacements[i].pfnReplacement;
+                        KWLDR_LOG(("replacing %s!%s\n",&pImpMod->pszPath[pImpMod->offFilename], g_aSandboxReplacements[i].pszFunction));
+                        if (!g_aSandboxReplacements[i].fCrtSlotArray)
+                            *puValue = g_aSandboxReplacements[i].pfnReplacement;
+                        else
+                        {
+                            if (pImpMod->iCrtSlot == KU8_MAX)
+                            {
+                                rc = kwLdrModuleCreateCrtSlot(pImpMod);
+                                if (rc)
+                                    KWLDR_LOG(("kwLdrModuleGetImportCallback: kwLdrModuleCreateCrtSlot failed: %d\n", rc));
+                            }
+                            *puValue = ((KUPTR *)g_aSandboxReplacements[i].pfnReplacement)[pImpMod->iCrtSlot];
+                        }
                     }
                     break;
@@ -2684 +2939 @@
 
     //printf("iImport=%u (%s) %*.*s rc=%d\n", iImport, &pImpMod->pszPath[pImpMod->offFilename], cchSymbol, cchSymbol, pchSymbol, rc);
-    KW_LOG(("iImport=%u (%s) %*.*s rc=%d\n", iImport, &pImpMod->pszPath[pImpMod->offFilename], cchSymbol, cchSymbol, pchSymbol, rc));
+    KWLDR_LOG(("iImport=%u (%s) %*.*s rc=%d\n", iImport, &pImpMod->pszPath[pImpMod->offFilename], cchSymbol, cchSymbol, pchSymbol, rc));
     return rc;
 
@@ -2721 +2976 @@
     if (enmLocation != KWLOCATION_SYSTEM32)
         return K_TRUE;
-    return kHlpStrNICompAscii(pszFilename, TUPLE("msvc"))   == 0
-        || kHlpStrNICompAscii(pszFilename, TUPLE("msdis"))  == 0
-        || kHlpStrNICompAscii(pszFilename, TUPLE("mspdb"))  == 0;
+    return kHlpStrNICompAscii(pszFilename, TUPLE("msvc"))            == 0
+        || kHlpStrNICompAscii(pszFilename, TUPLE("msdis"))           == 0
+        || kHlpStrNICompAscii(pszFilename, TUPLE("mspdb"))           == 0
+        || kHlpStrNICompAscii(pszFilename, TUPLE("vcruntime"))       == 0
+        || kHlpStrNICompAscii(pszFilename, TUPLE("ucrtbase"))        == 0
+        || kHlpStrNICompAscii(pszFilename, TUPLE("api-ms-win-crt-")) == 0
+        ;
 }
 
@@ -2775 +3034 @@
     if (enmLocation == KWLOCATION_UNKNOWN_NATIVE)
         return K_TRUE;
+    if (   enmLocation == KWLOCATION_UNKNOWN
+        && kwLdrIsVirtualApiModule(pszFilename, kHlpStrLen(pszFilename)))
+        return K_TRUE;
 
     /* If the location is unknown, we must check if it's some dynamic loading
@@ -2797 +3059 @@
     }
 
-    return kHlpStrNICompAscii(pszFilename, TUPLE("msvc"))   == 0
-        || kHlpStrNICompAscii(pszFilename, TUPLE("msdis"))  == 0
-        || kHlpStrNICompAscii(pszFilename, TUPLE("mspdb"))  == 0;
+    return kHlpStrNICompAscii(pszFilename, TUPLE("msvc"))      == 0
+        || kHlpStrNICompAscii(pszFilename, TUPLE("msdis"))     == 0
+        || kHlpStrNICompAscii(pszFilename, TUPLE("mspdb"))     == 0
+        || kHlpStrNICompAscii(pszFilename, TUPLE("tbbmalloc")) == 0
+        || kHlpStrNICompAscii(pszFilename, TUPLE("ucrtbase"))  == 0
+        || kHlpStrNICompAscii(pszFilename, TUPLE("vcruntime")) == 0
+        || (   enmLocation != KWLOCATION_UNKNOWN
+            && kwLdrIsVirtualApiModule(pszFilename, kHlpStrLen(pszFilename)));
 }
 
@@ -2929 +3196 @@
     KSIZE       cchSuffix   = fNeedSuffix ? 4 : 0;
 
+    /* Virtual API module.  Normalize and try load it. */
+    if (pMod == NULL && cchName > 7 && kwLdrIsVirtualApiModule(pszName, cchName))
+    {
+        if (cchName + cchSuffix >= sizeof(szPath))
+            return KERR_BUFFER_OVERFLOW;
+        kHlpMemCopy(szPath, pszName, cchName);
+        if (fNeedSuffix)
+            kHlpMemCopy(&szPath[cchName], ".dll", sizeof(".dll"));
+        szPath[cchName + cchSuffix] = '\0';
+        _strlwr(szPath);
+        pMod = kwLdrModuleTryLoadVirtualDll(szPath, cchName + cchSuffix);
+    }
 
     /* The import path. */
@@ -3043 +3322 @@
                 kwErrPrintf("Failed to resolved 'malloc' in '%s': %d\n", pModule->pszPath, rc);
 
+            rc = kLdrModQuerySymbol(pModule->pLdrMod, NULL /*pvBits*/, KLDRMOD_BASEADDRESS_MAP, KU32_MAX, "_beginthreadex", 14,
+                                    NULL /*pvszVersion*/, NULL /*pfnGetForwarder*/, NULL /*pvUser*/, &uAddr, NULL);
+            *(KUPTR *)&g_aCrtSlots[iSlot].pfnBeginThreadEx = rc == 0 ? (KUPTR)uAddr : 0;
+            //if (rc != 0)
+            //    kwErrPrintf("Failed to resolved '_beginthreadex' in '%s': %d\n", pModule->pszPath, rc);
+
             return 0;
         }
     kwErrPrintf("Out of CRT slots!\n");
     return KERR_NO_MEMORY;
+}
+
+
+/**
+ * Locates the module structure for an already loaded native module.
+ *
+ * This will create a module structure if needed.
+ *
+ * @returns Pointer to the module structure on success, NULL on failure.
+ * @param   hModule         The native module handle.
+ * @param   fEnsureCrtSlot  Whether to ensure that it has a valid CRT slot.
+ * @param   pszLogName      The name to use for logging/errors.
+ */
+static PKWMODULE kwLdrModuleForLoadedNativeByHandle(HMODULE hModule, KBOOL fEnsureCrtSlot, const char *pszLogName)
+{
+    /*
+     * Get a normalized path for it.
+     */
+    char szModPath[1024];
+    if (GetModuleFileNameA(hModule, szModPath, sizeof(szModPath)) > 0)
+    {
+        char szNormPath[1024];
+        int rc = kwPathNormalize(szModPath, szNormPath, sizeof(szNormPath));
+        if (rc == 0)
+        {
+            /*
+             * Hash the path and look it up.
+             */
+            KU32        uHashPath;
+            KSIZE const cchPath   = kwStrHashEx(szNormPath, &uHashPath);
+            unsigned    idxHash   = uHashPath % K_ELEMENTS(g_apModules);
+            PKWMODULE   pMod      = g_apModules[idxHash];
+            if (pMod)
+            {
+                do
+                {
+                    if (   pMod->uHashPath == uHashPath
+                        && kHlpStrComp(pMod->pszPath, szNormPath) == 0)
+                    {
+                        kwLdrModuleRetain(pMod);
+                        break;
+                    }
+                    pMod = pMod->pNextHash;
+                } while (pMod);
+            }
+
+            /*
+             * If not in the hash table, so create a module entry.
+             */
+            if (!pMod)
+            {
+                PKLDRMOD pLdrMod;
+                rc = kLdrModOpenNativeByHandle((KUPTR)hModule, KLDRMOD_OPEN_FLAGS_NATIVE_ALLOW_INIT_TERM, &pLdrMod);
+                if (rc == 0)
+                {
+                    /** @todo more accurately determine location  */
+                    const char *pszFilename = kHlpGetFilename(szNormPath);
+                    KBOOL fDoReplacements = kwLdrModuleShouldDoNativeReplacements(pszFilename, KWLOCATION_SYSTEM32);
+                    pMod = kwLdrModuleCreateForNativekLdrModule(pLdrMod, szNormPath, cchPath + 1, uHashPath,
+                                                                fDoReplacements, NULL /*pVirtualApiMod*/);
+                    if (!pMod)
+                    {
+                        kLdrModClose(pLdrMod);
+                        kwErrPrintf("out of memory\n");
+                    }
+                }
+                else
+                    kwErrPrintf("kLdrModOpenNativeByHandle failed for %p / '%s': %d\n", hModule, pszLogName, rc);
+            }
+            if (pMod)
+            {
+                /*
+                 * Create a CRT slot for the module if necessary.
+                 */
+                if (!fEnsureCrtSlot || pMod->iCrtSlot != KU8_MAX)
+                    return pMod;
+                rc = kwLdrModuleCreateCrtSlot(pMod);
+                if (rc == 0)
+                    return pMod;
+                kwLdrModuleRelease(pMod);
+            }
+        }
+        else
+            kwErrPrintf("kwPathNormalize failed for '%s' (%s): %u!\n", szModPath, pszLogName, GetLastError());
+    }
+    else
+        kwErrPrintf("GetModuleFileNameA failed for '%s': %u!\n", pszLogName, GetLastError());
+    return NULL;
 }
 
@@ -3064 +3437 @@
 {
     /*
-     * Locate the module and get a normalized path for it.
+     * Locate the module handle and pass it to kwLdrModuleForLoadedNativeByHandle.
      */
     HANDLE hModule = GetModuleHandleA(pszName);
     if (hModule)
-    {
-        char szModPath[1024];
-        if (GetModuleFileNameA(hModule, szModPath, sizeof(szModPath)) > 0)
-        {
-            char szNormPath[1024];
-            int rc = kwPathNormalize(szModPath, szNormPath, sizeof(szNormPath));
-            if (rc == 0)
-            {
-                /*
-                 * Hash the path and look it up.
-                 */
-                KU32        uHashPath;
-                KSIZE const cchPath   = kwStrHashEx(szNormPath, &uHashPath);
-                unsigned    idxHash   = uHashPath % K_ELEMENTS(g_apModules);
-                PKWMODULE   pMod      = g_apModules[idxHash];
-                if (pMod)
-                {
-                    do
-                    {
-                        if (   pMod->uHashPath == uHashPath
-                            && kHlpStrComp(pMod->pszPath, szNormPath) == 0)
-                        {
-                            kwLdrModuleRetain(pMod);
-                            break;
-                        }
-                        pMod = pMod->pNextHash;
-                    } while (pMod);
-                }
-
-                /*
-                 * If not in the hash table, so create a module entry.
-                 */
-                if (!pMod)
-                {
-                    PKLDRMOD pLdrMod;
-                    rc = kLdrModOpenNativeByHandle((KUPTR)hModule, KLDRMOD_OPEN_FLAGS_NATIVE_ALLOW_INIT_TERM, &pLdrMod);
-                    if (rc == 0)
-                    {
-                        pMod = kwLdrModuleCreateForNativekLdrModule(pLdrMod, szNormPath, cchPath + 1, uHashPath,
-                                                                    K_FALSE /*fDoReplacements*/);
-                        if (!pMod)
-                        {
-                            kLdrModClose(pLdrMod);
-                            kwErrPrintf("out of memory\n");
-                        }
-                    }
-                    else
-                        kwErrPrintf("kLdrModOpenNativeByHandle failed for %p / '%s': %d\n", hModule, pszName, rc);
-                }
-                if (pMod)
-                {
-                    /*
-                     * Create a CRT slot for the module if necessary.
-                     */
-                    if (!fEnsureCrtSlot || pMod->iCrtSlot != KU8_MAX)
-                        return pMod;
-                    rc = kwLdrModuleCreateCrtSlot(pMod);
-                    if (rc == 0)
-                        return pMod;
-                    kwLdrModuleRelease(pMod);
-                }
-            }
-            else
-                kwErrPrintf("kwPathNormalize failed for '%s' (%s): %u!\n", szModPath, pszName, GetLastError());
-        }
-        else
-            kwErrPrintf("GetModuleFileNameA failed for '%s': %u!\n", pszName, GetLastError());
-    }
-    else if (fAlwaysPresent)
+        return kwLdrModuleForLoadedNativeByHandle(hModule, fEnsureCrtSlot, pszName);
+    if (fAlwaysPresent)
         kwErrPrintf("Module '%s' was not found by GetModuleHandleA/W!\n", pszName);
     return NULL;
@@ -3346 +3652 @@
     PKWDYNLOAD      pDynLoad;
 
+    if (pTool)
+    { /* likely */ }
+    else
+        return NULL;
+
     /* The executable. */
     if (   hmod == NULL
-        || pTool->u.Sandboxed.pExe->hOurMod == hmod)
-        return kwLdrModuleRetain(pTool->u.Sandboxed.pExe);
+        || (pTool->u.Sandboxed.pExe && pTool->u.Sandboxed.pExe->hOurMod == hmod))
+    {
+        if (pTool->u.Sandboxed.pExe)
+            return kwLdrModuleRetain(pTool->u.Sandboxed.pExe);
+        return NULL;
+    }
 
     /*
@@ -3362 +3677 @@
         for (;;)
         {
-            KUPTR const uHModThis = (KUPTR)papMods[i]->hOurMod;
-            if (uHMod < uHModThis)
+            KUPTR const uHModCur = (KUPTR)papMods[i]->hOurMod;
+            if (uHMod < uHModCur)
             {
                 iEnd = i--;
@@ -3371 +3686 @@
                     break;
             }
-            else if (uHMod != uHModThis)
+            else if (uHMod != uHModCur)
             {
                 iStart = ++i;
@@ -3379 +3694 @@
                     break;
             }
+            /* We've got a match.  Always return the non-virtual module (first) when there is one. */
+            else if (!papMods[i]->pVirtualApiMod)
+                return kwLdrModuleRetain(papMods[i]);
             else
+            {
+                while (i > 0 && papMods[i - 1]->pVirtualApiMod && papMods[i - 1]->hOurMod == hmod)
+                    i--;
                 return kwLdrModuleRetain(papMods[i]);
+            }
 
             i = iStart + (iEnd - iStart) / 2;
@@ -3430 +3752 @@
         for (;;)
         {
-            KUPTR const uHModThis = (KUPTR)papMods[i]->hOurMod;
-            if (uHMod < uHModThis)
+            PKWMODULE   pCurMod  = papMods[i];
+            KUPTR const uHModCur = (KUPTR)pCurMod->hOurMod;
+            if (uHMod < uHModCur)
             {
                 iEnd = i;
@@ -3439 +3762 @@
                     break;
             }
-            else if (uHMod != uHModThis)
+            else if (uHMod != uHModCur)
             {
                 iStart = ++i;
@@ -3449 +3772 @@
             else
             {
-                /* Already there in the table. */
+                /* Already there in the table. The non-virtual module must be the first
+                   entry if we've got duplicate hmod values because of virtual modules.  */
+                if (pMod != pCurMod)
+                {
+                    /* Skip to the last module with the same hmod. */
+                    while (i + 1 < iEnd && (KUPTR)(pCurMod = papMods[i + 1])->hOurMod == uHMod)
+                    {
+                        if (pMod == pCurMod)
+                            return 0;
+                        i++;
+                    }
+
+                    /* Then scan backwards till the first one. */
+                    while (i > 0 && (KUPTR)(pCurMod = papMods[i - 1])->hOurMod == uHMod)
+                    {
+                        if (pMod == pCurMod)
+                            return 0;
+                        i--;
+                    }
+                    pCurMod = papMods[i];
+                    if (pMod != pCurMod)
+                    {
+                        if (pMod->pVirtualApiMod && !pCurMod->pVirtualApiMod)
+                            i++;
+                        break;
+                    }
+                }
                 return 0;
             }
@@ -3460 +3809 @@
         {
             kHlpAssert(papMods[iStart] != pMod);
-            kHlpAssert((KUPTR)papMods[iStart]->hOurMod != uHMod);
-        }
-        kHlpAssert(i == 0 || (KUPTR)papMods[i - 1]->hOurMod < uHMod);
-        kHlpAssert(i == pTool->u.Sandboxed.cModules || (KUPTR)papMods[i]->hOurMod > uHMod);
+            kHlpAssert(   (KUPTR)papMods[iStart]->hOurMod != uHMod
+                       || pMod->pVirtualApiMod
+                       || papMods[iStart]->pVirtualApiMod);
+        }
+        kHlpAssert(i == 0 || (KUPTR)papMods[i - 1]->hOurMod <= uHMod);
+        kHlpAssert(i == pTool->u.Sandboxed.cModules || (KUPTR)papMods[i]->hOurMod >= uHMod);
 #endif
     }
@@ -3485 +3836 @@
     papMods[i] = kwLdrModuleRetain(pMod);
     pTool->u.Sandboxed.cModules++;
-    KW_LOG(("kwToolAddModule: %u modules after adding %p=%s\n", pTool->u.Sandboxed.cModules, uHMod, pMod->pszPath));
+    KWLDR_LOG(("kwToolAddModule: %u modules after adding %p=%s\n", pTool->u.Sandboxed.cModules, uHMod, pMod->pszPath));
     return 0;
 }
@@ -3500 +3851 @@
 {
     int rc = kwToolAddModule(pTool, pMod);
+    if (pMod->pVirtualApiMod && rc == 0)
+        rc = kwToolAddModule(pTool, pMod->pVirtualApiMod);
     if (!pMod->fNative && rc == 0)
     {
@@ -3512 +3865 @@
         }
     }
+
     return 0;
 }
@@ -3817 +4171 @@
             break;
         pfFlags = pchPool;
-        PUTC((char)_ARG_NONZERO);
+        PUTC((unsigned char)_ARG_NONZERO);
         PUTV;
         bs = 0; chQuote = 0;
@@ -3979 +4333 @@
         return NULL;
     }
-    KW_LOG(("_onexit(%p) - IGNORED\n", pfnFunc));
-    return pfnFunc;
+    //KW_LOG(("_onexit(%p) - IGNORED\n", pfnFunc));
+    //return pfnFunc;
 }
 
@@ -4004 +4358 @@
         return -1;
     }
-    KW_LOG(("atexit(%p) - IGNORED!\n", pfnFunc));
-    return 0;
+    //KW_LOG(("atexit(%p) - IGNORED!\n", pfnFunc));
+    //return 0;
 }
 
@@ -4241 +4595 @@
 
 /** _beginthreadex - create a new thread. */
-static uintptr_t __cdecl kwSandbox_msvcrt__beginthreadex(void *pvSecAttr, unsigned cbStack,
-                                                         unsigned (__stdcall *pfnThreadProc)(void *), void *pvUser,
-                                                         unsigned fCreate, unsigned *pidThread)
-{
+static uintptr_t __cdecl kwSandbox_msvcrt__beginthreadex_wrapped(void *pvSecAttr, unsigned cbStack,
+                                                                 unsigned (__stdcall *pfnThreadProc)(void *), void *pvUser,
+                                                                 unsigned fCreate, unsigned *pidThread, PKWCRTSLOT pSlot)
+{
+    /*
+     * Since the VC++ 12 (VS 2013) compiler, the 2nd pass is now threaded.
+     * Let it do whatever it needs to.
+     */
+    KW_LOG(("kwSandbox_msvcrt__beginthreadex: pvSecAttr=%p (inh=%d) cbStack=%#x pfnThreadProc=%p pvUser=%p fCreate=%#x pidThread=%p\n",
+            pvSecAttr, pvSecAttr ? ((LPSECURITY_ATTRIBUTES)pvSecAttr)->bInheritHandle : 0, cbStack,
+            pfnThreadProc, pvUser, fCreate, pidThread));
+    if (   g_Sandbox.pTool->u.Sandboxed.enmHint == KWTOOLHINT_VISUAL_CPP_CL
+        && pSlot->pfnBeginThreadEx)
+    {
+        uintptr_t rcRet = pSlot->pfnBeginThreadEx(pvSecAttr, cbStack, pfnThreadProc, pvUser, fCreate, pidThread);
+        KW_LOG(("kwSandbox_msvcrt__beginthreadex: returns %p *pidThread=%#x\n", rcRet, pidThread ? *pidThread : -1));
+        return rcRet;
+    }
+
     kHlpAssert(GetCurrentThreadId() == g_Sandbox.idMainThread);
     KWFS_TODO();
     return 0;
 }
+
+CRT_SLOT_FUNCTION_WRAPPER(uintptr_t __cdecl, kwSandbox_msvcrt__beginthreadex,
+                          (void *pvSecAttr, unsigned cbStack, unsigned (__stdcall *pfnThreadProc)(void *),
+                           void *pvUser, unsigned fCreate, unsigned *pidThread),
+                          (pvSecAttr, cbStack, pfnThreadProc, pvUser, fCreate, pidThread, &g_aCrtSlots[iCrtSlot]));
+
 
 
@@ -4286 +4661 @@
         }
         *psz++ = '\0';
-        kHlpAssert(psz - pszzEnv == cbNeeded);
+        kHlpAssert((KUPTR)(psz - pszzEnv) == cbNeeded);
     }
 
@@ -4302 +4677 @@
     fprintf(stderr, "  %u:%p=<eos>\n\n", iVar, pszCur);
     pszCur++;
-    fprintf(stderr, "ended at %p, after %u bytes (exepcted %u)\n", pszCur, pszCur - pszzEnv, cbNeeded);
+    fprintf(stderr, "ended at %p, after %u bytes (expected %u)\n", pszCur, pszCur - pszzEnv, cbNeeded);
 #endif
     return pszzEnv;
@@ -4343 +4718 @@
         }
         *pwsz++ = '\0';
-        kHlpAssert(pwsz - pwszzEnv == cwcNeeded);
+        kHlpAssert((KUPTR)(pwsz - pwszzEnv) == cwcNeeded);
     }
 
@@ -5054 +5429 @@
         pDynLoad->pNext = g_Sandbox.pTool->u.Sandboxed.pDynLoadHead;
         g_Sandbox.pTool->u.Sandboxed.pDynLoadHead = pDynLoad;
-        KW_LOG(("LoadLibraryExA(%s,,[resource]) -> %p\n", pDynLoad->szRequest, pDynLoad->hmod));
+        KWLDR_LOG(("LoadLibraryExA(%s,,[resource]) -> %p\n", pDynLoad->szRequest, pDynLoad->hmod));
     }
     else
@@ -5067 +5442 @@
 static HMODULE WINAPI kwSandbox_Kernel32_LoadLibraryExA_VirtualApiModule(PKWDYNLOAD pDynLoad, DWORD fFlags)
 {
-    HMODULE     hmod;
+    static const char s_szDll[] = ".dll";
+    KSIZE       cbFilename = kHlpStrLen(pDynLoad->szRequest) + 1;
     PKWMODULE   pMod;
-    KU32        uHashPath;
-    KSIZE       idxHash;
     char        szNormPath[256];
-    KSIZE       cbFilename = kHlpStrLen(pDynLoad->szRequest) + 1;
 
     /*
-     * Lower case it and make it ends with .dll.
+     * Lower case it and make sure it ends with .dll.
      */
-    if (cbFilename <= sizeof(szNormPath))
-    {
-        static const char s_szDll[] = ".dll";
-        kHlpMemCopy(szNormPath, pDynLoad->szRequest, cbFilename);
-        _strlwr(szNormPath);
-        if (   (   cbFilename <= 4
-                || strcmp(&szNormPath[cbFilename - 5], s_szDll) != 0)
-            && cbFilename + sizeof(s_szDll) - 1 <= sizeof(szNormPath))
-        {
-            memcpy(&szNormPath[cbFilename - sizeof(s_szDll)], s_szDll, sizeof(s_szDll));
-            cbFilename += sizeof(s_szDll) - 1;
-        }
-    }
-    else
+    if (cbFilename > sizeof(szNormPath))
     {
         SetLastError(ERROR_FILENAME_EXCED_RANGE);
         return NULL;
     }
+    kHlpMemCopy(szNormPath, pDynLoad->szRequest, cbFilename);
+    _strlwr(szNormPath);
+    kHlpAssert(cbFilename > 7 /* api-ms- */ );
+    if (strcmp(&szNormPath[cbFilename - 5], s_szDll) != 0)
+    {
+        if (cbFilename + sizeof(s_szDll) - 1 > sizeof(szNormPath))
+        {
+            SetLastError(ERROR_FILENAME_EXCED_RANGE);
+            return NULL;
+        }
+
+        memcpy(&szNormPath[cbFilename - sizeof(s_szDll)], s_szDll, sizeof(s_szDll));
+        cbFilename += sizeof(s_szDll) - 1;
+    }
 
     /*
-     * Check if it has already been loaded so we don't create an unnecessary
-     * loader module for it.
+     * Try load it.
      */
-    uHashPath = kwStrHash(szNormPath);
-    idxHash   = uHashPath % K_ELEMENTS(g_apModules);
-    pMod      = g_apModules[idxHash];
+    pMod = kwLdrModuleTryLoadVirtualDll(szNormPath, cbFilename - 1);
     if (pMod)
     {
-        do
-        {
-            if (   pMod->uHashPath == uHashPath
-                && kHlpStrComp(pMod->pszPath, szNormPath) == 0)
-            {
-                pDynLoad->pMod = kwLdrModuleRetain(pMod);
-                pDynLoad->hmod = pMod->hOurMod;
-
-                pDynLoad->pNext = g_Sandbox.pTool->u.Sandboxed.pDynLoadHead;
-                g_Sandbox.pTool->u.Sandboxed.pDynLoadHead = pDynLoad;
-                KW_LOG(("LoadLibraryExA(%s,,) -> %p [virtual API module - already loaded]\n", pDynLoad->szRequest, pDynLoad->hmod));
-                return pDynLoad->hmod;
-            }
-            pMod = pMod->pNextHash;
-        } while (pMod);
-    }
-
-
-    /*
-     * Try load it and make a kLdr module for it.
-     */
-    hmod = LoadLibraryExA(szNormPath, NULL /*hFile*/, fFlags);
-    if (hmod)
-    {
-        PKLDRMOD pLdrMod;
-        int rc = kLdrModOpenNativeByHandle((KUPTR)hmod, KLDRMOD_OPEN_FLAGS_NATIVE_ALLOW_INIT_TERM, &pLdrMod);
-        if (rc == 0)
-        {
-            PKWMODULE pMod = kwLdrModuleCreateForNativekLdrModule(pLdrMod, szNormPath, cbFilename, uHashPath,
-                                                                  K_FALSE /*fDoReplacements*/);
-            if (pMod)
-            {
-                kwToolAddModuleAndImports(g_Sandbox.pTool, pMod);
-
-                pDynLoad->pMod = pMod;
-                pDynLoad->hmod = hmod;
-
-                pDynLoad->pNext = g_Sandbox.pTool->u.Sandboxed.pDynLoadHead;
-                g_Sandbox.pTool->u.Sandboxed.pDynLoadHead = pDynLoad;
-                KW_LOG(("LoadLibraryExA(%s,,) -> %p [virtual API module - new]\n", pDynLoad->szRequest, pDynLoad->hmod));
-                return hmod;
-            }
-            KWFS_TODO();
-        }
-        else
-            KWFS_TODO();
+        kwToolAddModuleAndImports(g_Sandbox.pTool, pMod);
+
+        pDynLoad->pMod = pMod;
+        pDynLoad->hmod = pMod->hOurMod;
+
+        pDynLoad->pNext = g_Sandbox.pTool->u.Sandboxed.pDynLoadHead;
+        g_Sandbox.pTool->u.Sandboxed.pDynLoadHead = pDynLoad;
+        KWLDR_LOG(("LoadLibraryExA(%s,,) -> %p [virtual API module - new]\n", pDynLoad->szRequest, pDynLoad->hmod));
+        return pDynLoad->hmod;
     }
     kHlpFree(pDynLoad);
-    return hmod;
+    return NULL;
 }
 
@@ -5166 +5500 @@
     int         rc;
     kHlpAssert(GetCurrentThreadId() == g_Sandbox.idMainThread);
+    //fprintf(stderr, "LoadLibraryExA: %s, %#x\n", pszFilename, fFlags);
 
     /*
@@ -5193 +5528 @@
             if (rc == 0)
             {
-                KW_LOG(("LoadLibraryExA(%s,,) -> %p [cached]\n", pszFilename, pDynLoad->hmod));
+                KWLDR_LOG(("LoadLibraryExA(%s,,) -> %p [cached]\n", pszFilename, pDynLoad->hmod));
                 return pDynLoad->hmod;
             }
@@ -5211 +5546 @@
     else
     {
-        KW_LOG(("LoadLibraryExA: Out of memory!\n"));
+        KWLDR_LOG(("LoadLibraryExA: Out of memory!\n"));
         SetLastError(ERROR_NOT_ENOUGH_MEMORY);
         return NULL;
@@ -5228 +5563 @@
      * Special case: api-ms-win-core-synch-l1-2-0 and friends (32-bit yasm, built with VS2015).
      */
-    if (   strnicmp(pszFilename, TUPLE("api-ms-")) == 0
+    if (   kwLdrIsVirtualApiModule(pszFilename, cchFilename)
         && kHlpIsFilenameOnly(pszFilename))
         return kwSandbox_Kernel32_LoadLibraryExA_VirtualApiModule(pDynLoad, fFlags);
@@ -5263 +5598 @@
         if (rc == 0)
         {
-            KW_LOG(("LoadLibraryExA(%s,,) -> %p\n", pszFilename, pDynLoad->hmod));
+            KWLDR_LOG(("LoadLibraryExA(%s,,) -> %p\n", pszFilename, pDynLoad->hmod));
             return pDynLoad->hmod;
         }
@@ -5479 +5814 @@
                     || pDynLoad->pMod->u.Manual.enmState == KWMODSTATE_READY)
                 {
-                    KW_LOG(("kwSandbox_Kernel32_GetModuleHandleA(%s,,) -> %p [dynload]\n", pszModule, pDynLoad->hmod));
+                    KWLDR_LOG(("kwSandbox_Kernel32_GetModuleHandleA(%s,,) -> %p [dynload]\n", pszModule, pDynLoad->hmod));
                     return pDynLoad->hmod;
                 }
@@ -5492 +5827 @@
      * to and go via the g_aGetModuleHandleCache cache.
      */
-    if (strnicmp(pszModule, "api-ms-win-", 11) == 0)
+/** @todo virtual api DLLs */
+    if (kHlpStrNICompAscii(pszModule, "api-ms-win-", 11) == 0)
     {
         HMODULE hmod = GetModuleHandleA(pszModule);
@@ -5504 +5840 @@
             if (hmod == GetModuleHandleW(L"NTDLL.DLL"))
                 return kwSandbox_Kernel32_GetModuleHandleA("NTDLL.DLL");
+            if (hmod == GetModuleHandleW(L"UCRTBASE.DLL"))
+                return kwSandbox_Kernel32_GetModuleHandleA("UCRTBASE.DLL");
         }
         else
@@ -5615 +5953 @@
     KWFS_TODO();
 #endif
-    return -1;
+    return ~(UINT)0;
 }
 
@@ -5722 +6060 @@
                                < g_Sandbox.pTool->u.Sandboxed.pExe->cbImage)
                         {
-                            uValue = g_aSandboxGetProcReplacements[i].pfnReplacement;
+                            if (!g_aSandboxReplacements[i].fCrtSlotArray)
+                                uValue = g_aSandboxGetProcReplacements[i].pfnReplacement;
+                            else
+                            {
+                                if (pMod->iCrtSlot == KU8_MAX)
+                                {
+                                    int rc = kwLdrModuleCreateCrtSlot(pMod);
+                                    if (rc)
+                                    {
+                                        KW_LOG(("GetProcAddress: kwLdrModuleCreateCrtSlot failed: %d\n", rc));
+                                        SetLastError(ERROR_INTERNAL_ERROR);
+                                        return NULL;
+                                    }
+                                }
+                                uValue = ((KUPTR *)g_aSandboxGetProcReplacements[i].pfnReplacement)[pMod->iCrtSlot];
+                            }
+
                             KW_LOG(("GetProcAddress(%s, %s) -> %p replaced\n", pMod->pszPath, pszProc, (KUPTR)uValue));
                         }
@@ -5839 +6193 @@
             for (;;)
             {
-                KUPTR const uHModThis = (KUPTR)papMods[i]->hOurMod;
-                if (uPC < uHModThis)
+                KUPTR const uHModCur = (KUPTR)papMods[i]->hOurMod;
+                if (uPC < uHModCur)
                 {
                     iEnd = i;
@@ -5848 +6202 @@
                         break;
                 }
-                else if (uPC != uHModThis)
+                else if (uPC != uHModCur)
                 {
                     iStart = ++i;
@@ -5972 +6326 @@
             return K_TRUE;
         }
+
+        /* In VC2019 there is also one {UUID} file in temp: */
+        if (pwszName[0] == '{')
+        {
+            KSIZE cwcName = kwUtf16Len(pwszName);
+            if (   cwcName == sizeof("{4465DDD9-E494-471B-996B-9B556E25AEF8}") - 1
+                && pwszName[37] == '}'
+                && iswalnum(pwszName[1])   // 4
+                && iswalnum(pwszName[2])   // 4
+                && iswalnum(pwszName[3])   // 6
+                && iswalnum(pwszName[4])   // 5
+                && iswalnum(pwszName[5])   // d
+                && iswalnum(pwszName[6])   // d
+                && iswalnum(pwszName[7])   // d
+                && iswalnum(pwszName[8])   // 9
+                && pwszName[9] == '-'      // -
+                && iswalnum(pwszName[10])  // e
+                && iswalnum(pwszName[11])  // 4
+                && iswalnum(pwszName[12])  // 9
+                && iswalnum(pwszName[13])  // 4
+                && pwszName[14] == '-'     // -
+                && iswalnum(pwszName[15])  // 4
+                && iswalnum(pwszName[16])  // 7
+                && iswalnum(pwszName[17])  // 1
+                && iswalnum(pwszName[18])  // b
+                && pwszName[19] == '-'     // -
+                && iswalnum(pwszName[20])  // 9
+                && iswalnum(pwszName[21])  // 9
+                && iswalnum(pwszName[22])  // 6
+                && iswalnum(pwszName[23])  // b
+                && pwszName[24] == '-'     // -
+                && iswalnum(pwszName[25])  // 9
+                && iswalnum(pwszName[26])  // b
+                && iswalnum(pwszName[27])  // 5
+                && iswalnum(pwszName[28])  // 5
+                && iswalnum(pwszName[29])  // 6
+                && iswalnum(pwszName[30])  // e
+                && iswalnum(pwszName[31])  // 2
+                && iswalnum(pwszName[32])  // 5
+                && iswalnum(pwszName[33])  // a
+                && iswalnum(pwszName[34])  // 3
+                && iswalnum(pwszName[35])  // f
+                && iswalnum(pwszName[36])) // 8
+                return K_TRUE;
+        }
     }
     return K_FALSE;
@@ -6029 +6428 @@
 
 
-static HANDLE kwFsTempFileCreateW(const wchar_t *pwszFilename, DWORD dwDesiredAccess, DWORD dwCreationDisposition)
+static HANDLE kwFsTempFileCreateW(const wchar_t *pwszFilename, DWORD dwDesiredAccess, DWORD dwCreationDisposition,
+                                  KBOOL *pfFallback)
 {
     HANDLE hFile;
@@ -6053 +6453 @@
      * Create a new temporary file instance if not found.
      */
+    *pfFallback = K_FALSE;
     if (pTempFile == NULL)
     {
@@ -6061 +6462 @@
             case CREATE_ALWAYS:
             case OPEN_ALWAYS:
+            case CREATE_NEW:
                 dwErr = NO_ERROR;
                 break;
 
-            case CREATE_NEW:
-                kHlpAssertFailed();
-                SetLastError(ERROR_ALREADY_EXISTS);
-                return INVALID_HANDLE_VALUE;
-
             case OPEN_EXISTING:
             case TRUNCATE_EXISTING:
+                *pfFallback = K_TRUE;
                 kHlpAssertFailed();
                 SetLastError(ERROR_FILE_NOT_FOUND);
@@ -6113 +6511 @@
                 break;
             case OPEN_ALWAYS:
-                dwErr = ERROR_ALREADY_EXISTS ;
+                dwErr = ERROR_ALREADY_EXISTS;
                 break;
 
@@ -6369 +6767 @@
     kHlpAssertReturn(pFsObj->pParent->hDir != INVALID_HANDLE_VALUE, NULL);
 
-    Ios.Information = -1;
+    Ios.Information = ~(ULONG_PTR)0;
     Ios.u.Status    = -1;
 
@@ -6627 +7025 @@
         && kwFsIsClTempFileW(pwszFilename))
     {
-        hFile = kwFsTempFileCreateW(pwszFilename, dwDesiredAccess, dwCreationDisposition);
-        KWFS_LOG(("CreateFileW(%ls) -> %p [temp]\n", pwszFilename, hFile));
-        return hFile;
+        KBOOL fFallback = K_FALSE;
+        hFile = kwFsTempFileCreateW(pwszFilename, dwDesiredAccess, dwCreationDisposition, &fFallback);
+        if (!fFallback)
+        {
+            KWFS_LOG(("CreateFileW(%ls) -> %p [temp]\n", pwszFilename, hFile));
+            return hFile;
+        }
     }
 #endif
@@ -7089 +7491 @@
             {
                 pOutBuf->u.Fully.cchBufAlloc = sizeof(pOutBuf->abPadding);
-                pOutBuf->u.Fully.pchBuf      = pOutBuf->abPadding;
+                pOutBuf->u.Fully.pchBuf      = (char *)&pOutBuf->abPadding[0];
             }
         }
@@ -8042 +8444 @@
             fRet = INVALID_FILE_ATTRIBUTES;
         }
-
+#ifndef NDEBUG
+        {
+            DWORD fCheck = GetFileAttributesW(pwszFilename);
+            kHlpAssertMsg(fCheck == fRet, ("fCheck=%x vs fRet=%#x diff=%#x; %ls\n", fCheck, fRet, fCheck ^ fRet, pwszFilename));
+        }
+#endif
         KWFS_LOG(("GetFileAttributesW(%ls) -> %#x [cached]\n", pwszFilename, fRet));
         return fRet;
@@ -8078 +8485 @@
                 *(KU64 *)&pData->ftLastWriteTime  = birdNtTimeFromTimeSpec(&pFsObj->Stats.st_mtim);
                 kFsCacheObjRelease(g_pFsCache, pFsObj);
+                fRet = TRUE;
             }
             else
@@ -8091 +8499 @@
         }
 
+#ifdef K_STRICT
+        {
+            WIN32_FILE_ATTRIBUTE_DATA CheckData = { 0 };
+            DWORD const dwErrSaved = GetLastError();
+            BOOL const fRetCheck = GetFileAttributesExA(pszFilename, enmLevel, &CheckData);
+            kHlpAssertMsg(fRet == fRetCheck, ("fRet=%d fRetCheck=%d; %s\n", fRet, fRetCheck, pszFilename));
+            if (fRetCheck && fRet)
+            {
+# define ASSERT_FS_FIELD_EQUAL_A(pResult, pExpected, pszFilename, Field, szFmt) \
+    kHlpAssertMsg((pResult)->Field == (pExpected)->Field, (#Field ": " szFmt " , expected " szFmt "; %s\n", (pResult)->Field, (pExpected)->Field, pszFilename))
+                ASSERT_FS_FIELD_EQUAL_A(pData, &CheckData, pszFilename, dwFileAttributes, "%#x");
+                ASSERT_FS_FIELD_EQUAL_A(pData, &CheckData, pszFilename, nFileSizeHigh, "%#x");
+                ASSERT_FS_FIELD_EQUAL_A(pData, &CheckData, pszFilename, nFileSizeLow, "%#x");
+                ASSERT_FS_FIELD_EQUAL_A(pData, &CheckData, pszFilename, ftCreationTime.dwHighDateTime, "%#x");
+                ASSERT_FS_FIELD_EQUAL_A(pData, &CheckData, pszFilename, ftCreationTime.dwLowDateTime, "%#x");
+                ASSERT_FS_FIELD_EQUAL_A(pData, &CheckData, pszFilename, ftLastWriteTime.dwHighDateTime, "%#x");
+                ASSERT_FS_FIELD_EQUAL_A(pData, &CheckData, pszFilename, ftLastWriteTime.dwLowDateTime, "%#x");
+            }
+            else
+                kHlpAssertMsg(dwErrSaved == GetLastError(), ("%u, expected %u; %s\n", dwErrSaved, GetLastError(), pszFilename));
+            SetLastError(dwErrSaved);
+        }
+#endif
         KWFS_LOG(("GetFileAttributesA(%s,%d,) -> %d [cached]\n", pszFilename, enmLevel, fRet));
         return fRet;
@@ -8125 +8556 @@
                 *(KU64 *)&pData->ftLastWriteTime  = birdNtTimeFromTimeSpec(&pFsObj->Stats.st_mtim);
                 kFsCacheObjRelease(g_pFsCache, pFsObj);
+                fRet = TRUE;
             }
             else
@@ -8138 +8570 @@
         }
 
-        KWFS_LOG(("GetFileAttributesW(%ls,%d,) -> %d [cached]\n", pwszFilename, enmLevel, fRet));
+#ifdef K_STRICT
+        {
+            WIN32_FILE_ATTRIBUTE_DATA CheckData = { 0 };
+            DWORD const dwErrSaved = GetLastError();
+            BOOL const fRetCheck = GetFileAttributesExW(pwszFilename, enmLevel, &CheckData);
+            kHlpAssertMsg(fRet == fRetCheck, ("fRet=%d fRetCheck=%d; %ls\n", fRet, fRetCheck, pwszFilename));
+            if (fRetCheck && fRet)
+            {
+# define ASSERT_FS_FIELD_EQUAL_W(pResult, pExpected, pszFilename, Field, szFmt) \
+    kHlpAssertMsg((pResult)->Field == (pExpected)->Field, (#Field ": " szFmt " , expected " szFmt "; %ls\n", (pResult)->Field, (pExpected)->Field, pwszFilename))
+                ASSERT_FS_FIELD_EQUAL_W(pData, &CheckData, pwszFilename, dwFileAttributes, "%#x");
+                ASSERT_FS_FIELD_EQUAL_W(pData, &CheckData, pwszFilename, nFileSizeHigh, "%#x");
+                ASSERT_FS_FIELD_EQUAL_W(pData, &CheckData, pwszFilename, nFileSizeLow, "%#x");
+                ASSERT_FS_FIELD_EQUAL_W(pData, &CheckData, pwszFilename, ftCreationTime.dwHighDateTime, "%#x");
+                ASSERT_FS_FIELD_EQUAL_W(pData, &CheckData, pwszFilename, ftCreationTime.dwLowDateTime, "%#x");
+                ASSERT_FS_FIELD_EQUAL_W(pData, &CheckData, pwszFilename, ftLastWriteTime.dwHighDateTime, "%#x");
+                ASSERT_FS_FIELD_EQUAL_W(pData, &CheckData, pwszFilename, ftLastWriteTime.dwLowDateTime, "%#x");
+            }
+            else
+                kHlpAssertMsg(dwErrSaved == GetLastError(), ("%u, expected %u; %ls\n", dwErrSaved, GetLastError(), pwszFilename));
+            SetLastError(dwErrSaved);
+        }
+#endif
+        KWFS_LOG(("GetFileAttributesExW(%ls,%d,) -> %d [cached]\n", pwszFilename, enmLevel, fRet));
         return fRet;
     }
@@ -8861 +9316 @@
              */
             PKWVIRTALLOC pTracker;
-            kHlpAssert(GetCurrentThreadId() == g_Sandbox.idMainThread);
-
+
+            EnterCriticalSection(&g_Sandbox.VirtualAllocLock);
             pTracker = g_Sandbox.pVirtualAllocHead;
             while (   pTracker
                    && (KUPTR)pvMem - (KUPTR)pTracker->pvAlloc >= pTracker->cbAlloc)
                 pTracker = pTracker->pNext;
+            LeaveCriticalSection(&g_Sandbox.VirtualAllocLock);
             if (!pTracker)
             {
@@ -8876 +9332 @@
                     pTracker->cbAlloc           = cb;
                     pTracker->idxPreAllocated   = idxPreAllocated;
+                    EnterCriticalSection(&g_Sandbox.VirtualAllocLock);
                     pTracker->pNext             = g_Sandbox.pVirtualAllocHead;
                     g_Sandbox.pVirtualAllocHead = pTracker;
+                    LeaveCriticalSection(&g_Sandbox.VirtualAllocLock);
                 }
                 SetLastError(dwErr);
@@ -8900 +9358 @@
         if (dwFreeType & MEM_RELEASE)
         {
-            PKWVIRTALLOC pTracker = g_Sandbox.pVirtualAllocHead;
+            PKWVIRTALLOC pTracker;
+            EnterCriticalSection(&g_Sandbox.VirtualAllocLock);
+            pTracker = g_Sandbox.pVirtualAllocHead;
             if (pTracker)
             {
@@ -8922 +9382 @@
                     {
                         kwSandboxResetFixedAllocation(pTracker->idxPreAllocated);
+                        LeaveCriticalSection(&g_Sandbox.VirtualAllocLock);
                         KW_LOG(("VirtualFree: pvAddr=%p cb=%p type=%#x -> TRUE [pre allocated #%u]\n",
                                 pvAddr, cb, dwFreeType, pTracker->idxPreAllocated));
@@ -8937 +9398 @@
                         g_Sandbox.pVirtualAllocHead = pTracker;
                     }
+                    LeaveCriticalSection(&g_Sandbox.VirtualAllocLock);
                     KW_LOG(("VirtualFree: pvAddr=%p cb=%p type=%#x -> %d\n", pvAddr, cb, dwFreeType, fRc));
                     return fRc;
@@ -8943 +9405 @@
                 KW_LOG(("VirtualFree: pvAddr=%p not found!\n", pvAddr));
             }
+            LeaveCriticalSection(&g_Sandbox.VirtualAllocLock);
         }
     }
@@ -9188 +9651 @@
     if (g_Sandbox.pTool->u.Sandboxed.enmHint == KWTOOLHINT_VISUAL_CPP_CL)
     {
+        /** @todo do generic caching of hash results.  Need SHA-1 and SHA-256 for
+         *        more recent compilers. */
         if (idAlg == CALG_MD5)
         {
@@ -9222 +9687 @@
                 kwErrPrintf("CryptCreateHash: hKey=%p is not supported with CALG_MD5\n", hKey);
         }
-        else
-            kwErrPrintf("CryptCreateHash: idAlg=%#x is not supported\n", idAlg);
+        //else
+        //    kwErrPrintf("CryptCreateHash: idAlg=%#x is not supported\n", idAlg);
     }
 
@@ -9337 +9802 @@
             else
                 kwErrPrintf("CryptHashData: Hash is already finalized!!\n");
-            SetLastError(NTE_BAD_HASH);
+            SetLastError((DWORD)NTE_BAD_HASH);
             fRc = FALSE;
         }
@@ -9447 +9912 @@
                     default:
                         kwErrPrintf("CryptGetHashParam: Unknown dwParam=%#x\n", dwParam);
-                        SetLastError(NTE_BAD_TYPE);
+                        SetLastError((DWORD)NTE_BAD_TYPE);
                         return FALSE;
                 }
@@ -9485 +9950 @@
             {
                 kwErrPrintf("CryptGetHashParam: dwFlags is not zero: %#x!\n", dwFlags);
-                SetLastError(NTE_BAD_FLAGS);
+                SetLastError((DWORD)NTE_BAD_FLAGS);
             }
         }
@@ -9491 +9956 @@
         {
             kwErrPrintf("CryptGetHashParam: Invalid cached hash handle!!\n");
-            SetLastError(NTE_BAD_HASH);
+            SetLastError((DWORD)NTE_BAD_HASH);
         }
         fRc = FALSE;
@@ -9919 +10384 @@
     KSIZE      cbLeft = cb;
     while (cbLeft-- > 0)
-        *pbDst++ = bFiller;
+        *pbDst++ = (KU8)bFiller;
     return pvDst;
 }
@@ -10046 +10511 @@
 
     { TUPLE("_beginthread"),                NULL,       (KUPTR)kwSandbox_msvcrt__beginthread },
-    { TUPLE("_beginthreadex"),              NULL,       (KUPTR)kwSandbox_msvcrt__beginthreadex },
+    { TUPLE("_beginthreadex"),              NULL,       (KUPTR)kwSandbox_msvcrt__beginthreadex, K_FALSE /*fOnlyExe*/, K_TRUE /*fCrtSlotArray*/ },
     { TUPLE("_beginthreadex"),          "msvcr120.dll", (KUPTR)kwSandbox_msvcr120__beginthreadex }, /* higher priority last */
 
@@ -10262 +10727 @@
 
 
+#if 0
 /**
  * Resets the KWMODULE::fVisited flag for _all_ known modules.
@@ -10326 +10792 @@
     }
 }
+#else
+/**
+ * Used by kwSandboxExec to reset the state of the module tree.
+ */
+static void kwSandboxResetModuleState(void)
+{
+    PKWMODULE pMod = g_pModuleHead;
+    while (pMod)
+    {
+        if (!pMod->fNative)
+            pMod->u.Manual.enmState = K_MIN(pMod->u.Manual.enmReInitState, pMod->u.Manual.enmState);
+        /* Hack: Re-init mspdbXXX.dll when we want to use a different mspdbsrv.exe instance. */
+        else if (   pMod->fReInitOnMsPdbSrvEndpointChange
+                 && (   g_Sandbox.pTool->u.Sandboxed.enmHint == KWTOOLHINT_VISUAL_CPP_CL
+                     || g_Sandbox.pTool->u.Sandboxed.enmHint == KWTOOLHINT_VISUAL_CPP_LINK))
+        {
+            const char *pszValue = kwSandboxDoGetEnvA(&g_Sandbox, TUPLE("_MSPDBSRV_ENDPOINT_"));
+            if (pMod->fReInitOnMsPdbSrvEndpointChange == 1)
+            {
+                pMod->fReInitOnMsPdbSrvEndpointChange = 2;
+                pMod->pszMsPdbSrvEndpoint = pszValue ? kHlpStrDup(pszValue) : NULL;
+                KWLDR_LOG(("Not re-initing '%s': first time used (_MSPDBSRV_ENDPOINT_ is '%s')\n",
+                           pMod->pszPath, pszValue ? pszValue : "<null>"));
+            }
+            else if (   (pszValue == NULL && pMod->pszMsPdbSrvEndpoint == NULL)
+                     || (pszValue != NULL && pMod->pszMsPdbSrvEndpoint != NULL && kHlpStrComp(pszValue, pMod->pszMsPdbSrvEndpoint) == 0))
+                KWLDR_LOG(("Not re-initing '%s': _MSPDBSRV_ENDPOINT_ unchanged ('%s')\n",
+                           pMod->pszPath, pszValue ? pszValue : "<null>"));
+            else
+            {
+                KWLDR_LOG(("Re-initing '%s': _MSPDBSRV_ENDPOINT_ changed from '%s' to '%s'\n", pMod->pszPath,
+                           pMod->pszMsPdbSrvEndpoint ? pMod->pszMsPdbSrvEndpoint : "<null>", pszValue ? pszValue : "<null>"));
+                kHlpFree(pMod->pszMsPdbSrvEndpoint);
+                if (pszValue != NULL)
+                    pMod->pszMsPdbSrvEndpoint = kHlpStrDup(pszValue);
+                else
+                    pMod->pszMsPdbSrvEndpoint = NULL;
+                pMod->fNeedReInit = K_TRUE;
+            }
+        }
+
+        pMod = pMod->pNextList;
+    }
+}
+#endif
 
 static PPEB kwSandboxGetProcessEnvironmentBlock(void)
@@ -10740 +11251 @@
 
     /* Free left behind VirtualAlloc leaks. */
+    EnterCriticalSection(&g_Sandbox.VirtualAllocLock);
     pTracker = g_Sandbox.pVirtualAllocHead;
     g_Sandbox.pVirtualAllocHead = NULL;
+    LeaveCriticalSection(&g_Sandbox.VirtualAllocLock);
     while (pTracker)
     {
@@ -10911 +11424 @@
          * Do module initialization.
          */
-        kwSandboxResetModuleVisited();
-        kwSandboxResetModuleState(pTool->u.Sandboxed.pExe);
+#if 0
+        //kwSandboxResetModuleVisited();
+        //kwSandboxResetModuleState(pTool->u.Sandboxed.pExe);
+#else
+        kwSandboxResetModuleState();
+#endif
         rc = kwLdrModuleInitTree(pTool->u.Sandboxed.pExe);
         if (rc == 0)
@@ -11877 +12394 @@
      * Do the job.
      */
+    rcExit = 0;
     for (j = 0; j < cRepeats; j++)
     {
@@ -12017 +12535 @@
  * @param   pszAppend       The string to append.
  */
-static int kwFullTestVectorAppend(const char ***ppapszVector, int *pcEntries, char const *pszAppend)
-{
-    unsigned cEntries = *pcEntries;
+static int kwFullTestVectorAppend(const char ***ppapszVector, KU32 *pcEntries, char const *pszAppend)
+{
+    KU32 cEntries = *pcEntries;
     if (!(cEntries & 15))
     {
@@ -12395 +12913 @@
         return kwErrPrintfRc(3, "VirtualProtect(%p, %#x, PAGE_EXECUTE_READWRITE,NULL) failed: %u\n",
                              g_abDefLdBuf, sizeof(g_abDefLdBuf), GetLastError());
+    InitializeCriticalSection(&g_Sandbox.VirtualAllocLock);
 
 #ifdef WITH_CONSOLE_OUTPUT_BUFFERING
@@ -12519 +13038 @@
                     && uValue <= 5)
                 {
-                    DWORD dwClass, dwPriority;
+                    DWORD dwClass;
+                    int   dwPriority;
                     switch (uValue)
                     {
                         case 1: dwClass = IDLE_PRIORITY_CLASS;         dwPriority = THREAD_PRIORITY_IDLE; break;
                         case 2: dwClass = BELOW_NORMAL_PRIORITY_CLASS; dwPriority = THREAD_PRIORITY_BELOW_NORMAL; break;
+                        default:
                         case 3: dwClass = NORMAL_PRIORITY_CLASS;       dwPriority = THREAD_PRIORITY_NORMAL; break;
-                        case 4: dwClass = HIGH_PRIORITY_CLASS;         dwPriority = 0xffffffff; break;
-                        case 5: dwClass = REALTIME_PRIORITY_CLASS;     dwPriority = 0xffffffff; break;
+                        case 4: dwClass = HIGH_PRIORITY_CLASS;         dwPriority = INT_MAX; break;
+                        case 5: dwClass = REALTIME_PRIORITY_CLASS;     dwPriority = INT_MAX; break;
                     }
                     SetPriorityClass(GetCurrentProcess(), dwClass);
-                    if (dwPriority != 0xffffffff)
+                    if (dwPriority != INT_MAX)
                         SetThreadPriority(GetCurrentThread(), dwPriority);
                 }
@@ -12692 +13213 @@
         if (g_fRestart)
         {
+            DWORD cbIgnored = 1;
             KU8 b;
             FlushFileBuffers(hPipe);
-            ReadFile(hPipe, &b, 1, &cbMsg, NULL);
+            ReadFile(hPipe, &b, 1, &cbIgnored, NULL);
         }
 

trunk/src/kWorker/kWorkerTlsXxxK.c

@@ -58 +58 @@
  * Initialization data.
  */
-static char const g_abDummy[TLS_SIZE] = {0x42};
+//static char const g_abDummy[TLS_SIZE] = {0x42};
+static char g_abDummy[TLS_SIZE] = {0};
 
 /**
@@ -88 +89 @@
     if (dwReason == DLL_PROCESS_ATTACH)
     {
-        HMODULE hModExe = (HMODULE)(ULONG_PTR)KWORKER_BASE;
+        //HMODULE hModExe = (HMODULE)(ULONG_PTR)KWORKER_BASE;
+        HMODULE hModExe = GetModuleHandleW(NULL);
         KWLDRTLSALLOCATIONHOOK *pfnHook = (KWLDRTLSALLOCATIONHOOK *)GetProcAddress(hModExe, "kwLdrTlsAllocationHook");
         if (pfnHook)