Changeset 3159 for trunk/src/kmk


Ignore:
Timestamp:
Mar 19, 2018, 2:37:13 PM (7 years ago)
Author:
bird
Message:

kmk/win: Some fixes & docs.

Location:
trunk/src/kmk
Files:
10 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/kmk/function.c

    r3156 r3159  
    30583058    {
    30593059      FILE *fp;
     3060#ifdef KMK_FOPEN_NO_INHERIT_MODE
     3061      const char *mode = "w" KMK_FOPEN_NO_INHERIT_MODE;
     3062#else
    30603063      const char *mode = "w";
     3064#endif
    30613065
    30623066      /* We are writing a file.  */
    … …  
    30643068      if (fn[0] == '>')
    30653069        {
     3070#ifdef KMK_FOPEN_NO_INHERIT_MODE
     3071          mode = "a" KMK_FOPEN_NO_INHERIT_MODE;
     3072#else
    30663073          mode = "a";
     3074#endif
    30673075          ++fn;
    30683076        }
    … …  
    31003108        O (fatal, *expanding_var, _("file: too many arguments"));
    31013109
     3110#ifdef KMK_FOPEN_NO_INHERIT_MODE
     3111      ENULLLOOP (fp, fopen (fn, "r" KMK_FOPEN_NO_INHERIT_MODE));
     3112#else
    31023113      ENULLLOOP (fp, fopen (fn, "r"));
     3114#endif
    31033115      if (fp == NULL)
    31043116        {

  • trunk/src/kmk/job.c

    r3156 r3159  
    4242int no_default_sh_exe = 1;
    4343int batch_mode_shell = 1;
     44# ifndef CONFIG_NEW_WIN32_CTRL_EVENT
    4445HANDLE main_thread;
     46# endif
    4547
    4648#elif defined (_AMIGA)
    … …  
    813815#endif /* _AMIGA */
    814816#ifdef WINDOWS32
     817          {
    815818# ifndef CONFIG_NEW_WIN_CHILDREN
    816           {
    817819            HANDLE hPID;
    818820            HANDLE hcTID, hcPID;
    … …  
    822824            coredump = 0;
    823825
     826#  ifndef CONFIG_NEW_WIN32_CTRL_EVENT
    824827            /* Record the thread ID of the main process, so that we
    825828               could suspend it in the signal handler.  */
    … …  
    839842                  DB (DB_VERBOSE, ("Main thread handle = %p\n", main_thread));
    840843              }
     844#  endif
    841845
    842846            /* wait for anything to finish */
    … …  
    875879
    876880            pid = (pid_t) hPID;
     881# else  /* CONFIG_NEW_WIN_CHILDREN */
     882#  ifndef CONFIG_NEW_WIN32_CTRL_EVENT
     883            /* Ctrl-C handler needs to suspend the main thread handle to
     884               prevent mayhem when concurrently calling reap_children.  */
     885            if (   !main_thread
     886                && !DuplicateHandle (GetCurrentProcess (), GetCurrentThread (),
     887                                     GetCurrentProcess (), &main_thread, 0,
     888                                     FALSE, DUPLICATE_SAME_ACCESS))
     889              fprintf (stderr, "Failed to duplicate main thread handle: %u\n",
     890                       GetLastError ());
     891#  endif
     892
     893            assert (!any_remote);
     894            pid = 0;
     895            coredump = exit_sig = exit_code = 0;
     896            {
     897              int rc = MkWinChildWait(block, &pid, &exit_code, &exit_sig, &coredump, &c);
     898              if (rc != 0)
     899                    ON (fatal, NILF, _("MkWinChildWait: %u"), rc);
     900            }
     901            if (pid == 0)
     902              {
     903                /* No more children, stop. */
     904                reap_more = 0;
     905                break;
     906              }
     907
     908            /* If we have started jobs in this second, remove one.  */
     909            if (job_counter)
     910              --job_counter;
     911# endif /* CONFIG_NEW_WIN_CHILDREN */
    877912          }
    878 # else  /* CONFIG_NEW_WIN_CHILDREN */
    879           assert (!any_remote);
    880           pid = 0;
    881           coredump = exit_sig = exit_code = 0;
    882           {
    883             int rc = MkWinChildWait(block, &pid, &exit_code, &exit_sig, &coredump, &c);
    884             if (rc != 0)
    885                   ON (fatal, NILF, _("MkWinChildWait: %u"), rc);
    886           }
    887           if (pid == 0)
    888             {
    889               /* No more children, stop. */
    890               reap_more = 0;
    891               break;
    892             }
    893 
    894           /* If we have started jobs in this second, remove one.  */
    895           if (job_counter)
    896             --job_counter;
    897 # endif /* CONFIG_NEW_WIN_CHILDREN */
    898913#endif /* WINDOWS32 */
    899914        }

  • trunk/src/kmk/kmkbuiltin.h

    r3108 r3159  
    4747#endif
    4848
     49/** This is for telling fopen() to get a close-on-exec handle.
     50 * @todo glibc 2.7+ and recent cygwin supports 'e' for doing this. */
     51#ifndef KMK_FOPEN_NO_INHERIT_MODE
     52# ifdef _MSC_VER
     53#  define KMK_FOPEN_NO_INHERIT_MODE "N"
     54# else
     55#  define KMK_FOPEN_NO_INHERIT_MODE ""
     56# endif
     57#endif
    4958
    5059#include "kbuild_version.h"

  • trunk/src/kmk/kmkbuiltin/append.c

    r3141 r3159  
    190190
    191191    /*
    192      * Open the output file.
     192     * Open the output file, preferrably with close-on-exec.
    193193     */
    194194    iFile = i;
    195     pFile = fopen(argv[i], fTruncate ? "w" : "a");
     195    pFile = fopen(argv[i],
     196                  fTruncate ? "w" KMK_FOPEN_NO_INHERIT_MODE
     197                            : "a" KMK_FOPEN_NO_INHERIT_MODE);
    196198    if (!pFile)
    197199        return err(1, "failed to open '%s'", argv[i]);

  • trunk/src/kmk/kmkbuiltin/kDepIDB.c

    r2955 r3159  
    732732                        pOutput = stdout;
    733733                    else
    734                         pOutput = fopen(pszOutput, "w");
     734                        pOutput = fopen(pszOutput, "w" KMK_FOPEN_NO_INHERIT_MODE);
    735735                    if (!pOutput)
    736736                    {
    … …  
    812812        else
    813813        {
    814             pInput = fopen(argv[i], "rb");
     814            pInput = fopen(argv[i], "rb" KMK_FOPEN_NO_INHERIT_MODE);
    815815            if (!pInput)
    816816            {

  • trunk/src/kmk/kmkbuiltin/md5sum.c

    r3131 r3159  
    156156
    157157    errno = 0;
    158     pFile = fopen(pszFilename, fText ? "r" : "rb");
     158    pFile = fopen(pszFilename,
     159                  fText ? "r"  KMK_FOPEN_NO_INHERIT_MODE
     160                        : "rb" KMK_FOPEN_NO_INHERIT_MODE);
    159161    if (!pFile && errno == EINVAL && !fText)
    160         pFile = fopen(pszFilename, "r");
     162        pFile = fopen(pszFilename, "r" KMK_FOPEN_NO_INHERIT_MODE);
    161163    return pFile;
    162164
    … …  
    472474     * Try open the md5.lst file and process it line by line.
    473475     */
    474     pFile = fopen(pszFilename, "r");
     476    pFile = fopen(pszFilename, "r" KMK_FOPEN_NO_INHERIT_MODE);
    475477    if (pFile)
    476478    {
    … …  
    837839                if (pOutput)
    838840                    fclose(pOutput);
    839                 pOutput = fopen(pszOutput, "w");
     841                pOutput = fopen(pszOutput, "w" KMK_FOPEN_NO_INHERIT_MODE);
    840842                if (!pOutput)
    841843                {
    842                     rc = err(1, "fopen(\"%s\", \"w\") failed", pszOutput);
     844                    rc = err(1, "fopen(\"%s\", \"w" KMK_FOPEN_NO_INHERIT_MODE "\") failed", pszOutput);
    843845                    break;
    844846                }

  • trunk/src/kmk/kmkbuiltin/redirect.c

    r3156 r3159  
    878878             */
    879879            FILE *pWorkingStdErr = NULL;
     880# if defined(CONFIG_NEW_WIN_CHILDREN) && defined(KMK)
     881            if (cOrders > 0)
     882                MkWinChildExclusiveAcquire();
     883# endif
    880884            rcExit = kRedirectExecFdOrders(cOrders, paOrders, &pWorkingStdErr);
    881885            if (rcExit == 0)
    … …  
    890894                HANDLE  hProcess = (HANDLE)_spawnvpe(_P_NOWAIT, pszExecutable, papszArgs, papszEnvVars);
    891895                kRedirectRestoreFdOrders(cOrders, paOrders, &pWorkingStdErr);
     896# ifdef CONFIG_NEW_WIN_CHILDREN
     897                if (cOrders > 0)
     898                    MkWinChildExclusiveRelease();
     899# endif
    892900                if ((intptr_t)hProcess != -1)
    893901                {
    … …  
    10151023#endif /* !KMK */
    10161024            }
     1025#if defined(CONFIG_NEW_WIN_CHILDREN) && defined(KBUILD_OS_WINDOWS) && defined(KMK)
     1026            else if (cOrders > 0)
     1027                MkWinChildExclusiveRelease();
     1028#endif
     1029
    10171030        }
    10181031

  • trunk/src/kmk/main.c

    r3156 r3159  
    31293129            setrlimit (RLIMIT_STACK, &stack_limit);
    31303130#endif
     3131fprintf(stderr, "respawning 1..\n");
     3132fprintf(stderr, "respawning 2..\n");
     3133fprintf(stderr, "respawning 3..!\n");
    31313134# if !defined(WINDOWS32) || !defined(CONFIG_NEW_WIN_CHILDREN)
    31323135          exec_command ((char **)nargv, environ);

  • trunk/src/kmk/w32/winchildren.c

    r3158 r3159  
    2525
    2626/* No GNU coding style here atm, convert if upstreamed. */
     27
     28/** @page pg_win_children   Windows child process creation and managment
     29 *
     30 * This new implementation aims at addressing the following:
     31 *
     32 *      1. Speed up process creation by doing the expensive CreateProcess call
     33 *         in a worker thread.
     34 *
     35 *      2. No 64 process limit imposed by WaitForMultipleObjects.
     36 *
     37 *      3. Better distribute jobs among processor groups.
     38 *
     39 *      4. Offloading more expensive kmkbuiltin operations to worker threads,
     40 *         making the main thread focus on managing child processes.
     41 *
     42 *      5. Output synchronization using reusable pipes [not yet implemented].
     43 *
     44 *
     45 * To be quite honest, the first item (CreateProcess expense) didn't occur to me
     46 * at first and was more of a sideeffect discovered along the way.  A test
     47 * rebuilding IPRT went from 4m52s to 3m19s on a 8 thread system.
     48 *
     49 * The 2nd and 3rd goals are related to newer build servers that have lots of
     50 * CPU threads and various Windows NT (aka NT OS/2 at the time) design choices
     51 * made in the late 1980ies.
     52 *
     53 * WaitForMultipleObjects does not support waiting for more than 64 objects,
     54 * unlike poll and select.  This is just something everyone ends up having to
     55 * work around in the end.
     56 *
     57 * Affinity masks are uintptr_t sized, so 64-bit hosts can only manage 64
     58 * processors and 32-bit only 32.  Workaround was introduced with Windows 7
     59 * (IIRC) and is called processor groups.  The CPU threads are grouped into 1 or
     60 * more groups of up to 64 processors.  Processes are generally scheduled to a
     61 * signle processor group at first, but threads may be changed to be scheduled
     62 * on different groups.  This code will try distribute children evenly among the
     63 * processor groups, using a very simple algorithm (see details in code).
     64 *
     65 */
     66
    2767
    2868/*********************************************************************************************************************************
    … …  
    66106    /** Normal child process. */
    67107    WINCHILDTYPE_PROCESS,
     108#ifdef KMK
    68109    /** kmkbuiltin command. */
    69110    WINCHILDTYPE_BUILTIN,
    … …  
    72113    /** kmk_redirect job. */
    73114    WINCHILDTYPE_REDIRECT,
     115#endif
    74116    /** End of valid child types. */
    75117    WINCHILDTYPE_END
    … …  
    247289static unsigned volatile    g_idxLastChildcareWorker = 0;
    248290
     291/** Temporary RW lock for serializing kmkbuiltin_redirect and CreateProcess. */
     292static SRWLOCK              g_RWLock;
    249293
    250294
    … …  
    320364        }
    321365    }
     366
     367    /* Temporary: */
     368    InitializeSRWLock(&g_RWLock);
    322369}
    323370
    … …  
    12051252    if (rc == 0)
    12061253    {
     1254        AcquireSRWLockShared(&g_RWLock); /* temporary */
     1255
    12071256        /*
    12081257         * Populate startup info.
    … …  
    12481297                              TRUE /*fInheritHandles*/, fFlags, pwszzEnvironment, NULL /*pwsz*/, &StartupInfo, &ProcInfo);
    12491298        rc = GetLastError();
     1299        ReleaseSRWLockShared(&g_RWLock); /* temporary */
    12501300        if (fRet)
    12511301        {
    … …  
    13151365}
    13161366
     1367#ifdef KMK
     1368
    13171369/**
    13181370 * Childcare worker: handle builtin command.
    … …  
    13661418    mkWinChildcareWorkerWaitForProcess(pWorker, pChild, pChild->u.Redirect.hProcess);
    13671419}
     1420
     1421#endif /* KMK */
    13681422
    13691423/**
    … …  
    14351489                        mkWinChildcareWorkerThreadHandleProcess(pWorker, pChild);
    14361490                        break;
     1491#ifdef KMK
    14371492                    case WINCHILDTYPE_BUILTIN:
    14381493                        mkWinChildcareWorkerThreadHandleBuiltin(pWorker, pChild);
    … …  
    14441499                        mkWinChildcareWorkerThreadHandleRedirect(pWorker, pChild);
    14451500                        break;
     1501#endif
     1502                    default:
     1503                        assert(0);
    14461504                }
    14471505
    … …  
    16381696        }
    16391697
     1698#ifdef KMK
     1699
    16401700        case WINCHILDTYPE_BUILTIN:
    16411701            assert(0);
    … …  
    16571717            }
    16581718            break;
     1719
     1720#endif /* KMK */
     1721
     1722        default:
     1723            assert(0);
    16591724    }
    16601725
    … …  
    18901955}
    18911956
     1957#ifdef KMK
     1958
    18921959/**
    18931960 * Interface used by kSubmit.c for registering stuff to wait on.
    … …  
    19191986    return mkWinChildPushToCareWorker(pChild, pPid);
    19201987}
     1988
     1989#endif /* CONFIG_NEW_WIN_CHILDREN */
    19211990
    19221991/**
    … …  
    19432012                    pChild->iSignal = iSignal;
    19442013                    break;
     2014
     2015#ifdef KMK
    19452016
    19462017                case WINCHILDTYPE_SUBMIT:
    … …  
    19582029                case WINCHILDTYPE_BUILTIN:
    19592030                    break;
     2031
     2032#endif /* KMK */
     2033
     2034                default:
     2035                    assert(0);
    19602036            }
    19612037        }
    … …  
    20152091        case WINCHILDTYPE_PROCESS:
    20162092            break;
     2093#ifdef KMK
    20172094        case WINCHILDTYPE_BUILTIN:
    20182095            break;
    … …  
    20212098        case WINCHILDTYPE_REDIRECT:
    20222099            break;
     2100#endif /* KMK */
    20232101        default:
    20242102            assert(0);
    20252103    }
    20262104    mkWinChildDelete(pChild);
     2105
     2106#ifdef KMK
     2107    /* Flush the volatile directory cache. */
     2108    dir_cache_invalid_after_job();
     2109#endif
    20272110    return 0;
    20282111}
    20292112
    2030 
     2113/**
     2114 * Get the child completed event handle.
     2115 *
     2116 * Needed when w32os.c is waiting for a job token to become available, given
     2117 * that completed children is the typical source of these tokens (esp. for kmk).
     2118 *
     2119 * @returns Event handle.
     2120 */
    20312121intptr_t MkWinChildGetCompleteEventHandle(void)
    20322122{
    20332123    return (intptr_t)g_hEvtWaitChildren;
    20342124}
    2035 
    20362125
    20372126/**
    … …  
    20532142    int                     rc;
    20542143
    2055 /** @todo this code needs testing... */
    2056 
    20572144    /*
    20582145     * Get the executable name.
    … …  
    20842171    GetStartupInfoW(&StartupInfo);
    20852172    if (!CreateProcessW(wszImageName, pwszCommandLine, NULL /*pProcSecAttr*/, NULL /*pThreadSecAttr*/,
    2086                         TRUE /*fInheritHandles*/, 0 /*fFlags*/, pwszzEnvironment, NULL /*pwsz*/,
     2173                        TRUE /*fInheritHandles*/, CREATE_UNICODE_ENVIRONMENT, pwszzEnvironment, NULL /*pwsz*/,
    20872174                        &StartupInfo, &ProcInfo))
    20882175        ON(fatal, NILF, _("MkWinChildReExecMake: CreateProcessW failed: %u\n"), GetLastError());
    … …  
    21032190        if (dwStatus == WAIT_OBJECT_0)
    21042191        {
    2105             if (GetExitCodeProcess(ProcInfo.hProcess, &dwExitCode))
     2192            if (!GetExitCodeProcess(ProcInfo.hProcess, &dwExitCode))
     2193            {
    21062194                ON(fatal, NILF, _("MkWinChildReExecMake: GetExitCodeProcess failed: %u\n"), GetLastError());
    2107             else
    21082195                dwExitCode = -2222;
     2196            }
    21092197        }
    21102198        else if (dwStatus)
    … …  
    21172205}
    21182206
    2119 
     2207/** Temporary serialization with kmkbuiltin_redirect. */
     2208void MkWinChildExclusiveAcquire(void)
     2209{
     2210    AcquireSRWLockExclusive(&g_RWLock);
     2211}
     2212
     2213/** Temporary serialization with kmkbuiltin_redirect. */
     2214void MkWinChildExclusiveRelease(void)
     2215{
     2216    ReleaseSRWLockExclusive(&g_RWLock);
     2217}
     2218
     2219/**
     2220 * Implementation of the CLOSE_ON_EXEC macro.
     2221 *
     2222 * @returns errno value.
     2223 * @param   fd          The file descriptor to hide from children.
     2224 */
    21202225int MkWinChildUnrelatedCloseOnExec(int fd)
    21212226{
    … …  
    21282233                return 0;
    21292234        }
     2235        return errno;
    21302236    }
    21312237    return EINVAL;
    21322238}
    21332239
    2134 
    2135 

  • trunk/src/kmk/w32/winchildren.h

    r3156 r3159  
    3636int     MkWinChildKill(pid_t pid, int iSignal, struct child *pMkChild);
    3737int     MkWinChildWait(int fBlock, pid_t *pPid, int *piExitCode, int *piSignal, int *pfCoreDumped, struct child **ppMkChild);
     38void    MkWinChildExclusiveAcquire(void);
     39void    MkWinChildExclusiveRelease(void);
    3840
    3941#undef  CLOSE_ON_EXEC
Note: See TracChangeset for help on using the changeset viewer.