source: trunk/src/kmk/incdep.c@ 1853

Last change on this file since 1853 was 1852, checked in by bird, 17 years ago

kmk/incdep: fixed the multithreaded heap issue on darwin. Only one worker thread for the pthread bunch, should suffice.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 42.4 KB
Line 
1#ifdef CONFIG_WITH_INCLUDEDEP
2/* $Id: incdep.c 1852 2008-10-13 00:16:34Z bird $ */
3/** @file
4 * incdep - Simple dependency files.
5 */
6
7/*
8 * Copyright (c) 2006-2008 knut st. osmundsen <bird-src-spam@anduin.net>
9 *
10 * This file is part of kBuild.
11 *
12 * kBuild is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * kBuild is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with kBuild; if not, write to the Free Software
24 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 *
26 */
27
28
29/*******************************************************************************
30* Header Files *
31*******************************************************************************/
32#ifdef __OS2__
33# define INCL_BASE
34# define INCL_ERRORS
35#endif
36
37#include "make.h"
38
39#if !defined(WINDOWS32) && !defined(__OS2__)
40# define HAVE_PTHREAD
41#endif
42
43#include <assert.h>
44
45#include <glob.h>
46
47#include "dep.h"
48#include "filedef.h"
49#include "job.h"
50#include "commands.h"
51#include "variable.h"
52#include "rule.h"
53#include "debug.h"
54#include "hash.h"
55
56#ifdef HAVE_FCNTL_H
57# include <fcntl.h>
58#else
59# include <sys/file.h>
60#endif
61
62#ifdef WINDOWS32
63# include <io.h>
64# include <process.h>
65# include <Windows.h>
66# define PARSE_IN_WORKER
67#endif
68
69#ifdef __OS2__
70# include <os2.h>
71# include <sys/fmutex.h>
72#endif
73
74#ifdef HAVE_PTHREAD
75# include <pthread.h>
76#endif
77
78#ifdef __APPLE__
79# include <malloc/malloc.h>
80# define PARSE_IN_WORKER
81#endif
82
83
84/*******************************************************************************
85* Structures and Typedefs *
86*******************************************************************************/
87
88struct incdep_variable_in_set
89{
90 struct incdep_variable_in_set *next;
91 /* the parameters */
92 char *name; /* xmalloc'ed -> strcache */
93 unsigned int name_length;
94 const char *value; /* xmalloc'ed */
95 unsigned int value_length;
96 int duplicate_value; /* 0 */
97 enum variable_origin origin;
98 int recursive;
99 struct variable_set *set;
100 const struct floc *flocp; /* NILF */
101};
102
103struct incdep_variable_def
104{
105 struct incdep_variable_def *next;
106 /* the parameters */
107 const struct floc *flocp; /* NILF */
108 char *name; /* xmalloc'ed -> strcache */
109 unsigned int name_length; /* (not an actual parameter) */
110 char *value; /* xmalloc'ed, free it */
111 unsigned int value_length;
112 enum variable_origin origin;
113 enum variable_flavor flavor;
114 int target_var;
115};
116
117struct incdep_recorded_files
118{
119 struct incdep_recorded_files *next;
120
121 /* the parameters */
122 struct nameseq *filenames; /* only one file? its name needs be strcache'ed */
123 const char *pattern; /* NULL */
124 const char *pattern_percent; /* NULL */
125 struct dep *deps; /* names need to be strcache'ed */
126 unsigned int cmds_started; /* 0 */
127 char *commands; /* NULL */
128 unsigned int commands_idx; /* 0 */
129 int two_colon; /* 0 */
130 const struct floc *flocp; /* NILF */
131};
132
133
134/* per dep file structure. */
135struct incdep
136{
137 struct incdep *next;
138 char *file_base;
139 char *file_end;
140
141 int is_worker;
142#ifdef PARSE_IN_WORKER
143 unsigned int err_line_no;
144 const char *err_msg;
145
146 struct incdep_variable_in_set *recorded_variables_in_set_head;
147 struct incdep_variable_in_set *recorded_variables_in_set_tail;
148
149 struct incdep_variable_def *recorded_variable_defs_head;
150 struct incdep_variable_def *recorded_variable_defs_tail;
151
152 struct incdep_recorded_files *recorded_files_head;
153 struct incdep_recorded_files *recorded_files_tail;
154#endif
155
156 char name[1];
157};
158
159
160/*******************************************************************************
161* Global Variables *
162*******************************************************************************/
163
164/* mutex protecting the globals and an associated condition/event. */
165#ifdef HAVE_PTHREAD
166static pthread_mutex_t incdep_mtx;
167static pthread_cond_t incdep_cond_todo;
168static pthread_cond_t incdep_cond_done;
169
170#elif defined (WINDOWS32)
171static CRITICAL_SECTION incdep_mtx;
172static HANDLE incdep_hev_todo;
173static HANDLE incdep_hev_done;
174static int volatile incdep_hev_todo_waiters;
175static int volatile incdep_hev_done_waiters;
176
177#elif defined (__OS2__)
178static fmutex incdep_mtx;
179static HEV incdep_hev_todo;
180static HEV incdep_hev_done;
181static int volatile incdep_hev_todo_waiters;
182static int volatile incdep_hev_done_waiters;
183#endif
184
185/* flag indicating whether the threads, lock and event/condvars has
186 been initialized or not. */
187static int incdep_initialized;
188
189/* the list of files that needs reading. */
190static struct incdep * volatile incdep_head_todo;
191static struct incdep * volatile incdep_tail_todo;
192
193/* the number of files that are currently being read. */
194static int volatile incdep_num_reading;
195
196/* the list of files that have been read. */
197static struct incdep * volatile incdep_head_done;
198static struct incdep * volatile incdep_tail_done;
199
200/* The handles to the worker threads. */
201#ifdef HAVE_PTHREAD
202static pthread_t incdep_threads[1];
203#elif defined (WINDOWS32)
204static HANDLE incdep_threads[2];
205#elif defined (__OS2__)
206static TID incdep_threads[2];
207#endif
208static unsigned incdep_num_threads;
209
210/* flag indicating whether the worker threads should terminate or not. */
211static int volatile incdep_terminate;
212
213#ifdef __APPLE__
214/* malloc zone for the incdep threads. */
215static malloc_zone_t *incdep_zone;
216#endif
217
218
219/*******************************************************************************
220* Internal Functions *
221*******************************************************************************/
222static void incdep_flush_it (struct floc *);
223static void eval_include_dep_file (struct incdep *, struct floc *);
224
225
226/* xmalloc wrapper.
227 For working around multithreaded performance problems found on Darwin,
228 Linux (glibc), and possibly other systems. */
229static void *incdep_xmalloc (struct incdep *cur, size_t size)
230{
231 void *ptr;
232
233#ifdef __APPLE__
234 if (cur && cur->is_worker)
235 {
236 ptr = malloc_zone_malloc (incdep_zone, size);
237 if (!ptr)
238 fatal (NILF, _("virtual memory exhausted"));
239 }
240 else
241 ptr = xmalloc (size);
242#else
243 ptr = xmalloc (size);
244#endif
245
246 (void)cur;
247 return ptr;
248}
249
250/* memset(malloc(sz),'\0',sz) wrapper. */
251static void *incdep_xcalloc (struct incdep *cur, size_t size)
252{
253 void *ptr;
254
255#ifdef __APPLE__
256 if (cur && cur->is_worker)
257 ptr = malloc_zone_calloc (incdep_zone, size, 1);
258 else
259 ptr = calloc (size, 1);
260#else
261 ptr = calloc (size, 1);
262#endif
263 if (!ptr)
264 fatal (NILF, _("virtual memory exhausted"));
265
266 (void)cur;
267 return ptr;
268}
269
270/* free wrapper */
271static void incdep_xfree (struct incdep *cur, void *ptr)
272{
273 /* free() *must* work for the allocation hacks above because
274 of free_dep_chain. */
275 free (ptr);
276 (void)cur;
277}
278
279
280
281/* acquires the lock */
282void
283incdep_lock(void)
284{
285#ifdef HAVE_PTHREAD
286 pthread_mutex_lock (&incdep_mtx);
287#elif defined (WINDOWS32)
288 EnterCriticalSection (&incdep_mtx);
289#elif defined (__OS2__)
290 _fmutex_request (&incdep_mtx, 0)
291#endif
292}
293
294/* releases the lock */
295void
296incdep_unlock(void)
297{
298#ifdef HAVE_PTHREAD
299 pthread_mutex_unlock (&incdep_mtx);
300#elif defined(WINDOWS32)
301 LeaveCriticalSection (&incdep_mtx);
302#elif defined(__OS2__)
303 _fmutex_release (&incdep_mtx)
304#endif
305}
306
307/* signals the main thread that there is stuff todo. caller owns the lock. */
308static void
309incdep_signal_done (void)
310{
311#ifdef HAVE_PTHREAD
312 pthread_cond_broadcast (&incdep_cond_done);
313#elif defined (WINDOWS32)
314 if (incdep_hev_done_waiters)
315 SetEvent (incdep_hev_done);
316#elif defined (__OS2__)
317 if (incdep_hev_done_waiters)
318 DosPostEventSem (incdep_hev_done);
319#endif
320}
321
322/* waits for a reader to finish reading. caller owns the lock. */
323static void
324incdep_wait_done (void)
325{
326#ifdef HAVE_PTHREAD
327 pthread_cond_wait (&incdep_cond_done, &incdep_mtx);
328
329#elif defined (WINDOWS32)
330 ResetEvent (incdep_hev_done);
331 incdep_hev_done_waiters++;
332 incdep_unlock ();
333 WaitForSingleObject (incdep_hev_done, INFINITE);
334 incdep_lock ();
335 incdep_hev_done_waiters--;
336
337#elif defined (__OS2__)
338 ULONG ulIgnore;
339 DosResetEventSem (incdep_hev_done, &ulIgnore);
340 incdep_hev_done_waiters++;
341 incdep_unlock ();
342 DosWaitEventSem (incdep_hev_done, SEM_INDEFINITE_WAIT);
343 incdep_lock ();
344 incdep_hev_done_waiters--;
345#endif
346}
347
348/* signals the worker threads. caller owns the lock. */
349static void
350incdep_signal_todo (void)
351{
352#ifdef HAVE_PTHREAD
353 pthread_cond_broadcast (&incdep_cond_todo);
354#elif defined(WINDOWS32)
355 if (incdep_hev_todo_waiters)
356 SetEvent (incdep_hev_todo);
357#elif defined(__OS2__)
358 if (incdep_hev_todo_waiters)
359 DosPostEventSem (incdep_hev_todo);
360#endif
361}
362
363/* waits for stuff to arrive in the todo list. caller owns the lock. */
364static void
365incdep_wait_todo (void)
366{
367#ifdef HAVE_PTHREAD
368 pthread_cond_wait (&incdep_cond_todo, &incdep_mtx);
369
370#elif defined (WINDOWS32)
371 ResetEvent (incdep_hev_todo);
372 incdep_hev_todo_waiters++;
373 incdep_unlock ();
374 WaitForSingleObject (incdep_hev_todo, INFINITE);
375 incdep_lock ();
376 incdep_hev_todo_waiters--;
377
378#elif defined (__OS2__)
379 ULONG ulIgnore;
380 DosResetEventSem (incdep_hev_todo, &ulIgnore);
381 incdep_hev_todo_waiters++;
382 incdep_unlock ();
383 DosWaitEventSem (incdep_hev_todo, SEM_INDEFINITE_WAIT);
384 incdep_lock ();
385 incdep_hev_todo_waiters--;
386#endif
387}
388
389/* Reads a dep file into memory. */
390static int
391incdep_read_file (struct incdep *cur, struct floc *f)
392{
393 int fd;
394 struct stat st;
395
396 errno = 0;
397#ifdef O_BINARY
398 fd = open (cur->name, O_RDONLY | O_BINARY, 0);
399#else
400 fd = open (cur->name, O_RDONLY, 0);
401#endif
402 if (fd < 0)
403 {
404 /* ignore non-existing dependency files. */
405 int err = errno;
406 if (err == ENOENT || stat (cur->name, &st) != 0)
407 return 1;
408 error (f, "%s: %s", cur->name, strerror (err));
409 return -1;
410 }
411 if (!fstat (fd, &st))
412 {
413 cur->file_base = incdep_xmalloc (cur, st.st_size + 1);
414 if (read (fd, cur->file_base, st.st_size) == st.st_size)
415 {
416 close (fd);
417 cur->file_end = cur->file_base + st.st_size;
418 cur->file_base[st.st_size] = '\0';
419 return 0;
420 }
421
422 /* bail out */
423
424 error (f, "%s: read: %s", cur->name, strerror (errno));
425 incdep_xfree (cur, cur->file_base);
426 }
427 else
428 error (f, "%s: fstat: %s", cur->name, strerror (errno));
429
430 close (fd);
431 cur->file_base = cur->file_end = NULL;
432 return -1;
433}
434
435/* Free the incdep structure. */
436static void
437incdep_freeit (struct incdep *cur)
438{
439#ifdef PARSE_IN_WORKER
440 assert (!cur->recorded_variables_in_set_head);
441 assert (!cur->recorded_variable_defs_head);
442 assert (!cur->recorded_files_head);
443#endif
444
445 incdep_xfree (cur, cur->file_base);
446 free (cur);
447}
448
449/* A worker thread. */
450void
451incdep_worker (void)
452{
453 incdep_lock ();
454
455 while (!incdep_terminate)
456 {
457 /* get job from the todo list. */
458
459 struct incdep *cur = incdep_head_todo;
460 if (!cur)
461 {
462 incdep_wait_todo ();
463 continue;
464 }
465 if (cur->next)
466 incdep_head_todo = cur->next;
467 else
468 incdep_head_todo = incdep_tail_todo = NULL;
469 incdep_num_reading++;
470
471 /* read the file. */
472
473 incdep_unlock ();
474 cur->is_worker = 1;
475 incdep_read_file (cur, NILF);
476#ifdef PARSE_IN_WORKER
477 eval_include_dep_file (cur, NILF);
478#endif
479 cur->is_worker = 0;
480 incdep_lock ();
481
482 /* insert finished job into the done list. */
483
484 incdep_num_reading--;
485 cur->next = NULL;
486 if (incdep_tail_done)
487 incdep_tail_done->next = cur;
488 else
489 incdep_head_done = cur;
490 incdep_tail_done = cur;
491
492 incdep_signal_done ();
493 }
494
495 incdep_unlock ();
496}
497
498/* Thread library specific thread functions wrapping incdep_wroker. */
499#ifdef HAVE_PTHREAD
500static void *
501incdep_worker_pthread (void *ignore)
502{
503 incdep_worker ();
504 (void)ignore;
505 return NULL;
506}
507
508#elif defined (WINDOWS32)
509static unsigned __stdcall
510incdep_worker_windows (void *ignore)
511{
512 incdep_worker ();
513 (void)ignore;
514 return 0;
515}
516
517#elif defined (__OS2__)
518static void
519incdep_worker_os2 (void *ignore)
520{
521 incdep_worker ();
522 (void)ignore;
523}
524#endif
525
526/* Creates the the worker threads. */
527static void
528incdep_init (struct floc *f)
529{
530 unsigned i;
531#ifdef HAVE_PTHREAD
532 int rc;
533 pthread_attr_t attr;
534
535#elif defined (WINDOWS32)
536 unsigned tid;
537 uintptr_t hThread;
538
539#elif defined (__OS2__)
540 int rc;
541 int tid;
542#endif
543
544 /* heap hacks */
545
546#ifdef __APPLE__
547 incdep_zone = malloc_create_zone (0, 0);
548 if (!incdep_zone)
549 incdep_zone = malloc_default_zone ();
550#endif
551
552
553 /* create the mutex and two condition variables / event objects. */
554
555#ifdef HAVE_PTHREAD
556 rc = pthread_mutex_init (&incdep_mtx, NULL);
557 if (rc)
558 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
559 rc = pthread_cond_init (&incdep_cond_todo, NULL);
560 if (rc)
561 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
562 rc = pthread_cond_init (&incdep_cond_done, NULL);
563 if (rc)
564 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
565
566#elif defined (WINDOWS32)
567 InitializeCriticalSection (&incdep_mtx);
568 incdep_hev_todo = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
569 if (!incdep_hev_todo)
570 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
571 incdep_hev_done = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
572 if (!incdep_hev_done)
573 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
574 incdep_hev_todo_waiters = 0;
575 incdep_hev_done_waiters = 0;
576
577#elif defined (__OS2__)
578 _fmutex_create (&incdep_mtx, 0)
579 rc = DosCreateEventSem (NULL, &incdep_hev_todo, 0, FALSE);
580 if (rc)
581 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
582 rc = DosCreateEventSem (NULL, &incdep_hev_done, 0, FALSE);
583 if (rc)
584 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
585 incdep_hev_todo_waiters = 0;
586 incdep_hev_done_waiters = 0;
587#endif
588
589 /* create the worker threads. */
590
591 incdep_terminate = 0;
592 incdep_num_threads = sizeof (incdep_threads) / sizeof (incdep_threads[0]);
593 if (incdep_num_threads + 1 > job_slots)
594 incdep_num_threads = job_slots <= 1 ? 1 : job_slots - 1;
595 for (i = 0; i < incdep_num_threads; i++)
596 {
597#ifdef HAVE_PTHREAD
598 rc = pthread_attr_init (&attr);
599 if (rc)
600 fatal (f, _("pthread_attr_init failed: err=%d"), rc);
601 /*rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE); */
602 rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
603 if (rc)
604 fatal (f, _("pthread_attr_setdetachstate failed: err=%d"), rc);
605 rc = pthread_create (&incdep_threads[i], &attr,
606 incdep_worker_pthread, f);
607 if (rc)
608 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
609 pthread_attr_destroy (&attr);
610
611#elif defined (WINDOWS32)
612 tid = 0;
613 hThread = _beginthreadex (NULL, 128*1024, incdep_worker_windows,
614 NULL, 0, &tid);
615 if (hThread == 0 || hThread == ~(uintptr_t)0)
616 fatal (f, _("_beginthreadex failed: err=%d"), errno);
617 incdep_threads[i] = (HANDLE)hThread;
618
619#elif defined (__OS2__)
620 tid = _beginthread (incdep_worker_os2, NULL, 128*1024, NULL);
621 if (tid <= 0)
622 fatal (f, _("_beginthread failed: err=%d"), errno);
623 incdep_threads[i] = tid;
624#endif
625 }
626
627 incdep_initialized = 1;
628}
629
630/* Flushes outstanding work and terminates the worker threads.
631 This is called from snap_deps(). */
632void
633incdep_flush_and_term (void)
634{
635 unsigned i;
636
637 if (!incdep_initialized)
638 return;
639
640 /* flush any out standing work */
641
642 incdep_flush_it (NILF);
643
644 /* tell the threads to terminate */
645
646 incdep_lock ();
647 incdep_terminate = 1;
648 incdep_signal_todo ();
649 incdep_unlock ();
650
651 /* wait for the threads to quit */
652
653 for (i = 0; i < incdep_num_threads; i++)
654 {
655 /* later? */
656 }
657 incdep_num_threads = 0;
658
659 /* destroy the lock and condition variables / event objects. */
660
661 /* later */
662
663 incdep_initialized = 0;
664}
665
666#ifdef PARSE_IN_WORKER
667/* Flushes the recorded instructions. */
668static void
669incdep_flush_recorded_instructions (struct incdep *cur)
670{
671 struct incdep_variable_in_set *rec_vis;
672 struct incdep_variable_def *rec_vd;
673 struct incdep_recorded_files *rec_f;
674
675 /* define_variable_in_set */
676
677 rec_vis = cur->recorded_variables_in_set_head;
678 cur->recorded_variables_in_set_head = cur->recorded_variables_in_set_tail = NULL;
679 if (rec_vis)
680 do
681 {
682 void *free_me = rec_vis;
683 define_variable_in_set (strcache_add_len (rec_vis->name, rec_vis->name_length),
684 rec_vis->name_length,
685 rec_vis->value,
686 rec_vis->value_length,
687 rec_vis->duplicate_value,
688 rec_vis->origin,
689 rec_vis->recursive,
690 rec_vis->set,
691 rec_vis->flocp);
692 incdep_xfree (cur, rec_vis->name);
693 rec_vis = rec_vis->next;
694 incdep_xfree (cur, free_me);
695 }
696 while (rec_vis);
697
698 /* do_variable_definition */
699
700 rec_vd = cur->recorded_variable_defs_head;
701 cur->recorded_variable_defs_head = cur->recorded_variable_defs_tail = NULL;
702 if (rec_vd)
703 do
704 {
705 void *free_me = rec_vd;
706 do_variable_definition_2 (rec_vd->flocp,
707 strcache_add_len(rec_vd->name, rec_vd->name_length),
708 rec_vd->value,
709 rec_vd->value_length,
710 0,
711 rec_vd->value,
712 rec_vd->origin,
713 rec_vd->flavor,
714 rec_vd->target_var);
715 incdep_xfree (cur, rec_vd->name);
716 rec_vd = rec_vd->next;
717 incdep_xfree (cur, free_me);
718 }
719 while (rec_vd);
720
721 /* record_files */
722
723 rec_f = cur->recorded_files_head;
724 cur->recorded_files_head = cur->recorded_files_tail = NULL;
725 if (rec_f)
726 do
727 {
728 void *free_me = rec_f;
729 struct dep *dep;
730 const char *newname;
731
732 for (dep = rec_f->deps; dep; dep = dep->next)
733 {
734 newname = strcache_add (dep->name);
735 free ((char *)dep->name);
736 dep->name = newname;
737 }
738
739 newname = strcache_add (rec_f->filenames->name);
740 incdep_xfree (cur, (char *)rec_f->filenames->name);
741 rec_f->filenames->name = newname;
742
743 record_files (rec_f->filenames,
744 rec_f->pattern,
745 rec_f->pattern_percent,
746 rec_f->deps,
747 rec_f->cmds_started,
748 rec_f->commands,
749 rec_f->commands_idx,
750 rec_f->two_colon,
751 rec_f->flocp);
752
753 rec_f = rec_f->next;
754 incdep_xfree (cur, free_me);
755 }
756 while (rec_f);
757}
758#endif /* PARSE_IN_WORKER */
759
760/* Record / issue a warning about a misformed dep file. */
761static void
762incdep_warn (struct incdep *cur, unsigned int line_no, const char *msg)
763{
764 if (!cur->is_worker)
765 error (NILF, "%s(%d): %s", cur->name, line_no, msg);
766#ifdef PARSE_IN_WORKER
767 else
768 {
769 cur->err_line_no = line_no;
770 cur->err_msg = msg;
771 }
772#endif
773}
774
775/* Record / execute a strcache add. */
776static const char *
777incdep_record_strcache (struct incdep *cur, const char *str, int len)
778{
779 const char *ret;
780 if (!cur->is_worker)
781 {
782 /* Make sure the string is terminated before we hand it to
783 strcache_add_len so it does have to make a temporary copy
784 of it on the stack. */
785 char ch = str[len];
786 ((char *)str)[len] = '\0';
787 ret = strcache_add_len (str, len);
788 ((char *)str)[len] = ch;
789 }
790 else
791 {
792 /* Duplicate the string. The other recorders knows which arguments
793 needs to be added to the string cache later. */
794 char *newstr = incdep_xmalloc (cur, len + 1);
795 memcpy (newstr, str, len);
796 newstr[len] = '\0';
797 ret = newstr;
798 }
799 return ret;
800}
801
802/* Record / perform a variable definition in a set.
803 The NAME is in the string cache.
804 The VALUE is on the heap.
805 The DUPLICATE_VALUE is always 0. */
806static void
807incdep_record_variable_in_set (struct incdep *cur,
808 const char *name, unsigned int name_length,
809 const char *value,
810 unsigned int value_length,
811 int duplicate_value,
812 enum variable_origin origin,
813 int recursive,
814 struct variable_set *set,
815 const struct floc *flocp)
816{
817 assert (!duplicate_value);
818 if (!cur->is_worker)
819 define_variable_in_set (name, name_length, value, value_length,
820 duplicate_value, origin, recursive, set, flocp);
821#ifdef PARSE_IN_WORKER
822 else
823 {
824 struct incdep_variable_in_set *rec = incdep_xmalloc (cur, sizeof (*rec));
825 rec->name = (char *)name;
826 rec->name_length = name_length;
827 rec->value = value;
828 rec->value_length = value_length;
829 rec->duplicate_value = duplicate_value;
830 rec->origin = origin;
831 rec->recursive = recursive;
832 rec->set = set;
833 rec->flocp = flocp;
834
835 rec->next = NULL;
836 if (cur->recorded_variables_in_set_tail)
837 cur->recorded_variables_in_set_tail->next = rec;
838 else
839 cur->recorded_variables_in_set_head = rec;
840 cur->recorded_variables_in_set_tail = rec;
841 }
842#endif
843}
844
845/* Record / perform a variable definition. The VALUE should be disposed of. */
846static void
847incdep_record_variable_def (struct incdep *cur,
848 const struct floc *flocp,
849 const char *name,
850 unsigned int name_length,
851 char *value,
852 unsigned int value_length,
853 enum variable_origin origin,
854 enum variable_flavor flavor,
855 int target_var)
856{
857 if (!cur->is_worker)
858 do_variable_definition_2 (flocp, name, value, value_length, 0, value,
859 origin, flavor, target_var);
860#ifdef PARSE_IN_WORKER
861 else
862 {
863 struct incdep_variable_def *rec = incdep_xmalloc (cur, sizeof (*rec));
864 rec->flocp = flocp;
865 rec->name = (char *)name;
866 rec->name_length = name_length;
867 rec->value = value;
868 rec->value_length = value_length;
869 rec->origin = origin;
870 rec->flavor = flavor;
871 rec->target_var = target_var;
872
873 rec->next = NULL;
874 if (cur->recorded_variable_defs_tail)
875 cur->recorded_variable_defs_tail->next = rec;
876 else
877 cur->recorded_variable_defs_head = rec;
878 cur->recorded_variable_defs_tail = rec;
879 }
880#else
881 (void)name_length;
882#endif
883}
884
885/* Record files.*/
886static void
887incdep_record_files (struct incdep *cur,
888 struct nameseq *filenames, const char *pattern,
889 const char *pattern_percent, struct dep *deps,
890 unsigned int cmds_started, char *commands,
891 unsigned int commands_idx, int two_colon,
892 const struct floc *flocp)
893{
894 if (!cur->is_worker)
895 record_files (filenames, pattern, pattern_percent, deps, cmds_started,
896 commands, commands_idx, two_colon, flocp);
897#ifdef PARSE_IN_WORKER
898 else
899 {
900 struct incdep_recorded_files *rec = incdep_xmalloc (cur, sizeof (*rec));
901
902 rec->filenames = filenames;
903 rec->pattern = pattern;
904 rec->pattern_percent = pattern_percent;
905 rec->deps = deps;
906 rec->cmds_started = cmds_started;
907 rec->commands = commands;
908 rec->commands_idx = commands_idx;
909 rec->two_colon = two_colon;
910 rec->flocp = flocp;
911
912 rec->next = NULL;
913 if (cur->recorded_files_tail)
914 cur->recorded_files_tail->next = rec;
915 else
916 cur->recorded_files_head = rec;
917 cur->recorded_files_tail = rec;
918 }
919#endif
920}
921
922
923/* no nonsense dependency file including.
924
925 Because nobody wants bogus dependency files to break their incremental
926 builds with hard to comprehend error messages, this function does not
927 use the normal eval routine but does all the parsing itself. This isn't,
928 as much work as it sounds, because the necessary feature set is very
929 limited.
930
931 eval_include_dep_file groks:
932
933 define var
934 endef
935
936 var [|:|?|>]= value [\]
937
938 [\]
939 file: [deps] [\]
940
941 */
942static void
943eval_include_dep_file (struct incdep *curdep, struct floc *f)
944{
945 unsigned line_no = 1;
946 const char *file_end = curdep->file_end;
947 const char *cur = curdep->file_base;
948 const char *endp;
949
950 /* if no file data, just return immediately. */
951 if (!cur)
952 return;
953
954 /* now parse the file. */
955 while (cur < file_end)
956 {
957 /* skip empty lines */
958 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
959 ++cur;
960 if (cur >= file_end)
961 break;
962 if (*cur == '#')
963 {
964 cur = memchr (cur, '\n', file_end - cur);
965 if (!cur)
966 break;
967 }
968 if (*cur == '\\')
969 {
970 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
971 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
972 : (file_end - cur == 1) ? 1 : 0;
973 if (eol_len)
974 {
975 cur += eol_len;
976 line_no++;
977 continue;
978 }
979 }
980 if (*cur == '\n')
981 {
982 cur++;
983 line_no++;
984 continue;
985 }
986
987 /* define var
988 ...
989 endef */
990 if (strneq (cur, "define ", 7))
991 {
992 const char *var;
993 unsigned var_len;
994 const char *value_start;
995 const char *value_end;
996 char *value;
997 unsigned value_len;
998 int found_endef = 0;
999
1000 /* extract the variable name. */
1001 cur += 7;
1002 while (isblank ((unsigned char)*cur))
1003 ++cur;
1004 value_start = endp = memchr (cur, '\n', file_end - cur);
1005 if (!endp)
1006 endp = cur;
1007 while (endp > cur && isspace ((unsigned char)endp[-1]))
1008 --endp;
1009 var_len = endp - cur;
1010 if (!var_len)
1011 {
1012 incdep_warn (curdep, line_no, "bogus define statement.");
1013 break;
1014 }
1015 var = incdep_record_strcache (curdep, cur, var_len);
1016
1017 /* find the end of the variable. */
1018 cur = value_end = value_start = value_start + 1;
1019 ++line_no;
1020 while (cur < file_end)
1021 {
1022 /* check for endef, don't bother with skipping leading spaces. */
1023 if ( file_end - cur >= 5
1024 && strneq (cur, "endef", 5))
1025 {
1026 endp = cur + 5;
1027 while (endp < file_end && isspace ((unsigned char)*endp) && *endp != '\n')
1028 endp++;
1029 if (endp >= file_end || *endp == '\n')
1030 {
1031 found_endef = 1;
1032 cur = endp >= file_end ? file_end : endp + 1;
1033 break;
1034 }
1035 }
1036
1037 /* skip a line ahead. */
1038 cur = value_end = memchr (cur, '\n', file_end - cur);
1039 if (cur != NULL)
1040 ++cur;
1041 else
1042 cur = value_end = file_end;
1043 ++line_no;
1044 }
1045
1046 if (!found_endef)
1047 {
1048 incdep_warn (curdep, line_no, "missing endef, dropping the rest of the file.");
1049 break;
1050 }
1051 value_len = value_end - value_start;
1052 if (memchr (value_start, '\0', value_len))
1053 {
1054 incdep_warn (curdep, line_no, "'\\0' in define, dropping the rest of the file.");
1055 break;
1056 }
1057
1058 /* make a copy of the value, converting \r\n to \n, and define it. */
1059 value = incdep_xmalloc (curdep, value_len + 1);
1060 endp = memchr (value_start, '\r', value_len);
1061 if (endp)
1062 {
1063 const char *src = value_start;
1064 char *dst = value;
1065 for (;;)
1066 {
1067 size_t len = endp - src;
1068 memcpy (dst, src, len);
1069 dst += len;
1070 src = endp;
1071 if (src + 1 < file_end && src[1] == '\n')
1072 src++; /* skip the '\r' */
1073 if (src >= value_end)
1074 break;
1075 endp = memchr (endp + 1, '\r', src - value_end);
1076 if (!endp)
1077 endp = value_end;
1078 }
1079 value_len = dst - value;
1080 }
1081 else
1082 memcpy (value, value_start, value_len);
1083 value [value_len] = '\0';
1084
1085 incdep_record_variable_in_set (curdep,
1086 var, var_len, value, value_len,
1087 0 /* don't duplicate */, o_file,
1088 0 /* defines are recursive but this is faster */,
1089 NULL /* global set */, f);
1090 }
1091
1092 /* file: deps
1093 OR
1094 variable [:]= value */
1095 else
1096 {
1097 const char *colonp;
1098 const char *equalp;
1099
1100 /* Look for a colon and an equal sign, optimize for colon.
1101 Only one file is support and the colon / equal must be on
1102 the same line. */
1103 colonp = memchr (cur, ':', file_end - cur);
1104#ifdef HAVE_DOS_PATHS
1105 while ( colonp
1106 && colonp + 1 < file_end
1107 && (colonp[1] == '/' || colonp[1] == '\\')
1108 && colonp > cur
1109 && isalpha ((unsigned char)colonp[-1])
1110 && ( colonp == cur + 1
1111 || strchr (" \t(", colonp[-2]) != 0))
1112 colonp = memchr (colonp + 1, ':', file_end - (colonp + 1));
1113#endif
1114 endp = NULL;
1115 if ( !colonp
1116 || (endp = memchr (cur, '\n', colonp - cur)))
1117 {
1118 colonp = NULL;
1119 equalp = memchr (cur, '=', (endp ? endp : file_end) - cur);
1120 if ( !equalp
1121 || (!endp && memchr (cur, '\n', equalp - cur)))
1122 {
1123 incdep_warn (curdep, line_no, "no colon.");
1124 break;
1125 }
1126 }
1127 else
1128 equalp = memchr (cur, '=', (colonp + 2 <= file_end
1129 ? colonp + 2 : file_end) - cur);
1130 if (equalp)
1131 {
1132 /* An assignment of some sort. */
1133 const char *var;
1134 unsigned var_len;
1135 const char *value_start;
1136 const char *value_end;
1137 char *value;
1138 unsigned value_len;
1139 unsigned multi_line = 0;
1140 enum variable_flavor flavor;
1141
1142 /* figure the flavor first. */
1143 flavor = f_recursive;
1144 if (equalp > cur)
1145 {
1146 if (equalp[-1] == ':')
1147 flavor = f_simple;
1148 else if (equalp[-1] == '?')
1149 flavor = f_conditional;
1150 else if (equalp[-1] == '+')
1151 flavor = f_append;
1152 else if (equalp[-1] == '>')
1153 flavor = f_prepend;
1154 }
1155
1156 /* extract the variable name. */
1157 endp = flavor == f_recursive ? equalp : equalp - 1;
1158 while (endp > cur && isblank ((unsigned char)endp[-1]))
1159 --endp;
1160 var_len = endp - cur;
1161 if (!var_len)
1162 {
1163 incdep_warn (curdep, line_no, "empty variable. (includedep)");
1164 break;
1165 }
1166 if ( memchr (cur, '$', var_len)
1167 || memchr (cur, ' ', var_len)
1168 || memchr (cur, '\t', var_len))
1169 {
1170 incdep_warn (curdep, line_no, "fancy variable name. (includedep)");
1171 break;
1172 }
1173 var = incdep_record_strcache (curdep, cur, var_len);
1174
1175 /* find the start of the value. */
1176 cur = equalp + 1;
1177 while (cur < file_end && isblank ((unsigned char)*cur))
1178 cur++;
1179 value_start = cur;
1180
1181 /* find the end of the value / line (this isn't 101% correct). */
1182 value_end = cur;
1183 while (cur < file_end)
1184 {
1185 endp = value_end = memchr (cur, '\n', file_end - cur);
1186 if (!value_end)
1187 value_end = file_end;
1188 if (value_end - 1 >= cur && value_end[-1] == '\r')
1189 --value_end;
1190 if (value_end - 1 < cur || value_end[-1] != '\\')
1191 {
1192 cur = endp ? endp + 1 : file_end;
1193 break;
1194 }
1195 --value_end;
1196 if (value_end - 1 >= cur && value_end[-1] == '\\')
1197 {
1198 incdep_warn (curdep, line_no, "fancy escaping! (includedep)");
1199 cur = NULL;
1200 break;
1201 }
1202 if (!endp)
1203 {
1204 cur = file_end;
1205 break;
1206 }
1207
1208 cur = endp + 1;
1209 ++multi_line;
1210 ++line_no;
1211 }
1212 if (!cur)
1213 break;
1214 ++line_no;
1215
1216 /* make a copy of the value, converting \r\n to \n, and define it. */
1217 value_len = value_end - value_start;
1218 value = incdep_xmalloc (curdep, value_len + 1);
1219 if (!multi_line)
1220 memcpy (value, value_start, value_len);
1221 else
1222 {
1223 /* unescape it */
1224 const char *src = value_start;
1225 char *dst = value;
1226 while (src < value_end)
1227 {
1228 const char *nextp;
1229
1230 endp = memchr (src, '\n', value_end - src);
1231 if (!endp)
1232 nextp = endp = value_end;
1233 else
1234 nextp = endp + 1;
1235 if (endp > src && endp[-1] == '\r')
1236 --endp;
1237 if (endp > src && endp[-1] == '\\')
1238 --endp;
1239
1240 if (src != value_start)
1241 *dst++ = ' ';
1242 memcpy (dst, src, endp - src);
1243 dst += endp - src;
1244 src = nextp;
1245 }
1246 value_len = dst - value;
1247 }
1248 value [value_len] = '\0';
1249
1250 /* do the definition */
1251 if (flavor == f_recursive
1252 || ( flavor == f_simple
1253 && !memchr (value, '$', value_len)))
1254 incdep_record_variable_in_set (curdep,
1255 var, var_len, value, value_len,
1256 0 /* don't duplicate */, o_file,
1257 flavor == f_recursive /* recursive */,
1258 NULL /* global set */, f);
1259 else
1260 incdep_record_variable_def (curdep,
1261 f, var, var_len, value, value_len,
1262 o_file, flavor, 0 /* not target var */);
1263 }
1264 else
1265 {
1266 /* file: dependencies */
1267
1268 struct nameseq *filenames = 0;
1269 struct dep *deps = 0;
1270 struct dep **nextdep = &deps;
1271 struct dep *dep;
1272
1273 /* extract the filename, ASSUME a single one. */
1274 endp = colonp;
1275 while (endp > cur && isblank ((unsigned char)endp[-1]))
1276 --endp;
1277 if (cur == endp)
1278 {
1279 incdep_warn (curdep, line_no, "empty filename.");
1280 break;
1281 }
1282 if ( memchr (cur, '$', endp - cur)
1283 || memchr (cur, ' ', endp - cur)
1284 || memchr (cur, '\t', endp - cur))
1285 {
1286 incdep_warn (curdep, line_no, "multiple / fancy file name. (includedep)");
1287 break;
1288 }
1289 filenames = incdep_xmalloc (curdep, sizeof (struct nameseq));
1290 memset (filenames, 0, sizeof (*filenames));
1291 filenames->name = incdep_record_strcache (curdep, cur, endp - cur);
1292
1293 /* parse any dependencies. */
1294 cur = colonp + 1;
1295 while (cur < file_end)
1296 {
1297 /* skip blanks and count lines. */
1298 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1299 ++cur;
1300 if (cur >= file_end)
1301 break;
1302 if (*cur == '\n')
1303 {
1304 cur++;
1305 line_no++;
1306 break;
1307 }
1308
1309 /* continuation + eol? */
1310 if (*cur == '\\')
1311 {
1312 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1313 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1314 : (file_end - cur == 1) ? 1 : 0;
1315 if (eol_len)
1316 {
1317 cur += eol_len;
1318 line_no++;
1319 continue;
1320 }
1321 }
1322
1323 /* find the end of the filename */
1324 endp = cur;
1325 while (endp < file_end && !isspace ((unsigned char)*endp))
1326 ++endp;
1327
1328 /* add it to the list. */
1329 *nextdep = dep = incdep_xcalloc (curdep, sizeof (*dep));
1330 dep->name = incdep_record_strcache (curdep, cur, endp - cur);
1331 dep->includedep = 1;
1332 nextdep = &dep->next;
1333
1334 cur = endp;
1335 }
1336
1337 /* enter the file with its dependencies. */
1338 incdep_record_files (curdep,
1339 filenames, NULL, NULL, deps, 0, NULL, 0, 0, f);
1340 }
1341 }
1342 }
1343
1344 /* free the file data */
1345 incdep_xfree (curdep, curdep->file_base);
1346 curdep->file_base = curdep->file_end = NULL;
1347}
1348
1349/* Flushes the incdep todo and done lists. */
1350static void
1351incdep_flush_it (struct floc *f)
1352{
1353 incdep_lock ();
1354 for (;;)
1355 {
1356 struct incdep *cur = incdep_head_done;
1357
1358 /* if the done list is empty, grab a todo list entry. */
1359 if (!cur && incdep_head_todo)
1360 {
1361 cur = incdep_head_todo;
1362 if (cur->next)
1363 incdep_head_todo = cur->next;
1364 else
1365 incdep_head_todo = incdep_tail_todo = NULL;
1366 incdep_unlock ();
1367
1368 incdep_read_file (cur, f);
1369 eval_include_dep_file (cur, f);
1370 incdep_freeit (cur);
1371
1372 incdep_lock ();
1373 continue;
1374 }
1375
1376 /* if the todo list and done list are empty we're either done
1377 or will have to wait for the thread(s) to finish. */
1378 if (!cur && !incdep_num_reading)
1379 break; /* done */
1380 if (!cur)
1381 {
1382 while (!incdep_head_done)
1383 incdep_wait_done ();
1384 cur = incdep_head_done;
1385 }
1386
1387 /* we grab the entire done list and work thru it. */
1388 incdep_head_done = incdep_tail_done = NULL;
1389 incdep_unlock ();
1390
1391 while (cur)
1392 {
1393 struct incdep *next = cur->next;
1394#ifdef PARSE_IN_WORKER
1395 incdep_flush_recorded_instructions (cur);
1396#else
1397 eval_include_dep_file (cur, f);
1398#endif
1399 incdep_freeit (cur);
1400 cur = next;
1401 }
1402
1403 incdep_lock ();
1404 } /* outer loop */
1405 incdep_unlock ();
1406}
1407
1408
1409/* splits up a list of file names and feeds it to eval_include_dep_file,
1410 employing threads to try speed up the file reading. */
1411void
1412eval_include_dep (const char *names, struct floc *f, enum incdep_op op)
1413{
1414 struct incdep *head = 0;
1415 struct incdep *tail = 0;
1416 struct incdep *cur;
1417 const char *names_iterator = names;
1418 const char *name;
1419 unsigned int name_len;
1420
1421 /* loop through NAMES, creating a todo list out of them. */
1422
1423 while ((name = find_next_token (&names_iterator, &name_len)) != 0)
1424 {
1425 cur = xmalloc (sizeof (*cur) + name_len); /* not incdep_xmalloc here */
1426 cur->file_base = cur->file_end = NULL;
1427 memcpy (cur->name, name, name_len);
1428 cur->name[name_len] = '\0';
1429 cur->is_worker = 0;
1430#ifdef PARSE_IN_WORKER
1431 cur->err_line_no = 0;
1432 cur->err_msg = NULL;
1433 cur->recorded_variables_in_set_head = NULL;
1434 cur->recorded_variables_in_set_tail = NULL;
1435 cur->recorded_variable_defs_head = NULL;
1436 cur->recorded_variable_defs_tail = NULL;
1437 cur->recorded_files_head = NULL;
1438 cur->recorded_files_tail = NULL;
1439#endif
1440
1441 cur->next = NULL;
1442 if (tail)
1443 tail->next = cur;
1444 else
1445 head = cur;
1446 tail = cur;
1447 }
1448
1449 if (op == incdep_read_it)
1450 {
1451 /* work our way thru the files directly */
1452
1453 cur = head;
1454 while (cur)
1455 {
1456 struct incdep *next = cur->next;
1457 incdep_read_file (cur, f);
1458 eval_include_dep_file (cur, f);
1459 incdep_freeit (cur);
1460 cur = next;
1461 }
1462 }
1463 else
1464 {
1465 /* initialize the worker threads and related stuff the first time around. */
1466
1467 if (!incdep_initialized)
1468 incdep_init (f);
1469
1470 /* queue the files and notify the worker threads. */
1471
1472 incdep_lock ();
1473
1474 if (incdep_tail_todo)
1475 incdep_tail_todo->next = head;
1476 else
1477 incdep_head_todo = head;
1478 incdep_tail_todo = tail;
1479
1480 incdep_signal_todo ();
1481 incdep_unlock ();
1482
1483 /* flush the todo queue if we're requested to do so. */
1484
1485 if (op == incdep_flush)
1486 incdep_flush_it (f);
1487 }
1488}
1489
1490#endif /* CONFIG_WITH_INCLUDEDEP */
1491
Note: See TracBrowser for help on using the repository browser.