source: trunk/src/kmk/incdep.c@ 1869

Last change on this file since 1869 was 1868, checked in by bird, 17 years ago

kmk/incdep: getting acceptable parse-in-worker performance on linux now.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 43.9 KB
Line 
1#ifdef CONFIG_WITH_INCLUDEDEP
2/* $Id: incdep.c 1868 2008-10-15 23:10:08Z bird $ */
3/** @file
4 * incdep - Simple dependency files.
5 */
6
7/*
8 * Copyright (c) 2006-2008 knut st. osmundsen <bird-src-spam@anduin.net>
9 *
10 * This file is part of kBuild.
11 *
12 * kBuild is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * kBuild is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with kBuild; if not, write to the Free Software
24 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 *
26 */
27
28
29/*******************************************************************************
30* Header Files *
31*******************************************************************************/
32#ifdef __OS2__
33# define INCL_BASE
34# define INCL_ERRORS
35#endif
36
37#include "make.h"
38
39#if !defined(WINDOWS32) && !defined(__OS2__)
40# define HAVE_PTHREAD
41#endif
42
43#include <assert.h>
44
45#include <glob.h>
46
47#include "dep.h"
48#include "filedef.h"
49#include "job.h"
50#include "commands.h"
51#include "variable.h"
52#include "rule.h"
53#include "debug.h"
54#include "hash.h"
55
56#ifdef HAVE_FCNTL_H
57# include <fcntl.h>
58#else
59# include <sys/file.h>
60#endif
61
62#ifdef WINDOWS32
63# include <io.h>
64# include <process.h>
65# include <Windows.h>
66# define PARSE_IN_WORKER
67#endif
68
69#ifdef __OS2__
70# include <os2.h>
71# include <sys/fmutex.h>
72#endif
73
74#ifdef HAVE_PTHREAD
75# include <pthread.h>
76#endif
77
78#ifdef __APPLE__
79# include <malloc/malloc.h>
80# define PARSE_IN_WORKER
81#endif
82
83#if defined(__gnu_linux__) || defined(__linux__)
84# define PARSE_IN_WORKER
85#endif
86
87
88/*******************************************************************************
89* Structures and Typedefs *
90*******************************************************************************/
91struct incdep_strcache_entry
92{
93 unsigned int length;
94 unsigned int alignment;
95 unsigned long hash1;
96 unsigned long hash2;
97 char str[1];
98};
99
100struct incdep_variable_in_set
101{
102 struct incdep_variable_in_set *next;
103 /* the parameters */
104 struct incdep_strcache_entry *name_entry;
105 const char *value; /* xmalloc'ed */
106 unsigned int value_length;
107 int duplicate_value; /* 0 */
108 enum variable_origin origin;
109 int recursive;
110 struct variable_set *set;
111 const struct floc *flocp; /* NILF */
112};
113
114struct incdep_variable_def
115{
116 struct incdep_variable_def *next;
117 /* the parameters */
118 const struct floc *flocp; /* NILF */
119 struct incdep_strcache_entry *name_entry;
120 char *value; /* xmalloc'ed, free it */
121 unsigned int value_length;
122 enum variable_origin origin;
123 enum variable_flavor flavor;
124 int target_var;
125};
126
127struct incdep_recorded_files
128{
129 struct incdep_recorded_files *next;
130
131 /* the parameters */
132 struct incdep_strcache_entry *filename_entry; /* Converted to a nameseq record. */
133 const char *pattern; /* NULL */
134 const char *pattern_percent; /* NULL */
135 struct dep *deps; /* All the names are strcache entries. */
136 unsigned int cmds_started; /* 0 */
137 char *commands; /* NULL */
138 unsigned int commands_idx; /* 0 */
139 int two_colon; /* 0 */
140 const struct floc *flocp; /* NILF */
141};
142
143
144/* per dep file structure. */
145struct incdep
146{
147 struct incdep *next;
148 char *file_base;
149 char *file_end;
150
151 int worker_tid;
152#ifdef PARSE_IN_WORKER
153 unsigned int err_line_no;
154 const char *err_msg;
155
156 struct incdep_variable_in_set *recorded_variables_in_set_head;
157 struct incdep_variable_in_set *recorded_variables_in_set_tail;
158
159 struct incdep_variable_def *recorded_variable_defs_head;
160 struct incdep_variable_def *recorded_variable_defs_tail;
161
162 struct incdep_recorded_files *recorded_files_head;
163 struct incdep_recorded_files *recorded_files_tail;
164#endif
165
166 char name[1];
167};
168
169
170/*******************************************************************************
171* Global Variables *
172*******************************************************************************/
173
174/* mutex protecting the globals and an associated condition/event. */
175#ifdef HAVE_PTHREAD
176static pthread_mutex_t incdep_mtx;
177static pthread_cond_t incdep_cond_todo;
178static pthread_cond_t incdep_cond_done;
179
180#elif defined (WINDOWS32)
181static CRITICAL_SECTION incdep_mtx;
182static HANDLE incdep_hev_todo;
183static HANDLE incdep_hev_done;
184static int volatile incdep_hev_todo_waiters;
185static int volatile incdep_hev_done_waiters;
186
187#elif defined (__OS2__)
188static fmutex incdep_mtx;
189static HEV incdep_hev_todo;
190static HEV incdep_hev_done;
191static int volatile incdep_hev_todo_waiters;
192static int volatile incdep_hev_done_waiters;
193#endif
194
195/* flag indicating whether the threads, lock and event/condvars has
196 been initialized or not. */
197static int incdep_initialized;
198
199/* the list of files that needs reading. */
200static struct incdep * volatile incdep_head_todo;
201static struct incdep * volatile incdep_tail_todo;
202
203/* the number of files that are currently being read. */
204static int volatile incdep_num_reading;
205
206/* the list of files that have been read. */
207static struct incdep * volatile incdep_head_done;
208static struct incdep * volatile incdep_tail_done;
209
210
211/* The handles to the worker threads. */
212#ifdef HAVE_PTHREAD
213static pthread_t incdep_threads[1];
214static struct alloccache incdep_dep_caches[1];
215
216#elif defined (WINDOWS32)
217static HANDLE incdep_threads[2];
218static struct alloccache incdep_dep_caches[2];
219
220#elif defined (__OS2__)
221static TID incdep_threads[2];
222static struct alloccache incdep_dep_caches[2];
223#endif
224static unsigned incdep_num_threads;
225
226/* flag indicating whether the worker threads should terminate or not. */
227static int volatile incdep_terminate;
228
229#ifdef __APPLE__
230/* malloc zone for the incdep threads. */
231static malloc_zone_t *incdep_zone;
232#endif
233
234
235/*******************************************************************************
236* Internal Functions *
237*******************************************************************************/
238static void incdep_flush_it (struct floc *);
239static void eval_include_dep_file (struct incdep *, struct floc *);
240
241
242/* xmalloc wrapper.
243 For working around multithreaded performance problems found on Darwin,
244 Linux (glibc), and possibly other systems. */
245static void *
246incdep_xmalloc (struct incdep *cur, size_t size)
247{
248 void *ptr;
249
250#ifdef __APPLE__
251 if (cur && cur->worker_tid != -1)
252 {
253 ptr = malloc_zone_malloc (incdep_zone, size);
254 if (!ptr)
255 fatal (NILF, _("virtual memory exhausted"));
256 }
257 else
258 ptr = xmalloc (size);
259#else
260 ptr = xmalloc (size);
261#endif
262
263 (void)cur;
264 return ptr;
265}
266
267#if 0
268/* memset(malloc(sz),'\0',sz) wrapper. */
269static void *
270incdep_xcalloc (struct incdep *cur, size_t size)
271{
272 void *ptr;
273
274#ifdef __APPLE__
275 if (cur && cur->worker_tid != -1)
276 ptr = malloc_zone_calloc (incdep_zone, size, 1);
277 else
278 ptr = calloc (size, 1);
279#else
280 ptr = calloc (size, 1);
281#endif
282 if (!ptr)
283 fatal (NILF, _("virtual memory exhausted"));
284
285 (void)cur;
286 return ptr;
287}
288#endif /* unused */
289
290/* free wrapper */
291static void
292incdep_xfree (struct incdep *cur, void *ptr)
293{
294 /* free() *must* work for the allocation hacks above because
295 of free_dep_chain. */
296 free (ptr);
297 (void)cur;
298}
299
300/* alloc a dep structure. These are allocated in bunches to save time. */
301struct dep *
302incdep_alloc_dep (struct incdep *cur)
303{
304 struct alloccache *cache;
305 if (cur->worker_tid != -1)
306 cache = &incdep_dep_caches[cur->worker_tid];
307 else
308 cache = &dep_cache;
309 return alloccache_calloc (cache);
310}
311
312/* grow a cache. */
313static void *
314incdep_cache_allocator (void *thrd, unsigned int size)
315{
316 (void)thrd;
317#ifdef __APPLE__
318 return malloc_zone_malloc (incdep_zone, size);
319#else
320 return xmalloc (size);
321#endif
322}
323
324/* acquires the lock */
325void
326incdep_lock(void)
327{
328#ifdef HAVE_PTHREAD
329 pthread_mutex_lock (&incdep_mtx);
330#elif defined (WINDOWS32)
331 EnterCriticalSection (&incdep_mtx);
332#elif defined (__OS2__)
333 _fmutex_request (&incdep_mtx, 0)
334#endif
335}
336
337/* releases the lock */
338void
339incdep_unlock(void)
340{
341#ifdef HAVE_PTHREAD
342 pthread_mutex_unlock (&incdep_mtx);
343#elif defined(WINDOWS32)
344 LeaveCriticalSection (&incdep_mtx);
345#elif defined(__OS2__)
346 _fmutex_release (&incdep_mtx)
347#endif
348}
349
350/* signals the main thread that there is stuff todo. caller owns the lock. */
351static void
352incdep_signal_done (void)
353{
354#ifdef HAVE_PTHREAD
355 pthread_cond_broadcast (&incdep_cond_done);
356#elif defined (WINDOWS32)
357 if (incdep_hev_done_waiters)
358 SetEvent (incdep_hev_done);
359#elif defined (__OS2__)
360 if (incdep_hev_done_waiters)
361 DosPostEventSem (incdep_hev_done);
362#endif
363}
364
365/* waits for a reader to finish reading. caller owns the lock. */
366static void
367incdep_wait_done (void)
368{
369#ifdef HAVE_PTHREAD
370 pthread_cond_wait (&incdep_cond_done, &incdep_mtx);
371
372#elif defined (WINDOWS32)
373 ResetEvent (incdep_hev_done);
374 incdep_hev_done_waiters++;
375 incdep_unlock ();
376 WaitForSingleObject (incdep_hev_done, INFINITE);
377 incdep_lock ();
378 incdep_hev_done_waiters--;
379
380#elif defined (__OS2__)
381 ULONG ulIgnore;
382 DosResetEventSem (incdep_hev_done, &ulIgnore);
383 incdep_hev_done_waiters++;
384 incdep_unlock ();
385 DosWaitEventSem (incdep_hev_done, SEM_INDEFINITE_WAIT);
386 incdep_lock ();
387 incdep_hev_done_waiters--;
388#endif
389}
390
391/* signals the worker threads. caller owns the lock. */
392static void
393incdep_signal_todo (void)
394{
395#ifdef HAVE_PTHREAD
396 pthread_cond_broadcast (&incdep_cond_todo);
397#elif defined(WINDOWS32)
398 if (incdep_hev_todo_waiters)
399 SetEvent (incdep_hev_todo);
400#elif defined(__OS2__)
401 if (incdep_hev_todo_waiters)
402 DosPostEventSem (incdep_hev_todo);
403#endif
404}
405
406/* waits for stuff to arrive in the todo list. caller owns the lock. */
407static void
408incdep_wait_todo (void)
409{
410#ifdef HAVE_PTHREAD
411 pthread_cond_wait (&incdep_cond_todo, &incdep_mtx);
412
413#elif defined (WINDOWS32)
414 ResetEvent (incdep_hev_todo);
415 incdep_hev_todo_waiters++;
416 incdep_unlock ();
417 WaitForSingleObject (incdep_hev_todo, INFINITE);
418 incdep_lock ();
419 incdep_hev_todo_waiters--;
420
421#elif defined (__OS2__)
422 ULONG ulIgnore;
423 DosResetEventSem (incdep_hev_todo, &ulIgnore);
424 incdep_hev_todo_waiters++;
425 incdep_unlock ();
426 DosWaitEventSem (incdep_hev_todo, SEM_INDEFINITE_WAIT);
427 incdep_lock ();
428 incdep_hev_todo_waiters--;
429#endif
430}
431
432/* Reads a dep file into memory. */
433static int
434incdep_read_file (struct incdep *cur, struct floc *f)
435{
436 int fd;
437 struct stat st;
438
439 errno = 0;
440#ifdef O_BINARY
441 fd = open (cur->name, O_RDONLY | O_BINARY, 0);
442#else
443 fd = open (cur->name, O_RDONLY, 0);
444#endif
445 if (fd < 0)
446 {
447 /* ignore non-existing dependency files. */
448 int err = errno;
449 if (err == ENOENT || stat (cur->name, &st) != 0)
450 return 1;
451 error (f, "%s: %s", cur->name, strerror (err));
452 return -1;
453 }
454 if (!fstat (fd, &st))
455 {
456 cur->file_base = incdep_xmalloc (cur, st.st_size + 1);
457 if (read (fd, cur->file_base, st.st_size) == st.st_size)
458 {
459 close (fd);
460 cur->file_end = cur->file_base + st.st_size;
461 cur->file_base[st.st_size] = '\0';
462 return 0;
463 }
464
465 /* bail out */
466
467 error (f, "%s: read: %s", cur->name, strerror (errno));
468 incdep_xfree (cur, cur->file_base);
469 }
470 else
471 error (f, "%s: fstat: %s", cur->name, strerror (errno));
472
473 close (fd);
474 cur->file_base = cur->file_end = NULL;
475 return -1;
476}
477
478/* Free the incdep structure. */
479static void
480incdep_freeit (struct incdep *cur)
481{
482#ifdef PARSE_IN_WORKER
483 assert (!cur->recorded_variables_in_set_head);
484 assert (!cur->recorded_variable_defs_head);
485 assert (!cur->recorded_files_head);
486#endif
487
488 incdep_xfree (cur, cur->file_base);
489 free (cur);
490}
491
492/* A worker thread. */
493void
494incdep_worker (int thrd)
495{
496 incdep_lock ();
497
498 while (!incdep_terminate)
499 {
500 /* get job from the todo list. */
501
502 struct incdep *cur = incdep_head_todo;
503 if (!cur)
504 {
505 incdep_wait_todo ();
506 continue;
507 }
508 if (cur->next)
509 incdep_head_todo = cur->next;
510 else
511 incdep_head_todo = incdep_tail_todo = NULL;
512 incdep_num_reading++;
513
514 /* read the file. */
515
516 incdep_unlock ();
517 cur->worker_tid = thrd;
518
519 incdep_read_file (cur, NILF);
520#ifdef PARSE_IN_WORKER
521 eval_include_dep_file (cur, NILF);
522#endif
523
524 cur->worker_tid = -1;
525 incdep_lock ();
526
527 /* insert finished job into the done list. */
528
529 incdep_num_reading--;
530 cur->next = NULL;
531 if (incdep_tail_done)
532 incdep_tail_done->next = cur;
533 else
534 incdep_head_done = cur;
535 incdep_tail_done = cur;
536
537 incdep_signal_done ();
538 }
539
540 incdep_unlock ();
541}
542
543/* Thread library specific thread functions wrapping incdep_wroker. */
544#ifdef HAVE_PTHREAD
545static void *
546incdep_worker_pthread (void *thrd)
547{
548 incdep_worker ((size_t)thrd);
549 return NULL;
550}
551
552#elif defined (WINDOWS32)
553static unsigned __stdcall
554incdep_worker_windows (void *thrd)
555{
556 incdep_worker ((size_t)thrd);
557 return 0;
558}
559
560#elif defined (__OS2__)
561static void
562incdep_worker_os2 (void *thrd)
563{
564 incdep_worker ((size_t)thrd);
565}
566#endif
567
568/* Creates the the worker threads. */
569static void
570incdep_init (struct floc *f)
571{
572 unsigned i;
573#ifdef HAVE_PTHREAD
574 int rc;
575 pthread_attr_t attr;
576
577#elif defined (WINDOWS32)
578 unsigned tid;
579 uintptr_t hThread;
580
581#elif defined (__OS2__)
582 int rc;
583 int tid;
584#endif
585
586 /* heap hacks */
587
588#ifdef __APPLE__
589 incdep_zone = malloc_create_zone (0, 0);
590 if (!incdep_zone)
591 incdep_zone = malloc_default_zone ();
592#endif
593
594
595 /* create the mutex and two condition variables / event objects. */
596
597#ifdef HAVE_PTHREAD
598 rc = pthread_mutex_init (&incdep_mtx, NULL);
599 if (rc)
600 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
601 rc = pthread_cond_init (&incdep_cond_todo, NULL);
602 if (rc)
603 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
604 rc = pthread_cond_init (&incdep_cond_done, NULL);
605 if (rc)
606 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
607
608#elif defined (WINDOWS32)
609 InitializeCriticalSection (&incdep_mtx);
610 incdep_hev_todo = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
611 if (!incdep_hev_todo)
612 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
613 incdep_hev_done = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
614 if (!incdep_hev_done)
615 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
616 incdep_hev_todo_waiters = 0;
617 incdep_hev_done_waiters = 0;
618
619#elif defined (__OS2__)
620 _fmutex_create (&incdep_mtx, 0)
621 rc = DosCreateEventSem (NULL, &incdep_hev_todo, 0, FALSE);
622 if (rc)
623 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
624 rc = DosCreateEventSem (NULL, &incdep_hev_done, 0, FALSE);
625 if (rc)
626 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
627 incdep_hev_todo_waiters = 0;
628 incdep_hev_done_waiters = 0;
629#endif
630
631 /* create the worker threads. */
632
633 incdep_terminate = 0;
634 incdep_num_threads = sizeof (incdep_threads) / sizeof (incdep_threads[0]);
635 if (incdep_num_threads + 1 > job_slots)
636 incdep_num_threads = job_slots <= 1 ? 1 : job_slots - 1;
637 for (i = 0; i < incdep_num_threads; i++)
638 {
639 alloccache_init (&incdep_dep_caches[i], sizeof(struct dep), "incdep dep",
640 incdep_cache_allocator, (void *)(size_t)i);
641
642#ifdef HAVE_PTHREAD
643 rc = pthread_attr_init (&attr);
644 if (rc)
645 fatal (f, _("pthread_attr_init failed: err=%d"), rc);
646 /*rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE); */
647 rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
648 if (rc)
649 fatal (f, _("pthread_attr_setdetachstate failed: err=%d"), rc);
650 rc = pthread_create(&incdep_threads[i], &attr,
651 incdep_worker_pthread, (void *)(size_t)i);
652 if (rc)
653 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
654 pthread_attr_destroy (&attr);
655
656#elif defined (WINDOWS32)
657 tid = 0;
658 hThread = _beginthreadex (NULL, 128*1024, incdep_worker_windows,
659 (void *)i, 0, &tid);
660 if (hThread == 0 || hThread == ~(uintptr_t)0)
661 fatal (f, _("_beginthreadex failed: err=%d"), errno);
662 incdep_threads[i] = (HANDLE)hThread;
663
664#elif defined (__OS2__)
665 tid = _beginthread (incdep_worker_os2, NULL, 128*1024, (void *)i);
666 if (tid <= 0)
667 fatal (f, _("_beginthread failed: err=%d"), errno);
668 incdep_threads[i] = tid;
669#endif
670 }
671
672 incdep_initialized = 1;
673}
674
675/* Flushes outstanding work and terminates the worker threads.
676 This is called from snap_deps(). */
677void
678incdep_flush_and_term (void)
679{
680 unsigned i;
681
682 if (!incdep_initialized)
683 return;
684
685 /* flush any out standing work */
686
687 incdep_flush_it (NILF);
688
689 /* tell the threads to terminate */
690
691 incdep_lock ();
692 incdep_terminate = 1;
693 incdep_signal_todo ();
694 incdep_unlock ();
695
696 /* wait for the threads to quit */
697
698 for (i = 0; i < incdep_num_threads; i++)
699 {
700 /* more later? */
701
702 alloccache_join (&dep_cache, &incdep_dep_caches[i]);
703 }
704 incdep_num_threads = 0;
705
706 /* destroy the lock and condition variables / event objects. */
707
708 /* later */
709
710 incdep_initialized = 0;
711}
712
713#ifdef PARSE_IN_WORKER
714/* Flushes a strcache entry returning the actual string cache entry.
715 The input is freed! */
716static const char *
717incdep_flush_strcache_entry (const void *pv_entry)
718{
719 struct incdep_strcache_entry *entry = (struct incdep_strcache_entry *)pv_entry;
720 const char *result;
721 result = strcache_add_prehashed (entry->str, entry->length, entry->hash1, entry->hash2);
722 free (entry);
723 return result;
724}
725
726/* Flushes the recorded instructions. */
727static void
728incdep_flush_recorded_instructions (struct incdep *cur)
729{
730 struct incdep_variable_in_set *rec_vis;
731 struct incdep_variable_def *rec_vd;
732 struct incdep_recorded_files *rec_f;
733
734 /* define_variable_in_set */
735
736 rec_vis = cur->recorded_variables_in_set_head;
737 cur->recorded_variables_in_set_head = cur->recorded_variables_in_set_tail = NULL;
738 if (rec_vis)
739 do
740 {
741 void *free_me = rec_vis;
742 unsigned int name_length = rec_vis->name_entry->length;
743 define_variable_in_set (incdep_flush_strcache_entry (rec_vis->name_entry),
744 name_length,
745 rec_vis->value,
746 rec_vis->value_length,
747 rec_vis->duplicate_value,
748 rec_vis->origin,
749 rec_vis->recursive,
750 rec_vis->set,
751 rec_vis->flocp);
752 rec_vis = rec_vis->next;
753 incdep_xfree (cur, free_me);
754 }
755 while (rec_vis);
756
757 /* do_variable_definition */
758
759 rec_vd = cur->recorded_variable_defs_head;
760 cur->recorded_variable_defs_head = cur->recorded_variable_defs_tail = NULL;
761 if (rec_vd)
762 do
763 {
764 void *free_me = rec_vd;
765 do_variable_definition_2 (rec_vd->flocp,
766 incdep_flush_strcache_entry (rec_vd->name_entry),
767 rec_vd->value,
768 rec_vd->value_length,
769 0,
770 rec_vd->value,
771 rec_vd->origin,
772 rec_vd->flavor,
773 rec_vd->target_var);
774 rec_vd = rec_vd->next;
775 incdep_xfree (cur, free_me);
776 }
777 while (rec_vd);
778
779 /* record_files */
780
781 rec_f = cur->recorded_files_head;
782 cur->recorded_files_head = cur->recorded_files_tail = NULL;
783 if (rec_f)
784 do
785 {
786 void *free_me = rec_f;
787 struct dep *dep;
788 struct nameseq *filenames;
789
790 for (dep = rec_f->deps; dep; dep = dep->next)
791 dep->name = incdep_flush_strcache_entry (dep->name);
792
793 filenames = (struct nameseq *) alloccache_alloc (&nameseq_cache);
794 filenames->next = 0;
795 filenames->name = incdep_flush_strcache_entry (rec_f->filename_entry);
796
797 record_files (filenames,
798 rec_f->pattern,
799 rec_f->pattern_percent,
800 rec_f->deps,
801 rec_f->cmds_started,
802 rec_f->commands,
803 rec_f->commands_idx,
804 rec_f->two_colon,
805 rec_f->flocp);
806
807 rec_f = rec_f->next;
808 incdep_xfree (cur, free_me);
809 }
810 while (rec_f);
811}
812#endif /* PARSE_IN_WORKER */
813
814/* Record / issue a warning about a misformed dep file. */
815static void
816incdep_warn (struct incdep *cur, unsigned int line_no, const char *msg)
817{
818 if (cur->worker_tid == -1)
819 error (NILF, "%s(%d): %s", cur->name, line_no, msg);
820#ifdef PARSE_IN_WORKER
821 else
822 {
823 cur->err_line_no = line_no;
824 cur->err_msg = msg;
825 }
826#endif
827}
828
829/* Record / execute a strcache add. */
830static const char *
831incdep_record_strcache (struct incdep *cur, const char *str, int len)
832{
833 const char *ret;
834 if (cur->worker_tid == -1)
835 {
836 /* Make sure the string is terminated before we hand it to
837 strcache_add_len so it does have to make a temporary copy
838 of it on the stack. */
839 char ch = str[len];
840 ((char *)str)[len] = '\0';
841 ret = strcache_add_len (str, len);
842 ((char *)str)[len] = ch;
843 }
844 else
845 {
846 /* Allocate a strcache record for it, pre-hashing the string to save
847 time later on in the main thread. */
848 struct incdep_strcache_entry *entry = incdep_xmalloc (cur, sizeof (*entry) + len);
849 memcpy (entry->str, str, len);
850 entry->str[len] = '\0';
851 entry->length = len;
852 strcache_prehash_str (entry->str, &entry->hash1, &entry->hash2);
853
854 ret = (const char *)entry;
855 }
856 return ret;
857}
858
859/* Record / perform a variable definition in a set.
860 The NAME is in the string cache.
861 The VALUE is on the heap.
862 The DUPLICATE_VALUE is always 0. */
863static void
864incdep_record_variable_in_set (struct incdep *cur,
865 const char *name, unsigned int name_length,
866 const char *value,
867 unsigned int value_length,
868 int duplicate_value,
869 enum variable_origin origin,
870 int recursive,
871 struct variable_set *set,
872 const struct floc *flocp)
873{
874 assert (!duplicate_value);
875 if (cur->worker_tid == -1)
876 define_variable_in_set (name, name_length, value, value_length,
877 duplicate_value, origin, recursive, set, flocp);
878#ifdef PARSE_IN_WORKER
879 else
880 {
881 struct incdep_variable_in_set *rec = incdep_xmalloc (cur, sizeof (*rec));
882 rec->name_entry = (struct incdep_strcache_entry *)name;
883 rec->value = value;
884 rec->value_length = value_length;
885 rec->duplicate_value = duplicate_value;
886 rec->origin = origin;
887 rec->recursive = recursive;
888 rec->set = set;
889 rec->flocp = flocp;
890
891 rec->next = NULL;
892 if (cur->recorded_variables_in_set_tail)
893 cur->recorded_variables_in_set_tail->next = rec;
894 else
895 cur->recorded_variables_in_set_head = rec;
896 cur->recorded_variables_in_set_tail = rec;
897 }
898#endif
899}
900
901/* Record / perform a variable definition. The VALUE should be disposed of. */
902static void
903incdep_record_variable_def (struct incdep *cur,
904 const struct floc *flocp,
905 const char *name,
906 unsigned int name_length,
907 char *value,
908 unsigned int value_length,
909 enum variable_origin origin,
910 enum variable_flavor flavor,
911 int target_var)
912{
913 if (cur->worker_tid == -1)
914 do_variable_definition_2 (flocp, name, value, value_length, 0, value,
915 origin, flavor, target_var);
916#ifdef PARSE_IN_WORKER
917 else
918 {
919 struct incdep_variable_def *rec = incdep_xmalloc (cur, sizeof (*rec));
920 rec->flocp = flocp;
921 rec->name_entry = (struct incdep_strcache_entry *)name;
922 rec->value = value;
923 rec->value_length = value_length;
924 rec->origin = origin;
925 rec->flavor = flavor;
926 rec->target_var = target_var;
927
928 rec->next = NULL;
929 if (cur->recorded_variable_defs_tail)
930 cur->recorded_variable_defs_tail->next = rec;
931 else
932 cur->recorded_variable_defs_head = rec;
933 cur->recorded_variable_defs_tail = rec;
934 }
935#else
936 (void)name_length;
937#endif
938}
939
940/* Record files.*/
941static void
942incdep_record_files (struct incdep *cur,
943 const char *filename, const char *pattern,
944 const char *pattern_percent, struct dep *deps,
945 unsigned int cmds_started, char *commands,
946 unsigned int commands_idx, int two_colon,
947 const struct floc *flocp)
948{
949 if (cur->worker_tid == -1)
950 {
951 struct nameseq *filenames = (struct nameseq *) alloccache_alloc (&nameseq_cache);
952 filenames->next = 0;
953 filenames->name = filename;
954 record_files (filenames, pattern, pattern_percent, deps, cmds_started,
955 commands, commands_idx, two_colon, flocp);
956 }
957#ifdef PARSE_IN_WORKER
958 else
959 {
960 struct incdep_recorded_files *rec = incdep_xmalloc (cur, sizeof (*rec));
961
962 rec->filename_entry = (struct incdep_strcache_entry *)filename;
963 rec->pattern = pattern;
964 rec->pattern_percent = pattern_percent;
965 rec->deps = deps;
966 rec->cmds_started = cmds_started;
967 rec->commands = commands;
968 rec->commands_idx = commands_idx;
969 rec->two_colon = two_colon;
970 rec->flocp = flocp;
971
972 rec->next = NULL;
973 if (cur->recorded_files_tail)
974 cur->recorded_files_tail->next = rec;
975 else
976 cur->recorded_files_head = rec;
977 cur->recorded_files_tail = rec;
978 }
979#endif
980}
981
982
983/* no nonsense dependency file including.
984
985 Because nobody wants bogus dependency files to break their incremental
986 builds with hard to comprehend error messages, this function does not
987 use the normal eval routine but does all the parsing itself. This isn't,
988 as much work as it sounds, because the necessary feature set is very
989 limited.
990
991 eval_include_dep_file groks:
992
993 define var
994 endef
995
996 var [|:|?|>]= value [\]
997
998 [\]
999 file: [deps] [\]
1000
1001 */
1002static void
1003eval_include_dep_file (struct incdep *curdep, struct floc *f)
1004{
1005 unsigned line_no = 1;
1006 const char *file_end = curdep->file_end;
1007 const char *cur = curdep->file_base;
1008 const char *endp;
1009
1010 /* if no file data, just return immediately. */
1011 if (!cur)
1012 return;
1013
1014 /* now parse the file. */
1015 while (cur < file_end)
1016 {
1017 /* skip empty lines */
1018 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1019 ++cur;
1020 if (cur >= file_end)
1021 break;
1022 if (*cur == '#')
1023 {
1024 cur = memchr (cur, '\n', file_end - cur);
1025 if (!cur)
1026 break;
1027 }
1028 if (*cur == '\\')
1029 {
1030 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1031 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1032 : (file_end - cur == 1) ? 1 : 0;
1033 if (eol_len)
1034 {
1035 cur += eol_len;
1036 line_no++;
1037 continue;
1038 }
1039 }
1040 if (*cur == '\n')
1041 {
1042 cur++;
1043 line_no++;
1044 continue;
1045 }
1046
1047 /* define var
1048 ...
1049 endef */
1050 if (strneq (cur, "define ", 7))
1051 {
1052 const char *var;
1053 unsigned var_len;
1054 const char *value_start;
1055 const char *value_end;
1056 char *value;
1057 unsigned value_len;
1058 int found_endef = 0;
1059
1060 /* extract the variable name. */
1061 cur += 7;
1062 while (isblank ((unsigned char)*cur))
1063 ++cur;
1064 value_start = endp = memchr (cur, '\n', file_end - cur);
1065 if (!endp)
1066 endp = cur;
1067 while (endp > cur && isspace ((unsigned char)endp[-1]))
1068 --endp;
1069 var_len = endp - cur;
1070 if (!var_len)
1071 {
1072 incdep_warn (curdep, line_no, "bogus define statement.");
1073 break;
1074 }
1075 var = incdep_record_strcache (curdep, cur, var_len);
1076
1077 /* find the end of the variable. */
1078 cur = value_end = value_start = value_start + 1;
1079 ++line_no;
1080 while (cur < file_end)
1081 {
1082 /* check for endef, don't bother with skipping leading spaces. */
1083 if ( file_end - cur >= 5
1084 && strneq (cur, "endef", 5))
1085 {
1086 endp = cur + 5;
1087 while (endp < file_end && isspace ((unsigned char)*endp) && *endp != '\n')
1088 endp++;
1089 if (endp >= file_end || *endp == '\n')
1090 {
1091 found_endef = 1;
1092 cur = endp >= file_end ? file_end : endp + 1;
1093 break;
1094 }
1095 }
1096
1097 /* skip a line ahead. */
1098 cur = value_end = memchr (cur, '\n', file_end - cur);
1099 if (cur != NULL)
1100 ++cur;
1101 else
1102 cur = value_end = file_end;
1103 ++line_no;
1104 }
1105
1106 if (!found_endef)
1107 {
1108 incdep_warn (curdep, line_no, "missing endef, dropping the rest of the file.");
1109 break;
1110 }
1111 value_len = value_end - value_start;
1112 if (memchr (value_start, '\0', value_len))
1113 {
1114 incdep_warn (curdep, line_no, "'\\0' in define, dropping the rest of the file.");
1115 break;
1116 }
1117
1118 /* make a copy of the value, converting \r\n to \n, and define it. */
1119 value = incdep_xmalloc (curdep, value_len + 1);
1120 endp = memchr (value_start, '\r', value_len);
1121 if (endp)
1122 {
1123 const char *src = value_start;
1124 char *dst = value;
1125 for (;;)
1126 {
1127 size_t len = endp - src;
1128 memcpy (dst, src, len);
1129 dst += len;
1130 src = endp;
1131 if (src + 1 < file_end && src[1] == '\n')
1132 src++; /* skip the '\r' */
1133 if (src >= value_end)
1134 break;
1135 endp = memchr (endp + 1, '\r', src - value_end);
1136 if (!endp)
1137 endp = value_end;
1138 }
1139 value_len = dst - value;
1140 }
1141 else
1142 memcpy (value, value_start, value_len);
1143 value [value_len] = '\0';
1144
1145 incdep_record_variable_in_set (curdep,
1146 var, var_len, value, value_len,
1147 0 /* don't duplicate */, o_file,
1148 0 /* defines are recursive but this is faster */,
1149 NULL /* global set */, f);
1150 }
1151
1152 /* file: deps
1153 OR
1154 variable [:]= value */
1155 else
1156 {
1157 const char *colonp;
1158 const char *equalp;
1159
1160 /* Look for a colon and an equal sign, optimize for colon.
1161 Only one file is support and the colon / equal must be on
1162 the same line. */
1163 colonp = memchr (cur, ':', file_end - cur);
1164#ifdef HAVE_DOS_PATHS
1165 while ( colonp
1166 && colonp + 1 < file_end
1167 && (colonp[1] == '/' || colonp[1] == '\\')
1168 && colonp > cur
1169 && isalpha ((unsigned char)colonp[-1])
1170 && ( colonp == cur + 1
1171 || strchr (" \t(", colonp[-2]) != 0))
1172 colonp = memchr (colonp + 1, ':', file_end - (colonp + 1));
1173#endif
1174 endp = NULL;
1175 if ( !colonp
1176 || (endp = memchr (cur, '\n', colonp - cur)))
1177 {
1178 colonp = NULL;
1179 equalp = memchr (cur, '=', (endp ? endp : file_end) - cur);
1180 if ( !equalp
1181 || (!endp && memchr (cur, '\n', equalp - cur)))
1182 {
1183 incdep_warn (curdep, line_no, "no colon.");
1184 break;
1185 }
1186 }
1187 else
1188 equalp = memchr (cur, '=', (colonp + 2 <= file_end
1189 ? colonp + 2 : file_end) - cur);
1190 if (equalp)
1191 {
1192 /* An assignment of some sort. */
1193 const char *var;
1194 unsigned var_len;
1195 const char *value_start;
1196 const char *value_end;
1197 char *value;
1198 unsigned value_len;
1199 unsigned multi_line = 0;
1200 enum variable_flavor flavor;
1201
1202 /* figure the flavor first. */
1203 flavor = f_recursive;
1204 if (equalp > cur)
1205 {
1206 if (equalp[-1] == ':')
1207 flavor = f_simple;
1208 else if (equalp[-1] == '?')
1209 flavor = f_conditional;
1210 else if (equalp[-1] == '+')
1211 flavor = f_append;
1212 else if (equalp[-1] == '>')
1213 flavor = f_prepend;
1214 }
1215
1216 /* extract the variable name. */
1217 endp = flavor == f_recursive ? equalp : equalp - 1;
1218 while (endp > cur && isblank ((unsigned char)endp[-1]))
1219 --endp;
1220 var_len = endp - cur;
1221 if (!var_len)
1222 {
1223 incdep_warn (curdep, line_no, "empty variable. (includedep)");
1224 break;
1225 }
1226 if ( memchr (cur, '$', var_len)
1227 || memchr (cur, ' ', var_len)
1228 || memchr (cur, '\t', var_len))
1229 {
1230 incdep_warn (curdep, line_no, "fancy variable name. (includedep)");
1231 break;
1232 }
1233 var = incdep_record_strcache (curdep, cur, var_len);
1234
1235 /* find the start of the value. */
1236 cur = equalp + 1;
1237 while (cur < file_end && isblank ((unsigned char)*cur))
1238 cur++;
1239 value_start = cur;
1240
1241 /* find the end of the value / line (this isn't 101% correct). */
1242 value_end = cur;
1243 while (cur < file_end)
1244 {
1245 endp = value_end = memchr (cur, '\n', file_end - cur);
1246 if (!value_end)
1247 value_end = file_end;
1248 if (value_end - 1 >= cur && value_end[-1] == '\r')
1249 --value_end;
1250 if (value_end - 1 < cur || value_end[-1] != '\\')
1251 {
1252 cur = endp ? endp + 1 : file_end;
1253 break;
1254 }
1255 --value_end;
1256 if (value_end - 1 >= cur && value_end[-1] == '\\')
1257 {
1258 incdep_warn (curdep, line_no, "fancy escaping! (includedep)");
1259 cur = NULL;
1260 break;
1261 }
1262 if (!endp)
1263 {
1264 cur = file_end;
1265 break;
1266 }
1267
1268 cur = endp + 1;
1269 ++multi_line;
1270 ++line_no;
1271 }
1272 if (!cur)
1273 break;
1274 ++line_no;
1275
1276 /* make a copy of the value, converting \r\n to \n, and define it. */
1277 value_len = value_end - value_start;
1278 value = incdep_xmalloc (curdep, value_len + 1);
1279 if (!multi_line)
1280 memcpy (value, value_start, value_len);
1281 else
1282 {
1283 /* unescape it */
1284 const char *src = value_start;
1285 char *dst = value;
1286 while (src < value_end)
1287 {
1288 const char *nextp;
1289
1290 endp = memchr (src, '\n', value_end - src);
1291 if (!endp)
1292 nextp = endp = value_end;
1293 else
1294 nextp = endp + 1;
1295 if (endp > src && endp[-1] == '\r')
1296 --endp;
1297 if (endp > src && endp[-1] == '\\')
1298 --endp;
1299
1300 if (src != value_start)
1301 *dst++ = ' ';
1302 memcpy (dst, src, endp - src);
1303 dst += endp - src;
1304 src = nextp;
1305 }
1306 value_len = dst - value;
1307 }
1308 value [value_len] = '\0';
1309
1310 /* do the definition */
1311 if (flavor == f_recursive
1312 || ( flavor == f_simple
1313 && !memchr (value, '$', value_len)))
1314 incdep_record_variable_in_set (curdep,
1315 var, var_len, value, value_len,
1316 0 /* don't duplicate */, o_file,
1317 flavor == f_recursive /* recursive */,
1318 NULL /* global set */, f);
1319 else
1320 incdep_record_variable_def (curdep,
1321 f, var, var_len, value, value_len,
1322 o_file, flavor, 0 /* not target var */);
1323 }
1324 else
1325 {
1326 /* file: dependencies */
1327
1328 const char *filename;
1329 struct dep *deps = 0;
1330 struct dep **nextdep = &deps;
1331 struct dep *dep;
1332
1333 /* extract the filename, ASSUME a single one. */
1334 endp = colonp;
1335 while (endp > cur && isblank ((unsigned char)endp[-1]))
1336 --endp;
1337 if (cur == endp)
1338 {
1339 incdep_warn (curdep, line_no, "empty filename.");
1340 break;
1341 }
1342 if ( memchr (cur, '$', endp - cur)
1343 || memchr (cur, ' ', endp - cur)
1344 || memchr (cur, '\t', endp - cur))
1345 {
1346 incdep_warn (curdep, line_no, "multiple / fancy file name. (includedep)");
1347 break;
1348 }
1349 filename = incdep_record_strcache (curdep, cur, endp - cur);
1350
1351 /* parse any dependencies. */
1352 cur = colonp + 1;
1353 while (cur < file_end)
1354 {
1355 /* skip blanks and count lines. */
1356 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1357 ++cur;
1358 if (cur >= file_end)
1359 break;
1360 if (*cur == '\n')
1361 {
1362 cur++;
1363 line_no++;
1364 break;
1365 }
1366
1367 /* continuation + eol? */
1368 if (*cur == '\\')
1369 {
1370 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1371 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1372 : (file_end - cur == 1) ? 1 : 0;
1373 if (eol_len)
1374 {
1375 cur += eol_len;
1376 line_no++;
1377 continue;
1378 }
1379 }
1380
1381 /* find the end of the filename */
1382 endp = cur;
1383 while (endp < file_end && !isspace ((unsigned char)*endp))
1384 ++endp;
1385
1386 /* add it to the list. */
1387 *nextdep = dep = incdep_alloc_dep (curdep);
1388 dep->name = incdep_record_strcache (curdep, cur, endp - cur);
1389 dep->includedep = 1;
1390 nextdep = &dep->next;
1391
1392 cur = endp;
1393 }
1394
1395 /* enter the file with its dependencies. */
1396 incdep_record_files (curdep,
1397 filename, NULL, NULL, deps, 0, NULL, 0, 0, f);
1398 }
1399 }
1400 }
1401
1402 /* free the file data */
1403 incdep_xfree (curdep, curdep->file_base);
1404 curdep->file_base = curdep->file_end = NULL;
1405}
1406
1407/* Flushes the incdep todo and done lists. */
1408static void
1409incdep_flush_it (struct floc *f)
1410{
1411 incdep_lock ();
1412 for (;;)
1413 {
1414 struct incdep *cur = incdep_head_done;
1415
1416 /* if the done list is empty, grab a todo list entry. */
1417 if (!cur && incdep_head_todo)
1418 {
1419 cur = incdep_head_todo;
1420 if (cur->next)
1421 incdep_head_todo = cur->next;
1422 else
1423 incdep_head_todo = incdep_tail_todo = NULL;
1424 incdep_unlock ();
1425
1426 incdep_read_file (cur, f);
1427 eval_include_dep_file (cur, f);
1428 incdep_freeit (cur);
1429
1430 incdep_lock ();
1431 continue;
1432 }
1433
1434 /* if the todo list and done list are empty we're either done
1435 or will have to wait for the thread(s) to finish. */
1436 if (!cur && !incdep_num_reading)
1437 break; /* done */
1438 if (!cur)
1439 {
1440 while (!incdep_head_done)
1441 incdep_wait_done ();
1442 cur = incdep_head_done;
1443 }
1444
1445 /* we grab the entire done list and work thru it. */
1446 incdep_head_done = incdep_tail_done = NULL;
1447 incdep_unlock ();
1448
1449 while (cur)
1450 {
1451 struct incdep *next = cur->next;
1452#ifdef PARSE_IN_WORKER
1453 incdep_flush_recorded_instructions (cur);
1454#else
1455 eval_include_dep_file (cur, f);
1456#endif
1457 incdep_freeit (cur);
1458 cur = next;
1459 }
1460
1461 incdep_lock ();
1462 } /* outer loop */
1463 incdep_unlock ();
1464}
1465
1466
1467/* splits up a list of file names and feeds it to eval_include_dep_file,
1468 employing threads to try speed up the file reading. */
1469void
1470eval_include_dep (const char *names, struct floc *f, enum incdep_op op)
1471{
1472 struct incdep *head = 0;
1473 struct incdep *tail = 0;
1474 struct incdep *cur;
1475 const char *names_iterator = names;
1476 const char *name;
1477 unsigned int name_len;
1478
1479 /* loop through NAMES, creating a todo list out of them. */
1480
1481 while ((name = find_next_token (&names_iterator, &name_len)) != 0)
1482 {
1483 cur = xmalloc (sizeof (*cur) + name_len); /* not incdep_xmalloc here */
1484 cur->file_base = cur->file_end = NULL;
1485 memcpy (cur->name, name, name_len);
1486 cur->name[name_len] = '\0';
1487 cur->worker_tid = -1;
1488#ifdef PARSE_IN_WORKER
1489 cur->err_line_no = 0;
1490 cur->err_msg = NULL;
1491 cur->recorded_variables_in_set_head = NULL;
1492 cur->recorded_variables_in_set_tail = NULL;
1493 cur->recorded_variable_defs_head = NULL;
1494 cur->recorded_variable_defs_tail = NULL;
1495 cur->recorded_files_head = NULL;
1496 cur->recorded_files_tail = NULL;
1497#endif
1498
1499 cur->next = NULL;
1500 if (tail)
1501 tail->next = cur;
1502 else
1503 head = cur;
1504 tail = cur;
1505 }
1506
1507#ifdef ELECTRIC_HEAP
1508 if (1)
1509#else
1510 if (op == incdep_read_it)
1511#endif
1512 {
1513 /* work our way thru the files directly */
1514
1515 cur = head;
1516 while (cur)
1517 {
1518 struct incdep *next = cur->next;
1519 incdep_read_file (cur, f);
1520 eval_include_dep_file (cur, f);
1521 incdep_freeit (cur);
1522 cur = next;
1523 }
1524 }
1525 else
1526 {
1527 /* initialize the worker threads and related stuff the first time around. */
1528
1529 if (!incdep_initialized)
1530 incdep_init (f);
1531
1532 /* queue the files and notify the worker threads. */
1533
1534 incdep_lock ();
1535
1536 if (incdep_tail_todo)
1537 incdep_tail_todo->next = head;
1538 else
1539 incdep_head_todo = head;
1540 incdep_tail_todo = tail;
1541
1542 incdep_signal_todo ();
1543 incdep_unlock ();
1544
1545 /* flush the todo queue if we're requested to do so. */
1546
1547 if (op == incdep_flush)
1548 incdep_flush_it (f);
1549 }
1550}
1551
1552#endif /* CONFIG_WITH_INCLUDEDEP */
1553
Note: See TracBrowser for help on using the repository browser.