source: trunk/src/kmk/incdep.c@ 1862

Last change on this file since 1862 was 1862, checked in by bird, 17 years ago

kmk: some preliminary allocation caching. seems dep, variables, variable sets and files would be good candidates for generic alloc caches.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 43.5 KB
Line 
1#ifdef CONFIG_WITH_INCLUDEDEP
2/* $Id: incdep.c 1862 2008-10-14 01:04:03Z bird $ */
3/** @file
4 * incdep - Simple dependency files.
5 */
6
7/*
8 * Copyright (c) 2006-2008 knut st. osmundsen <bird-src-spam@anduin.net>
9 *
10 * This file is part of kBuild.
11 *
12 * kBuild is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * kBuild is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with kBuild; if not, write to the Free Software
24 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 *
26 */
27
28
29/*******************************************************************************
30* Header Files *
31*******************************************************************************/
32#ifdef __OS2__
33# define INCL_BASE
34# define INCL_ERRORS
35#endif
36
37#include "make.h"
38
39#if !defined(WINDOWS32) && !defined(__OS2__)
40# define HAVE_PTHREAD
41#endif
42
43#include <assert.h>
44
45#include <glob.h>
46
47#include "dep.h"
48#include "filedef.h"
49#include "job.h"
50#include "commands.h"
51#include "variable.h"
52#include "rule.h"
53#include "debug.h"
54#include "hash.h"
55
56#ifdef HAVE_FCNTL_H
57# include <fcntl.h>
58#else
59# include <sys/file.h>
60#endif
61
62#ifdef WINDOWS32
63# include <io.h>
64# include <process.h>
65# include <Windows.h>
66# define PARSE_IN_WORKER
67#endif
68
69#ifdef __OS2__
70# include <os2.h>
71# include <sys/fmutex.h>
72#endif
73
74#ifdef HAVE_PTHREAD
75# include <pthread.h>
76#endif
77
78#ifdef __APPLE__
79# include <malloc/malloc.h>
80# define PARSE_IN_WORKER
81#endif
82
83
84/*******************************************************************************
85* Structures and Typedefs *
86*******************************************************************************/
87struct incdep_strcache_entry
88{
89 unsigned int length;
90 unsigned int alignment;
91 unsigned long hash1;
92 unsigned long hash2;
93 char str[1];
94};
95
96struct incdep_variable_in_set
97{
98 struct incdep_variable_in_set *next;
99 /* the parameters */
100 struct incdep_strcache_entry *name_entry;
101 const char *value; /* xmalloc'ed */
102 unsigned int value_length;
103 int duplicate_value; /* 0 */
104 enum variable_origin origin;
105 int recursive;
106 struct variable_set *set;
107 const struct floc *flocp; /* NILF */
108};
109
110struct incdep_variable_def
111{
112 struct incdep_variable_def *next;
113 /* the parameters */
114 const struct floc *flocp; /* NILF */
115 struct incdep_strcache_entry *name_entry;
116 char *value; /* xmalloc'ed, free it */
117 unsigned int value_length;
118 enum variable_origin origin;
119 enum variable_flavor flavor;
120 int target_var;
121};
122
123struct incdep_recorded_files
124{
125 struct incdep_recorded_files *next;
126
127 /* the parameters */
128 struct nameseq *filenames; /* One only, its name is a strcache entry. */
129 const char *pattern; /* NULL */
130 const char *pattern_percent; /* NULL */
131 struct dep *deps; /* All the names are strcache entries. */
132 unsigned int cmds_started; /* 0 */
133 char *commands; /* NULL */
134 unsigned int commands_idx; /* 0 */
135 int two_colon; /* 0 */
136 const struct floc *flocp; /* NILF */
137};
138
139
140/* per dep file structure. */
141struct incdep
142{
143 struct incdep *next;
144 char *file_base;
145 char *file_end;
146
147 int is_worker;
148#ifdef PARSE_IN_WORKER
149 unsigned int err_line_no;
150 const char *err_msg;
151
152 struct dep *dep_start; /* start of the current dep block. */
153 struct dep *dep_end; /* end of the current dep block. */
154
155 struct incdep_variable_in_set *recorded_variables_in_set_head;
156 struct incdep_variable_in_set *recorded_variables_in_set_tail;
157
158 struct incdep_variable_def *recorded_variable_defs_head;
159 struct incdep_variable_def *recorded_variable_defs_tail;
160
161 struct incdep_recorded_files *recorded_files_head;
162 struct incdep_recorded_files *recorded_files_tail;
163#endif
164
165 char name[1];
166};
167
168
169/*******************************************************************************
170* Global Variables *
171*******************************************************************************/
172
173/* mutex protecting the globals and an associated condition/event. */
174#ifdef HAVE_PTHREAD
175static pthread_mutex_t incdep_mtx;
176static pthread_cond_t incdep_cond_todo;
177static pthread_cond_t incdep_cond_done;
178
179#elif defined (WINDOWS32)
180static CRITICAL_SECTION incdep_mtx;
181static HANDLE incdep_hev_todo;
182static HANDLE incdep_hev_done;
183static int volatile incdep_hev_todo_waiters;
184static int volatile incdep_hev_done_waiters;
185
186#elif defined (__OS2__)
187static fmutex incdep_mtx;
188static HEV incdep_hev_todo;
189static HEV incdep_hev_done;
190static int volatile incdep_hev_todo_waiters;
191static int volatile incdep_hev_done_waiters;
192#endif
193
194/* flag indicating whether the threads, lock and event/condvars has
195 been initialized or not. */
196static int incdep_initialized;
197
198/* the list of files that needs reading. */
199static struct incdep * volatile incdep_head_todo;
200static struct incdep * volatile incdep_tail_todo;
201
202/* the number of files that are currently being read. */
203static int volatile incdep_num_reading;
204
205/* the list of files that have been read. */
206static struct incdep * volatile incdep_head_done;
207static struct incdep * volatile incdep_tail_done;
208
209/* The handles to the worker threads. */
210#ifdef HAVE_PTHREAD
211static pthread_t incdep_threads[1];
212#elif defined (WINDOWS32)
213static HANDLE incdep_threads[2];
214#elif defined (__OS2__)
215static TID incdep_threads[2];
216#endif
217static unsigned incdep_num_threads;
218
219/* flag indicating whether the worker threads should terminate or not. */
220static int volatile incdep_terminate;
221
222#ifdef __APPLE__
223/* malloc zone for the incdep threads. */
224static malloc_zone_t *incdep_zone;
225#endif
226
227
228/*******************************************************************************
229* Internal Functions *
230*******************************************************************************/
231static void incdep_flush_it (struct floc *);
232static void eval_include_dep_file (struct incdep *, struct floc *);
233
234
235/* xmalloc wrapper.
236 For working around multithreaded performance problems found on Darwin,
237 Linux (glibc), and possibly other systems. */
238static void *
239incdep_xmalloc (struct incdep *cur, size_t size)
240{
241 void *ptr;
242
243#ifdef __APPLE__
244 if (cur && cur->is_worker)
245 {
246 ptr = malloc_zone_malloc (incdep_zone, size);
247 if (!ptr)
248 fatal (NILF, _("virtual memory exhausted"));
249 }
250 else
251 ptr = xmalloc (size);
252#else
253 ptr = xmalloc (size);
254#endif
255
256 (void)cur;
257 return ptr;
258}
259
260/* memset(malloc(sz),'\0',sz) wrapper. */
261static void *
262incdep_xcalloc (struct incdep *cur, size_t size)
263{
264 void *ptr;
265
266#ifdef __APPLE__
267 if (cur && cur->is_worker)
268 ptr = malloc_zone_calloc (incdep_zone, size, 1);
269 else
270 ptr = calloc (size, 1);
271#else
272 ptr = calloc (size, 1);
273#endif
274 if (!ptr)
275 fatal (NILF, _("virtual memory exhausted"));
276
277 (void)cur;
278 return ptr;
279}
280
281/* free wrapper */
282static void
283incdep_xfree (struct incdep *cur, void *ptr)
284{
285 /* free() *must* work for the allocation hacks above because
286 of free_dep_chain. */
287 free (ptr);
288 (void)cur;
289}
290
291/* alloc a dep structure. These are allocated in bunches to save time. */
292struct dep *
293incdep_alloc_dep (struct incdep *cur)
294{
295 if (cur->dep_start != cur->dep_end)
296 return cur->dep_start++;
297
298 cur->dep_start = (struct dep *)incdep_xcalloc (cur, sizeof(struct dep) * 256);
299 cur->dep_end = cur->dep_start + 256;
300 return cur->dep_start++;
301}
302
303
304/* acquires the lock */
305void
306incdep_lock(void)
307{
308#ifdef HAVE_PTHREAD
309 pthread_mutex_lock (&incdep_mtx);
310#elif defined (WINDOWS32)
311 EnterCriticalSection (&incdep_mtx);
312#elif defined (__OS2__)
313 _fmutex_request (&incdep_mtx, 0)
314#endif
315}
316
317/* releases the lock */
318void
319incdep_unlock(void)
320{
321#ifdef HAVE_PTHREAD
322 pthread_mutex_unlock (&incdep_mtx);
323#elif defined(WINDOWS32)
324 LeaveCriticalSection (&incdep_mtx);
325#elif defined(__OS2__)
326 _fmutex_release (&incdep_mtx)
327#endif
328}
329
330/* signals the main thread that there is stuff todo. caller owns the lock. */
331static void
332incdep_signal_done (void)
333{
334#ifdef HAVE_PTHREAD
335 pthread_cond_broadcast (&incdep_cond_done);
336#elif defined (WINDOWS32)
337 if (incdep_hev_done_waiters)
338 SetEvent (incdep_hev_done);
339#elif defined (__OS2__)
340 if (incdep_hev_done_waiters)
341 DosPostEventSem (incdep_hev_done);
342#endif
343}
344
345/* waits for a reader to finish reading. caller owns the lock. */
346static void
347incdep_wait_done (void)
348{
349#ifdef HAVE_PTHREAD
350 pthread_cond_wait (&incdep_cond_done, &incdep_mtx);
351
352#elif defined (WINDOWS32)
353 ResetEvent (incdep_hev_done);
354 incdep_hev_done_waiters++;
355 incdep_unlock ();
356 WaitForSingleObject (incdep_hev_done, INFINITE);
357 incdep_lock ();
358 incdep_hev_done_waiters--;
359
360#elif defined (__OS2__)
361 ULONG ulIgnore;
362 DosResetEventSem (incdep_hev_done, &ulIgnore);
363 incdep_hev_done_waiters++;
364 incdep_unlock ();
365 DosWaitEventSem (incdep_hev_done, SEM_INDEFINITE_WAIT);
366 incdep_lock ();
367 incdep_hev_done_waiters--;
368#endif
369}
370
371/* signals the worker threads. caller owns the lock. */
372static void
373incdep_signal_todo (void)
374{
375#ifdef HAVE_PTHREAD
376 pthread_cond_broadcast (&incdep_cond_todo);
377#elif defined(WINDOWS32)
378 if (incdep_hev_todo_waiters)
379 SetEvent (incdep_hev_todo);
380#elif defined(__OS2__)
381 if (incdep_hev_todo_waiters)
382 DosPostEventSem (incdep_hev_todo);
383#endif
384}
385
386/* waits for stuff to arrive in the todo list. caller owns the lock. */
387static void
388incdep_wait_todo (void)
389{
390#ifdef HAVE_PTHREAD
391 pthread_cond_wait (&incdep_cond_todo, &incdep_mtx);
392
393#elif defined (WINDOWS32)
394 ResetEvent (incdep_hev_todo);
395 incdep_hev_todo_waiters++;
396 incdep_unlock ();
397 WaitForSingleObject (incdep_hev_todo, INFINITE);
398 incdep_lock ();
399 incdep_hev_todo_waiters--;
400
401#elif defined (__OS2__)
402 ULONG ulIgnore;
403 DosResetEventSem (incdep_hev_todo, &ulIgnore);
404 incdep_hev_todo_waiters++;
405 incdep_unlock ();
406 DosWaitEventSem (incdep_hev_todo, SEM_INDEFINITE_WAIT);
407 incdep_lock ();
408 incdep_hev_todo_waiters--;
409#endif
410}
411
412/* Reads a dep file into memory. */
413static int
414incdep_read_file (struct incdep *cur, struct floc *f)
415{
416 int fd;
417 struct stat st;
418
419 errno = 0;
420#ifdef O_BINARY
421 fd = open (cur->name, O_RDONLY | O_BINARY, 0);
422#else
423 fd = open (cur->name, O_RDONLY, 0);
424#endif
425 if (fd < 0)
426 {
427 /* ignore non-existing dependency files. */
428 int err = errno;
429 if (err == ENOENT || stat (cur->name, &st) != 0)
430 return 1;
431 error (f, "%s: %s", cur->name, strerror (err));
432 return -1;
433 }
434 if (!fstat (fd, &st))
435 {
436 cur->file_base = incdep_xmalloc (cur, st.st_size + 1);
437 if (read (fd, cur->file_base, st.st_size) == st.st_size)
438 {
439 close (fd);
440 cur->file_end = cur->file_base + st.st_size;
441 cur->file_base[st.st_size] = '\0';
442 return 0;
443 }
444
445 /* bail out */
446
447 error (f, "%s: read: %s", cur->name, strerror (errno));
448 incdep_xfree (cur, cur->file_base);
449 }
450 else
451 error (f, "%s: fstat: %s", cur->name, strerror (errno));
452
453 close (fd);
454 cur->file_base = cur->file_end = NULL;
455 return -1;
456}
457
458/* Free the incdep structure. */
459static void
460incdep_freeit (struct incdep *cur)
461{
462#ifdef PARSE_IN_WORKER
463 assert (!cur->recorded_variables_in_set_head);
464 assert (!cur->recorded_variable_defs_head);
465 assert (!cur->recorded_files_head);
466#endif
467
468 incdep_xfree (cur, cur->file_base);
469 free (cur);
470}
471
472/* A worker thread. */
473void
474incdep_worker (void)
475{
476#ifdef PARSE_IN_WORKER
477 struct dep *dep_start = NULL;
478 struct dep *dep_end = NULL;
479#endif
480
481 incdep_lock ();
482
483 while (!incdep_terminate)
484 {
485 /* get job from the todo list. */
486
487 struct incdep *cur = incdep_head_todo;
488 if (!cur)
489 {
490 incdep_wait_todo ();
491 continue;
492 }
493 if (cur->next)
494 incdep_head_todo = cur->next;
495 else
496 incdep_head_todo = incdep_tail_todo = NULL;
497 incdep_num_reading++;
498
499 /* read the file. */
500
501 incdep_unlock ();
502 cur->is_worker = 1;
503 incdep_read_file (cur, NILF);
504
505#ifdef PARSE_IN_WORKER
506 cur->dep_start = dep_start;
507 cur->dep_end = dep_end;
508
509 eval_include_dep_file (cur, NILF);
510
511 dep_start = cur->dep_start;
512 cur->dep_start = NULL;
513 dep_end = cur->dep_end;
514 cur->dep_end = NULL;
515#endif
516
517 cur->is_worker = 0;
518 incdep_lock ();
519
520 /* insert finished job into the done list. */
521
522 incdep_num_reading--;
523 cur->next = NULL;
524 if (incdep_tail_done)
525 incdep_tail_done->next = cur;
526 else
527 incdep_head_done = cur;
528 incdep_tail_done = cur;
529
530 incdep_signal_done ();
531 }
532
533 incdep_unlock ();
534}
535
536/* Thread library specific thread functions wrapping incdep_wroker. */
537#ifdef HAVE_PTHREAD
538static void *
539incdep_worker_pthread (void *ignore)
540{
541 incdep_worker ();
542 (void)ignore;
543 return NULL;
544}
545
546#elif defined (WINDOWS32)
547static unsigned __stdcall
548incdep_worker_windows (void *ignore)
549{
550 incdep_worker ();
551 (void)ignore;
552 return 0;
553}
554
555#elif defined (__OS2__)
556static void
557incdep_worker_os2 (void *ignore)
558{
559 incdep_worker ();
560 (void)ignore;
561}
562#endif
563
564/* Creates the the worker threads. */
565static void
566incdep_init (struct floc *f)
567{
568 unsigned i;
569#ifdef HAVE_PTHREAD
570 int rc;
571 pthread_attr_t attr;
572
573#elif defined (WINDOWS32)
574 unsigned tid;
575 uintptr_t hThread;
576
577#elif defined (__OS2__)
578 int rc;
579 int tid;
580#endif
581
582 /* heap hacks */
583
584#ifdef __APPLE__
585 incdep_zone = malloc_create_zone (0, 0);
586 if (!incdep_zone)
587 incdep_zone = malloc_default_zone ();
588#endif
589
590
591 /* create the mutex and two condition variables / event objects. */
592
593#ifdef HAVE_PTHREAD
594 rc = pthread_mutex_init (&incdep_mtx, NULL);
595 if (rc)
596 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
597 rc = pthread_cond_init (&incdep_cond_todo, NULL);
598 if (rc)
599 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
600 rc = pthread_cond_init (&incdep_cond_done, NULL);
601 if (rc)
602 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
603
604#elif defined (WINDOWS32)
605 InitializeCriticalSection (&incdep_mtx);
606 incdep_hev_todo = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
607 if (!incdep_hev_todo)
608 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
609 incdep_hev_done = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
610 if (!incdep_hev_done)
611 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
612 incdep_hev_todo_waiters = 0;
613 incdep_hev_done_waiters = 0;
614
615#elif defined (__OS2__)
616 _fmutex_create (&incdep_mtx, 0)
617 rc = DosCreateEventSem (NULL, &incdep_hev_todo, 0, FALSE);
618 if (rc)
619 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
620 rc = DosCreateEventSem (NULL, &incdep_hev_done, 0, FALSE);
621 if (rc)
622 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
623 incdep_hev_todo_waiters = 0;
624 incdep_hev_done_waiters = 0;
625#endif
626
627 /* create the worker threads. */
628
629 incdep_terminate = 0;
630 incdep_num_threads = sizeof (incdep_threads) / sizeof (incdep_threads[0]);
631 if (incdep_num_threads + 1 > job_slots)
632 incdep_num_threads = job_slots <= 1 ? 1 : job_slots - 1;
633 for (i = 0; i < incdep_num_threads; i++)
634 {
635#ifdef HAVE_PTHREAD
636 rc = pthread_attr_init (&attr);
637 if (rc)
638 fatal (f, _("pthread_attr_init failed: err=%d"), rc);
639 /*rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE); */
640 rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
641 if (rc)
642 fatal (f, _("pthread_attr_setdetachstate failed: err=%d"), rc);
643 rc = pthread_create (&incdep_threads[i], &attr,
644 incdep_worker_pthread, f);
645 if (rc)
646 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
647 pthread_attr_destroy (&attr);
648
649#elif defined (WINDOWS32)
650 tid = 0;
651 hThread = _beginthreadex (NULL, 128*1024, incdep_worker_windows,
652 NULL, 0, &tid);
653 if (hThread == 0 || hThread == ~(uintptr_t)0)
654 fatal (f, _("_beginthreadex failed: err=%d"), errno);
655 incdep_threads[i] = (HANDLE)hThread;
656
657#elif defined (__OS2__)
658 tid = _beginthread (incdep_worker_os2, NULL, 128*1024, NULL);
659 if (tid <= 0)
660 fatal (f, _("_beginthread failed: err=%d"), errno);
661 incdep_threads[i] = tid;
662#endif
663 }
664
665 incdep_initialized = 1;
666}
667
668/* Flushes outstanding work and terminates the worker threads.
669 This is called from snap_deps(). */
670void
671incdep_flush_and_term (void)
672{
673 unsigned i;
674
675 if (!incdep_initialized)
676 return;
677
678 /* flush any out standing work */
679
680 incdep_flush_it (NILF);
681
682 /* tell the threads to terminate */
683
684 incdep_lock ();
685 incdep_terminate = 1;
686 incdep_signal_todo ();
687 incdep_unlock ();
688
689 /* wait for the threads to quit */
690
691 for (i = 0; i < incdep_num_threads; i++)
692 {
693 /* later? */
694 }
695 incdep_num_threads = 0;
696
697 /* destroy the lock and condition variables / event objects. */
698
699 /* later */
700
701 incdep_initialized = 0;
702}
703
704#ifdef PARSE_IN_WORKER
705/* Flushes a strcache entry returning the actual string cache entry.
706 The input is freed! */
707static const char *
708incdep_flush_strcache_entry (const void *pv_entry)
709{
710 struct incdep_strcache_entry *entry = (struct incdep_strcache_entry *)pv_entry;
711 const char *result;
712 result = strcache_add_prehashed (entry->str, entry->length, entry->hash1, entry->hash2);
713 free (entry);
714 return result;
715}
716
717/* Flushes the recorded instructions. */
718static void
719incdep_flush_recorded_instructions (struct incdep *cur)
720{
721 struct incdep_variable_in_set *rec_vis;
722 struct incdep_variable_def *rec_vd;
723 struct incdep_recorded_files *rec_f;
724
725 /* define_variable_in_set */
726
727 rec_vis = cur->recorded_variables_in_set_head;
728 cur->recorded_variables_in_set_head = cur->recorded_variables_in_set_tail = NULL;
729 if (rec_vis)
730 do
731 {
732 void *free_me = rec_vis;
733 unsigned int name_length = rec_vis->name_entry->length;
734 define_variable_in_set (incdep_flush_strcache_entry (rec_vis->name_entry),
735 name_length,
736 rec_vis->value,
737 rec_vis->value_length,
738 rec_vis->duplicate_value,
739 rec_vis->origin,
740 rec_vis->recursive,
741 rec_vis->set,
742 rec_vis->flocp);
743 rec_vis = rec_vis->next;
744 incdep_xfree (cur, free_me);
745 }
746 while (rec_vis);
747
748 /* do_variable_definition */
749
750 rec_vd = cur->recorded_variable_defs_head;
751 cur->recorded_variable_defs_head = cur->recorded_variable_defs_tail = NULL;
752 if (rec_vd)
753 do
754 {
755 void *free_me = rec_vd;
756 do_variable_definition_2 (rec_vd->flocp,
757 incdep_flush_strcache_entry (rec_vd->name_entry),
758 rec_vd->value,
759 rec_vd->value_length,
760 0,
761 rec_vd->value,
762 rec_vd->origin,
763 rec_vd->flavor,
764 rec_vd->target_var);
765 rec_vd = rec_vd->next;
766 incdep_xfree (cur, free_me);
767 }
768 while (rec_vd);
769
770 /* record_files */
771
772 rec_f = cur->recorded_files_head;
773 cur->recorded_files_head = cur->recorded_files_tail = NULL;
774 if (rec_f)
775 do
776 {
777 void *free_me = rec_f;
778 struct dep *dep;
779
780 for (dep = rec_f->deps; dep; dep = dep->next)
781 dep->name = incdep_flush_strcache_entry (dep->name);
782 rec_f->filenames->name = incdep_flush_strcache_entry (rec_f->filenames->name);
783
784 record_files (rec_f->filenames,
785 rec_f->pattern,
786 rec_f->pattern_percent,
787 rec_f->deps,
788 rec_f->cmds_started,
789 rec_f->commands,
790 rec_f->commands_idx,
791 rec_f->two_colon,
792 rec_f->flocp);
793
794 rec_f = rec_f->next;
795 incdep_xfree (cur, free_me);
796 }
797 while (rec_f);
798}
799#endif /* PARSE_IN_WORKER */
800
801/* Record / issue a warning about a misformed dep file. */
802static void
803incdep_warn (struct incdep *cur, unsigned int line_no, const char *msg)
804{
805 if (!cur->is_worker)
806 error (NILF, "%s(%d): %s", cur->name, line_no, msg);
807#ifdef PARSE_IN_WORKER
808 else
809 {
810 cur->err_line_no = line_no;
811 cur->err_msg = msg;
812 }
813#endif
814}
815
816/* Record / execute a strcache add. */
817static const char *
818incdep_record_strcache (struct incdep *cur, const char *str, int len)
819{
820 const char *ret;
821 if (!cur->is_worker)
822 {
823 /* Make sure the string is terminated before we hand it to
824 strcache_add_len so it does have to make a temporary copy
825 of it on the stack. */
826 char ch = str[len];
827 ((char *)str)[len] = '\0';
828 ret = strcache_add_len (str, len);
829 ((char *)str)[len] = ch;
830 }
831 else
832 {
833 /* Allocate a strcache record for it, pre-hashing the string to save
834 time later on in the main thread. */
835 struct incdep_strcache_entry *entry = incdep_xmalloc (cur, sizeof (*entry) + len);
836 memcpy (entry->str, str, len);
837 entry->str[len] = '\0';
838 entry->length = len;
839 strcache_prehash_str (entry->str, &entry->hash1, &entry->hash2);
840
841 ret = (const char *)entry;
842 }
843 return ret;
844}
845
846/* Record / perform a variable definition in a set.
847 The NAME is in the string cache.
848 The VALUE is on the heap.
849 The DUPLICATE_VALUE is always 0. */
850static void
851incdep_record_variable_in_set (struct incdep *cur,
852 const char *name, unsigned int name_length,
853 const char *value,
854 unsigned int value_length,
855 int duplicate_value,
856 enum variable_origin origin,
857 int recursive,
858 struct variable_set *set,
859 const struct floc *flocp)
860{
861 assert (!duplicate_value);
862 if (!cur->is_worker)
863 define_variable_in_set (name, name_length, value, value_length,
864 duplicate_value, origin, recursive, set, flocp);
865#ifdef PARSE_IN_WORKER
866 else
867 {
868 struct incdep_variable_in_set *rec = incdep_xmalloc (cur, sizeof (*rec));
869 rec->name_entry = (struct incdep_strcache_entry *)name;
870 rec->value = value;
871 rec->value_length = value_length;
872 rec->duplicate_value = duplicate_value;
873 rec->origin = origin;
874 rec->recursive = recursive;
875 rec->set = set;
876 rec->flocp = flocp;
877
878 rec->next = NULL;
879 if (cur->recorded_variables_in_set_tail)
880 cur->recorded_variables_in_set_tail->next = rec;
881 else
882 cur->recorded_variables_in_set_head = rec;
883 cur->recorded_variables_in_set_tail = rec;
884 }
885#endif
886}
887
888/* Record / perform a variable definition. The VALUE should be disposed of. */
889static void
890incdep_record_variable_def (struct incdep *cur,
891 const struct floc *flocp,
892 const char *name,
893 unsigned int name_length,
894 char *value,
895 unsigned int value_length,
896 enum variable_origin origin,
897 enum variable_flavor flavor,
898 int target_var)
899{
900 if (!cur->is_worker)
901 do_variable_definition_2 (flocp, name, value, value_length, 0, value,
902 origin, flavor, target_var);
903#ifdef PARSE_IN_WORKER
904 else
905 {
906 struct incdep_variable_def *rec = incdep_xmalloc (cur, sizeof (*rec));
907 rec->flocp = flocp;
908 rec->name_entry = (struct incdep_strcache_entry *)name;
909 rec->value = value;
910 rec->value_length = value_length;
911 rec->origin = origin;
912 rec->flavor = flavor;
913 rec->target_var = target_var;
914
915 rec->next = NULL;
916 if (cur->recorded_variable_defs_tail)
917 cur->recorded_variable_defs_tail->next = rec;
918 else
919 cur->recorded_variable_defs_head = rec;
920 cur->recorded_variable_defs_tail = rec;
921 }
922#else
923 (void)name_length;
924#endif
925}
926
927/* Record files.*/
928static void
929incdep_record_files (struct incdep *cur,
930 struct nameseq *filenames, const char *pattern,
931 const char *pattern_percent, struct dep *deps,
932 unsigned int cmds_started, char *commands,
933 unsigned int commands_idx, int two_colon,
934 const struct floc *flocp)
935{
936 if (!cur->is_worker)
937 record_files (filenames, pattern, pattern_percent, deps, cmds_started,
938 commands, commands_idx, two_colon, flocp);
939#ifdef PARSE_IN_WORKER
940 else
941 {
942 struct incdep_recorded_files *rec = incdep_xmalloc (cur, sizeof (*rec));
943
944 rec->filenames = filenames;
945 rec->pattern = pattern;
946 rec->pattern_percent = pattern_percent;
947 rec->deps = deps;
948 rec->cmds_started = cmds_started;
949 rec->commands = commands;
950 rec->commands_idx = commands_idx;
951 rec->two_colon = two_colon;
952 rec->flocp = flocp;
953
954 rec->next = NULL;
955 if (cur->recorded_files_tail)
956 cur->recorded_files_tail->next = rec;
957 else
958 cur->recorded_files_head = rec;
959 cur->recorded_files_tail = rec;
960 }
961#endif
962}
963
964
965/* no nonsense dependency file including.
966
967 Because nobody wants bogus dependency files to break their incremental
968 builds with hard to comprehend error messages, this function does not
969 use the normal eval routine but does all the parsing itself. This isn't,
970 as much work as it sounds, because the necessary feature set is very
971 limited.
972
973 eval_include_dep_file groks:
974
975 define var
976 endef
977
978 var [|:|?|>]= value [\]
979
980 [\]
981 file: [deps] [\]
982
983 */
984static void
985eval_include_dep_file (struct incdep *curdep, struct floc *f)
986{
987 unsigned line_no = 1;
988 const char *file_end = curdep->file_end;
989 const char *cur = curdep->file_base;
990 const char *endp;
991
992 /* if no file data, just return immediately. */
993 if (!cur)
994 return;
995
996 /* now parse the file. */
997 while (cur < file_end)
998 {
999 /* skip empty lines */
1000 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1001 ++cur;
1002 if (cur >= file_end)
1003 break;
1004 if (*cur == '#')
1005 {
1006 cur = memchr (cur, '\n', file_end - cur);
1007 if (!cur)
1008 break;
1009 }
1010 if (*cur == '\\')
1011 {
1012 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1013 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1014 : (file_end - cur == 1) ? 1 : 0;
1015 if (eol_len)
1016 {
1017 cur += eol_len;
1018 line_no++;
1019 continue;
1020 }
1021 }
1022 if (*cur == '\n')
1023 {
1024 cur++;
1025 line_no++;
1026 continue;
1027 }
1028
1029 /* define var
1030 ...
1031 endef */
1032 if (strneq (cur, "define ", 7))
1033 {
1034 const char *var;
1035 unsigned var_len;
1036 const char *value_start;
1037 const char *value_end;
1038 char *value;
1039 unsigned value_len;
1040 int found_endef = 0;
1041
1042 /* extract the variable name. */
1043 cur += 7;
1044 while (isblank ((unsigned char)*cur))
1045 ++cur;
1046 value_start = endp = memchr (cur, '\n', file_end - cur);
1047 if (!endp)
1048 endp = cur;
1049 while (endp > cur && isspace ((unsigned char)endp[-1]))
1050 --endp;
1051 var_len = endp - cur;
1052 if (!var_len)
1053 {
1054 incdep_warn (curdep, line_no, "bogus define statement.");
1055 break;
1056 }
1057 var = incdep_record_strcache (curdep, cur, var_len);
1058
1059 /* find the end of the variable. */
1060 cur = value_end = value_start = value_start + 1;
1061 ++line_no;
1062 while (cur < file_end)
1063 {
1064 /* check for endef, don't bother with skipping leading spaces. */
1065 if ( file_end - cur >= 5
1066 && strneq (cur, "endef", 5))
1067 {
1068 endp = cur + 5;
1069 while (endp < file_end && isspace ((unsigned char)*endp) && *endp != '\n')
1070 endp++;
1071 if (endp >= file_end || *endp == '\n')
1072 {
1073 found_endef = 1;
1074 cur = endp >= file_end ? file_end : endp + 1;
1075 break;
1076 }
1077 }
1078
1079 /* skip a line ahead. */
1080 cur = value_end = memchr (cur, '\n', file_end - cur);
1081 if (cur != NULL)
1082 ++cur;
1083 else
1084 cur = value_end = file_end;
1085 ++line_no;
1086 }
1087
1088 if (!found_endef)
1089 {
1090 incdep_warn (curdep, line_no, "missing endef, dropping the rest of the file.");
1091 break;
1092 }
1093 value_len = value_end - value_start;
1094 if (memchr (value_start, '\0', value_len))
1095 {
1096 incdep_warn (curdep, line_no, "'\\0' in define, dropping the rest of the file.");
1097 break;
1098 }
1099
1100 /* make a copy of the value, converting \r\n to \n, and define it. */
1101 value = incdep_xmalloc (curdep, value_len + 1);
1102 endp = memchr (value_start, '\r', value_len);
1103 if (endp)
1104 {
1105 const char *src = value_start;
1106 char *dst = value;
1107 for (;;)
1108 {
1109 size_t len = endp - src;
1110 memcpy (dst, src, len);
1111 dst += len;
1112 src = endp;
1113 if (src + 1 < file_end && src[1] == '\n')
1114 src++; /* skip the '\r' */
1115 if (src >= value_end)
1116 break;
1117 endp = memchr (endp + 1, '\r', src - value_end);
1118 if (!endp)
1119 endp = value_end;
1120 }
1121 value_len = dst - value;
1122 }
1123 else
1124 memcpy (value, value_start, value_len);
1125 value [value_len] = '\0';
1126
1127 incdep_record_variable_in_set (curdep,
1128 var, var_len, value, value_len,
1129 0 /* don't duplicate */, o_file,
1130 0 /* defines are recursive but this is faster */,
1131 NULL /* global set */, f);
1132 }
1133
1134 /* file: deps
1135 OR
1136 variable [:]= value */
1137 else
1138 {
1139 const char *colonp;
1140 const char *equalp;
1141
1142 /* Look for a colon and an equal sign, optimize for colon.
1143 Only one file is support and the colon / equal must be on
1144 the same line. */
1145 colonp = memchr (cur, ':', file_end - cur);
1146#ifdef HAVE_DOS_PATHS
1147 while ( colonp
1148 && colonp + 1 < file_end
1149 && (colonp[1] == '/' || colonp[1] == '\\')
1150 && colonp > cur
1151 && isalpha ((unsigned char)colonp[-1])
1152 && ( colonp == cur + 1
1153 || strchr (" \t(", colonp[-2]) != 0))
1154 colonp = memchr (colonp + 1, ':', file_end - (colonp + 1));
1155#endif
1156 endp = NULL;
1157 if ( !colonp
1158 || (endp = memchr (cur, '\n', colonp - cur)))
1159 {
1160 colonp = NULL;
1161 equalp = memchr (cur, '=', (endp ? endp : file_end) - cur);
1162 if ( !equalp
1163 || (!endp && memchr (cur, '\n', equalp - cur)))
1164 {
1165 incdep_warn (curdep, line_no, "no colon.");
1166 break;
1167 }
1168 }
1169 else
1170 equalp = memchr (cur, '=', (colonp + 2 <= file_end
1171 ? colonp + 2 : file_end) - cur);
1172 if (equalp)
1173 {
1174 /* An assignment of some sort. */
1175 const char *var;
1176 unsigned var_len;
1177 const char *value_start;
1178 const char *value_end;
1179 char *value;
1180 unsigned value_len;
1181 unsigned multi_line = 0;
1182 enum variable_flavor flavor;
1183
1184 /* figure the flavor first. */
1185 flavor = f_recursive;
1186 if (equalp > cur)
1187 {
1188 if (equalp[-1] == ':')
1189 flavor = f_simple;
1190 else if (equalp[-1] == '?')
1191 flavor = f_conditional;
1192 else if (equalp[-1] == '+')
1193 flavor = f_append;
1194 else if (equalp[-1] == '>')
1195 flavor = f_prepend;
1196 }
1197
1198 /* extract the variable name. */
1199 endp = flavor == f_recursive ? equalp : equalp - 1;
1200 while (endp > cur && isblank ((unsigned char)endp[-1]))
1201 --endp;
1202 var_len = endp - cur;
1203 if (!var_len)
1204 {
1205 incdep_warn (curdep, line_no, "empty variable. (includedep)");
1206 break;
1207 }
1208 if ( memchr (cur, '$', var_len)
1209 || memchr (cur, ' ', var_len)
1210 || memchr (cur, '\t', var_len))
1211 {
1212 incdep_warn (curdep, line_no, "fancy variable name. (includedep)");
1213 break;
1214 }
1215 var = incdep_record_strcache (curdep, cur, var_len);
1216
1217 /* find the start of the value. */
1218 cur = equalp + 1;
1219 while (cur < file_end && isblank ((unsigned char)*cur))
1220 cur++;
1221 value_start = cur;
1222
1223 /* find the end of the value / line (this isn't 101% correct). */
1224 value_end = cur;
1225 while (cur < file_end)
1226 {
1227 endp = value_end = memchr (cur, '\n', file_end - cur);
1228 if (!value_end)
1229 value_end = file_end;
1230 if (value_end - 1 >= cur && value_end[-1] == '\r')
1231 --value_end;
1232 if (value_end - 1 < cur || value_end[-1] != '\\')
1233 {
1234 cur = endp ? endp + 1 : file_end;
1235 break;
1236 }
1237 --value_end;
1238 if (value_end - 1 >= cur && value_end[-1] == '\\')
1239 {
1240 incdep_warn (curdep, line_no, "fancy escaping! (includedep)");
1241 cur = NULL;
1242 break;
1243 }
1244 if (!endp)
1245 {
1246 cur = file_end;
1247 break;
1248 }
1249
1250 cur = endp + 1;
1251 ++multi_line;
1252 ++line_no;
1253 }
1254 if (!cur)
1255 break;
1256 ++line_no;
1257
1258 /* make a copy of the value, converting \r\n to \n, and define it. */
1259 value_len = value_end - value_start;
1260 value = incdep_xmalloc (curdep, value_len + 1);
1261 if (!multi_line)
1262 memcpy (value, value_start, value_len);
1263 else
1264 {
1265 /* unescape it */
1266 const char *src = value_start;
1267 char *dst = value;
1268 while (src < value_end)
1269 {
1270 const char *nextp;
1271
1272 endp = memchr (src, '\n', value_end - src);
1273 if (!endp)
1274 nextp = endp = value_end;
1275 else
1276 nextp = endp + 1;
1277 if (endp > src && endp[-1] == '\r')
1278 --endp;
1279 if (endp > src && endp[-1] == '\\')
1280 --endp;
1281
1282 if (src != value_start)
1283 *dst++ = ' ';
1284 memcpy (dst, src, endp - src);
1285 dst += endp - src;
1286 src = nextp;
1287 }
1288 value_len = dst - value;
1289 }
1290 value [value_len] = '\0';
1291
1292 /* do the definition */
1293 if (flavor == f_recursive
1294 || ( flavor == f_simple
1295 && !memchr (value, '$', value_len)))
1296 incdep_record_variable_in_set (curdep,
1297 var, var_len, value, value_len,
1298 0 /* don't duplicate */, o_file,
1299 flavor == f_recursive /* recursive */,
1300 NULL /* global set */, f);
1301 else
1302 incdep_record_variable_def (curdep,
1303 f, var, var_len, value, value_len,
1304 o_file, flavor, 0 /* not target var */);
1305 }
1306 else
1307 {
1308 /* file: dependencies */
1309
1310 struct nameseq *filenames = 0;
1311 struct dep *deps = 0;
1312 struct dep **nextdep = &deps;
1313 struct dep *dep;
1314
1315 /* extract the filename, ASSUME a single one. */
1316 endp = colonp;
1317 while (endp > cur && isblank ((unsigned char)endp[-1]))
1318 --endp;
1319 if (cur == endp)
1320 {
1321 incdep_warn (curdep, line_no, "empty filename.");
1322 break;
1323 }
1324 if ( memchr (cur, '$', endp - cur)
1325 || memchr (cur, ' ', endp - cur)
1326 || memchr (cur, '\t', endp - cur))
1327 {
1328 incdep_warn (curdep, line_no, "multiple / fancy file name. (includedep)");
1329 break;
1330 }
1331 filenames = incdep_xmalloc (curdep, sizeof (struct nameseq));
1332 memset (filenames, 0, sizeof (*filenames));
1333 filenames->name = incdep_record_strcache (curdep, cur, endp - cur);
1334
1335 /* parse any dependencies. */
1336 cur = colonp + 1;
1337 while (cur < file_end)
1338 {
1339 /* skip blanks and count lines. */
1340 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1341 ++cur;
1342 if (cur >= file_end)
1343 break;
1344 if (*cur == '\n')
1345 {
1346 cur++;
1347 line_no++;
1348 break;
1349 }
1350
1351 /* continuation + eol? */
1352 if (*cur == '\\')
1353 {
1354 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1355 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1356 : (file_end - cur == 1) ? 1 : 0;
1357 if (eol_len)
1358 {
1359 cur += eol_len;
1360 line_no++;
1361 continue;
1362 }
1363 }
1364
1365 /* find the end of the filename */
1366 endp = cur;
1367 while (endp < file_end && !isspace ((unsigned char)*endp))
1368 ++endp;
1369
1370 /* add it to the list. */
1371 *nextdep = dep = incdep_alloc_dep (curdep);
1372 dep->name = incdep_record_strcache (curdep, cur, endp - cur);
1373 dep->includedep = 1;
1374 nextdep = &dep->next;
1375
1376 cur = endp;
1377 }
1378
1379 /* enter the file with its dependencies. */
1380 incdep_record_files (curdep,
1381 filenames, NULL, NULL, deps, 0, NULL, 0, 0, f);
1382 }
1383 }
1384 }
1385
1386 /* free the file data */
1387 incdep_xfree (curdep, curdep->file_base);
1388 curdep->file_base = curdep->file_end = NULL;
1389}
1390
1391/* Flushes the incdep todo and done lists. */
1392static void
1393incdep_flush_it (struct floc *f)
1394{
1395 incdep_lock ();
1396 for (;;)
1397 {
1398 struct incdep *cur = incdep_head_done;
1399
1400 /* if the done list is empty, grab a todo list entry. */
1401 if (!cur && incdep_head_todo)
1402 {
1403 cur = incdep_head_todo;
1404 if (cur->next)
1405 incdep_head_todo = cur->next;
1406 else
1407 incdep_head_todo = incdep_tail_todo = NULL;
1408 incdep_unlock ();
1409
1410 incdep_read_file (cur, f);
1411 eval_include_dep_file (cur, f);
1412 incdep_freeit (cur);
1413
1414 incdep_lock ();
1415 continue;
1416 }
1417
1418 /* if the todo list and done list are empty we're either done
1419 or will have to wait for the thread(s) to finish. */
1420 if (!cur && !incdep_num_reading)
1421 break; /* done */
1422 if (!cur)
1423 {
1424 while (!incdep_head_done)
1425 incdep_wait_done ();
1426 cur = incdep_head_done;
1427 }
1428
1429 /* we grab the entire done list and work thru it. */
1430 incdep_head_done = incdep_tail_done = NULL;
1431 incdep_unlock ();
1432
1433 while (cur)
1434 {
1435 struct incdep *next = cur->next;
1436#ifdef PARSE_IN_WORKER
1437 incdep_flush_recorded_instructions (cur);
1438#else
1439 eval_include_dep_file (cur, f);
1440#endif
1441 incdep_freeit (cur);
1442 cur = next;
1443 }
1444
1445 incdep_lock ();
1446 } /* outer loop */
1447 incdep_unlock ();
1448}
1449
1450
1451/* splits up a list of file names and feeds it to eval_include_dep_file,
1452 employing threads to try speed up the file reading. */
1453void
1454eval_include_dep (const char *names, struct floc *f, enum incdep_op op)
1455{
1456 struct incdep *head = 0;
1457 struct incdep *tail = 0;
1458 struct incdep *cur;
1459 const char *names_iterator = names;
1460 const char *name;
1461 unsigned int name_len;
1462
1463 /* loop through NAMES, creating a todo list out of them. */
1464
1465 while ((name = find_next_token (&names_iterator, &name_len)) != 0)
1466 {
1467 cur = xmalloc (sizeof (*cur) + name_len); /* not incdep_xmalloc here */
1468 cur->file_base = cur->file_end = NULL;
1469 memcpy (cur->name, name, name_len);
1470 cur->name[name_len] = '\0';
1471 cur->is_worker = 0;
1472#ifdef PARSE_IN_WORKER
1473 cur->err_line_no = 0;
1474 cur->err_msg = NULL;
1475 cur->dep_start = NULL;
1476 cur->dep_end = NULL;
1477 cur->recorded_variables_in_set_head = NULL;
1478 cur->recorded_variables_in_set_tail = NULL;
1479 cur->recorded_variable_defs_head = NULL;
1480 cur->recorded_variable_defs_tail = NULL;
1481 cur->recorded_files_head = NULL;
1482 cur->recorded_files_tail = NULL;
1483#endif
1484
1485 cur->next = NULL;
1486 if (tail)
1487 tail->next = cur;
1488 else
1489 head = cur;
1490 tail = cur;
1491 }
1492
1493 if (op == incdep_read_it)
1494 {
1495 /* work our way thru the files directly */
1496
1497 cur = head;
1498 while (cur)
1499 {
1500 struct incdep *next = cur->next;
1501 incdep_read_file (cur, f);
1502 eval_include_dep_file (cur, f);
1503 incdep_freeit (cur);
1504 cur = next;
1505 }
1506 }
1507 else
1508 {
1509 /* initialize the worker threads and related stuff the first time around. */
1510
1511 if (!incdep_initialized)
1512 incdep_init (f);
1513
1514 /* queue the files and notify the worker threads. */
1515
1516 incdep_lock ();
1517
1518 if (incdep_tail_todo)
1519 incdep_tail_todo->next = head;
1520 else
1521 incdep_head_todo = head;
1522 incdep_tail_todo = tail;
1523
1524 incdep_signal_todo ();
1525 incdep_unlock ();
1526
1527 /* flush the todo queue if we're requested to do so. */
1528
1529 if (op == incdep_flush)
1530 incdep_flush_it (f);
1531 }
1532}
1533
1534#endif /* CONFIG_WITH_INCLUDEDEP */
1535
Note: See TracBrowser for help on using the repository browser.