source: trunk/src/kmk/incdep.c@ 1867

Last change on this file since 1867 was 1866, checked in by bird, 17 years ago

kmk/incdep: don't allocate nameseq records, there will be a whole lot of them.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 43.8 KB
Line 
1#ifdef CONFIG_WITH_INCLUDEDEP
2/* $Id: incdep.c 1866 2008-10-15 01:11:41Z bird $ */
3/** @file
4 * incdep - Simple dependency files.
5 */
6
7/*
8 * Copyright (c) 2006-2008 knut st. osmundsen <bird-src-spam@anduin.net>
9 *
10 * This file is part of kBuild.
11 *
12 * kBuild is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * kBuild is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with kBuild; if not, write to the Free Software
24 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 *
26 */
27
28
29/*******************************************************************************
30* Header Files *
31*******************************************************************************/
32#ifdef __OS2__
33# define INCL_BASE
34# define INCL_ERRORS
35#endif
36
37#include "make.h"
38
39#if !defined(WINDOWS32) && !defined(__OS2__)
40# define HAVE_PTHREAD
41#endif
42
43#include <assert.h>
44
45#include <glob.h>
46
47#include "dep.h"
48#include "filedef.h"
49#include "job.h"
50#include "commands.h"
51#include "variable.h"
52#include "rule.h"
53#include "debug.h"
54#include "hash.h"
55
56#ifdef HAVE_FCNTL_H
57# include <fcntl.h>
58#else
59# include <sys/file.h>
60#endif
61
62#ifdef WINDOWS32
63# include <io.h>
64# include <process.h>
65# include <Windows.h>
66# define PARSE_IN_WORKER
67#endif
68
69#ifdef __OS2__
70# include <os2.h>
71# include <sys/fmutex.h>
72#endif
73
74#ifdef HAVE_PTHREAD
75# include <pthread.h>
76#endif
77
78#ifdef __APPLE__
79# include <malloc/malloc.h>
80# define PARSE_IN_WORKER
81#endif
82
83
84/*******************************************************************************
85* Structures and Typedefs *
86*******************************************************************************/
87struct incdep_strcache_entry
88{
89 unsigned int length;
90 unsigned int alignment;
91 unsigned long hash1;
92 unsigned long hash2;
93 char str[1];
94};
95
96struct incdep_variable_in_set
97{
98 struct incdep_variable_in_set *next;
99 /* the parameters */
100 struct incdep_strcache_entry *name_entry;
101 const char *value; /* xmalloc'ed */
102 unsigned int value_length;
103 int duplicate_value; /* 0 */
104 enum variable_origin origin;
105 int recursive;
106 struct variable_set *set;
107 const struct floc *flocp; /* NILF */
108};
109
110struct incdep_variable_def
111{
112 struct incdep_variable_def *next;
113 /* the parameters */
114 const struct floc *flocp; /* NILF */
115 struct incdep_strcache_entry *name_entry;
116 char *value; /* xmalloc'ed, free it */
117 unsigned int value_length;
118 enum variable_origin origin;
119 enum variable_flavor flavor;
120 int target_var;
121};
122
123struct incdep_recorded_files
124{
125 struct incdep_recorded_files *next;
126
127 /* the parameters */
128 struct incdep_strcache_entry *filename_entry; /* Converted to a nameseq record. */
129 const char *pattern; /* NULL */
130 const char *pattern_percent; /* NULL */
131 struct dep *deps; /* All the names are strcache entries. */
132 unsigned int cmds_started; /* 0 */
133 char *commands; /* NULL */
134 unsigned int commands_idx; /* 0 */
135 int two_colon; /* 0 */
136 const struct floc *flocp; /* NILF */
137};
138
139
140/* per dep file structure. */
141struct incdep
142{
143 struct incdep *next;
144 char *file_base;
145 char *file_end;
146
147 int worker_tid;
148#ifdef PARSE_IN_WORKER
149 unsigned int err_line_no;
150 const char *err_msg;
151
152 struct incdep_variable_in_set *recorded_variables_in_set_head;
153 struct incdep_variable_in_set *recorded_variables_in_set_tail;
154
155 struct incdep_variable_def *recorded_variable_defs_head;
156 struct incdep_variable_def *recorded_variable_defs_tail;
157
158 struct incdep_recorded_files *recorded_files_head;
159 struct incdep_recorded_files *recorded_files_tail;
160#endif
161
162 char name[1];
163};
164
165
166/*******************************************************************************
167* Global Variables *
168*******************************************************************************/
169
170/* mutex protecting the globals and an associated condition/event. */
171#ifdef HAVE_PTHREAD
172static pthread_mutex_t incdep_mtx;
173static pthread_cond_t incdep_cond_todo;
174static pthread_cond_t incdep_cond_done;
175
176#elif defined (WINDOWS32)
177static CRITICAL_SECTION incdep_mtx;
178static HANDLE incdep_hev_todo;
179static HANDLE incdep_hev_done;
180static int volatile incdep_hev_todo_waiters;
181static int volatile incdep_hev_done_waiters;
182
183#elif defined (__OS2__)
184static fmutex incdep_mtx;
185static HEV incdep_hev_todo;
186static HEV incdep_hev_done;
187static int volatile incdep_hev_todo_waiters;
188static int volatile incdep_hev_done_waiters;
189#endif
190
191/* flag indicating whether the threads, lock and event/condvars has
192 been initialized or not. */
193static int incdep_initialized;
194
195/* the list of files that needs reading. */
196static struct incdep * volatile incdep_head_todo;
197static struct incdep * volatile incdep_tail_todo;
198
199/* the number of files that are currently being read. */
200static int volatile incdep_num_reading;
201
202/* the list of files that have been read. */
203static struct incdep * volatile incdep_head_done;
204static struct incdep * volatile incdep_tail_done;
205
206
207/* The handles to the worker threads. */
208#ifdef HAVE_PTHREAD
209static pthread_t incdep_threads[1];
210static struct alloccache incdep_dep_caches[1];
211
212#elif defined (WINDOWS32)
213static HANDLE incdep_threads[2];
214static struct alloccache incdep_dep_caches[2];
215
216#elif defined (__OS2__)
217static TID incdep_threads[2];
218static struct alloccache incdep_dep_caches[2];
219#endif
220static unsigned incdep_num_threads;
221
222/* flag indicating whether the worker threads should terminate or not. */
223static int volatile incdep_terminate;
224
225#ifdef __APPLE__
226/* malloc zone for the incdep threads. */
227static malloc_zone_t *incdep_zone;
228#endif
229
230
231/*******************************************************************************
232* Internal Functions *
233*******************************************************************************/
234static void incdep_flush_it (struct floc *);
235static void eval_include_dep_file (struct incdep *, struct floc *);
236
237
238/* xmalloc wrapper.
239 For working around multithreaded performance problems found on Darwin,
240 Linux (glibc), and possibly other systems. */
241static void *
242incdep_xmalloc (struct incdep *cur, size_t size)
243{
244 void *ptr;
245
246#ifdef __APPLE__
247 if (cur && cur->worker_tid != -1)
248 {
249 ptr = malloc_zone_malloc (incdep_zone, size);
250 if (!ptr)
251 fatal (NILF, _("virtual memory exhausted"));
252 }
253 else
254 ptr = xmalloc (size);
255#else
256 ptr = xmalloc (size);
257#endif
258
259 (void)cur;
260 return ptr;
261}
262
263#if 0
264/* memset(malloc(sz),'\0',sz) wrapper. */
265static void *
266incdep_xcalloc (struct incdep *cur, size_t size)
267{
268 void *ptr;
269
270#ifdef __APPLE__
271 if (cur && cur->worker_tid != -1)
272 ptr = malloc_zone_calloc (incdep_zone, size, 1);
273 else
274 ptr = calloc (size, 1);
275#else
276 ptr = calloc (size, 1);
277#endif
278 if (!ptr)
279 fatal (NILF, _("virtual memory exhausted"));
280
281 (void)cur;
282 return ptr;
283}
284#endif /* unused */
285
286/* free wrapper */
287static void
288incdep_xfree (struct incdep *cur, void *ptr)
289{
290 /* free() *must* work for the allocation hacks above because
291 of free_dep_chain. */
292 free (ptr);
293 (void)cur;
294}
295
296/* alloc a dep structure. These are allocated in bunches to save time. */
297struct dep *
298incdep_alloc_dep (struct incdep *cur)
299{
300 struct alloccache *cache;
301 if (cur->worker_tid != -1)
302 cache = &incdep_dep_caches[cur->worker_tid];
303 else
304 cache = &dep_cache;
305 return alloccache_calloc (cache);
306}
307
308/* grow a cache. */
309static void *
310incdep_cache_allocator (void *thrd, unsigned int size)
311{
312 (void)thrd;
313#ifdef __APPLE__
314 return malloc_zone_malloc (incdep_zone, size);
315#else
316 return xmalloc (size);
317#endif
318}
319
320/* acquires the lock */
321void
322incdep_lock(void)
323{
324#ifdef HAVE_PTHREAD
325 pthread_mutex_lock (&incdep_mtx);
326#elif defined (WINDOWS32)
327 EnterCriticalSection (&incdep_mtx);
328#elif defined (__OS2__)
329 _fmutex_request (&incdep_mtx, 0)
330#endif
331}
332
333/* releases the lock */
334void
335incdep_unlock(void)
336{
337#ifdef HAVE_PTHREAD
338 pthread_mutex_unlock (&incdep_mtx);
339#elif defined(WINDOWS32)
340 LeaveCriticalSection (&incdep_mtx);
341#elif defined(__OS2__)
342 _fmutex_release (&incdep_mtx)
343#endif
344}
345
346/* signals the main thread that there is stuff todo. caller owns the lock. */
347static void
348incdep_signal_done (void)
349{
350#ifdef HAVE_PTHREAD
351 pthread_cond_broadcast (&incdep_cond_done);
352#elif defined (WINDOWS32)
353 if (incdep_hev_done_waiters)
354 SetEvent (incdep_hev_done);
355#elif defined (__OS2__)
356 if (incdep_hev_done_waiters)
357 DosPostEventSem (incdep_hev_done);
358#endif
359}
360
361/* waits for a reader to finish reading. caller owns the lock. */
362static void
363incdep_wait_done (void)
364{
365#ifdef HAVE_PTHREAD
366 pthread_cond_wait (&incdep_cond_done, &incdep_mtx);
367
368#elif defined (WINDOWS32)
369 ResetEvent (incdep_hev_done);
370 incdep_hev_done_waiters++;
371 incdep_unlock ();
372 WaitForSingleObject (incdep_hev_done, INFINITE);
373 incdep_lock ();
374 incdep_hev_done_waiters--;
375
376#elif defined (__OS2__)
377 ULONG ulIgnore;
378 DosResetEventSem (incdep_hev_done, &ulIgnore);
379 incdep_hev_done_waiters++;
380 incdep_unlock ();
381 DosWaitEventSem (incdep_hev_done, SEM_INDEFINITE_WAIT);
382 incdep_lock ();
383 incdep_hev_done_waiters--;
384#endif
385}
386
387/* signals the worker threads. caller owns the lock. */
388static void
389incdep_signal_todo (void)
390{
391#ifdef HAVE_PTHREAD
392 pthread_cond_broadcast (&incdep_cond_todo);
393#elif defined(WINDOWS32)
394 if (incdep_hev_todo_waiters)
395 SetEvent (incdep_hev_todo);
396#elif defined(__OS2__)
397 if (incdep_hev_todo_waiters)
398 DosPostEventSem (incdep_hev_todo);
399#endif
400}
401
402/* waits for stuff to arrive in the todo list. caller owns the lock. */
403static void
404incdep_wait_todo (void)
405{
406#ifdef HAVE_PTHREAD
407 pthread_cond_wait (&incdep_cond_todo, &incdep_mtx);
408
409#elif defined (WINDOWS32)
410 ResetEvent (incdep_hev_todo);
411 incdep_hev_todo_waiters++;
412 incdep_unlock ();
413 WaitForSingleObject (incdep_hev_todo, INFINITE);
414 incdep_lock ();
415 incdep_hev_todo_waiters--;
416
417#elif defined (__OS2__)
418 ULONG ulIgnore;
419 DosResetEventSem (incdep_hev_todo, &ulIgnore);
420 incdep_hev_todo_waiters++;
421 incdep_unlock ();
422 DosWaitEventSem (incdep_hev_todo, SEM_INDEFINITE_WAIT);
423 incdep_lock ();
424 incdep_hev_todo_waiters--;
425#endif
426}
427
428/* Reads a dep file into memory. */
429static int
430incdep_read_file (struct incdep *cur, struct floc *f)
431{
432 int fd;
433 struct stat st;
434
435 errno = 0;
436#ifdef O_BINARY
437 fd = open (cur->name, O_RDONLY | O_BINARY, 0);
438#else
439 fd = open (cur->name, O_RDONLY, 0);
440#endif
441 if (fd < 0)
442 {
443 /* ignore non-existing dependency files. */
444 int err = errno;
445 if (err == ENOENT || stat (cur->name, &st) != 0)
446 return 1;
447 error (f, "%s: %s", cur->name, strerror (err));
448 return -1;
449 }
450 if (!fstat (fd, &st))
451 {
452 cur->file_base = incdep_xmalloc (cur, st.st_size + 1);
453 if (read (fd, cur->file_base, st.st_size) == st.st_size)
454 {
455 close (fd);
456 cur->file_end = cur->file_base + st.st_size;
457 cur->file_base[st.st_size] = '\0';
458 return 0;
459 }
460
461 /* bail out */
462
463 error (f, "%s: read: %s", cur->name, strerror (errno));
464 incdep_xfree (cur, cur->file_base);
465 }
466 else
467 error (f, "%s: fstat: %s", cur->name, strerror (errno));
468
469 close (fd);
470 cur->file_base = cur->file_end = NULL;
471 return -1;
472}
473
474/* Free the incdep structure. */
475static void
476incdep_freeit (struct incdep *cur)
477{
478#ifdef PARSE_IN_WORKER
479 assert (!cur->recorded_variables_in_set_head);
480 assert (!cur->recorded_variable_defs_head);
481 assert (!cur->recorded_files_head);
482#endif
483
484 incdep_xfree (cur, cur->file_base);
485 free (cur);
486}
487
488/* A worker thread. */
489void
490incdep_worker (int thrd)
491{
492 incdep_lock ();
493
494 while (!incdep_terminate)
495 {
496 /* get job from the todo list. */
497
498 struct incdep *cur = incdep_head_todo;
499 if (!cur)
500 {
501 incdep_wait_todo ();
502 continue;
503 }
504 if (cur->next)
505 incdep_head_todo = cur->next;
506 else
507 incdep_head_todo = incdep_tail_todo = NULL;
508 incdep_num_reading++;
509
510 /* read the file. */
511
512 incdep_unlock ();
513 cur->worker_tid = thrd;
514
515 incdep_read_file (cur, NILF);
516#ifdef PARSE_IN_WORKER
517 eval_include_dep_file (cur, NILF);
518#endif
519
520 cur->worker_tid = -1;
521 incdep_lock ();
522
523 /* insert finished job into the done list. */
524
525 incdep_num_reading--;
526 cur->next = NULL;
527 if (incdep_tail_done)
528 incdep_tail_done->next = cur;
529 else
530 incdep_head_done = cur;
531 incdep_tail_done = cur;
532
533 incdep_signal_done ();
534 }
535
536 incdep_unlock ();
537}
538
539/* Thread library specific thread functions wrapping incdep_wroker. */
540#ifdef HAVE_PTHREAD
541static void *
542incdep_worker_pthread (void *thrd)
543{
544 incdep_worker ((size_t)thrd);
545 return NULL;
546}
547
548#elif defined (WINDOWS32)
549static unsigned __stdcall
550incdep_worker_windows (void *thrd)
551{
552 incdep_worker ((size_t)thrd);
553 return 0;
554}
555
556#elif defined (__OS2__)
557static void
558incdep_worker_os2 (void *thrd)
559{
560 incdep_worker ((size_t)thrd);
561}
562#endif
563
564/* Creates the the worker threads. */
565static void
566incdep_init (struct floc *f)
567{
568 unsigned i;
569#ifdef HAVE_PTHREAD
570 int rc;
571 pthread_attr_t attr;
572
573#elif defined (WINDOWS32)
574 unsigned tid;
575 uintptr_t hThread;
576
577#elif defined (__OS2__)
578 int rc;
579 int tid;
580#endif
581
582 /* heap hacks */
583
584#ifdef __APPLE__
585 incdep_zone = malloc_create_zone (0, 0);
586 if (!incdep_zone)
587 incdep_zone = malloc_default_zone ();
588#endif
589
590
591 /* create the mutex and two condition variables / event objects. */
592
593#ifdef HAVE_PTHREAD
594 rc = pthread_mutex_init (&incdep_mtx, NULL);
595 if (rc)
596 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
597 rc = pthread_cond_init (&incdep_cond_todo, NULL);
598 if (rc)
599 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
600 rc = pthread_cond_init (&incdep_cond_done, NULL);
601 if (rc)
602 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
603
604#elif defined (WINDOWS32)
605 InitializeCriticalSection (&incdep_mtx);
606 incdep_hev_todo = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
607 if (!incdep_hev_todo)
608 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
609 incdep_hev_done = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
610 if (!incdep_hev_done)
611 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
612 incdep_hev_todo_waiters = 0;
613 incdep_hev_done_waiters = 0;
614
615#elif defined (__OS2__)
616 _fmutex_create (&incdep_mtx, 0)
617 rc = DosCreateEventSem (NULL, &incdep_hev_todo, 0, FALSE);
618 if (rc)
619 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
620 rc = DosCreateEventSem (NULL, &incdep_hev_done, 0, FALSE);
621 if (rc)
622 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
623 incdep_hev_todo_waiters = 0;
624 incdep_hev_done_waiters = 0;
625#endif
626
627 /* create the worker threads. */
628
629 incdep_terminate = 0;
630 incdep_num_threads = sizeof (incdep_threads) / sizeof (incdep_threads[0]);
631 if (incdep_num_threads + 1 > job_slots)
632 incdep_num_threads = job_slots <= 1 ? 1 : job_slots - 1;
633 for (i = 0; i < incdep_num_threads; i++)
634 {
635 alloccache_init (&incdep_dep_caches[i], sizeof(struct dep), "incdep dep",
636 incdep_cache_allocator, (void *)i);
637
638#ifdef HAVE_PTHREAD
639 rc = pthread_attr_init (&attr);
640 if (rc)
641 fatal (f, _("pthread_attr_init failed: err=%d"), rc);
642 /*rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE); */
643 rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
644 if (rc)
645 fatal (f, _("pthread_attr_setdetachstate failed: err=%d"), rc);
646 rc = pthread_create(&incdep_threads[i], &attr,
647 incdep_worker_pthread, (void *)i);
648 if (rc)
649 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
650 pthread_attr_destroy (&attr);
651
652#elif defined (WINDOWS32)
653 tid = 0;
654 hThread = _beginthreadex (NULL, 128*1024, incdep_worker_windows,
655 (void *)i, 0, &tid);
656 if (hThread == 0 || hThread == ~(uintptr_t)0)
657 fatal (f, _("_beginthreadex failed: err=%d"), errno);
658 incdep_threads[i] = (HANDLE)hThread;
659
660#elif defined (__OS2__)
661 tid = _beginthread (incdep_worker_os2, NULL, 128*1024, (void *)i);
662 if (tid <= 0)
663 fatal (f, _("_beginthread failed: err=%d"), errno);
664 incdep_threads[i] = tid;
665#endif
666 }
667
668 incdep_initialized = 1;
669}
670
671/* Flushes outstanding work and terminates the worker threads.
672 This is called from snap_deps(). */
673void
674incdep_flush_and_term (void)
675{
676 unsigned i;
677
678 if (!incdep_initialized)
679 return;
680
681 /* flush any out standing work */
682
683 incdep_flush_it (NILF);
684
685 /* tell the threads to terminate */
686
687 incdep_lock ();
688 incdep_terminate = 1;
689 incdep_signal_todo ();
690 incdep_unlock ();
691
692 /* wait for the threads to quit */
693
694 for (i = 0; i < incdep_num_threads; i++)
695 {
696 /* more later? */
697
698 alloccache_join (&dep_cache, &incdep_dep_caches[i]);
699 }
700 incdep_num_threads = 0;
701
702 /* destroy the lock and condition variables / event objects. */
703
704 /* later */
705
706 incdep_initialized = 0;
707}
708
709#ifdef PARSE_IN_WORKER
710/* Flushes a strcache entry returning the actual string cache entry.
711 The input is freed! */
712static const char *
713incdep_flush_strcache_entry (const void *pv_entry)
714{
715 struct incdep_strcache_entry *entry = (struct incdep_strcache_entry *)pv_entry;
716 const char *result;
717 result = strcache_add_prehashed (entry->str, entry->length, entry->hash1, entry->hash2);
718 free (entry);
719 return result;
720}
721
722/* Flushes the recorded instructions. */
723static void
724incdep_flush_recorded_instructions (struct incdep *cur)
725{
726 struct incdep_variable_in_set *rec_vis;
727 struct incdep_variable_def *rec_vd;
728 struct incdep_recorded_files *rec_f;
729
730 /* define_variable_in_set */
731
732 rec_vis = cur->recorded_variables_in_set_head;
733 cur->recorded_variables_in_set_head = cur->recorded_variables_in_set_tail = NULL;
734 if (rec_vis)
735 do
736 {
737 void *free_me = rec_vis;
738 unsigned int name_length = rec_vis->name_entry->length;
739 define_variable_in_set (incdep_flush_strcache_entry (rec_vis->name_entry),
740 name_length,
741 rec_vis->value,
742 rec_vis->value_length,
743 rec_vis->duplicate_value,
744 rec_vis->origin,
745 rec_vis->recursive,
746 rec_vis->set,
747 rec_vis->flocp);
748 rec_vis = rec_vis->next;
749 incdep_xfree (cur, free_me);
750 }
751 while (rec_vis);
752
753 /* do_variable_definition */
754
755 rec_vd = cur->recorded_variable_defs_head;
756 cur->recorded_variable_defs_head = cur->recorded_variable_defs_tail = NULL;
757 if (rec_vd)
758 do
759 {
760 void *free_me = rec_vd;
761 do_variable_definition_2 (rec_vd->flocp,
762 incdep_flush_strcache_entry (rec_vd->name_entry),
763 rec_vd->value,
764 rec_vd->value_length,
765 0,
766 rec_vd->value,
767 rec_vd->origin,
768 rec_vd->flavor,
769 rec_vd->target_var);
770 rec_vd = rec_vd->next;
771 incdep_xfree (cur, free_me);
772 }
773 while (rec_vd);
774
775 /* record_files */
776
777 rec_f = cur->recorded_files_head;
778 cur->recorded_files_head = cur->recorded_files_tail = NULL;
779 if (rec_f)
780 do
781 {
782 void *free_me = rec_f;
783 struct dep *dep;
784 struct nameseq *filenames;
785
786 for (dep = rec_f->deps; dep; dep = dep->next)
787 dep->name = incdep_flush_strcache_entry (dep->name);
788
789 filenames = (struct nameseq *) alloccache_alloc (&nameseq_cache);
790 filenames->next = 0;
791 filenames->name = incdep_flush_strcache_entry (rec_f->filename_entry);
792
793 record_files (filenames,
794 rec_f->pattern,
795 rec_f->pattern_percent,
796 rec_f->deps,
797 rec_f->cmds_started,
798 rec_f->commands,
799 rec_f->commands_idx,
800 rec_f->two_colon,
801 rec_f->flocp);
802
803 rec_f = rec_f->next;
804 incdep_xfree (cur, free_me);
805 }
806 while (rec_f);
807}
808#endif /* PARSE_IN_WORKER */
809
810/* Record / issue a warning about a misformed dep file. */
811static void
812incdep_warn (struct incdep *cur, unsigned int line_no, const char *msg)
813{
814 if (cur->worker_tid == -1)
815 error (NILF, "%s(%d): %s", cur->name, line_no, msg);
816#ifdef PARSE_IN_WORKER
817 else
818 {
819 cur->err_line_no = line_no;
820 cur->err_msg = msg;
821 }
822#endif
823}
824
825/* Record / execute a strcache add. */
826static const char *
827incdep_record_strcache (struct incdep *cur, const char *str, int len)
828{
829 const char *ret;
830 if (cur->worker_tid == -1)
831 {
832 /* Make sure the string is terminated before we hand it to
833 strcache_add_len so it does have to make a temporary copy
834 of it on the stack. */
835 char ch = str[len];
836 ((char *)str)[len] = '\0';
837 ret = strcache_add_len (str, len);
838 ((char *)str)[len] = ch;
839 }
840 else
841 {
842 /* Allocate a strcache record for it, pre-hashing the string to save
843 time later on in the main thread. */
844 struct incdep_strcache_entry *entry = incdep_xmalloc (cur, sizeof (*entry) + len);
845 memcpy (entry->str, str, len);
846 entry->str[len] = '\0';
847 entry->length = len;
848 strcache_prehash_str (entry->str, &entry->hash1, &entry->hash2);
849
850 ret = (const char *)entry;
851 }
852 return ret;
853}
854
855/* Record / perform a variable definition in a set.
856 The NAME is in the string cache.
857 The VALUE is on the heap.
858 The DUPLICATE_VALUE is always 0. */
859static void
860incdep_record_variable_in_set (struct incdep *cur,
861 const char *name, unsigned int name_length,
862 const char *value,
863 unsigned int value_length,
864 int duplicate_value,
865 enum variable_origin origin,
866 int recursive,
867 struct variable_set *set,
868 const struct floc *flocp)
869{
870 assert (!duplicate_value);
871 if (cur->worker_tid == -1)
872 define_variable_in_set (name, name_length, value, value_length,
873 duplicate_value, origin, recursive, set, flocp);
874#ifdef PARSE_IN_WORKER
875 else
876 {
877 struct incdep_variable_in_set *rec = incdep_xmalloc (cur, sizeof (*rec));
878 rec->name_entry = (struct incdep_strcache_entry *)name;
879 rec->value = value;
880 rec->value_length = value_length;
881 rec->duplicate_value = duplicate_value;
882 rec->origin = origin;
883 rec->recursive = recursive;
884 rec->set = set;
885 rec->flocp = flocp;
886
887 rec->next = NULL;
888 if (cur->recorded_variables_in_set_tail)
889 cur->recorded_variables_in_set_tail->next = rec;
890 else
891 cur->recorded_variables_in_set_head = rec;
892 cur->recorded_variables_in_set_tail = rec;
893 }
894#endif
895}
896
897/* Record / perform a variable definition. The VALUE should be disposed of. */
898static void
899incdep_record_variable_def (struct incdep *cur,
900 const struct floc *flocp,
901 const char *name,
902 unsigned int name_length,
903 char *value,
904 unsigned int value_length,
905 enum variable_origin origin,
906 enum variable_flavor flavor,
907 int target_var)
908{
909 if (cur->worker_tid == -1)
910 do_variable_definition_2 (flocp, name, value, value_length, 0, value,
911 origin, flavor, target_var);
912#ifdef PARSE_IN_WORKER
913 else
914 {
915 struct incdep_variable_def *rec = incdep_xmalloc (cur, sizeof (*rec));
916 rec->flocp = flocp;
917 rec->name_entry = (struct incdep_strcache_entry *)name;
918 rec->value = value;
919 rec->value_length = value_length;
920 rec->origin = origin;
921 rec->flavor = flavor;
922 rec->target_var = target_var;
923
924 rec->next = NULL;
925 if (cur->recorded_variable_defs_tail)
926 cur->recorded_variable_defs_tail->next = rec;
927 else
928 cur->recorded_variable_defs_head = rec;
929 cur->recorded_variable_defs_tail = rec;
930 }
931#else
932 (void)name_length;
933#endif
934}
935
936/* Record files.*/
937static void
938incdep_record_files (struct incdep *cur,
939 const char *filename, const char *pattern,
940 const char *pattern_percent, struct dep *deps,
941 unsigned int cmds_started, char *commands,
942 unsigned int commands_idx, int two_colon,
943 const struct floc *flocp)
944{
945 if (cur->worker_tid == -1)
946 {
947 struct nameseq *filenames = (struct nameseq *) alloccache_alloc (&nameseq_cache);
948 filenames->next = 0;
949 filenames->name = filename;
950 record_files (filenames, pattern, pattern_percent, deps, cmds_started,
951 commands, commands_idx, two_colon, flocp);
952 }
953#ifdef PARSE_IN_WORKER
954 else
955 {
956 struct incdep_recorded_files *rec = incdep_xmalloc (cur, sizeof (*rec));
957
958 rec->filename_entry = (struct incdep_strcache_entry *)filename;
959 rec->pattern = pattern;
960 rec->pattern_percent = pattern_percent;
961 rec->deps = deps;
962 rec->cmds_started = cmds_started;
963 rec->commands = commands;
964 rec->commands_idx = commands_idx;
965 rec->two_colon = two_colon;
966 rec->flocp = flocp;
967
968 rec->next = NULL;
969 if (cur->recorded_files_tail)
970 cur->recorded_files_tail->next = rec;
971 else
972 cur->recorded_files_head = rec;
973 cur->recorded_files_tail = rec;
974 }
975#endif
976}
977
978
979/* no nonsense dependency file including.
980
981 Because nobody wants bogus dependency files to break their incremental
982 builds with hard to comprehend error messages, this function does not
983 use the normal eval routine but does all the parsing itself. This isn't,
984 as much work as it sounds, because the necessary feature set is very
985 limited.
986
987 eval_include_dep_file groks:
988
989 define var
990 endef
991
992 var [|:|?|>]= value [\]
993
994 [\]
995 file: [deps] [\]
996
997 */
998static void
999eval_include_dep_file (struct incdep *curdep, struct floc *f)
1000{
1001 unsigned line_no = 1;
1002 const char *file_end = curdep->file_end;
1003 const char *cur = curdep->file_base;
1004 const char *endp;
1005
1006 /* if no file data, just return immediately. */
1007 if (!cur)
1008 return;
1009
1010 /* now parse the file. */
1011 while (cur < file_end)
1012 {
1013 /* skip empty lines */
1014 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1015 ++cur;
1016 if (cur >= file_end)
1017 break;
1018 if (*cur == '#')
1019 {
1020 cur = memchr (cur, '\n', file_end - cur);
1021 if (!cur)
1022 break;
1023 }
1024 if (*cur == '\\')
1025 {
1026 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1027 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1028 : (file_end - cur == 1) ? 1 : 0;
1029 if (eol_len)
1030 {
1031 cur += eol_len;
1032 line_no++;
1033 continue;
1034 }
1035 }
1036 if (*cur == '\n')
1037 {
1038 cur++;
1039 line_no++;
1040 continue;
1041 }
1042
1043 /* define var
1044 ...
1045 endef */
1046 if (strneq (cur, "define ", 7))
1047 {
1048 const char *var;
1049 unsigned var_len;
1050 const char *value_start;
1051 const char *value_end;
1052 char *value;
1053 unsigned value_len;
1054 int found_endef = 0;
1055
1056 /* extract the variable name. */
1057 cur += 7;
1058 while (isblank ((unsigned char)*cur))
1059 ++cur;
1060 value_start = endp = memchr (cur, '\n', file_end - cur);
1061 if (!endp)
1062 endp = cur;
1063 while (endp > cur && isspace ((unsigned char)endp[-1]))
1064 --endp;
1065 var_len = endp - cur;
1066 if (!var_len)
1067 {
1068 incdep_warn (curdep, line_no, "bogus define statement.");
1069 break;
1070 }
1071 var = incdep_record_strcache (curdep, cur, var_len);
1072
1073 /* find the end of the variable. */
1074 cur = value_end = value_start = value_start + 1;
1075 ++line_no;
1076 while (cur < file_end)
1077 {
1078 /* check for endef, don't bother with skipping leading spaces. */
1079 if ( file_end - cur >= 5
1080 && strneq (cur, "endef", 5))
1081 {
1082 endp = cur + 5;
1083 while (endp < file_end && isspace ((unsigned char)*endp) && *endp != '\n')
1084 endp++;
1085 if (endp >= file_end || *endp == '\n')
1086 {
1087 found_endef = 1;
1088 cur = endp >= file_end ? file_end : endp + 1;
1089 break;
1090 }
1091 }
1092
1093 /* skip a line ahead. */
1094 cur = value_end = memchr (cur, '\n', file_end - cur);
1095 if (cur != NULL)
1096 ++cur;
1097 else
1098 cur = value_end = file_end;
1099 ++line_no;
1100 }
1101
1102 if (!found_endef)
1103 {
1104 incdep_warn (curdep, line_no, "missing endef, dropping the rest of the file.");
1105 break;
1106 }
1107 value_len = value_end - value_start;
1108 if (memchr (value_start, '\0', value_len))
1109 {
1110 incdep_warn (curdep, line_no, "'\\0' in define, dropping the rest of the file.");
1111 break;
1112 }
1113
1114 /* make a copy of the value, converting \r\n to \n, and define it. */
1115 value = incdep_xmalloc (curdep, value_len + 1);
1116 endp = memchr (value_start, '\r', value_len);
1117 if (endp)
1118 {
1119 const char *src = value_start;
1120 char *dst = value;
1121 for (;;)
1122 {
1123 size_t len = endp - src;
1124 memcpy (dst, src, len);
1125 dst += len;
1126 src = endp;
1127 if (src + 1 < file_end && src[1] == '\n')
1128 src++; /* skip the '\r' */
1129 if (src >= value_end)
1130 break;
1131 endp = memchr (endp + 1, '\r', src - value_end);
1132 if (!endp)
1133 endp = value_end;
1134 }
1135 value_len = dst - value;
1136 }
1137 else
1138 memcpy (value, value_start, value_len);
1139 value [value_len] = '\0';
1140
1141 incdep_record_variable_in_set (curdep,
1142 var, var_len, value, value_len,
1143 0 /* don't duplicate */, o_file,
1144 0 /* defines are recursive but this is faster */,
1145 NULL /* global set */, f);
1146 }
1147
1148 /* file: deps
1149 OR
1150 variable [:]= value */
1151 else
1152 {
1153 const char *colonp;
1154 const char *equalp;
1155
1156 /* Look for a colon and an equal sign, optimize for colon.
1157 Only one file is support and the colon / equal must be on
1158 the same line. */
1159 colonp = memchr (cur, ':', file_end - cur);
1160#ifdef HAVE_DOS_PATHS
1161 while ( colonp
1162 && colonp + 1 < file_end
1163 && (colonp[1] == '/' || colonp[1] == '\\')
1164 && colonp > cur
1165 && isalpha ((unsigned char)colonp[-1])
1166 && ( colonp == cur + 1
1167 || strchr (" \t(", colonp[-2]) != 0))
1168 colonp = memchr (colonp + 1, ':', file_end - (colonp + 1));
1169#endif
1170 endp = NULL;
1171 if ( !colonp
1172 || (endp = memchr (cur, '\n', colonp - cur)))
1173 {
1174 colonp = NULL;
1175 equalp = memchr (cur, '=', (endp ? endp : file_end) - cur);
1176 if ( !equalp
1177 || (!endp && memchr (cur, '\n', equalp - cur)))
1178 {
1179 incdep_warn (curdep, line_no, "no colon.");
1180 break;
1181 }
1182 }
1183 else
1184 equalp = memchr (cur, '=', (colonp + 2 <= file_end
1185 ? colonp + 2 : file_end) - cur);
1186 if (equalp)
1187 {
1188 /* An assignment of some sort. */
1189 const char *var;
1190 unsigned var_len;
1191 const char *value_start;
1192 const char *value_end;
1193 char *value;
1194 unsigned value_len;
1195 unsigned multi_line = 0;
1196 enum variable_flavor flavor;
1197
1198 /* figure the flavor first. */
1199 flavor = f_recursive;
1200 if (equalp > cur)
1201 {
1202 if (equalp[-1] == ':')
1203 flavor = f_simple;
1204 else if (equalp[-1] == '?')
1205 flavor = f_conditional;
1206 else if (equalp[-1] == '+')
1207 flavor = f_append;
1208 else if (equalp[-1] == '>')
1209 flavor = f_prepend;
1210 }
1211
1212 /* extract the variable name. */
1213 endp = flavor == f_recursive ? equalp : equalp - 1;
1214 while (endp > cur && isblank ((unsigned char)endp[-1]))
1215 --endp;
1216 var_len = endp - cur;
1217 if (!var_len)
1218 {
1219 incdep_warn (curdep, line_no, "empty variable. (includedep)");
1220 break;
1221 }
1222 if ( memchr (cur, '$', var_len)
1223 || memchr (cur, ' ', var_len)
1224 || memchr (cur, '\t', var_len))
1225 {
1226 incdep_warn (curdep, line_no, "fancy variable name. (includedep)");
1227 break;
1228 }
1229 var = incdep_record_strcache (curdep, cur, var_len);
1230
1231 /* find the start of the value. */
1232 cur = equalp + 1;
1233 while (cur < file_end && isblank ((unsigned char)*cur))
1234 cur++;
1235 value_start = cur;
1236
1237 /* find the end of the value / line (this isn't 101% correct). */
1238 value_end = cur;
1239 while (cur < file_end)
1240 {
1241 endp = value_end = memchr (cur, '\n', file_end - cur);
1242 if (!value_end)
1243 value_end = file_end;
1244 if (value_end - 1 >= cur && value_end[-1] == '\r')
1245 --value_end;
1246 if (value_end - 1 < cur || value_end[-1] != '\\')
1247 {
1248 cur = endp ? endp + 1 : file_end;
1249 break;
1250 }
1251 --value_end;
1252 if (value_end - 1 >= cur && value_end[-1] == '\\')
1253 {
1254 incdep_warn (curdep, line_no, "fancy escaping! (includedep)");
1255 cur = NULL;
1256 break;
1257 }
1258 if (!endp)
1259 {
1260 cur = file_end;
1261 break;
1262 }
1263
1264 cur = endp + 1;
1265 ++multi_line;
1266 ++line_no;
1267 }
1268 if (!cur)
1269 break;
1270 ++line_no;
1271
1272 /* make a copy of the value, converting \r\n to \n, and define it. */
1273 value_len = value_end - value_start;
1274 value = incdep_xmalloc (curdep, value_len + 1);
1275 if (!multi_line)
1276 memcpy (value, value_start, value_len);
1277 else
1278 {
1279 /* unescape it */
1280 const char *src = value_start;
1281 char *dst = value;
1282 while (src < value_end)
1283 {
1284 const char *nextp;
1285
1286 endp = memchr (src, '\n', value_end - src);
1287 if (!endp)
1288 nextp = endp = value_end;
1289 else
1290 nextp = endp + 1;
1291 if (endp > src && endp[-1] == '\r')
1292 --endp;
1293 if (endp > src && endp[-1] == '\\')
1294 --endp;
1295
1296 if (src != value_start)
1297 *dst++ = ' ';
1298 memcpy (dst, src, endp - src);
1299 dst += endp - src;
1300 src = nextp;
1301 }
1302 value_len = dst - value;
1303 }
1304 value [value_len] = '\0';
1305
1306 /* do the definition */
1307 if (flavor == f_recursive
1308 || ( flavor == f_simple
1309 && !memchr (value, '$', value_len)))
1310 incdep_record_variable_in_set (curdep,
1311 var, var_len, value, value_len,
1312 0 /* don't duplicate */, o_file,
1313 flavor == f_recursive /* recursive */,
1314 NULL /* global set */, f);
1315 else
1316 incdep_record_variable_def (curdep,
1317 f, var, var_len, value, value_len,
1318 o_file, flavor, 0 /* not target var */);
1319 }
1320 else
1321 {
1322 /* file: dependencies */
1323
1324 const char *filename;
1325 struct dep *deps = 0;
1326 struct dep **nextdep = &deps;
1327 struct dep *dep;
1328
1329 /* extract the filename, ASSUME a single one. */
1330 endp = colonp;
1331 while (endp > cur && isblank ((unsigned char)endp[-1]))
1332 --endp;
1333 if (cur == endp)
1334 {
1335 incdep_warn (curdep, line_no, "empty filename.");
1336 break;
1337 }
1338 if ( memchr (cur, '$', endp - cur)
1339 || memchr (cur, ' ', endp - cur)
1340 || memchr (cur, '\t', endp - cur))
1341 {
1342 incdep_warn (curdep, line_no, "multiple / fancy file name. (includedep)");
1343 break;
1344 }
1345 filename = incdep_record_strcache (curdep, cur, endp - cur);
1346
1347 /* parse any dependencies. */
1348 cur = colonp + 1;
1349 while (cur < file_end)
1350 {
1351 /* skip blanks and count lines. */
1352 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1353 ++cur;
1354 if (cur >= file_end)
1355 break;
1356 if (*cur == '\n')
1357 {
1358 cur++;
1359 line_no++;
1360 break;
1361 }
1362
1363 /* continuation + eol? */
1364 if (*cur == '\\')
1365 {
1366 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1367 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1368 : (file_end - cur == 1) ? 1 : 0;
1369 if (eol_len)
1370 {
1371 cur += eol_len;
1372 line_no++;
1373 continue;
1374 }
1375 }
1376
1377 /* find the end of the filename */
1378 endp = cur;
1379 while (endp < file_end && !isspace ((unsigned char)*endp))
1380 ++endp;
1381
1382 /* add it to the list. */
1383 *nextdep = dep = incdep_alloc_dep (curdep);
1384 dep->name = incdep_record_strcache (curdep, cur, endp - cur);
1385 dep->includedep = 1;
1386 nextdep = &dep->next;
1387
1388 cur = endp;
1389 }
1390
1391 /* enter the file with its dependencies. */
1392 incdep_record_files (curdep,
1393 filename, NULL, NULL, deps, 0, NULL, 0, 0, f);
1394 }
1395 }
1396 }
1397
1398 /* free the file data */
1399 incdep_xfree (curdep, curdep->file_base);
1400 curdep->file_base = curdep->file_end = NULL;
1401}
1402
1403/* Flushes the incdep todo and done lists. */
1404static void
1405incdep_flush_it (struct floc *f)
1406{
1407 incdep_lock ();
1408 for (;;)
1409 {
1410 struct incdep *cur = incdep_head_done;
1411
1412 /* if the done list is empty, grab a todo list entry. */
1413 if (!cur && incdep_head_todo)
1414 {
1415 cur = incdep_head_todo;
1416 if (cur->next)
1417 incdep_head_todo = cur->next;
1418 else
1419 incdep_head_todo = incdep_tail_todo = NULL;
1420 incdep_unlock ();
1421
1422 incdep_read_file (cur, f);
1423 eval_include_dep_file (cur, f);
1424 incdep_freeit (cur);
1425
1426 incdep_lock ();
1427 continue;
1428 }
1429
1430 /* if the todo list and done list are empty we're either done
1431 or will have to wait for the thread(s) to finish. */
1432 if (!cur && !incdep_num_reading)
1433 break; /* done */
1434 if (!cur)
1435 {
1436 while (!incdep_head_done)
1437 incdep_wait_done ();
1438 cur = incdep_head_done;
1439 }
1440
1441 /* we grab the entire done list and work thru it. */
1442 incdep_head_done = incdep_tail_done = NULL;
1443 incdep_unlock ();
1444
1445 while (cur)
1446 {
1447 struct incdep *next = cur->next;
1448#ifdef PARSE_IN_WORKER
1449 incdep_flush_recorded_instructions (cur);
1450#else
1451 eval_include_dep_file (cur, f);
1452#endif
1453 incdep_freeit (cur);
1454 cur = next;
1455 }
1456
1457 incdep_lock ();
1458 } /* outer loop */
1459 incdep_unlock ();
1460}
1461
1462
1463/* splits up a list of file names and feeds it to eval_include_dep_file,
1464 employing threads to try speed up the file reading. */
1465void
1466eval_include_dep (const char *names, struct floc *f, enum incdep_op op)
1467{
1468 struct incdep *head = 0;
1469 struct incdep *tail = 0;
1470 struct incdep *cur;
1471 const char *names_iterator = names;
1472 const char *name;
1473 unsigned int name_len;
1474
1475 /* loop through NAMES, creating a todo list out of them. */
1476
1477 while ((name = find_next_token (&names_iterator, &name_len)) != 0)
1478 {
1479 cur = xmalloc (sizeof (*cur) + name_len); /* not incdep_xmalloc here */
1480 cur->file_base = cur->file_end = NULL;
1481 memcpy (cur->name, name, name_len);
1482 cur->name[name_len] = '\0';
1483 cur->worker_tid = -1;
1484#ifdef PARSE_IN_WORKER
1485 cur->err_line_no = 0;
1486 cur->err_msg = NULL;
1487 cur->recorded_variables_in_set_head = NULL;
1488 cur->recorded_variables_in_set_tail = NULL;
1489 cur->recorded_variable_defs_head = NULL;
1490 cur->recorded_variable_defs_tail = NULL;
1491 cur->recorded_files_head = NULL;
1492 cur->recorded_files_tail = NULL;
1493#endif
1494
1495 cur->next = NULL;
1496 if (tail)
1497 tail->next = cur;
1498 else
1499 head = cur;
1500 tail = cur;
1501 }
1502
1503#ifdef ELECTRIC_HEAP
1504 if (1)
1505#else
1506 if (op == incdep_read_it)
1507#endif
1508 {
1509 /* work our way thru the files directly */
1510
1511 cur = head;
1512 while (cur)
1513 {
1514 struct incdep *next = cur->next;
1515 incdep_read_file (cur, f);
1516 eval_include_dep_file (cur, f);
1517 incdep_freeit (cur);
1518 cur = next;
1519 }
1520 }
1521 else
1522 {
1523 /* initialize the worker threads and related stuff the first time around. */
1524
1525 if (!incdep_initialized)
1526 incdep_init (f);
1527
1528 /* queue the files and notify the worker threads. */
1529
1530 incdep_lock ();
1531
1532 if (incdep_tail_todo)
1533 incdep_tail_todo->next = head;
1534 else
1535 incdep_head_todo = head;
1536 incdep_tail_todo = tail;
1537
1538 incdep_signal_todo ();
1539 incdep_unlock ();
1540
1541 /* flush the todo queue if we're requested to do so. */
1542
1543 if (op == incdep_flush)
1544 incdep_flush_it (f);
1545 }
1546}
1547
1548#endif /* CONFIG_WITH_INCLUDEDEP */
1549
Note: See TracBrowser for help on using the repository browser.