source: trunk/src/kmk/incdep.c@ 1871

Last change on this file since 1871 was 1871, checked in by bird, 17 years ago

incdep: Use strcache2 for the dep and file names.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 44.4 KB
Line 
1#ifdef CONFIG_WITH_INCLUDEDEP
2/* $Id: incdep.c 1871 2008-10-16 23:38:45Z bird $ */
3/** @file
4 * incdep - Simple dependency files.
5 */
6
7/*
8 * Copyright (c) 2006-2008 knut st. osmundsen <bird-src-spam@anduin.net>
9 *
10 * This file is part of kBuild.
11 *
12 * kBuild is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * kBuild is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with kBuild; if not, write to the Free Software
24 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 *
26 */
27
28
29/*******************************************************************************
30* Header Files *
31*******************************************************************************/
32#ifdef __OS2__
33# define INCL_BASE
34# define INCL_ERRORS
35#endif
36
37#include "make.h"
38
39#if !defined(WINDOWS32) && !defined(__OS2__)
40# define HAVE_PTHREAD
41#endif
42
43#include <assert.h>
44
45#include <glob.h>
46
47#include "dep.h"
48#include "filedef.h"
49#include "job.h"
50#include "commands.h"
51#include "variable.h"
52#include "rule.h"
53#include "debug.h"
54#include "strcache2.h"
55
56#ifdef HAVE_FCNTL_H
57# include <fcntl.h>
58#else
59# include <sys/file.h>
60#endif
61
62#ifdef WINDOWS32
63# include <io.h>
64# include <process.h>
65# include <Windows.h>
66# define PARSE_IN_WORKER
67#endif
68
69#ifdef __OS2__
70# include <os2.h>
71# include <sys/fmutex.h>
72#endif
73
74#ifdef HAVE_PTHREAD
75# include <pthread.h>
76#endif
77
78#ifdef __APPLE__
79# include <malloc/malloc.h>
80# define PARSE_IN_WORKER
81#endif
82
83#if defined(__gnu_linux__) || defined(__linux__)
84# define PARSE_IN_WORKER
85#endif
86
87
88/*******************************************************************************
89* Structures and Typedefs *
90*******************************************************************************/
91struct incdep_variable_in_set
92{
93 struct incdep_variable_in_set *next;
94 /* the parameters */
95 struct strcache2_entry *name_entry; /* dep strcache */
96 const char *value; /* xmalloc'ed */
97 unsigned int value_length;
98 int duplicate_value; /* 0 */
99 enum variable_origin origin;
100 int recursive;
101 struct variable_set *set;
102 const struct floc *flocp; /* NILF */
103};
104
105struct incdep_variable_def
106{
107 struct incdep_variable_def *next;
108 /* the parameters */
109 const struct floc *flocp; /* NILF */
110 struct strcache2_entry *name_entry; /* dep strcache */
111 char *value; /* xmalloc'ed, free it */
112 unsigned int value_length;
113 enum variable_origin origin;
114 enum variable_flavor flavor;
115 int target_var;
116};
117
118struct incdep_recorded_files
119{
120 struct incdep_recorded_files *next;
121
122 /* the parameters */
123 struct strcache2_entry *filename_entry; /* dep strcache; converted to a nameseq record. */
124 const char *pattern; /* NULL */
125 const char *pattern_percent; /* NULL */
126 struct dep *deps; /* All the names are dep strcache entries. */
127 unsigned int cmds_started; /* 0 */
128 char *commands; /* NULL */
129 unsigned int commands_idx; /* 0 */
130 int two_colon; /* 0 */
131 const struct floc *flocp; /* NILF */
132};
133
134
135/* per dep file structure. */
136struct incdep
137{
138 struct incdep *next;
139 char *file_base;
140 char *file_end;
141
142 int worker_tid;
143#ifdef PARSE_IN_WORKER
144 unsigned int err_line_no;
145 const char *err_msg;
146
147 struct incdep_variable_in_set *recorded_variables_in_set_head;
148 struct incdep_variable_in_set *recorded_variables_in_set_tail;
149
150 struct incdep_variable_def *recorded_variable_defs_head;
151 struct incdep_variable_def *recorded_variable_defs_tail;
152
153 struct incdep_recorded_files *recorded_files_head;
154 struct incdep_recorded_files *recorded_files_tail;
155#endif
156
157 char name[1];
158};
159
160
161/*******************************************************************************
162* Global Variables *
163*******************************************************************************/
164
165/* mutex protecting the globals and an associated condition/event. */
166#ifdef HAVE_PTHREAD
167static pthread_mutex_t incdep_mtx;
168static pthread_cond_t incdep_cond_todo;
169static pthread_cond_t incdep_cond_done;
170
171#elif defined (WINDOWS32)
172static CRITICAL_SECTION incdep_mtx;
173static HANDLE incdep_hev_todo;
174static HANDLE incdep_hev_done;
175static int volatile incdep_hev_todo_waiters;
176static int volatile incdep_hev_done_waiters;
177
178#elif defined (__OS2__)
179static fmutex incdep_mtx;
180static HEV incdep_hev_todo;
181static HEV incdep_hev_done;
182static int volatile incdep_hev_todo_waiters;
183static int volatile incdep_hev_done_waiters;
184#endif
185
186/* flag indicating whether the threads, lock and event/condvars has
187 been initialized or not. */
188static int incdep_initialized;
189
190/* the list of files that needs reading. */
191static struct incdep * volatile incdep_head_todo;
192static struct incdep * volatile incdep_tail_todo;
193
194/* the number of files that are currently being read. */
195static int volatile incdep_num_reading;
196
197/* the list of files that have been read. */
198static struct incdep * volatile incdep_head_done;
199static struct incdep * volatile incdep_tail_done;
200
201
202/* The handles to the worker threads. */
203#ifdef HAVE_PTHREAD
204static pthread_t incdep_threads[1];
205static struct alloccache incdep_dep_caches[1];
206static struct strcache2 incdep_dep_strcaches[1];
207
208#elif defined (WINDOWS32)
209static HANDLE incdep_threads[2];
210static struct alloccache incdep_dep_caches[2];
211static struct strcache2 incdep_dep_strcaches[2];
212
213#elif defined (__OS2__)
214static TID incdep_threads[2];
215static struct alloccache incdep_dep_caches[2];
216static struct strcache2 incdep_dep_strcaches[2];
217#endif
218static unsigned incdep_num_threads;
219
220/* flag indicating whether the worker threads should terminate or not. */
221static int volatile incdep_terminate;
222
223#ifdef __APPLE__
224/* malloc zone for the incdep threads. */
225static malloc_zone_t *incdep_zone;
226#endif
227
228
229/*******************************************************************************
230* Internal Functions *
231*******************************************************************************/
232static void incdep_flush_it (struct floc *);
233static void eval_include_dep_file (struct incdep *, struct floc *);
234
235
236/* xmalloc wrapper.
237 For working around multithreaded performance problems found on Darwin,
238 Linux (glibc), and possibly other systems. */
239static void *
240incdep_xmalloc (struct incdep *cur, size_t size)
241{
242 void *ptr;
243
244#ifdef __APPLE__
245 if (cur && cur->worker_tid != -1)
246 {
247 ptr = malloc_zone_malloc (incdep_zone, size);
248 if (!ptr)
249 fatal (NILF, _("virtual memory exhausted"));
250 }
251 else
252 ptr = xmalloc (size);
253#else
254 ptr = xmalloc (size);
255#endif
256
257 (void)cur;
258 return ptr;
259}
260
261#if 0
262/* memset(malloc(sz),'\0',sz) wrapper. */
263static void *
264incdep_xcalloc (struct incdep *cur, size_t size)
265{
266 void *ptr;
267
268#ifdef __APPLE__
269 if (cur && cur->worker_tid != -1)
270 ptr = malloc_zone_calloc (incdep_zone, size, 1);
271 else
272 ptr = calloc (size, 1);
273#else
274 ptr = calloc (size, 1);
275#endif
276 if (!ptr)
277 fatal (NILF, _("virtual memory exhausted"));
278
279 (void)cur;
280 return ptr;
281}
282#endif /* unused */
283
284/* free wrapper */
285static void
286incdep_xfree (struct incdep *cur, void *ptr)
287{
288 /* free() *must* work for the allocation hacks above because
289 of free_dep_chain. */
290 free (ptr);
291 (void)cur;
292}
293
294/* alloc a dep structure. These are allocated in bunches to save time. */
295struct dep *
296incdep_alloc_dep (struct incdep *cur)
297{
298 struct alloccache *cache;
299 if (cur->worker_tid != -1)
300 cache = &incdep_dep_caches[cur->worker_tid];
301 else
302 cache = &dep_cache;
303 return alloccache_calloc (cache);
304}
305
306/* grow a cache. */
307static void *
308incdep_cache_allocator (void *thrd, unsigned int size)
309{
310 (void)thrd;
311#ifdef __APPLE__
312 return malloc_zone_malloc (incdep_zone, size);
313#else
314 return xmalloc (size);
315#endif
316}
317
318/* acquires the lock */
319void
320incdep_lock(void)
321{
322#ifdef HAVE_PTHREAD
323 pthread_mutex_lock (&incdep_mtx);
324#elif defined (WINDOWS32)
325 EnterCriticalSection (&incdep_mtx);
326#elif defined (__OS2__)
327 _fmutex_request (&incdep_mtx, 0)
328#endif
329}
330
331/* releases the lock */
332void
333incdep_unlock(void)
334{
335#ifdef HAVE_PTHREAD
336 pthread_mutex_unlock (&incdep_mtx);
337#elif defined(WINDOWS32)
338 LeaveCriticalSection (&incdep_mtx);
339#elif defined(__OS2__)
340 _fmutex_release (&incdep_mtx)
341#endif
342}
343
344/* signals the main thread that there is stuff todo. caller owns the lock. */
345static void
346incdep_signal_done (void)
347{
348#ifdef HAVE_PTHREAD
349 pthread_cond_broadcast (&incdep_cond_done);
350#elif defined (WINDOWS32)
351 if (incdep_hev_done_waiters)
352 SetEvent (incdep_hev_done);
353#elif defined (__OS2__)
354 if (incdep_hev_done_waiters)
355 DosPostEventSem (incdep_hev_done);
356#endif
357}
358
359/* waits for a reader to finish reading. caller owns the lock. */
360static void
361incdep_wait_done (void)
362{
363#ifdef HAVE_PTHREAD
364 pthread_cond_wait (&incdep_cond_done, &incdep_mtx);
365
366#elif defined (WINDOWS32)
367 ResetEvent (incdep_hev_done);
368 incdep_hev_done_waiters++;
369 incdep_unlock ();
370 WaitForSingleObject (incdep_hev_done, INFINITE);
371 incdep_lock ();
372 incdep_hev_done_waiters--;
373
374#elif defined (__OS2__)
375 ULONG ulIgnore;
376 DosResetEventSem (incdep_hev_done, &ulIgnore);
377 incdep_hev_done_waiters++;
378 incdep_unlock ();
379 DosWaitEventSem (incdep_hev_done, SEM_INDEFINITE_WAIT);
380 incdep_lock ();
381 incdep_hev_done_waiters--;
382#endif
383}
384
385/* signals the worker threads. caller owns the lock. */
386static void
387incdep_signal_todo (void)
388{
389#ifdef HAVE_PTHREAD
390 pthread_cond_broadcast (&incdep_cond_todo);
391#elif defined(WINDOWS32)
392 if (incdep_hev_todo_waiters)
393 SetEvent (incdep_hev_todo);
394#elif defined(__OS2__)
395 if (incdep_hev_todo_waiters)
396 DosPostEventSem (incdep_hev_todo);
397#endif
398}
399
400/* waits for stuff to arrive in the todo list. caller owns the lock. */
401static void
402incdep_wait_todo (void)
403{
404#ifdef HAVE_PTHREAD
405 pthread_cond_wait (&incdep_cond_todo, &incdep_mtx);
406
407#elif defined (WINDOWS32)
408 ResetEvent (incdep_hev_todo);
409 incdep_hev_todo_waiters++;
410 incdep_unlock ();
411 WaitForSingleObject (incdep_hev_todo, INFINITE);
412 incdep_lock ();
413 incdep_hev_todo_waiters--;
414
415#elif defined (__OS2__)
416 ULONG ulIgnore;
417 DosResetEventSem (incdep_hev_todo, &ulIgnore);
418 incdep_hev_todo_waiters++;
419 incdep_unlock ();
420 DosWaitEventSem (incdep_hev_todo, SEM_INDEFINITE_WAIT);
421 incdep_lock ();
422 incdep_hev_todo_waiters--;
423#endif
424}
425
426/* Reads a dep file into memory. */
427static int
428incdep_read_file (struct incdep *cur, struct floc *f)
429{
430 int fd;
431 struct stat st;
432
433 errno = 0;
434#ifdef O_BINARY
435 fd = open (cur->name, O_RDONLY | O_BINARY, 0);
436#else
437 fd = open (cur->name, O_RDONLY, 0);
438#endif
439 if (fd < 0)
440 {
441 /* ignore non-existing dependency files. */
442 int err = errno;
443 if (err == ENOENT || stat (cur->name, &st) != 0)
444 return 1;
445 error (f, "%s: %s", cur->name, strerror (err));
446 return -1;
447 }
448 if (!fstat (fd, &st))
449 {
450 cur->file_base = incdep_xmalloc (cur, st.st_size + 1);
451 if (read (fd, cur->file_base, st.st_size) == st.st_size)
452 {
453 close (fd);
454 cur->file_end = cur->file_base + st.st_size;
455 cur->file_base[st.st_size] = '\0';
456 return 0;
457 }
458
459 /* bail out */
460
461 error (f, "%s: read: %s", cur->name, strerror (errno));
462 incdep_xfree (cur, cur->file_base);
463 }
464 else
465 error (f, "%s: fstat: %s", cur->name, strerror (errno));
466
467 close (fd);
468 cur->file_base = cur->file_end = NULL;
469 return -1;
470}
471
472/* Free the incdep structure. */
473static void
474incdep_freeit (struct incdep *cur)
475{
476#ifdef PARSE_IN_WORKER
477 assert (!cur->recorded_variables_in_set_head);
478 assert (!cur->recorded_variable_defs_head);
479 assert (!cur->recorded_files_head);
480#endif
481
482 incdep_xfree (cur, cur->file_base);
483 free (cur);
484}
485
486/* A worker thread. */
487void
488incdep_worker (int thrd)
489{
490 incdep_lock ();
491
492 while (!incdep_terminate)
493 {
494 /* get job from the todo list. */
495
496 struct incdep *cur = incdep_head_todo;
497 if (!cur)
498 {
499 incdep_wait_todo ();
500 continue;
501 }
502 if (cur->next)
503 incdep_head_todo = cur->next;
504 else
505 incdep_head_todo = incdep_tail_todo = NULL;
506 incdep_num_reading++;
507
508 /* read the file. */
509
510 incdep_unlock ();
511 cur->worker_tid = thrd;
512
513 incdep_read_file (cur, NILF);
514#ifdef PARSE_IN_WORKER
515 eval_include_dep_file (cur, NILF);
516#endif
517
518 cur->worker_tid = -1;
519 incdep_lock ();
520
521 /* insert finished job into the done list. */
522
523 incdep_num_reading--;
524 cur->next = NULL;
525 if (incdep_tail_done)
526 incdep_tail_done->next = cur;
527 else
528 incdep_head_done = cur;
529 incdep_tail_done = cur;
530
531 incdep_signal_done ();
532 }
533
534 incdep_unlock ();
535}
536
537/* Thread library specific thread functions wrapping incdep_wroker. */
538#ifdef HAVE_PTHREAD
539static void *
540incdep_worker_pthread (void *thrd)
541{
542 incdep_worker ((size_t)thrd);
543 return NULL;
544}
545
546#elif defined (WINDOWS32)
547static unsigned __stdcall
548incdep_worker_windows (void *thrd)
549{
550 incdep_worker ((size_t)thrd);
551 return 0;
552}
553
554#elif defined (__OS2__)
555static void
556incdep_worker_os2 (void *thrd)
557{
558 incdep_worker ((size_t)thrd);
559}
560#endif
561
562/* Creates the the worker threads. */
563static void
564incdep_init (struct floc *f)
565{
566 unsigned i;
567#ifdef HAVE_PTHREAD
568 int rc;
569 pthread_attr_t attr;
570
571#elif defined (WINDOWS32)
572 unsigned tid;
573 uintptr_t hThread;
574
575#elif defined (__OS2__)
576 int rc;
577 int tid;
578#endif
579
580 /* heap hacks */
581
582#ifdef __APPLE__
583 incdep_zone = malloc_create_zone (0, 0);
584 if (!incdep_zone)
585 incdep_zone = malloc_default_zone ();
586#endif
587
588
589 /* create the mutex and two condition variables / event objects. */
590
591#ifdef HAVE_PTHREAD
592 rc = pthread_mutex_init (&incdep_mtx, NULL);
593 if (rc)
594 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
595 rc = pthread_cond_init (&incdep_cond_todo, NULL);
596 if (rc)
597 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
598 rc = pthread_cond_init (&incdep_cond_done, NULL);
599 if (rc)
600 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
601
602#elif defined (WINDOWS32)
603 InitializeCriticalSection (&incdep_mtx);
604 incdep_hev_todo = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
605 if (!incdep_hev_todo)
606 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
607 incdep_hev_done = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
608 if (!incdep_hev_done)
609 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
610 incdep_hev_todo_waiters = 0;
611 incdep_hev_done_waiters = 0;
612
613#elif defined (__OS2__)
614 _fmutex_create (&incdep_mtx, 0)
615 rc = DosCreateEventSem (NULL, &incdep_hev_todo, 0, FALSE);
616 if (rc)
617 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
618 rc = DosCreateEventSem (NULL, &incdep_hev_done, 0, FALSE);
619 if (rc)
620 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
621 incdep_hev_todo_waiters = 0;
622 incdep_hev_done_waiters = 0;
623#endif
624
625 /* create the worker threads and associated per thread data. */
626
627 incdep_terminate = 0;
628 incdep_num_threads = sizeof (incdep_threads) / sizeof (incdep_threads[0]);
629 if (incdep_num_threads + 1 > job_slots)
630 incdep_num_threads = job_slots <= 1 ? 1 : job_slots - 1;
631 for (i = 0; i < incdep_num_threads; i++)
632 {
633 /* init caches */
634 alloccache_init (&incdep_dep_caches[i], sizeof(struct dep), "incdep dep",
635 incdep_cache_allocator, (void *)(size_t)i);
636 strcache2_init(&incdep_dep_strcaches[i],
637 "incdep dep", /* name */
638 65536, /* hash size */
639 0, /* default segment size*/
640#ifdef HAVE_CASE_INSENSITIVE_FS
641 1, /* case insensitive */
642#else
643 0, /* case insensitive */
644#endif
645 0); /* thread safe */
646
647 /* create the thread. */
648#ifdef HAVE_PTHREAD
649 rc = pthread_attr_init (&attr);
650 if (rc)
651 fatal (f, _("pthread_attr_init failed: err=%d"), rc);
652 /*rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE); */
653 rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
654 if (rc)
655 fatal (f, _("pthread_attr_setdetachstate failed: err=%d"), rc);
656 rc = pthread_create(&incdep_threads[i], &attr,
657 incdep_worker_pthread, (void *)(size_t)i);
658 if (rc)
659 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
660 pthread_attr_destroy (&attr);
661
662#elif defined (WINDOWS32)
663 tid = 0;
664 hThread = _beginthreadex (NULL, 128*1024, incdep_worker_windows,
665 (void *)i, 0, &tid);
666 if (hThread == 0 || hThread == ~(uintptr_t)0)
667 fatal (f, _("_beginthreadex failed: err=%d"), errno);
668 incdep_threads[i] = (HANDLE)hThread;
669
670#elif defined (__OS2__)
671 tid = _beginthread (incdep_worker_os2, NULL, 128*1024, (void *)i);
672 if (tid <= 0)
673 fatal (f, _("_beginthread failed: err=%d"), errno);
674 incdep_threads[i] = tid;
675#endif
676 }
677
678 incdep_initialized = 1;
679}
680
681/* Flushes outstanding work and terminates the worker threads.
682 This is called from snap_deps(). */
683void
684incdep_flush_and_term (void)
685{
686 unsigned i;
687
688 if (!incdep_initialized)
689 return;
690
691 /* flush any out standing work */
692
693 incdep_flush_it (NILF);
694
695 /* tell the threads to terminate */
696
697 incdep_lock ();
698 incdep_terminate = 1;
699 incdep_signal_todo ();
700 incdep_unlock ();
701
702 /* wait for the threads to quit */
703
704 for (i = 0; i < incdep_num_threads; i++)
705 {
706 /* more later? */
707
708 alloccache_join (&dep_cache, &incdep_dep_caches[i]);
709 strcache2_term (&incdep_dep_strcaches[i]);
710 }
711 incdep_num_threads = 0;
712
713 /* destroy the lock and condition variables / event objects. */
714
715 /* later */
716
717 incdep_initialized = 0;
718}
719
720#ifdef PARSE_IN_WORKER
721/* Flushes a strcache entry returning the actual string cache entry.
722 The input is freed! */
723static const char *
724incdep_flush_strcache_entry (struct strcache2_entry *entry)
725{
726 if (!entry->user)
727 entry->user = (void *)strcache_add_prehashed ((const char *)(entry + 1),
728 entry->length,
729 entry->hash1,
730 entry->hash2);
731 return (const char *)entry->user;
732}
733
734/* Flushes the recorded instructions. */
735static void
736incdep_flush_recorded_instructions (struct incdep *cur)
737{
738 struct incdep_variable_in_set *rec_vis;
739 struct incdep_variable_def *rec_vd;
740 struct incdep_recorded_files *rec_f;
741
742 /* define_variable_in_set */
743
744 rec_vis = cur->recorded_variables_in_set_head;
745 cur->recorded_variables_in_set_head = cur->recorded_variables_in_set_tail = NULL;
746 if (rec_vis)
747 do
748 {
749 void *free_me = rec_vis;
750 unsigned int name_length = rec_vis->name_entry->length;
751 define_variable_in_set (incdep_flush_strcache_entry (rec_vis->name_entry),
752 name_length,
753 rec_vis->value,
754 rec_vis->value_length,
755 rec_vis->duplicate_value,
756 rec_vis->origin,
757 rec_vis->recursive,
758 rec_vis->set,
759 rec_vis->flocp);
760 rec_vis = rec_vis->next;
761 incdep_xfree (cur, free_me);
762 }
763 while (rec_vis);
764
765 /* do_variable_definition */
766
767 rec_vd = cur->recorded_variable_defs_head;
768 cur->recorded_variable_defs_head = cur->recorded_variable_defs_tail = NULL;
769 if (rec_vd)
770 do
771 {
772 void *free_me = rec_vd;
773 do_variable_definition_2 (rec_vd->flocp,
774 incdep_flush_strcache_entry (rec_vd->name_entry),
775 rec_vd->value,
776 rec_vd->value_length,
777 0,
778 rec_vd->value,
779 rec_vd->origin,
780 rec_vd->flavor,
781 rec_vd->target_var);
782 rec_vd = rec_vd->next;
783 incdep_xfree (cur, free_me);
784 }
785 while (rec_vd);
786
787 /* record_files */
788
789 rec_f = cur->recorded_files_head;
790 cur->recorded_files_head = cur->recorded_files_tail = NULL;
791 if (rec_f)
792 do
793 {
794 void *free_me = rec_f;
795 struct dep *dep;
796 struct nameseq *filenames;
797
798 for (dep = rec_f->deps; dep; dep = dep->next)
799 dep->name = incdep_flush_strcache_entry ((struct strcache2_entry *)dep->name);
800
801 filenames = (struct nameseq *) alloccache_alloc (&nameseq_cache);
802 filenames->next = 0;
803 filenames->name = incdep_flush_strcache_entry (rec_f->filename_entry);
804
805 record_files (filenames,
806 rec_f->pattern,
807 rec_f->pattern_percent,
808 rec_f->deps,
809 rec_f->cmds_started,
810 rec_f->commands,
811 rec_f->commands_idx,
812 rec_f->two_colon,
813 rec_f->flocp);
814
815 rec_f = rec_f->next;
816 incdep_xfree (cur, free_me);
817 }
818 while (rec_f);
819}
820#endif /* PARSE_IN_WORKER */
821
822/* Record / issue a warning about a misformed dep file. */
823static void
824incdep_warn (struct incdep *cur, unsigned int line_no, const char *msg)
825{
826 if (cur->worker_tid == -1)
827 error (NILF, "%s(%d): %s", cur->name, line_no, msg);
828#ifdef PARSE_IN_WORKER
829 else
830 {
831 cur->err_line_no = line_no;
832 cur->err_msg = msg;
833 }
834#endif
835}
836
837/* Record / execute a strcache add. */
838static const char *
839incdep_record_strcache (struct incdep *cur, const char *str, int len)
840{
841 const char *ret;
842 if (cur->worker_tid == -1)
843 {
844 /* Make sure the string is terminated before we hand it to
845 strcache_add_len so it does have to make a temporary copy
846 of it on the stack. */
847 char ch = str[len];
848 ((char *)str)[len] = '\0';
849 ret = strcache_add_len (str, len);
850 ((char *)str)[len] = ch;
851 }
852 else
853 {
854 /* Add it out the strcache of the thread. */
855 ret = strcache2_add (&incdep_dep_strcaches[cur->worker_tid], str, len);
856 ret = (const char *)strcache2_get_entry(&incdep_dep_strcaches[cur->worker_tid], ret);
857 }
858 return ret;
859}
860
861/* Record / perform a variable definition in a set.
862 The NAME is in the string cache.
863 The VALUE is on the heap.
864 The DUPLICATE_VALUE is always 0. */
865static void
866incdep_record_variable_in_set (struct incdep *cur,
867 const char *name, unsigned int name_length,
868 const char *value,
869 unsigned int value_length,
870 int duplicate_value,
871 enum variable_origin origin,
872 int recursive,
873 struct variable_set *set,
874 const struct floc *flocp)
875{
876 assert (!duplicate_value);
877 if (cur->worker_tid == -1)
878 define_variable_in_set (name, name_length, value, value_length,
879 duplicate_value, origin, recursive, set, flocp);
880#ifdef PARSE_IN_WORKER
881 else
882 {
883 struct incdep_variable_in_set *rec = incdep_xmalloc (cur, sizeof (*rec));
884 rec->name_entry = (struct strcache2_entry *)name;
885 rec->value = value;
886 rec->value_length = value_length;
887 rec->duplicate_value = duplicate_value;
888 rec->origin = origin;
889 rec->recursive = recursive;
890 rec->set = set;
891 rec->flocp = flocp;
892
893 rec->next = NULL;
894 if (cur->recorded_variables_in_set_tail)
895 cur->recorded_variables_in_set_tail->next = rec;
896 else
897 cur->recorded_variables_in_set_head = rec;
898 cur->recorded_variables_in_set_tail = rec;
899 }
900#endif
901}
902
903/* Record / perform a variable definition. The VALUE should be disposed of. */
904static void
905incdep_record_variable_def (struct incdep *cur,
906 const struct floc *flocp,
907 const char *name,
908 unsigned int name_length,
909 char *value,
910 unsigned int value_length,
911 enum variable_origin origin,
912 enum variable_flavor flavor,
913 int target_var)
914{
915 if (cur->worker_tid == -1)
916 do_variable_definition_2 (flocp, name, value, value_length, 0, value,
917 origin, flavor, target_var);
918#ifdef PARSE_IN_WORKER
919 else
920 {
921 struct incdep_variable_def *rec = incdep_xmalloc (cur, sizeof (*rec));
922 rec->flocp = flocp;
923 rec->name_entry = (struct strcache2_entry *)name;
924 rec->value = value;
925 rec->value_length = value_length;
926 rec->origin = origin;
927 rec->flavor = flavor;
928 rec->target_var = target_var;
929
930 rec->next = NULL;
931 if (cur->recorded_variable_defs_tail)
932 cur->recorded_variable_defs_tail->next = rec;
933 else
934 cur->recorded_variable_defs_head = rec;
935 cur->recorded_variable_defs_tail = rec;
936 }
937#else
938 (void)name_length;
939#endif
940}
941
942/* Record files.*/
943static void
944incdep_record_files (struct incdep *cur,
945 const char *filename, const char *pattern,
946 const char *pattern_percent, struct dep *deps,
947 unsigned int cmds_started, char *commands,
948 unsigned int commands_idx, int two_colon,
949 const struct floc *flocp)
950{
951 if (cur->worker_tid == -1)
952 {
953 struct nameseq *filenames = (struct nameseq *) alloccache_alloc (&nameseq_cache);
954 filenames->next = 0;
955 filenames->name = filename;
956 record_files (filenames, pattern, pattern_percent, deps, cmds_started,
957 commands, commands_idx, two_colon, flocp);
958 }
959#ifdef PARSE_IN_WORKER
960 else
961 {
962 struct incdep_recorded_files *rec = incdep_xmalloc (cur, sizeof (*rec));
963
964 rec->filename_entry = (struct strcache2_entry *)filename;
965 rec->pattern = pattern;
966 rec->pattern_percent = pattern_percent;
967 rec->deps = deps;
968 rec->cmds_started = cmds_started;
969 rec->commands = commands;
970 rec->commands_idx = commands_idx;
971 rec->two_colon = two_colon;
972 rec->flocp = flocp;
973
974 rec->next = NULL;
975 if (cur->recorded_files_tail)
976 cur->recorded_files_tail->next = rec;
977 else
978 cur->recorded_files_head = rec;
979 cur->recorded_files_tail = rec;
980 }
981#endif
982}
983
984
985/* no nonsense dependency file including.
986
987 Because nobody wants bogus dependency files to break their incremental
988 builds with hard to comprehend error messages, this function does not
989 use the normal eval routine but does all the parsing itself. This isn't,
990 as much work as it sounds, because the necessary feature set is very
991 limited.
992
993 eval_include_dep_file groks:
994
995 define var
996 endef
997
998 var [|:|?|>]= value [\]
999
1000 [\]
1001 file: [deps] [\]
1002
1003 */
1004static void
1005eval_include_dep_file (struct incdep *curdep, struct floc *f)
1006{
1007 unsigned line_no = 1;
1008 const char *file_end = curdep->file_end;
1009 const char *cur = curdep->file_base;
1010 const char *endp;
1011
1012 /* if no file data, just return immediately. */
1013 if (!cur)
1014 return;
1015
1016 /* now parse the file. */
1017 while (cur < file_end)
1018 {
1019 /* skip empty lines */
1020 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1021 ++cur;
1022 if (cur >= file_end)
1023 break;
1024 if (*cur == '#')
1025 {
1026 cur = memchr (cur, '\n', file_end - cur);
1027 if (!cur)
1028 break;
1029 }
1030 if (*cur == '\\')
1031 {
1032 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1033 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1034 : (file_end - cur == 1) ? 1 : 0;
1035 if (eol_len)
1036 {
1037 cur += eol_len;
1038 line_no++;
1039 continue;
1040 }
1041 }
1042 if (*cur == '\n')
1043 {
1044 cur++;
1045 line_no++;
1046 continue;
1047 }
1048
1049 /* define var
1050 ...
1051 endef */
1052 if (strneq (cur, "define ", 7))
1053 {
1054 const char *var;
1055 unsigned var_len;
1056 const char *value_start;
1057 const char *value_end;
1058 char *value;
1059 unsigned value_len;
1060 int found_endef = 0;
1061
1062 /* extract the variable name. */
1063 cur += 7;
1064 while (isblank ((unsigned char)*cur))
1065 ++cur;
1066 value_start = endp = memchr (cur, '\n', file_end - cur);
1067 if (!endp)
1068 endp = cur;
1069 while (endp > cur && isspace ((unsigned char)endp[-1]))
1070 --endp;
1071 var_len = endp - cur;
1072 if (!var_len)
1073 {
1074 incdep_warn (curdep, line_no, "bogus define statement.");
1075 break;
1076 }
1077 var = incdep_record_strcache (curdep, cur, var_len);
1078
1079 /* find the end of the variable. */
1080 cur = value_end = value_start = value_start + 1;
1081 ++line_no;
1082 while (cur < file_end)
1083 {
1084 /* check for endef, don't bother with skipping leading spaces. */
1085 if ( file_end - cur >= 5
1086 && strneq (cur, "endef", 5))
1087 {
1088 endp = cur + 5;
1089 while (endp < file_end && isspace ((unsigned char)*endp) && *endp != '\n')
1090 endp++;
1091 if (endp >= file_end || *endp == '\n')
1092 {
1093 found_endef = 1;
1094 cur = endp >= file_end ? file_end : endp + 1;
1095 break;
1096 }
1097 }
1098
1099 /* skip a line ahead. */
1100 cur = value_end = memchr (cur, '\n', file_end - cur);
1101 if (cur != NULL)
1102 ++cur;
1103 else
1104 cur = value_end = file_end;
1105 ++line_no;
1106 }
1107
1108 if (!found_endef)
1109 {
1110 incdep_warn (curdep, line_no, "missing endef, dropping the rest of the file.");
1111 break;
1112 }
1113 value_len = value_end - value_start;
1114 if (memchr (value_start, '\0', value_len))
1115 {
1116 incdep_warn (curdep, line_no, "'\\0' in define, dropping the rest of the file.");
1117 break;
1118 }
1119
1120 /* make a copy of the value, converting \r\n to \n, and define it. */
1121 value = incdep_xmalloc (curdep, value_len + 1);
1122 endp = memchr (value_start, '\r', value_len);
1123 if (endp)
1124 {
1125 const char *src = value_start;
1126 char *dst = value;
1127 for (;;)
1128 {
1129 size_t len = endp - src;
1130 memcpy (dst, src, len);
1131 dst += len;
1132 src = endp;
1133 if (src + 1 < file_end && src[1] == '\n')
1134 src++; /* skip the '\r' */
1135 if (src >= value_end)
1136 break;
1137 endp = memchr (endp + 1, '\r', src - value_end);
1138 if (!endp)
1139 endp = value_end;
1140 }
1141 value_len = dst - value;
1142 }
1143 else
1144 memcpy (value, value_start, value_len);
1145 value [value_len] = '\0';
1146
1147 incdep_record_variable_in_set (curdep,
1148 var, var_len, value, value_len,
1149 0 /* don't duplicate */, o_file,
1150 0 /* defines are recursive but this is faster */,
1151 NULL /* global set */, f);
1152 }
1153
1154 /* file: deps
1155 OR
1156 variable [:]= value */
1157 else
1158 {
1159 const char *colonp;
1160 const char *equalp;
1161
1162 /* Look for a colon and an equal sign, optimize for colon.
1163 Only one file is support and the colon / equal must be on
1164 the same line. */
1165 colonp = memchr (cur, ':', file_end - cur);
1166#ifdef HAVE_DOS_PATHS
1167 while ( colonp
1168 && colonp + 1 < file_end
1169 && (colonp[1] == '/' || colonp[1] == '\\')
1170 && colonp > cur
1171 && isalpha ((unsigned char)colonp[-1])
1172 && ( colonp == cur + 1
1173 || strchr (" \t(", colonp[-2]) != 0))
1174 colonp = memchr (colonp + 1, ':', file_end - (colonp + 1));
1175#endif
1176 endp = NULL;
1177 if ( !colonp
1178 || (endp = memchr (cur, '\n', colonp - cur)))
1179 {
1180 colonp = NULL;
1181 equalp = memchr (cur, '=', (endp ? endp : file_end) - cur);
1182 if ( !equalp
1183 || (!endp && memchr (cur, '\n', equalp - cur)))
1184 {
1185 incdep_warn (curdep, line_no, "no colon.");
1186 break;
1187 }
1188 }
1189 else
1190 equalp = memchr (cur, '=', (colonp + 2 <= file_end
1191 ? colonp + 2 : file_end) - cur);
1192 if (equalp)
1193 {
1194 /* An assignment of some sort. */
1195 const char *var;
1196 unsigned var_len;
1197 const char *value_start;
1198 const char *value_end;
1199 char *value;
1200 unsigned value_len;
1201 unsigned multi_line = 0;
1202 enum variable_flavor flavor;
1203
1204 /* figure the flavor first. */
1205 flavor = f_recursive;
1206 if (equalp > cur)
1207 {
1208 if (equalp[-1] == ':')
1209 flavor = f_simple;
1210 else if (equalp[-1] == '?')
1211 flavor = f_conditional;
1212 else if (equalp[-1] == '+')
1213 flavor = f_append;
1214 else if (equalp[-1] == '>')
1215 flavor = f_prepend;
1216 }
1217
1218 /* extract the variable name. */
1219 endp = flavor == f_recursive ? equalp : equalp - 1;
1220 while (endp > cur && isblank ((unsigned char)endp[-1]))
1221 --endp;
1222 var_len = endp - cur;
1223 if (!var_len)
1224 {
1225 incdep_warn (curdep, line_no, "empty variable. (includedep)");
1226 break;
1227 }
1228 if ( memchr (cur, '$', var_len)
1229 || memchr (cur, ' ', var_len)
1230 || memchr (cur, '\t', var_len))
1231 {
1232 incdep_warn (curdep, line_no, "fancy variable name. (includedep)");
1233 break;
1234 }
1235 var = incdep_record_strcache (curdep, cur, var_len);
1236
1237 /* find the start of the value. */
1238 cur = equalp + 1;
1239 while (cur < file_end && isblank ((unsigned char)*cur))
1240 cur++;
1241 value_start = cur;
1242
1243 /* find the end of the value / line (this isn't 101% correct). */
1244 value_end = cur;
1245 while (cur < file_end)
1246 {
1247 endp = value_end = memchr (cur, '\n', file_end - cur);
1248 if (!value_end)
1249 value_end = file_end;
1250 if (value_end - 1 >= cur && value_end[-1] == '\r')
1251 --value_end;
1252 if (value_end - 1 < cur || value_end[-1] != '\\')
1253 {
1254 cur = endp ? endp + 1 : file_end;
1255 break;
1256 }
1257 --value_end;
1258 if (value_end - 1 >= cur && value_end[-1] == '\\')
1259 {
1260 incdep_warn (curdep, line_no, "fancy escaping! (includedep)");
1261 cur = NULL;
1262 break;
1263 }
1264 if (!endp)
1265 {
1266 cur = file_end;
1267 break;
1268 }
1269
1270 cur = endp + 1;
1271 ++multi_line;
1272 ++line_no;
1273 }
1274 if (!cur)
1275 break;
1276 ++line_no;
1277
1278 /* make a copy of the value, converting \r\n to \n, and define it. */
1279 value_len = value_end - value_start;
1280 value = incdep_xmalloc (curdep, value_len + 1);
1281 if (!multi_line)
1282 memcpy (value, value_start, value_len);
1283 else
1284 {
1285 /* unescape it */
1286 const char *src = value_start;
1287 char *dst = value;
1288 while (src < value_end)
1289 {
1290 const char *nextp;
1291
1292 endp = memchr (src, '\n', value_end - src);
1293 if (!endp)
1294 nextp = endp = value_end;
1295 else
1296 nextp = endp + 1;
1297 if (endp > src && endp[-1] == '\r')
1298 --endp;
1299 if (endp > src && endp[-1] == '\\')
1300 --endp;
1301
1302 if (src != value_start)
1303 *dst++ = ' ';
1304 memcpy (dst, src, endp - src);
1305 dst += endp - src;
1306 src = nextp;
1307 }
1308 value_len = dst - value;
1309 }
1310 value [value_len] = '\0';
1311
1312 /* do the definition */
1313 if (flavor == f_recursive
1314 || ( flavor == f_simple
1315 && !memchr (value, '$', value_len)))
1316 incdep_record_variable_in_set (curdep,
1317 var, var_len, value, value_len,
1318 0 /* don't duplicate */, o_file,
1319 flavor == f_recursive /* recursive */,
1320 NULL /* global set */, f);
1321 else
1322 incdep_record_variable_def (curdep,
1323 f, var, var_len, value, value_len,
1324 o_file, flavor, 0 /* not target var */);
1325 }
1326 else
1327 {
1328 /* file: dependencies */
1329
1330 const char *filename;
1331 struct dep *deps = 0;
1332 struct dep **nextdep = &deps;
1333 struct dep *dep;
1334
1335 /* extract the filename, ASSUME a single one. */
1336 endp = colonp;
1337 while (endp > cur && isblank ((unsigned char)endp[-1]))
1338 --endp;
1339 if (cur == endp)
1340 {
1341 incdep_warn (curdep, line_no, "empty filename.");
1342 break;
1343 }
1344 if ( memchr (cur, '$', endp - cur)
1345 || memchr (cur, ' ', endp - cur)
1346 || memchr (cur, '\t', endp - cur))
1347 {
1348 incdep_warn (curdep, line_no, "multiple / fancy file name. (includedep)");
1349 break;
1350 }
1351 filename = incdep_record_strcache (curdep, cur, endp - cur);
1352
1353 /* parse any dependencies. */
1354 cur = colonp + 1;
1355 while (cur < file_end)
1356 {
1357 /* skip blanks and count lines. */
1358 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1359 ++cur;
1360 if (cur >= file_end)
1361 break;
1362 if (*cur == '\n')
1363 {
1364 cur++;
1365 line_no++;
1366 break;
1367 }
1368
1369 /* continuation + eol? */
1370 if (*cur == '\\')
1371 {
1372 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1373 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1374 : (file_end - cur == 1) ? 1 : 0;
1375 if (eol_len)
1376 {
1377 cur += eol_len;
1378 line_no++;
1379 continue;
1380 }
1381 }
1382
1383 /* find the end of the filename */
1384 endp = cur;
1385 while (endp < file_end && !isspace ((unsigned char)*endp))
1386 ++endp;
1387
1388 /* add it to the list. */
1389 *nextdep = dep = incdep_alloc_dep (curdep);
1390 dep->name = incdep_record_strcache (curdep, cur, endp - cur);
1391 dep->includedep = 1;
1392 nextdep = &dep->next;
1393
1394 cur = endp;
1395 }
1396
1397 /* enter the file with its dependencies. */
1398 incdep_record_files (curdep,
1399 filename, NULL, NULL, deps, 0, NULL, 0, 0, f);
1400 }
1401 }
1402 }
1403
1404 /* free the file data */
1405 incdep_xfree (curdep, curdep->file_base);
1406 curdep->file_base = curdep->file_end = NULL;
1407}
1408
1409/* Flushes the incdep todo and done lists. */
1410static void
1411incdep_flush_it (struct floc *f)
1412{
1413 incdep_lock ();
1414 for (;;)
1415 {
1416 struct incdep *cur = incdep_head_done;
1417
1418 /* if the done list is empty, grab a todo list entry. */
1419 if (!cur && incdep_head_todo)
1420 {
1421 cur = incdep_head_todo;
1422 if (cur->next)
1423 incdep_head_todo = cur->next;
1424 else
1425 incdep_head_todo = incdep_tail_todo = NULL;
1426 incdep_unlock ();
1427
1428 incdep_read_file (cur, f);
1429 eval_include_dep_file (cur, f);
1430 incdep_freeit (cur);
1431
1432 incdep_lock ();
1433 continue;
1434 }
1435
1436 /* if the todo list and done list are empty we're either done
1437 or will have to wait for the thread(s) to finish. */
1438 if (!cur && !incdep_num_reading)
1439 break; /* done */
1440 if (!cur)
1441 {
1442 while (!incdep_head_done)
1443 incdep_wait_done ();
1444 cur = incdep_head_done;
1445 }
1446
1447 /* we grab the entire done list and work thru it. */
1448 incdep_head_done = incdep_tail_done = NULL;
1449 incdep_unlock ();
1450
1451 while (cur)
1452 {
1453 struct incdep *next = cur->next;
1454#ifdef PARSE_IN_WORKER
1455 incdep_flush_recorded_instructions (cur);
1456#else
1457 eval_include_dep_file (cur, f);
1458#endif
1459 incdep_freeit (cur);
1460 cur = next;
1461 }
1462
1463 incdep_lock ();
1464 } /* outer loop */
1465 incdep_unlock ();
1466}
1467
1468
1469/* splits up a list of file names and feeds it to eval_include_dep_file,
1470 employing threads to try speed up the file reading. */
1471void
1472eval_include_dep (const char *names, struct floc *f, enum incdep_op op)
1473{
1474 struct incdep *head = 0;
1475 struct incdep *tail = 0;
1476 struct incdep *cur;
1477 const char *names_iterator = names;
1478 const char *name;
1479 unsigned int name_len;
1480
1481 /* loop through NAMES, creating a todo list out of them. */
1482
1483 while ((name = find_next_token (&names_iterator, &name_len)) != 0)
1484 {
1485 cur = xmalloc (sizeof (*cur) + name_len); /* not incdep_xmalloc here */
1486 cur->file_base = cur->file_end = NULL;
1487 memcpy (cur->name, name, name_len);
1488 cur->name[name_len] = '\0';
1489 cur->worker_tid = -1;
1490#ifdef PARSE_IN_WORKER
1491 cur->err_line_no = 0;
1492 cur->err_msg = NULL;
1493 cur->recorded_variables_in_set_head = NULL;
1494 cur->recorded_variables_in_set_tail = NULL;
1495 cur->recorded_variable_defs_head = NULL;
1496 cur->recorded_variable_defs_tail = NULL;
1497 cur->recorded_files_head = NULL;
1498 cur->recorded_files_tail = NULL;
1499#endif
1500
1501 cur->next = NULL;
1502 if (tail)
1503 tail->next = cur;
1504 else
1505 head = cur;
1506 tail = cur;
1507 }
1508
1509#ifdef ELECTRIC_HEAP
1510 if (1)
1511#else
1512 if (op == incdep_read_it)
1513#endif
1514 {
1515 /* work our way thru the files directly */
1516
1517 cur = head;
1518 while (cur)
1519 {
1520 struct incdep *next = cur->next;
1521 incdep_read_file (cur, f);
1522 eval_include_dep_file (cur, f);
1523 incdep_freeit (cur);
1524 cur = next;
1525 }
1526 }
1527 else
1528 {
1529 /* initialize the worker threads and related stuff the first time around. */
1530
1531 if (!incdep_initialized)
1532 incdep_init (f);
1533
1534 /* queue the files and notify the worker threads. */
1535
1536 incdep_lock ();
1537
1538 if (incdep_tail_todo)
1539 incdep_tail_todo->next = head;
1540 else
1541 incdep_head_todo = head;
1542 incdep_tail_todo = tail;
1543
1544 incdep_signal_todo ();
1545 incdep_unlock ();
1546
1547 /* flush the todo queue if we're requested to do so. */
1548
1549 if (op == incdep_flush)
1550 incdep_flush_it (f);
1551 }
1552}
1553
1554#endif /* CONFIG_WITH_INCLUDEDEP */
1555
Note: See TracBrowser for help on using the repository browser.