source: trunk/src/kmk/incdep.c@ 1857

Last change on this file since 1857 was 1854, checked in by bird, 17 years ago

kmk: offload hashing of strcache entries to the includedep thread(s).
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 42.7 KB
Line 
1#ifdef CONFIG_WITH_INCLUDEDEP
2/* $Id: incdep.c 1854 2008-10-13 01:50:33Z bird $ */
3/** @file
4 * incdep - Simple dependency files.
5 */
6
7/*
8 * Copyright (c) 2006-2008 knut st. osmundsen <bird-src-spam@anduin.net>
9 *
10 * This file is part of kBuild.
11 *
12 * kBuild is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * kBuild is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with kBuild; if not, write to the Free Software
24 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 *
26 */
27
28
29/*******************************************************************************
30* Header Files *
31*******************************************************************************/
32#ifdef __OS2__
33# define INCL_BASE
34# define INCL_ERRORS
35#endif
36
37#include "make.h"
38
39#if !defined(WINDOWS32) && !defined(__OS2__)
40# define HAVE_PTHREAD
41#endif
42
43#include <assert.h>
44
45#include <glob.h>
46
47#include "dep.h"
48#include "filedef.h"
49#include "job.h"
50#include "commands.h"
51#include "variable.h"
52#include "rule.h"
53#include "debug.h"
54#include "hash.h"
55
56#ifdef HAVE_FCNTL_H
57# include <fcntl.h>
58#else
59# include <sys/file.h>
60#endif
61
62#ifdef WINDOWS32
63# include <io.h>
64# include <process.h>
65# include <Windows.h>
66# define PARSE_IN_WORKER
67#endif
68
69#ifdef __OS2__
70# include <os2.h>
71# include <sys/fmutex.h>
72#endif
73
74#ifdef HAVE_PTHREAD
75# include <pthread.h>
76#endif
77
78#ifdef __APPLE__
79# include <malloc/malloc.h>
80# define PARSE_IN_WORKER
81#endif
82
83
84/*******************************************************************************
85* Structures and Typedefs *
86*******************************************************************************/
87struct incdep_strcache_entry
88{
89 unsigned int length;
90 unsigned int alignment;
91 unsigned long hash1;
92 unsigned long hash2;
93 char str[1];
94};
95
96struct incdep_variable_in_set
97{
98 struct incdep_variable_in_set *next;
99 /* the parameters */
100 struct incdep_strcache_entry *name_entry;
101 const char *value; /* xmalloc'ed */
102 unsigned int value_length;
103 int duplicate_value; /* 0 */
104 enum variable_origin origin;
105 int recursive;
106 struct variable_set *set;
107 const struct floc *flocp; /* NILF */
108};
109
110struct incdep_variable_def
111{
112 struct incdep_variable_def *next;
113 /* the parameters */
114 const struct floc *flocp; /* NILF */
115 struct incdep_strcache_entry *name_entry;
116 char *value; /* xmalloc'ed, free it */
117 unsigned int value_length;
118 enum variable_origin origin;
119 enum variable_flavor flavor;
120 int target_var;
121};
122
123struct incdep_recorded_files
124{
125 struct incdep_recorded_files *next;
126
127 /* the parameters */
128 struct nameseq *filenames; /* One only, its name is a strcache entry. */
129 const char *pattern; /* NULL */
130 const char *pattern_percent; /* NULL */
131 struct dep *deps; /* All the names are strcache entries. */
132 unsigned int cmds_started; /* 0 */
133 char *commands; /* NULL */
134 unsigned int commands_idx; /* 0 */
135 int two_colon; /* 0 */
136 const struct floc *flocp; /* NILF */
137};
138
139
140/* per dep file structure. */
141struct incdep
142{
143 struct incdep *next;
144 char *file_base;
145 char *file_end;
146
147 int is_worker;
148#ifdef PARSE_IN_WORKER
149 unsigned int err_line_no;
150 const char *err_msg;
151
152 struct incdep_variable_in_set *recorded_variables_in_set_head;
153 struct incdep_variable_in_set *recorded_variables_in_set_tail;
154
155 struct incdep_variable_def *recorded_variable_defs_head;
156 struct incdep_variable_def *recorded_variable_defs_tail;
157
158 struct incdep_recorded_files *recorded_files_head;
159 struct incdep_recorded_files *recorded_files_tail;
160#endif
161
162 char name[1];
163};
164
165
166/*******************************************************************************
167* Global Variables *
168*******************************************************************************/
169
170/* mutex protecting the globals and an associated condition/event. */
171#ifdef HAVE_PTHREAD
172static pthread_mutex_t incdep_mtx;
173static pthread_cond_t incdep_cond_todo;
174static pthread_cond_t incdep_cond_done;
175
176#elif defined (WINDOWS32)
177static CRITICAL_SECTION incdep_mtx;
178static HANDLE incdep_hev_todo;
179static HANDLE incdep_hev_done;
180static int volatile incdep_hev_todo_waiters;
181static int volatile incdep_hev_done_waiters;
182
183#elif defined (__OS2__)
184static fmutex incdep_mtx;
185static HEV incdep_hev_todo;
186static HEV incdep_hev_done;
187static int volatile incdep_hev_todo_waiters;
188static int volatile incdep_hev_done_waiters;
189#endif
190
191/* flag indicating whether the threads, lock and event/condvars has
192 been initialized or not. */
193static int incdep_initialized;
194
195/* the list of files that needs reading. */
196static struct incdep * volatile incdep_head_todo;
197static struct incdep * volatile incdep_tail_todo;
198
199/* the number of files that are currently being read. */
200static int volatile incdep_num_reading;
201
202/* the list of files that have been read. */
203static struct incdep * volatile incdep_head_done;
204static struct incdep * volatile incdep_tail_done;
205
206/* The handles to the worker threads. */
207#ifdef HAVE_PTHREAD
208static pthread_t incdep_threads[1];
209#elif defined (WINDOWS32)
210static HANDLE incdep_threads[2];
211#elif defined (__OS2__)
212static TID incdep_threads[2];
213#endif
214static unsigned incdep_num_threads;
215
216/* flag indicating whether the worker threads should terminate or not. */
217static int volatile incdep_terminate;
218
219#ifdef __APPLE__
220/* malloc zone for the incdep threads. */
221static malloc_zone_t *incdep_zone;
222#endif
223
224
225/*******************************************************************************
226* Internal Functions *
227*******************************************************************************/
228static void incdep_flush_it (struct floc *);
229static void eval_include_dep_file (struct incdep *, struct floc *);
230
231
232/* xmalloc wrapper.
233 For working around multithreaded performance problems found on Darwin,
234 Linux (glibc), and possibly other systems. */
235static void *incdep_xmalloc (struct incdep *cur, size_t size)
236{
237 void *ptr;
238
239#ifdef __APPLE__
240 if (cur && cur->is_worker)
241 {
242 ptr = malloc_zone_malloc (incdep_zone, size);
243 if (!ptr)
244 fatal (NILF, _("virtual memory exhausted"));
245 }
246 else
247 ptr = xmalloc (size);
248#else
249 ptr = xmalloc (size);
250#endif
251
252 (void)cur;
253 return ptr;
254}
255
256/* memset(malloc(sz),'\0',sz) wrapper. */
257static void *incdep_xcalloc (struct incdep *cur, size_t size)
258{
259 void *ptr;
260
261#ifdef __APPLE__
262 if (cur && cur->is_worker)
263 ptr = malloc_zone_calloc (incdep_zone, size, 1);
264 else
265 ptr = calloc (size, 1);
266#else
267 ptr = calloc (size, 1);
268#endif
269 if (!ptr)
270 fatal (NILF, _("virtual memory exhausted"));
271
272 (void)cur;
273 return ptr;
274}
275
276/* free wrapper */
277static void incdep_xfree (struct incdep *cur, void *ptr)
278{
279 /* free() *must* work for the allocation hacks above because
280 of free_dep_chain. */
281 free (ptr);
282 (void)cur;
283}
284
285
286
287/* acquires the lock */
288void
289incdep_lock(void)
290{
291#ifdef HAVE_PTHREAD
292 pthread_mutex_lock (&incdep_mtx);
293#elif defined (WINDOWS32)
294 EnterCriticalSection (&incdep_mtx);
295#elif defined (__OS2__)
296 _fmutex_request (&incdep_mtx, 0)
297#endif
298}
299
300/* releases the lock */
301void
302incdep_unlock(void)
303{
304#ifdef HAVE_PTHREAD
305 pthread_mutex_unlock (&incdep_mtx);
306#elif defined(WINDOWS32)
307 LeaveCriticalSection (&incdep_mtx);
308#elif defined(__OS2__)
309 _fmutex_release (&incdep_mtx)
310#endif
311}
312
313/* signals the main thread that there is stuff todo. caller owns the lock. */
314static void
315incdep_signal_done (void)
316{
317#ifdef HAVE_PTHREAD
318 pthread_cond_broadcast (&incdep_cond_done);
319#elif defined (WINDOWS32)
320 if (incdep_hev_done_waiters)
321 SetEvent (incdep_hev_done);
322#elif defined (__OS2__)
323 if (incdep_hev_done_waiters)
324 DosPostEventSem (incdep_hev_done);
325#endif
326}
327
328/* waits for a reader to finish reading. caller owns the lock. */
329static void
330incdep_wait_done (void)
331{
332#ifdef HAVE_PTHREAD
333 pthread_cond_wait (&incdep_cond_done, &incdep_mtx);
334
335#elif defined (WINDOWS32)
336 ResetEvent (incdep_hev_done);
337 incdep_hev_done_waiters++;
338 incdep_unlock ();
339 WaitForSingleObject (incdep_hev_done, INFINITE);
340 incdep_lock ();
341 incdep_hev_done_waiters--;
342
343#elif defined (__OS2__)
344 ULONG ulIgnore;
345 DosResetEventSem (incdep_hev_done, &ulIgnore);
346 incdep_hev_done_waiters++;
347 incdep_unlock ();
348 DosWaitEventSem (incdep_hev_done, SEM_INDEFINITE_WAIT);
349 incdep_lock ();
350 incdep_hev_done_waiters--;
351#endif
352}
353
354/* signals the worker threads. caller owns the lock. */
355static void
356incdep_signal_todo (void)
357{
358#ifdef HAVE_PTHREAD
359 pthread_cond_broadcast (&incdep_cond_todo);
360#elif defined(WINDOWS32)
361 if (incdep_hev_todo_waiters)
362 SetEvent (incdep_hev_todo);
363#elif defined(__OS2__)
364 if (incdep_hev_todo_waiters)
365 DosPostEventSem (incdep_hev_todo);
366#endif
367}
368
369/* waits for stuff to arrive in the todo list. caller owns the lock. */
370static void
371incdep_wait_todo (void)
372{
373#ifdef HAVE_PTHREAD
374 pthread_cond_wait (&incdep_cond_todo, &incdep_mtx);
375
376#elif defined (WINDOWS32)
377 ResetEvent (incdep_hev_todo);
378 incdep_hev_todo_waiters++;
379 incdep_unlock ();
380 WaitForSingleObject (incdep_hev_todo, INFINITE);
381 incdep_lock ();
382 incdep_hev_todo_waiters--;
383
384#elif defined (__OS2__)
385 ULONG ulIgnore;
386 DosResetEventSem (incdep_hev_todo, &ulIgnore);
387 incdep_hev_todo_waiters++;
388 incdep_unlock ();
389 DosWaitEventSem (incdep_hev_todo, SEM_INDEFINITE_WAIT);
390 incdep_lock ();
391 incdep_hev_todo_waiters--;
392#endif
393}
394
395/* Reads a dep file into memory. */
396static int
397incdep_read_file (struct incdep *cur, struct floc *f)
398{
399 int fd;
400 struct stat st;
401
402 errno = 0;
403#ifdef O_BINARY
404 fd = open (cur->name, O_RDONLY | O_BINARY, 0);
405#else
406 fd = open (cur->name, O_RDONLY, 0);
407#endif
408 if (fd < 0)
409 {
410 /* ignore non-existing dependency files. */
411 int err = errno;
412 if (err == ENOENT || stat (cur->name, &st) != 0)
413 return 1;
414 error (f, "%s: %s", cur->name, strerror (err));
415 return -1;
416 }
417 if (!fstat (fd, &st))
418 {
419 cur->file_base = incdep_xmalloc (cur, st.st_size + 1);
420 if (read (fd, cur->file_base, st.st_size) == st.st_size)
421 {
422 close (fd);
423 cur->file_end = cur->file_base + st.st_size;
424 cur->file_base[st.st_size] = '\0';
425 return 0;
426 }
427
428 /* bail out */
429
430 error (f, "%s: read: %s", cur->name, strerror (errno));
431 incdep_xfree (cur, cur->file_base);
432 }
433 else
434 error (f, "%s: fstat: %s", cur->name, strerror (errno));
435
436 close (fd);
437 cur->file_base = cur->file_end = NULL;
438 return -1;
439}
440
441/* Free the incdep structure. */
442static void
443incdep_freeit (struct incdep *cur)
444{
445#ifdef PARSE_IN_WORKER
446 assert (!cur->recorded_variables_in_set_head);
447 assert (!cur->recorded_variable_defs_head);
448 assert (!cur->recorded_files_head);
449#endif
450
451 incdep_xfree (cur, cur->file_base);
452 free (cur);
453}
454
455/* A worker thread. */
456void
457incdep_worker (void)
458{
459 incdep_lock ();
460
461 while (!incdep_terminate)
462 {
463 /* get job from the todo list. */
464
465 struct incdep *cur = incdep_head_todo;
466 if (!cur)
467 {
468 incdep_wait_todo ();
469 continue;
470 }
471 if (cur->next)
472 incdep_head_todo = cur->next;
473 else
474 incdep_head_todo = incdep_tail_todo = NULL;
475 incdep_num_reading++;
476
477 /* read the file. */
478
479 incdep_unlock ();
480 cur->is_worker = 1;
481 incdep_read_file (cur, NILF);
482#ifdef PARSE_IN_WORKER
483 eval_include_dep_file (cur, NILF);
484#endif
485 cur->is_worker = 0;
486 incdep_lock ();
487
488 /* insert finished job into the done list. */
489
490 incdep_num_reading--;
491 cur->next = NULL;
492 if (incdep_tail_done)
493 incdep_tail_done->next = cur;
494 else
495 incdep_head_done = cur;
496 incdep_tail_done = cur;
497
498 incdep_signal_done ();
499 }
500
501 incdep_unlock ();
502}
503
504/* Thread library specific thread functions wrapping incdep_wroker. */
505#ifdef HAVE_PTHREAD
506static void *
507incdep_worker_pthread (void *ignore)
508{
509 incdep_worker ();
510 (void)ignore;
511 return NULL;
512}
513
514#elif defined (WINDOWS32)
515static unsigned __stdcall
516incdep_worker_windows (void *ignore)
517{
518 incdep_worker ();
519 (void)ignore;
520 return 0;
521}
522
523#elif defined (__OS2__)
524static void
525incdep_worker_os2 (void *ignore)
526{
527 incdep_worker ();
528 (void)ignore;
529}
530#endif
531
532/* Creates the the worker threads. */
533static void
534incdep_init (struct floc *f)
535{
536 unsigned i;
537#ifdef HAVE_PTHREAD
538 int rc;
539 pthread_attr_t attr;
540
541#elif defined (WINDOWS32)
542 unsigned tid;
543 uintptr_t hThread;
544
545#elif defined (__OS2__)
546 int rc;
547 int tid;
548#endif
549
550 /* heap hacks */
551
552#ifdef __APPLE__
553 incdep_zone = malloc_create_zone (0, 0);
554 if (!incdep_zone)
555 incdep_zone = malloc_default_zone ();
556#endif
557
558
559 /* create the mutex and two condition variables / event objects. */
560
561#ifdef HAVE_PTHREAD
562 rc = pthread_mutex_init (&incdep_mtx, NULL);
563 if (rc)
564 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
565 rc = pthread_cond_init (&incdep_cond_todo, NULL);
566 if (rc)
567 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
568 rc = pthread_cond_init (&incdep_cond_done, NULL);
569 if (rc)
570 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
571
572#elif defined (WINDOWS32)
573 InitializeCriticalSection (&incdep_mtx);
574 incdep_hev_todo = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
575 if (!incdep_hev_todo)
576 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
577 incdep_hev_done = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
578 if (!incdep_hev_done)
579 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
580 incdep_hev_todo_waiters = 0;
581 incdep_hev_done_waiters = 0;
582
583#elif defined (__OS2__)
584 _fmutex_create (&incdep_mtx, 0)
585 rc = DosCreateEventSem (NULL, &incdep_hev_todo, 0, FALSE);
586 if (rc)
587 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
588 rc = DosCreateEventSem (NULL, &incdep_hev_done, 0, FALSE);
589 if (rc)
590 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
591 incdep_hev_todo_waiters = 0;
592 incdep_hev_done_waiters = 0;
593#endif
594
595 /* create the worker threads. */
596
597 incdep_terminate = 0;
598 incdep_num_threads = sizeof (incdep_threads) / sizeof (incdep_threads[0]);
599 if (incdep_num_threads + 1 > job_slots)
600 incdep_num_threads = job_slots <= 1 ? 1 : job_slots - 1;
601 for (i = 0; i < incdep_num_threads; i++)
602 {
603#ifdef HAVE_PTHREAD
604 rc = pthread_attr_init (&attr);
605 if (rc)
606 fatal (f, _("pthread_attr_init failed: err=%d"), rc);
607 /*rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE); */
608 rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
609 if (rc)
610 fatal (f, _("pthread_attr_setdetachstate failed: err=%d"), rc);
611 rc = pthread_create (&incdep_threads[i], &attr,
612 incdep_worker_pthread, f);
613 if (rc)
614 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
615 pthread_attr_destroy (&attr);
616
617#elif defined (WINDOWS32)
618 tid = 0;
619 hThread = _beginthreadex (NULL, 128*1024, incdep_worker_windows,
620 NULL, 0, &tid);
621 if (hThread == 0 || hThread == ~(uintptr_t)0)
622 fatal (f, _("_beginthreadex failed: err=%d"), errno);
623 incdep_threads[i] = (HANDLE)hThread;
624
625#elif defined (__OS2__)
626 tid = _beginthread (incdep_worker_os2, NULL, 128*1024, NULL);
627 if (tid <= 0)
628 fatal (f, _("_beginthread failed: err=%d"), errno);
629 incdep_threads[i] = tid;
630#endif
631 }
632
633 incdep_initialized = 1;
634}
635
636/* Flushes outstanding work and terminates the worker threads.
637 This is called from snap_deps(). */
638void
639incdep_flush_and_term (void)
640{
641 unsigned i;
642
643 if (!incdep_initialized)
644 return;
645
646 /* flush any out standing work */
647
648 incdep_flush_it (NILF);
649
650 /* tell the threads to terminate */
651
652 incdep_lock ();
653 incdep_terminate = 1;
654 incdep_signal_todo ();
655 incdep_unlock ();
656
657 /* wait for the threads to quit */
658
659 for (i = 0; i < incdep_num_threads; i++)
660 {
661 /* later? */
662 }
663 incdep_num_threads = 0;
664
665 /* destroy the lock and condition variables / event objects. */
666
667 /* later */
668
669 incdep_initialized = 0;
670}
671
672#ifdef PARSE_IN_WORKER
673/* Flushes a strcache entry returning the actual string cache entry.
674 The input is freed! */
675static const char *
676incdep_flush_strcache_entry (const void *pv_entry)
677{
678 struct incdep_strcache_entry *entry = (struct incdep_strcache_entry *)pv_entry;
679 const char *result;
680 result = strcache_add_prehashed (entry->str, entry->length, entry->hash1, entry->hash2);
681 free (entry);
682 return result;
683}
684
685/* Flushes the recorded instructions. */
686static void
687incdep_flush_recorded_instructions (struct incdep *cur)
688{
689 struct incdep_variable_in_set *rec_vis;
690 struct incdep_variable_def *rec_vd;
691 struct incdep_recorded_files *rec_f;
692
693 /* define_variable_in_set */
694
695 rec_vis = cur->recorded_variables_in_set_head;
696 cur->recorded_variables_in_set_head = cur->recorded_variables_in_set_tail = NULL;
697 if (rec_vis)
698 do
699 {
700 void *free_me = rec_vis;
701 unsigned int name_length = rec_vis->name_entry->length;
702 define_variable_in_set (incdep_flush_strcache_entry (rec_vis->name_entry),
703 name_length,
704 rec_vis->value,
705 rec_vis->value_length,
706 rec_vis->duplicate_value,
707 rec_vis->origin,
708 rec_vis->recursive,
709 rec_vis->set,
710 rec_vis->flocp);
711 rec_vis = rec_vis->next;
712 incdep_xfree (cur, free_me);
713 }
714 while (rec_vis);
715
716 /* do_variable_definition */
717
718 rec_vd = cur->recorded_variable_defs_head;
719 cur->recorded_variable_defs_head = cur->recorded_variable_defs_tail = NULL;
720 if (rec_vd)
721 do
722 {
723 void *free_me = rec_vd;
724 do_variable_definition_2 (rec_vd->flocp,
725 incdep_flush_strcache_entry (rec_vd->name_entry),
726 rec_vd->value,
727 rec_vd->value_length,
728 0,
729 rec_vd->value,
730 rec_vd->origin,
731 rec_vd->flavor,
732 rec_vd->target_var);
733 rec_vd = rec_vd->next;
734 incdep_xfree (cur, free_me);
735 }
736 while (rec_vd);
737
738 /* record_files */
739
740 rec_f = cur->recorded_files_head;
741 cur->recorded_files_head = cur->recorded_files_tail = NULL;
742 if (rec_f)
743 do
744 {
745 void *free_me = rec_f;
746 struct dep *dep;
747
748 for (dep = rec_f->deps; dep; dep = dep->next)
749 dep->name = incdep_flush_strcache_entry (dep->name);
750 rec_f->filenames->name = incdep_flush_strcache_entry (rec_f->filenames->name);
751
752 record_files (rec_f->filenames,
753 rec_f->pattern,
754 rec_f->pattern_percent,
755 rec_f->deps,
756 rec_f->cmds_started,
757 rec_f->commands,
758 rec_f->commands_idx,
759 rec_f->two_colon,
760 rec_f->flocp);
761
762 rec_f = rec_f->next;
763 incdep_xfree (cur, free_me);
764 }
765 while (rec_f);
766}
767#endif /* PARSE_IN_WORKER */
768
769/* Record / issue a warning about a misformed dep file. */
770static void
771incdep_warn (struct incdep *cur, unsigned int line_no, const char *msg)
772{
773 if (!cur->is_worker)
774 error (NILF, "%s(%d): %s", cur->name, line_no, msg);
775#ifdef PARSE_IN_WORKER
776 else
777 {
778 cur->err_line_no = line_no;
779 cur->err_msg = msg;
780 }
781#endif
782}
783
784/* Record / execute a strcache add. */
785static const char *
786incdep_record_strcache (struct incdep *cur, const char *str, int len)
787{
788 const char *ret;
789 if (!cur->is_worker)
790 {
791 /* Make sure the string is terminated before we hand it to
792 strcache_add_len so it does have to make a temporary copy
793 of it on the stack. */
794 char ch = str[len];
795 ((char *)str)[len] = '\0';
796 ret = strcache_add_len (str, len);
797 ((char *)str)[len] = ch;
798 }
799 else
800 {
801 /* Allocate a strcache record for it, pre-hashing the string to save
802 time later on in the main thread. */
803 struct incdep_strcache_entry *entry = incdep_xmalloc (cur, sizeof (*entry) + len);
804 memcpy (entry->str, str, len);
805 entry->str[len] = '\0';
806 entry->length = len;
807 strcache_prehash_str (entry->str, &entry->hash1, &entry->hash2);
808
809 ret = (const char *)entry;
810 }
811 return ret;
812}
813
814/* Record / perform a variable definition in a set.
815 The NAME is in the string cache.
816 The VALUE is on the heap.
817 The DUPLICATE_VALUE is always 0. */
818static void
819incdep_record_variable_in_set (struct incdep *cur,
820 const char *name, unsigned int name_length,
821 const char *value,
822 unsigned int value_length,
823 int duplicate_value,
824 enum variable_origin origin,
825 int recursive,
826 struct variable_set *set,
827 const struct floc *flocp)
828{
829 assert (!duplicate_value);
830 if (!cur->is_worker)
831 define_variable_in_set (name, name_length, value, value_length,
832 duplicate_value, origin, recursive, set, flocp);
833#ifdef PARSE_IN_WORKER
834 else
835 {
836 struct incdep_variable_in_set *rec = incdep_xmalloc (cur, sizeof (*rec));
837 rec->name_entry = (struct incdep_strcache_entry *)name;
838 rec->value = value;
839 rec->value_length = value_length;
840 rec->duplicate_value = duplicate_value;
841 rec->origin = origin;
842 rec->recursive = recursive;
843 rec->set = set;
844 rec->flocp = flocp;
845
846 rec->next = NULL;
847 if (cur->recorded_variables_in_set_tail)
848 cur->recorded_variables_in_set_tail->next = rec;
849 else
850 cur->recorded_variables_in_set_head = rec;
851 cur->recorded_variables_in_set_tail = rec;
852 }
853#endif
854}
855
856/* Record / perform a variable definition. The VALUE should be disposed of. */
857static void
858incdep_record_variable_def (struct incdep *cur,
859 const struct floc *flocp,
860 const char *name,
861 unsigned int name_length,
862 char *value,
863 unsigned int value_length,
864 enum variable_origin origin,
865 enum variable_flavor flavor,
866 int target_var)
867{
868 if (!cur->is_worker)
869 do_variable_definition_2 (flocp, name, value, value_length, 0, value,
870 origin, flavor, target_var);
871#ifdef PARSE_IN_WORKER
872 else
873 {
874 struct incdep_variable_def *rec = incdep_xmalloc (cur, sizeof (*rec));
875 rec->flocp = flocp;
876 rec->name_entry = (struct incdep_strcache_entry *)name;
877 rec->value = value;
878 rec->value_length = value_length;
879 rec->origin = origin;
880 rec->flavor = flavor;
881 rec->target_var = target_var;
882
883 rec->next = NULL;
884 if (cur->recorded_variable_defs_tail)
885 cur->recorded_variable_defs_tail->next = rec;
886 else
887 cur->recorded_variable_defs_head = rec;
888 cur->recorded_variable_defs_tail = rec;
889 }
890#else
891 (void)name_length;
892#endif
893}
894
895/* Record files.*/
896static void
897incdep_record_files (struct incdep *cur,
898 struct nameseq *filenames, const char *pattern,
899 const char *pattern_percent, struct dep *deps,
900 unsigned int cmds_started, char *commands,
901 unsigned int commands_idx, int two_colon,
902 const struct floc *flocp)
903{
904 if (!cur->is_worker)
905 record_files (filenames, pattern, pattern_percent, deps, cmds_started,
906 commands, commands_idx, two_colon, flocp);
907#ifdef PARSE_IN_WORKER
908 else
909 {
910 struct incdep_recorded_files *rec = incdep_xmalloc (cur, sizeof (*rec));
911
912 rec->filenames = filenames;
913 rec->pattern = pattern;
914 rec->pattern_percent = pattern_percent;
915 rec->deps = deps;
916 rec->cmds_started = cmds_started;
917 rec->commands = commands;
918 rec->commands_idx = commands_idx;
919 rec->two_colon = two_colon;
920 rec->flocp = flocp;
921
922 rec->next = NULL;
923 if (cur->recorded_files_tail)
924 cur->recorded_files_tail->next = rec;
925 else
926 cur->recorded_files_head = rec;
927 cur->recorded_files_tail = rec;
928 }
929#endif
930}
931
932
933/* no nonsense dependency file including.
934
935 Because nobody wants bogus dependency files to break their incremental
936 builds with hard to comprehend error messages, this function does not
937 use the normal eval routine but does all the parsing itself. This isn't,
938 as much work as it sounds, because the necessary feature set is very
939 limited.
940
941 eval_include_dep_file groks:
942
943 define var
944 endef
945
946 var [|:|?|>]= value [\]
947
948 [\]
949 file: [deps] [\]
950
951 */
952static void
953eval_include_dep_file (struct incdep *curdep, struct floc *f)
954{
955 unsigned line_no = 1;
956 const char *file_end = curdep->file_end;
957 const char *cur = curdep->file_base;
958 const char *endp;
959
960 /* if no file data, just return immediately. */
961 if (!cur)
962 return;
963
964 /* now parse the file. */
965 while (cur < file_end)
966 {
967 /* skip empty lines */
968 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
969 ++cur;
970 if (cur >= file_end)
971 break;
972 if (*cur == '#')
973 {
974 cur = memchr (cur, '\n', file_end - cur);
975 if (!cur)
976 break;
977 }
978 if (*cur == '\\')
979 {
980 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
981 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
982 : (file_end - cur == 1) ? 1 : 0;
983 if (eol_len)
984 {
985 cur += eol_len;
986 line_no++;
987 continue;
988 }
989 }
990 if (*cur == '\n')
991 {
992 cur++;
993 line_no++;
994 continue;
995 }
996
997 /* define var
998 ...
999 endef */
1000 if (strneq (cur, "define ", 7))
1001 {
1002 const char *var;
1003 unsigned var_len;
1004 const char *value_start;
1005 const char *value_end;
1006 char *value;
1007 unsigned value_len;
1008 int found_endef = 0;
1009
1010 /* extract the variable name. */
1011 cur += 7;
1012 while (isblank ((unsigned char)*cur))
1013 ++cur;
1014 value_start = endp = memchr (cur, '\n', file_end - cur);
1015 if (!endp)
1016 endp = cur;
1017 while (endp > cur && isspace ((unsigned char)endp[-1]))
1018 --endp;
1019 var_len = endp - cur;
1020 if (!var_len)
1021 {
1022 incdep_warn (curdep, line_no, "bogus define statement.");
1023 break;
1024 }
1025 var = incdep_record_strcache (curdep, cur, var_len);
1026
1027 /* find the end of the variable. */
1028 cur = value_end = value_start = value_start + 1;
1029 ++line_no;
1030 while (cur < file_end)
1031 {
1032 /* check for endef, don't bother with skipping leading spaces. */
1033 if ( file_end - cur >= 5
1034 && strneq (cur, "endef", 5))
1035 {
1036 endp = cur + 5;
1037 while (endp < file_end && isspace ((unsigned char)*endp) && *endp != '\n')
1038 endp++;
1039 if (endp >= file_end || *endp == '\n')
1040 {
1041 found_endef = 1;
1042 cur = endp >= file_end ? file_end : endp + 1;
1043 break;
1044 }
1045 }
1046
1047 /* skip a line ahead. */
1048 cur = value_end = memchr (cur, '\n', file_end - cur);
1049 if (cur != NULL)
1050 ++cur;
1051 else
1052 cur = value_end = file_end;
1053 ++line_no;
1054 }
1055
1056 if (!found_endef)
1057 {
1058 incdep_warn (curdep, line_no, "missing endef, dropping the rest of the file.");
1059 break;
1060 }
1061 value_len = value_end - value_start;
1062 if (memchr (value_start, '\0', value_len))
1063 {
1064 incdep_warn (curdep, line_no, "'\\0' in define, dropping the rest of the file.");
1065 break;
1066 }
1067
1068 /* make a copy of the value, converting \r\n to \n, and define it. */
1069 value = incdep_xmalloc (curdep, value_len + 1);
1070 endp = memchr (value_start, '\r', value_len);
1071 if (endp)
1072 {
1073 const char *src = value_start;
1074 char *dst = value;
1075 for (;;)
1076 {
1077 size_t len = endp - src;
1078 memcpy (dst, src, len);
1079 dst += len;
1080 src = endp;
1081 if (src + 1 < file_end && src[1] == '\n')
1082 src++; /* skip the '\r' */
1083 if (src >= value_end)
1084 break;
1085 endp = memchr (endp + 1, '\r', src - value_end);
1086 if (!endp)
1087 endp = value_end;
1088 }
1089 value_len = dst - value;
1090 }
1091 else
1092 memcpy (value, value_start, value_len);
1093 value [value_len] = '\0';
1094
1095 incdep_record_variable_in_set (curdep,
1096 var, var_len, value, value_len,
1097 0 /* don't duplicate */, o_file,
1098 0 /* defines are recursive but this is faster */,
1099 NULL /* global set */, f);
1100 }
1101
1102 /* file: deps
1103 OR
1104 variable [:]= value */
1105 else
1106 {
1107 const char *colonp;
1108 const char *equalp;
1109
1110 /* Look for a colon and an equal sign, optimize for colon.
1111 Only one file is support and the colon / equal must be on
1112 the same line. */
1113 colonp = memchr (cur, ':', file_end - cur);
1114#ifdef HAVE_DOS_PATHS
1115 while ( colonp
1116 && colonp + 1 < file_end
1117 && (colonp[1] == '/' || colonp[1] == '\\')
1118 && colonp > cur
1119 && isalpha ((unsigned char)colonp[-1])
1120 && ( colonp == cur + 1
1121 || strchr (" \t(", colonp[-2]) != 0))
1122 colonp = memchr (colonp + 1, ':', file_end - (colonp + 1));
1123#endif
1124 endp = NULL;
1125 if ( !colonp
1126 || (endp = memchr (cur, '\n', colonp - cur)))
1127 {
1128 colonp = NULL;
1129 equalp = memchr (cur, '=', (endp ? endp : file_end) - cur);
1130 if ( !equalp
1131 || (!endp && memchr (cur, '\n', equalp - cur)))
1132 {
1133 incdep_warn (curdep, line_no, "no colon.");
1134 break;
1135 }
1136 }
1137 else
1138 equalp = memchr (cur, '=', (colonp + 2 <= file_end
1139 ? colonp + 2 : file_end) - cur);
1140 if (equalp)
1141 {
1142 /* An assignment of some sort. */
1143 const char *var;
1144 unsigned var_len;
1145 const char *value_start;
1146 const char *value_end;
1147 char *value;
1148 unsigned value_len;
1149 unsigned multi_line = 0;
1150 enum variable_flavor flavor;
1151
1152 /* figure the flavor first. */
1153 flavor = f_recursive;
1154 if (equalp > cur)
1155 {
1156 if (equalp[-1] == ':')
1157 flavor = f_simple;
1158 else if (equalp[-1] == '?')
1159 flavor = f_conditional;
1160 else if (equalp[-1] == '+')
1161 flavor = f_append;
1162 else if (equalp[-1] == '>')
1163 flavor = f_prepend;
1164 }
1165
1166 /* extract the variable name. */
1167 endp = flavor == f_recursive ? equalp : equalp - 1;
1168 while (endp > cur && isblank ((unsigned char)endp[-1]))
1169 --endp;
1170 var_len = endp - cur;
1171 if (!var_len)
1172 {
1173 incdep_warn (curdep, line_no, "empty variable. (includedep)");
1174 break;
1175 }
1176 if ( memchr (cur, '$', var_len)
1177 || memchr (cur, ' ', var_len)
1178 || memchr (cur, '\t', var_len))
1179 {
1180 incdep_warn (curdep, line_no, "fancy variable name. (includedep)");
1181 break;
1182 }
1183 var = incdep_record_strcache (curdep, cur, var_len);
1184
1185 /* find the start of the value. */
1186 cur = equalp + 1;
1187 while (cur < file_end && isblank ((unsigned char)*cur))
1188 cur++;
1189 value_start = cur;
1190
1191 /* find the end of the value / line (this isn't 101% correct). */
1192 value_end = cur;
1193 while (cur < file_end)
1194 {
1195 endp = value_end = memchr (cur, '\n', file_end - cur);
1196 if (!value_end)
1197 value_end = file_end;
1198 if (value_end - 1 >= cur && value_end[-1] == '\r')
1199 --value_end;
1200 if (value_end - 1 < cur || value_end[-1] != '\\')
1201 {
1202 cur = endp ? endp + 1 : file_end;
1203 break;
1204 }
1205 --value_end;
1206 if (value_end - 1 >= cur && value_end[-1] == '\\')
1207 {
1208 incdep_warn (curdep, line_no, "fancy escaping! (includedep)");
1209 cur = NULL;
1210 break;
1211 }
1212 if (!endp)
1213 {
1214 cur = file_end;
1215 break;
1216 }
1217
1218 cur = endp + 1;
1219 ++multi_line;
1220 ++line_no;
1221 }
1222 if (!cur)
1223 break;
1224 ++line_no;
1225
1226 /* make a copy of the value, converting \r\n to \n, and define it. */
1227 value_len = value_end - value_start;
1228 value = incdep_xmalloc (curdep, value_len + 1);
1229 if (!multi_line)
1230 memcpy (value, value_start, value_len);
1231 else
1232 {
1233 /* unescape it */
1234 const char *src = value_start;
1235 char *dst = value;
1236 while (src < value_end)
1237 {
1238 const char *nextp;
1239
1240 endp = memchr (src, '\n', value_end - src);
1241 if (!endp)
1242 nextp = endp = value_end;
1243 else
1244 nextp = endp + 1;
1245 if (endp > src && endp[-1] == '\r')
1246 --endp;
1247 if (endp > src && endp[-1] == '\\')
1248 --endp;
1249
1250 if (src != value_start)
1251 *dst++ = ' ';
1252 memcpy (dst, src, endp - src);
1253 dst += endp - src;
1254 src = nextp;
1255 }
1256 value_len = dst - value;
1257 }
1258 value [value_len] = '\0';
1259
1260 /* do the definition */
1261 if (flavor == f_recursive
1262 || ( flavor == f_simple
1263 && !memchr (value, '$', value_len)))
1264 incdep_record_variable_in_set (curdep,
1265 var, var_len, value, value_len,
1266 0 /* don't duplicate */, o_file,
1267 flavor == f_recursive /* recursive */,
1268 NULL /* global set */, f);
1269 else
1270 incdep_record_variable_def (curdep,
1271 f, var, var_len, value, value_len,
1272 o_file, flavor, 0 /* not target var */);
1273 }
1274 else
1275 {
1276 /* file: dependencies */
1277
1278 struct nameseq *filenames = 0;
1279 struct dep *deps = 0;
1280 struct dep **nextdep = &deps;
1281 struct dep *dep;
1282
1283 /* extract the filename, ASSUME a single one. */
1284 endp = colonp;
1285 while (endp > cur && isblank ((unsigned char)endp[-1]))
1286 --endp;
1287 if (cur == endp)
1288 {
1289 incdep_warn (curdep, line_no, "empty filename.");
1290 break;
1291 }
1292 if ( memchr (cur, '$', endp - cur)
1293 || memchr (cur, ' ', endp - cur)
1294 || memchr (cur, '\t', endp - cur))
1295 {
1296 incdep_warn (curdep, line_no, "multiple / fancy file name. (includedep)");
1297 break;
1298 }
1299 filenames = incdep_xmalloc (curdep, sizeof (struct nameseq));
1300 memset (filenames, 0, sizeof (*filenames));
1301 filenames->name = incdep_record_strcache (curdep, cur, endp - cur);
1302
1303 /* parse any dependencies. */
1304 cur = colonp + 1;
1305 while (cur < file_end)
1306 {
1307 /* skip blanks and count lines. */
1308 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1309 ++cur;
1310 if (cur >= file_end)
1311 break;
1312 if (*cur == '\n')
1313 {
1314 cur++;
1315 line_no++;
1316 break;
1317 }
1318
1319 /* continuation + eol? */
1320 if (*cur == '\\')
1321 {
1322 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1323 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1324 : (file_end - cur == 1) ? 1 : 0;
1325 if (eol_len)
1326 {
1327 cur += eol_len;
1328 line_no++;
1329 continue;
1330 }
1331 }
1332
1333 /* find the end of the filename */
1334 endp = cur;
1335 while (endp < file_end && !isspace ((unsigned char)*endp))
1336 ++endp;
1337
1338 /* add it to the list. */
1339 *nextdep = dep = incdep_xcalloc (curdep, sizeof (*dep));
1340 dep->name = incdep_record_strcache (curdep, cur, endp - cur);
1341 dep->includedep = 1;
1342 nextdep = &dep->next;
1343
1344 cur = endp;
1345 }
1346
1347 /* enter the file with its dependencies. */
1348 incdep_record_files (curdep,
1349 filenames, NULL, NULL, deps, 0, NULL, 0, 0, f);
1350 }
1351 }
1352 }
1353
1354 /* free the file data */
1355 incdep_xfree (curdep, curdep->file_base);
1356 curdep->file_base = curdep->file_end = NULL;
1357}
1358
1359/* Flushes the incdep todo and done lists. */
1360static void
1361incdep_flush_it (struct floc *f)
1362{
1363 incdep_lock ();
1364 for (;;)
1365 {
1366 struct incdep *cur = incdep_head_done;
1367
1368 /* if the done list is empty, grab a todo list entry. */
1369 if (!cur && incdep_head_todo)
1370 {
1371 cur = incdep_head_todo;
1372 if (cur->next)
1373 incdep_head_todo = cur->next;
1374 else
1375 incdep_head_todo = incdep_tail_todo = NULL;
1376 incdep_unlock ();
1377
1378 incdep_read_file (cur, f);
1379 eval_include_dep_file (cur, f);
1380 incdep_freeit (cur);
1381
1382 incdep_lock ();
1383 continue;
1384 }
1385
1386 /* if the todo list and done list are empty we're either done
1387 or will have to wait for the thread(s) to finish. */
1388 if (!cur && !incdep_num_reading)
1389 break; /* done */
1390 if (!cur)
1391 {
1392 while (!incdep_head_done)
1393 incdep_wait_done ();
1394 cur = incdep_head_done;
1395 }
1396
1397 /* we grab the entire done list and work thru it. */
1398 incdep_head_done = incdep_tail_done = NULL;
1399 incdep_unlock ();
1400
1401 while (cur)
1402 {
1403 struct incdep *next = cur->next;
1404#ifdef PARSE_IN_WORKER
1405 incdep_flush_recorded_instructions (cur);
1406#else
1407 eval_include_dep_file (cur, f);
1408#endif
1409 incdep_freeit (cur);
1410 cur = next;
1411 }
1412
1413 incdep_lock ();
1414 } /* outer loop */
1415 incdep_unlock ();
1416}
1417
1418
1419/* splits up a list of file names and feeds it to eval_include_dep_file,
1420 employing threads to try speed up the file reading. */
1421void
1422eval_include_dep (const char *names, struct floc *f, enum incdep_op op)
1423{
1424 struct incdep *head = 0;
1425 struct incdep *tail = 0;
1426 struct incdep *cur;
1427 const char *names_iterator = names;
1428 const char *name;
1429 unsigned int name_len;
1430
1431 /* loop through NAMES, creating a todo list out of them. */
1432
1433 while ((name = find_next_token (&names_iterator, &name_len)) != 0)
1434 {
1435 cur = xmalloc (sizeof (*cur) + name_len); /* not incdep_xmalloc here */
1436 cur->file_base = cur->file_end = NULL;
1437 memcpy (cur->name, name, name_len);
1438 cur->name[name_len] = '\0';
1439 cur->is_worker = 0;
1440#ifdef PARSE_IN_WORKER
1441 cur->err_line_no = 0;
1442 cur->err_msg = NULL;
1443 cur->recorded_variables_in_set_head = NULL;
1444 cur->recorded_variables_in_set_tail = NULL;
1445 cur->recorded_variable_defs_head = NULL;
1446 cur->recorded_variable_defs_tail = NULL;
1447 cur->recorded_files_head = NULL;
1448 cur->recorded_files_tail = NULL;
1449#endif
1450
1451 cur->next = NULL;
1452 if (tail)
1453 tail->next = cur;
1454 else
1455 head = cur;
1456 tail = cur;
1457 }
1458
1459 if (op == incdep_read_it)
1460 {
1461 /* work our way thru the files directly */
1462
1463 cur = head;
1464 while (cur)
1465 {
1466 struct incdep *next = cur->next;
1467 incdep_read_file (cur, f);
1468 eval_include_dep_file (cur, f);
1469 incdep_freeit (cur);
1470 cur = next;
1471 }
1472 }
1473 else
1474 {
1475 /* initialize the worker threads and related stuff the first time around. */
1476
1477 if (!incdep_initialized)
1478 incdep_init (f);
1479
1480 /* queue the files and notify the worker threads. */
1481
1482 incdep_lock ();
1483
1484 if (incdep_tail_todo)
1485 incdep_tail_todo->next = head;
1486 else
1487 incdep_head_todo = head;
1488 incdep_tail_todo = tail;
1489
1490 incdep_signal_todo ();
1491 incdep_unlock ();
1492
1493 /* flush the todo queue if we're requested to do so. */
1494
1495 if (op == incdep_flush)
1496 incdep_flush_it (f);
1497 }
1498}
1499
1500#endif /* CONFIG_WITH_INCLUDEDEP */
1501
Note: See TracBrowser for help on using the repository browser.