source: trunk/kBuild/footer-pass2-fetches.kmk@ 3340

Last change on this file since 3340 was 3121, checked in by bird, 8 years ago

(C) year)

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 14.7 KB
Line 
1# $Id: footer-pass2-fetches.kmk 3121 2017-10-31 10:58:59Z bird $
2## @file
3# kBuild - Footer - Target lists - Pass 2 - Fetches.
4#
5
6#
7# Copyright (c) 2004-2017 knut st. osmundsen <bird-kBuild-spam-xviiv@anduin.net>
8#
9# This file is part of kBuild.
10#
11# kBuild is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version source of the License, or
14# (at your option) any later version.
15#
16# kBuild is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with kBuild; if not, write to the Free Software
23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24#
25#
26# As a special exception you are granted permission to include this file, via
27# the kmk include directive, as you wish without this in itself causing the
28# resulting makefile, program or whatever to be covered by the GPL license.
29# This exception does not however invalidate any other reasons why the makefile,
30# program, whatever should not be covered the GPL.
31#
32#
33
34
35## @page pg_fetches Fetching Tools, Sources and Similar.
36#
37# The targets listed in the the FETCHES target list have the following attributes:
38# SOURCES
39# INST
40# FETCHTOOL
41# FETCHFLAGS
42# FETCHDIR
43# UNPACKTOOL
44# UNPACKFLAGS
45#
46# As usual the target name is an alias for 'creating' the target. Other
47# aliases are:
48# pass_fetches
49# fetch
50# unfetch
51# download
52# unpack
53#
54# @remark
55#
56# This is a little bit complex because we must guarantee that if a source file
57# changes only sligtly we must refetch it and to a proper unpacking of it. It
58# is also a desire that fetched archives and unpacked files can be deleted to
59# save space.
60#
61# Thus, we must be able to cleanup what we've unpacked should any of the
62# sources be removed. We do this by maintaining a file listing the files
63# and directories that was unpacked. This operation is named 'unfetch'.
64#
65# We make use of the SIZE and MD5 attributes for each of the sources to
66# create a digest that is stored in the primary target file. Subsequent
67# runswill compare their digest with it to decide if a refetch is required.
68# When a refetch is found necessary, an 'unfetch' is performed first to
69# clean out old files and directores. Note even changes in source order
70# will cause a refetch due to the way the digest is constructed and
71# evaluated.
72#
73# By not depending directly on the archives (nor on any unpacked files)
74# but on a goal made up from the archive name, size and md5, we allow
75# the user to delete the archives. Naturally, this means we'll have to
76# check and fetch missing archives before attempting to unpack them.
77#
78# @remark
79#
80# This feature will *NOT* work correctly with vanilla GNU make becuase
81# it makes use of includedep to avoid too many unnecessary files.
82#
83# @todo
84# 0. Move the fetches out into a unit.
85# 1. Download corruption / continuation.
86# 2. It's quite possible that there is one too many indirect dependency now...
87#
88
89## generates the fetch rule
90define def_fetch_src_fetch_rule
91# Indirect goal for downloading something.
92.PRECIOUS: $(out)
93$(out) + $($(target)_$(srcname)_FETCH_2_OUTPUT) +| $($(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE) : \
94 | $($(target)_$(srcname)_FETCH_2_DEPORD)
95 %$$(call MSG_FETCH_DL,$(target),$(source),$(out))
96 @## @todo do fancy stuff like download continuation.
97 $$(QUIET)$$(RM) -f -- $(out)
98 $(cmds)
99 $$(QUIET)$(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out))
100
101
102
103# Intermediate goal for making sure the md5 and size matches. it will (re) fetch the archive if necessary.
104$(out).checked_$(md5)_$(size): $($(target)_$(srcname)_FETCH_2_DEPEND) | $($(target)_$(srcname)_FETCH_2_DEPORD)
105 %$$(call MSG_FETCH_CHK,$(target),$(source),$(out))
106 $$(QUIET)$$(RM) -f -- $$@
107 @# (re)fetch the file if it doesn't exist or if it doesn't matches the md5.
108 @## @todo do fancy stuff like download continuation.
109 $$(QUIET)( test -f $(out) && $(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out), true) ) \
110 || ( $$(RM_EXT) -f $(out) \
111 && $$(MAKE) $(out) -f $(MAKEFILE) --no-print-directory )
112 $$(QUIET2)$$(APPEND) $$@
113
114_TARGET_$(target)_FETCHED += $(out) $(out).checked_$(md5)_$(size)
115
116# Just a little precaution.
117.NOTPARALLEL: $(out) $(out).checked_$(md5)_$(size)
118
119endef # def_fetch_src_fetch_rule
120$(eval-opt-var def_fetch_src_fetch_rule)
121
122## generates the unpack rule
123define def_fetch_src_unpack_rule
124# This is the unpack rule. it has an order-only dependency on the download check.
125$(out) + $($(target)_$(srcname)_UNPACK_2_OUTPUT) +| $($(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE) : \
126 $($(target)_$(srcname)_UNPACK_2_DEPEND) \
127 | $($(target)_$(srcname)_UNPACK_2_DEPORD) \
128 $(archive).checked_$(md5)_$(size) \
129 $(dir $(out))
130 %$$(call MSG_FETCH_UP,$(target),$(archive),$(inst))
131 $$(QUIET)$$(RM) -f -- $(out)
132 $$(QUIET)$$(MKDIR) -p -- $(dir $(out))
133 @# if the source archive doesn't exist fetch it (may have been deleted to save space).
134 $$(QUIET)test -f $(archive) \
135 || ( $$(RM_EXT) -f $(archive).checked_$(md5)_$(size) \
136 && $$(MAKE) $(archive).checked_$(md5)_$(size) -f $(MAKEFILE) --no-print-directory )
137 $(cmds)
138 $$(QUIET2)$$(APPEND) $(out) $(notdir $(archive).checked_$(md5)_$(size))
139 $$(QUIET2)$$(APPEND) $(out) $(notdir $(out))
140
141$(eval _TARGET_$(target)_UNPACKED += $(out))
142_TARGET_$(target)_DIGEST := $(_TARGET_$(target)_DIGEST)-$(srcname)_$(md5)_$(size)
143
144.NOTPARALLEL: $(out)
145
146endef # def_fetch_src_unpack_rule
147$(eval-opt-var def_fetch_src_unpack_rule)
148
149## Processes a fetch source
150#
151define def_fetch_src
152#$ (warning dbg: def_fetch_src: source='$(source)' target='$(target)')
153
154# common
155local srcname := $(notdir $(source))
156local inst := $(firstword \
157 $($(target)_$(source)_INST)\
158 $($(target)_$(srcname)_INST)\
159 $($(source)_INST)\
160 $($(srcname)_INST)\
161 $($(target)_INST)\
162)
163ifneq ($(patsubst %/,ok,$(inst)),ok)
164$(error kBuild: Bad or missing INST property for source '$(source)' in target '$(target)': $(inst))
165endif
166## @todo Install-revamp: FIXME
167INSTARGET_$(target)_$(srcname) := $(inst)
168local fetchdir := $(firstword \
169 $($(target)_$(source)_FETCHDIR)\
170 $($(target)_$(srcname)_FETCHDIR)\
171 $($(source)_FETCHDIR)\
172 $($(srcname)_FETCHDIR)\
173 $($(target)_FETCHDIR)\
174 $(FETCHDIR)\
175 $(PATH_TARGET)\
176)
177local deps := \
178 $($(target)_$(source)_DEPS)\
179 $($(target)_$(srcname)_DEPS)\
180 $($(source)_DEPS)\
181 $($(srcname)_DEPS)\
182 $($(target)_DEPS)
183local orderdeps := \
184 $($(target)_$(source)_ORDERDEPS)\
185 $($(target)_$(srcname)_ORDERDEPS)\
186 $($(source)_ORDERDEPS)\
187 $($(srcname)_ORDERDEPS)\
188 $($(target)_ORDERDEPS)
189local md5 := $(firstword \
190 $($(target)_$(source)_MD5)\
191 $($(target)_$(srcname)_MD5)\
192 $($(source)_MD5)\
193 $($(srcname)_MD5)\
194 $($(target)_MD5)\
195)
196local size := $(firstword \
197 $($(target)_$(source)_SIZE)\
198 $($(target)_$(srcname)_SIZE)\
199 $($(source)_SIZE)\
200 $($(srcname)_SIZE)\
201 $($(target)_SIZE)\
202)
203clean_files += \
204 $($(target)_$(source)_CLEAN)\
205 $($(target)_$(srcname)_CLEAN)\
206 $($(source)_CLEAN)\
207 $($(srcname)_CLEAN)
208local dep := # not legal for fetch and unpack tools
209
210
211#
212# The fetching.
213#
214local out := $(fetchdir)/$(srcname)
215local archive := $(out)
216$(target)_$(srcname)_1_TARGET = $(TARGET_$(target)_$(srcname))
217$(call KB_FN_ASSIGN_DEPRECATED,TARGET_$(target)_$(srcname),$(TARGET_$(target)_$(srcname)),TARGET_$(target)_$(srcname))
218
219local dirdep := $(call DIRDEP,$(fetchdir))
220local tool := $(firstword \
221 $($(target)_$(source)_FETCHTOOL)\
222 $($(target)_$(srcname)_FETCHTOOL)\
223 $($(target)_$(source)_TOOL)\
224 $($(target)_$(srcname)_TOOL)\
225 $($(source)_FETCHTOOL)\
226 $($(srcname)_FETCHTOOL)\
227 $($(source)_TOOL)\
228 $($(srcname)_TOOL)\
229 $($(target)_FETCHTOOL)\
230 $($(target)_TOOL)\
231 )
232local flags :=\
233 $(TOOL_$(tool)_FETCHFLAGS)\
234 $(FETCHFLAGS)\
235 $($(target)_FETCHFLAGS)\
236 $($(srcname)_FETCHFLAGS)\
237 $($(source)_FETCHFLAGS)\
238 $($(target)_$(srcname)_FETCHFLAGS)\
239 $($(target)_$(source)_FETCHFLAGS)
240
241#$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) fetchdir=$(fetchdir) md5=$(md5) size=$(size))
242
243ifndef TOOL_$(tool)_FETCH_CMDS
244$(warning kBuild: tools: \
245 1 $($(target)_$(source)_FETCHTOOL)\
246 2 $($(target)_$(srcname)_FETCHTOOL)\
247 3 $($(target)_$(source)_TOOL)\
248 4 $($(target)_$(srcname)_TOOL)\
249 5 $($(source)_FETCHTOOL)\
250 6 $($(srcname)_FETCHTOOL)\
251 7 $($(source)_TOOL)\
252 8 $($(srcname)_TOOL)\
253 9 $($(target)_FETCHTOOL)\
254 10 $($(target)_TOOL) )
255$(error kBuild: TOOL_$(tool)_FETCH_CMDS is not defined. source=$(source) target=$(target) )
256endif
257
258# call the tool
259local cmds := $(TOOL_$(tool)_FETCH_CMDS)
260$(target)_$(srcname)_FETCH_2_OUTPUT := $(TOOL_$(tool)_FETCH_OUTPUT)
261$(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE := $(TOOL_$(tool)_FETCH_OUTPUT_MAYBE)
262$(target)_$(srcname)_FETCH_2_DEPEND := $(TOOL_$(tool)_FETCH_DEPEND) $(deps)
263$(target)_$(srcname)_FETCH_2_DEPORD := $(TOOL_$(tool)_FETCH_DEPORD) $(dirdep) $(orderdeps)
264
265# generate the fetch rule.
266$(eval $(def_fetch_src_fetch_rule))
267
268
269#
270# The unpacking / installing.
271#
272local out := $(inst)_kBuild_$(target)_$(srcname)_unpacked.lst
273local dirdep := $(call DIRDEP,$(inst))
274local tool := $(firstword \
275 $($(target)_$(source)_UNPACKTOOL)\
276 $($(target)_$(srcname)_UNPACKTOOL)\
277 $($(target)_$(source)_TOOL)\
278 $($(target)_$(srcname)_TOOL)\
279 $($(source)_UNPACKTOOL)\
280 $($(srcname)_UNPACKTOOL)\
281 $($(source)_TOOL)\
282 $($(srcname)_TOOL)\
283 $($(target)_UNPACKTOOL)\
284 $($(target)_TOOL) \
285 )
286ifeq ($(tool),)
287local tool := $(toupper $(subst .,,$(suffix $(subst tar.,TAR,$(srcname)))))
288$(evalval def_tools_include)
289endif
290local flags :=\
291 $(TOOL_$(tool)_UNPACKFLAGS)\
292 $(UNPACKFLAGS)\
293 $($(target)_UNPACKFLAGS)\
294 $($(srcname)_UNPACKFLAGS)\
295 $($(source)_UNPACKFLAGS)\
296 $($(target)_$(srcname)_UNPACKFLAGS)\
297 $($(target)_$(source)_UNPACKFLAGS)
298
299#$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) inst=$(inst) md5=$(md5) size=$(size))
300ifndef TOOL_$(tool)_UNPACK_CMDS
301$(warning kBuild: tools: \
302 1 $($(target)_$(source)_UNPACKTOOL)\
303 2 $($(target)_$(srcname)_UNPACKTOOL)\
304 3 $($(target)_$(source)_TOOL)\
305 4 $($(target)_$(srcname)_TOOL)\
306 5 $($(source)_UNPACKTOOL)\
307 6 $($(srcname)_UNPACKTOOL)\
308 7 $($(source)_TOOL)\
309 8 $($(srcname)_TOOL)\
310 9 $($(target)_UNPACKTOOL)\
311 10 $($(target)_TOOL) \
312 11 $(toupper $(subst tar.,TAR,$(ext $(srcname)))) \
313 )
314$(error kBuild: TOOL_$(tool)_UNPACK_CMDS is not defined. source=$(source) target=$(target) )
315endif
316
317# call the tool
318local cmds := $(TOOL_$(tool)_UNPACK_CMDS)
319$(target)_$(srcname)_UNPACK_2_OUTPUT := $(TOOL_$(tool)_UNPACK_OUTPUT)
320$(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE := $(TOOL_$(tool)_UNPACK_OUTPUT_MAYBE)
321$(target)_$(srcname)_UNPACK_2_DEPEND := $(TOOL_$(tool)_UNPACK_DEPEND) $(deps)
322$(target)_$(srcname)_UNPACK_2_DEPORD := $(TOOL_$(tool)_UNPACK_DEPORD) $(dirdep) $(orderdeps)
323
324# generate the fetch rule.
325$(eval $(def_fetch_src_unpack_rule))
326
327_DIRS += $(inst) $(fetchdir)
328
329endef # def_fetch_src
330$(eval-opt-var def_fetch_src)
331
332
333##
334# Define the target level rules for a fetch.
335# @param target
336# @param out
337# @param inst
338# @param _TARGET_$(target)_UNPACKED
339# @param _TARGET_$(target)_DIGEST
340# @param bld_trg
341# @param bld_trg_arch
342define def_fetch_rules
343
344$(out).lst: $(_TARGET_$(target)_UNPACKED) | $(call DIRDEP,$(inst))
345 %$$(call MSG_FETCH_OK,$(target))
346 $$(QUIET)$$(RM) -f -- $$@ $$@.tmp
347 $$(QUIET2)$$(APPEND) $$@.tmp '$(notdir $(out))'
348 $$(QUIET)$(if $(_TARGET_$(target)_UNPACKED),$$(CAT_EXT) $(_TARGET_$(target)_UNPACKED) >> $$@.tmp)
349 $$(QUIET)$$(MV) -f -- $$@.tmp $$@
350
351$(out)_unfetched:
352 %$$(call MSG_UNFETCH,$(target))
353 $$(QUIET)$$(RM) -f -- $$(addprefix $(inst),$$(shell $$(CAT_EXT) $(out).lst 2> /dev/null | $$(SED) -e '/\/$$$$/d'))
354 $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@) \
355 $$(addprefix $(inst),$$(sort $$(dir $$(shell $$(CAT_EXT) $(out).lst 2> /dev/null))))
356 $$(QUIET)$$(RM) -f -- $(out).lst $(out)
357 $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@)
358
359$(out): $(comp-vars _TARGET_$(target)_DIGEST_PREV,_TARGET_$(target)_DIGEST,FORCE) | $(call DIRDEP,$(inst))
360 $$(QUIET)$$(RM) -f -- $$@
361 %$$(if $$(_TARGET_$(target)_DIGEST),$$(if $$(eq $$(file-size $(out).lst),-1)\
362 ,$$(call MSG_REFETCH,$(target)),$$(call MSG_FETCH,$(target))),$$(call MSG_UNFETCH,$(target)))
363 $$(QUIET)$(TEST_EXT) -f $(out).lst -- $$(MAKE) -f $(MAKEFILE) --no-print-directory $(out)_unfetched
364if $(KBUILD_KMK_REVISION) > 2911
365 $$(QUIET)kmk_builtin_dircache deleted "$(dir $(out))"
366endif
367 $$(QUIET)$$(if $$(_TARGET_$(target)_DIGEST),$$(MAKE) -f $(MAKEFILE) --no-print-directory $(out).lst,$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@))
368 $$(QUIET2)$$(if $$(_TARGET_$(target)_DIGEST),$$(APPEND) $$@ "_TARGET_$(target)_DIGEST_PREV := $(_TARGET_$(target)_DIGEST)")
369
370.NOTPARALLEL: $(out).lst $(out)_unfetched $(out)
371
372endef
373
374
375##
376# Deal with one fetch target.
377# @param target
378# @param bld_trg
379# @param bld_trg_arch
380define def_fetch
381# common
382## @todo Install-revamp: FIXME
383INSTARGET_$(target) := $($(target)_INST)
384ifneq ($(patsubst %/,ok,$(INSTARGET_$(target))),ok)
385$(error kBuild: Bad or missing INST property for target '$(target)'. \
386 $(target)_INST='$($(target)_INST)' ($(origin $(target)_INST)))
387endif
388_TARGET_$(target)_FETCHED :=
389_TARGET_$(target)_UNPACKED :=
390_TARGET_$(target)_DIGEST :=
391local clean_files := $($(target)_CLEAN) $($(target)_CLEAN.$(bld_trg)) $($(target)_CLEAN.$(bld_trg).$(bld_trg_arch)) $($(target)_CLEAN.$(bld_trg_arch)) $($(target)_CLEAN.$(KBUILD_TYPE))
392
393# The 'sources'.
394#$ (warning dbg fetch: target=$(target) sources=$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(KBUILD_TARGET)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(KBUILD_TARGET).$(bld_trg_arch)))
395$(foreach source,$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(bld_trg)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(bld_trg).$(bld_trg_arch)),\
396 $(evalval def_fetch_src))
397
398# The target.
399local inst := $(INSTARGET_$(target))
400local out := $(inst)_kBuild_fetch_$(target)
401
402$(eval includedep $(out))
403
404$(eval $(def_fetch_rules))
405
406# Define the aliases here (doesn't work if defined in def_fetch_rules, just like includedep).
407$(target): $(out)
408$(target)_unfetch: $(out)_unfetched
409
410_FETCHES += $(out)
411_DOWNLOADS += $(_TARGET_$(target)_FETCHED)
412_UNPACKS += $(_TARGET_$(target)_UNPACKED)
413_UNFETCHES += $(out)_unfetched
414_DIRS += $(inst)
415_CLEAN_FILES += $(clean_files)
416
417endef
418$(eval-opt-var def_fetch)
419
420# Walk the FETCH target lists.
421bld_trg := $(KBUILD_TARGET)
422bld_trg_arch := $(KBUILD_TARGET_ARCH)
423$(foreach target, $(_ALL_FETCHES), \
424 $(evalvalctx def_fetch))
425
426# some aliases.
427download: $(_DOWNLOADS)
428unpack: $(_UNPACKS)
429fetch: $(_FETCHES)
430unfetch: $(_UNFETCHES)
431
432ifdef KBUILD_PROFILE_SELF
433 $(evalcall def_profile_self, done fetching targets)
434endif
Note: See TracBrowser for help on using the repository browser.