source: trunk/kBuild/footer-pass2-fetches.kmk@ 2690

Last change on this file since 2690 was 2521, checked in by bird, 14 years ago

footer.kmk: Split it up into several files.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 14.6 KB
RevLine 
[72]1# $Id: footer-pass2-fetches.kmk 2521 2011-07-26 18:18:19Z bird $
2## @file
[2521]3# kBuild - Footer - Target lists - Pass 2 - Fetches.
[72]4#
[1547]5
6#
[2521]7# Copyright (c) 2004-2011 knut st. osmundsen <bird-kBuild-spamx@anduin.net>
[72]8#
9# This file is part of kBuild.
10#
11# kBuild is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
[106]13# the Free Software Foundation; either version source of the License, or
[72]14# (at your option) any later version.
15#
16# kBuild is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with kBuild; if not, write to the Free Software
[353]23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
[72]24#
25#
[1547]26# As a special exception you are granted permission to include this file, via
[1599]27# the kmk include directive, as you wish without this in itself causing the
28# resulting makefile, program or whatever to be covered by the GPL license.
[1547]29# This exception does not however invalidate any other reasons why the makefile,
30# program, whatever should not be covered the GPL.
31#
32#
[72]33
34
[688]35## @page pg_fetches Fetching Tools, Sources and Similar.
[73]36#
[688]37# The targets listed in the the FETCHES target list have the following attributes:
[2265]38# SOURCES
39# INST
40# FETCHTOOL
41# FETCHFLAGS
42# FETCHDIR
43# UNPACKTOOL
44# UNPACKFLAGS
[640]45#
[719]46# As usual the target name is an alias for 'creating' the target. Other
[688]47# aliases are:
[2265]48# pass_fetches
49# fetch
50# unfetch
51# download
52# unpack
[688]53#
54# @remark
55#
56# This is a little bit complex because we must guarantee that if a source file
[719]57# changes only sligtly we must refetch it and to a proper unpacking of it. It
58# is also a desire that fetched archives and unpacked files can be deleted to
[688]59# save space.
60#
[719]61# Thus, we must be able to cleanup what we've unpacked should any of the
[688]62# sources be removed. We do this by maintaining a file listing the files
63# and directories that was unpacked. This operation is named 'unfetch'.
64#
[719]65# We make use of the SIZE and MD5 attributes for each of the sources to
[689]66# create a digest that is stored in the primary target file. Subsequent
67# runswill compare their digest with it to decide if a refetch is required.
[719]68# When a refetch is found necessary, an 'unfetch' is performed first to
[689]69# clean out old files and directores. Note even changes in source order
[719]70# will cause a refetch due to the way the digest is constructed and
[689]71# evaluated.
[688]72#
73# By not depending directly on the archives (nor on any unpacked files)
74# but on a goal made up from the archive name, size and md5, we allow
75# the user to delete the archives. Naturally, this means we'll have to
76# check and fetch missing archives before attempting to unpack them.
[689]77#
78# @remark
79#
[719]80# This feature will *NOT* work correctly with vanilla GNU make becuase
[689]81# it makes use of includedep to avoid too many unnecessary files.
[719]82#
[689]83# @todo
[2265]84# 0. Move the fetches out into a unit.
[1255]85# 1. Download corruption / continuation.
86# 2. It's quite possible that there is one too many indirect dependency now...
[688]87#
88
[640]89## generates the fetch rule
90define def_fetch_src_fetch_rule
91# Indirect goal for downloading something.
[682]92.PRECIOUS: $(out)
[2234]93$(out) + $($(target)_$(srcname)_FETCH_2_OUTPUT) +| $($(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE) : \
94 | $($(target)_$(srcname)_FETCH_2_DEPORD)
[1496]95 %$$(call MSG_FETCH_DL,$(target),$(source),$(out))
[696]96 @## @todo do fancy stuff like download continuation.
[1599]97 $$(QUIET)$$(RM) -f -- $(out)
[2234]98 $(cmds)
[1255]99 $$(QUIET)$(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out))
[640]100
101
[682]102
[640]103# Intermediate goal for making sure the md5 and size matches. it will (re) fetch the archive if necessary.
[2234]104$(out).checked_$(md5)_$(size): $($(target)_$(srcname)_FETCH_2_DEPEND) | $($(target)_$(srcname)_FETCH_2_DEPORD)
[1496]105 %$$(call MSG_FETCH_CHK,$(target),$(source),$(out))
[1599]106 $$(QUIET)$$(RM) -f -- $$@
[688]107 @# (re)fetch the file if it doesn't exist or if it doesn't matches the md5.
[696]108 @## @todo do fancy stuff like download continuation.
[1255]109 $$(QUIET)( test -f $(out) && $(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out), true) ) \
[696]110 || ( $$(RM_EXT) -f $(out) \
111 && $$(MAKE) $(out) -f $(MAKEFILE) --no-print-directory )
112 $$(QUIET2)$$(APPEND) $$@
[640]113
[719]114_TARGET_$(target)_FETCHED += $(out) $(out).checked_$(md5)_$(size)
[640]115
[688]116# Just a little precaution.
117.NOTPARALLEL: $(out) $(out).checked_$(md5)_$(size)
118
[1936]119endef # def_fetch_src_fetch_rule
120$(eval-opt-var def_fetch_src_fetch_rule)
[640]121
122## generates the unpack rule
123define def_fetch_src_unpack_rule
124# This is the unpack rule. it has an order-only dependency on the download check.
[2234]125$(out) + $($(target)_$(srcname)_UNPACK_2_OUTPUT) +| $($(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE) : \
126 $($(target)_$(srcname)_UNPACK_2_DEPEND) \
127 | $($(target)_$(srcname)_UNPACK_2_DEPORD) \
[2094]128 $(archive).checked_$(md5)_$(size) \
129 $(dir $(out))
[1496]130 %$$(call MSG_FETCH_UP,$(target),$(archive),$(inst))
[1599]131 $$(QUIET)$$(RM) -f -- $(out)
[2094]132 $$(QUIET)$$(MKDIR) -p -- $(dir $(out))
133 @# if the source archive doesn't exist fetch it (may have been deleted to save space).
[696]134 $$(QUIET)test -f $(archive) \
135 || ( $$(RM_EXT) -f $(archive).checked_$(md5)_$(size) \
136 && $$(MAKE) $(archive).checked_$(md5)_$(size) -f $(MAKEFILE) --no-print-directory )
[2234]137 $(cmds)
[696]138 $$(QUIET2)$$(APPEND) $(out) $(notdir $(archive).checked_$(md5)_$(size))
139 $$(QUIET2)$$(APPEND) $(out) $(notdir $(out))
[640]140
141$(eval _TARGET_$(target)_UNPACKED += $(out))
142_TARGET_$(target)_DIGEST := $(_TARGET_$(target)_DIGEST)-$(srcname)_$(md5)_$(size)
143
[688]144.NOTPARALLEL: $(out)
145
[1936]146endef # def_fetch_src_unpack_rule
147$(eval-opt-var def_fetch_src_unpack_rule)
[640]148
149## Processes a fetch source
150#
151define def_fetch_src
152#$ (warning dbg: def_fetch_src: source='$(source)' target='$(target)')
153
154# common
[1496]155local srcname := $(notdir $(source))
156local inst := $(firstword \
[640]157 $($(target)_$(source)_INST)\
158 $($(target)_$(srcname)_INST)\
159 $($(source)_INST)\
160 $($(srcname)_INST)\
161 $($(target)_INST)\
162)
163ifneq ($(patsubst %/,ok,$(inst)),ok)
164$(error kBuild: Bad or missing INST property for source '$(source)' in target '$(target)': $(inst))
165endif
[2475]166## @todo Install-revamp: FIXME
[640]167INSTARGET_$(target)_$(srcname) := $(inst)
[1496]168local fetchdir := $(firstword \
[640]169 $($(target)_$(source)_FETCHDIR)\
170 $($(target)_$(srcname)_FETCHDIR)\
171 $($(source)_FETCHDIR)\
172 $($(srcname)_FETCHDIR)\
173 $($(target)_FETCHDIR)\
174 $(FETCHDIR)\
175 $(PATH_TARGET)\
176)
[1496]177local deps := \
[640]178 $($(target)_$(source)_DEPS)\
179 $($(target)_$(srcname)_DEPS)\
180 $($(source)_DEPS)\
181 $($(srcname)_DEPS)\
182 $($(target)_DEPS)
[1496]183local orderdeps := \
[827]184 $($(target)_$(source)_ORDERDEPS)\
185 $($(target)_$(srcname)_ORDERDEPS)\
186 $($(source)_ORDERDEPS)\
187 $($(srcname)_ORDERDEPS)\
188 $($(target)_ORDERDEPS)
[1496]189local md5 := $(firstword \
[640]190 $($(target)_$(source)_MD5)\
191 $($(target)_$(srcname)_MD5)\
192 $($(source)_MD5)\
193 $($(srcname)_MD5)\
194 $($(target)_MD5)\
195)
[1496]196local size := $(firstword \
[640]197 $($(target)_$(source)_SIZE)\
198 $($(target)_$(srcname)_SIZE)\
199 $($(source)_SIZE)\
200 $($(srcname)_SIZE)\
201 $($(target)_SIZE)\
202)
[1613]203clean_files += \
204 $($(target)_$(source)_CLEAN)\
205 $($(target)_$(srcname)_CLEAN)\
206 $($(source)_CLEAN)\
207 $($(srcname)_CLEAN)
[1496]208local dep := # not legal for fetch and unpack tools
[640]209
210
211#
212# The fetching.
213#
[1496]214local out := $(fetchdir)/$(srcname)
215local archive := $(out)
[2238]216$(target)_$(srcname)_1_TARGET = $(TARGET_$(target)_$(srcname))
[2436]217$(call KB_FN_ASSIGN_DEPRECATED,TARGET_$(target)_$(srcname),$(TARGET_$(target)_$(srcname)),TARGET_$(target)_$(srcname))
218
[1496]219local dirdep := $(call DIRDEP,$(fetchdir))
220local tool := $(firstword \
[640]221 $($(target)_$(source)_FETCHTOOL)\
222 $($(target)_$(srcname)_FETCHTOOL)\
223 $($(target)_$(source)_TOOL)\
224 $($(target)_$(srcname)_TOOL)\
225 $($(source)_FETCHTOOL)\
226 $($(srcname)_FETCHTOOL)\
227 $($(source)_TOOL)\
228 $($(srcname)_TOOL)\
229 $($(target)_FETCHTOOL)\
230 $($(target)_TOOL)\
231 )
[1496]232local flags :=\
[640]233 $(TOOL_$(tool)_FETCHFLAGS)\
234 $(FETCHFLAGS)\
235 $($(target)_FETCHFLAGS)\
236 $($(srcname)_FETCHFLAGS)\
237 $($(source)_FETCHFLAGS)\
238 $($(target)_$(srcname)_FETCHFLAGS)\
239 $($(target)_$(source)_FETCHFLAGS)
240
241#$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) fetchdir=$(fetchdir) md5=$(md5) size=$(size))
242
243ifndef TOOL_$(tool)_FETCH_CMDS
244$(warning kBuild: tools: \
245 1 $($(target)_$(source)_FETCHTOOL)\
246 2 $($(target)_$(srcname)_FETCHTOOL)\
247 3 $($(target)_$(source)_TOOL)\
248 4 $($(target)_$(srcname)_TOOL)\
249 5 $($(source)_FETCHTOOL)\
250 6 $($(srcname)_FETCHTOOL)\
251 7 $($(source)_TOOL)\
252 8 $($(srcname)_TOOL)\
253 9 $($(target)_FETCHTOOL)\
254 10 $($(target)_TOOL) )
255$(error kBuild: TOOL_$(tool)_FETCH_CMDS is not defined. source=$(source) target=$(target) )
256endif
257
258# call the tool
[2234]259local cmds := $(TOOL_$(tool)_FETCH_CMDS)
260$(target)_$(srcname)_FETCH_2_OUTPUT := $(TOOL_$(tool)_FETCH_OUTPUT)
261$(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE := $(TOOL_$(tool)_FETCH_OUTPUT_MAYBE)
262$(target)_$(srcname)_FETCH_2_DEPEND := $(TOOL_$(tool)_FETCH_DEPEND) $(deps)
263$(target)_$(srcname)_FETCH_2_DEPORD := $(TOOL_$(tool)_FETCH_DEPORD) $(dirdep) $(orderdeps)
[640]264
265# generate the fetch rule.
266$(eval $(def_fetch_src_fetch_rule))
267
268
269#
270# The unpacking / installing.
271#
[1496]272local out := $(inst)_kBuild_$(target)_$(srcname)_unpacked.lst
273local dirdep := $(call DIRDEP,$(inst))
274local tool := $(firstword \
[640]275 $($(target)_$(source)_UNPACKTOOL)\
276 $($(target)_$(srcname)_UNPACKTOOL)\
277 $($(target)_$(source)_TOOL)\
278 $($(target)_$(srcname)_TOOL)\
279 $($(source)_UNPACKTOOL)\
280 $($(srcname)_UNPACKTOOL)\
281 $($(source)_TOOL)\
282 $($(srcname)_TOOL)\
283 $($(target)_UNPACKTOOL)\
284 $($(target)_TOOL) \
285 )
286ifeq ($(tool),)
[1496]287local tool := $(toupper $(subst .,,$(suffix $(subst tar.,TAR,$(srcname)))))
288$(evalval def_tools_include)
[640]289endif
[1496]290local flags :=\
[640]291 $(TOOL_$(tool)_UNPACKFLAGS)\
292 $(UNPACKFLAGS)\
293 $($(target)_UNPACKFLAGS)\
294 $($(srcname)_UNPACKFLAGS)\
295 $($(source)_UNPACKFLAGS)\
296 $($(target)_$(srcname)_UNPACKFLAGS)\
297 $($(target)_$(source)_UNPACKFLAGS)
298
299#$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) inst=$(inst) md5=$(md5) size=$(size))
300ifndef TOOL_$(tool)_UNPACK_CMDS
301$(warning kBuild: tools: \
302 1 $($(target)_$(source)_UNPACKTOOL)\
303 2 $($(target)_$(srcname)_UNPACKTOOL)\
304 3 $($(target)_$(source)_TOOL)\
305 4 $($(target)_$(srcname)_TOOL)\
306 5 $($(source)_UNPACKTOOL)\
307 6 $($(srcname)_UNPACKTOOL)\
308 7 $($(source)_TOOL)\
309 8 $($(srcname)_TOOL)\
310 9 $($(target)_UNPACKTOOL)\
311 10 $($(target)_TOOL) \
312 11 $(toupper $(subst tar.,TAR,$(ext $(srcname)))) \
313 )
314$(error kBuild: TOOL_$(tool)_UNPACK_CMDS is not defined. source=$(source) target=$(target) )
315endif
316
317# call the tool
[2234]318local cmds := $(TOOL_$(tool)_UNPACK_CMDS)
319$(target)_$(srcname)_UNPACK_2_OUTPUT := $(TOOL_$(tool)_UNPACK_OUTPUT)
320$(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE := $(TOOL_$(tool)_UNPACK_OUTPUT_MAYBE)
321$(target)_$(srcname)_UNPACK_2_DEPEND := $(TOOL_$(tool)_UNPACK_DEPEND) $(deps)
322$(target)_$(srcname)_UNPACK_2_DEPORD := $(TOOL_$(tool)_UNPACK_DEPORD) $(dirdep) $(orderdeps)
[640]323
324# generate the fetch rule.
325$(eval $(def_fetch_src_unpack_rule))
326
327_DIRS += $(inst) $(fetchdir)
328
[1936]329endef # def_fetch_src
330$(eval-opt-var def_fetch_src)
[640]331
332
333##
334# Define the target level rules for a fetch.
335# @param target
336# @param out
337# @param inst
[688]338# @param _TARGET_$(target)_UNPACKED
339# @param _TARGET_$(target)_DIGEST
[640]340# @param bld_trg
341# @param bld_trg_arch
342define def_fetch_rules
343
[689]344$(out).lst: $(_TARGET_$(target)_UNPACKED) | $(call DIRDEP,$(inst))
[1496]345 %$$(call MSG_FETCH_OK,$(target))
[1599]346 $$(QUIET)$$(RM) -f -- $$@ $$@.tmp
[1311]347 $$(QUIET2)$$(APPEND) $$@.tmp '$(notdir $(out))'
[696]348 $$(QUIET)$(if $(_TARGET_$(target)_UNPACKED),$$(CAT_EXT) $(_TARGET_$(target)_UNPACKED) >> $$@.tmp)
[1599]349 $$(QUIET)$$(MV) -f -- $$@.tmp $$@
[640]350
351$(out)_unfetched:
[1496]352 %$$(call MSG_UNFETCH,$(target))
[1599]353 $$(QUIET)$$(RM) -f -- $$(addprefix $(inst),$$(shell $$(CAT_EXT) $(out).lst 2> /dev/null | $$(SED) -e '/\/$$$$/d'))
354 $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@) \
[689]355 $$(addprefix $(inst),$$(sort $$(dir $$(shell $$(CAT_EXT) $(out).lst 2> /dev/null))))
[1599]356 $$(QUIET)$$(RM) -f -- $(out).lst $(out)
357 $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@)
[640]358
[689]359$(out): $(comp-vars _TARGET_$(target)_DIGEST_PREV,_TARGET_$(target)_DIGEST,FORCE) | $(call DIRDEP,$(inst))
[1599]360 $$(QUIET)$$(RM) -f -- $$@
[2095]361 %$$(if $$(_TARGET_$(target)_DIGEST),$$(if $$(eq $$(file-size $(out).lst),-1)\
[776]362 ,$$(call MSG_REFETCH,$(target)),$$(call MSG_FETCH,$(target))),$$(call MSG_UNFETCH,$(target)))
[2094]363 $$(QUIET)$(TEST_EXT) -f $(out).lst -- $$(MAKE) -f $(MAKEFILE) --no-print-directory $(out)_unfetched
[1599]364 $$(QUIET)$$(if $$(_TARGET_$(target)_DIGEST),$$(MAKE) -f $(MAKEFILE) --no-print-directory $(out).lst,$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@))
[696]365 $$(QUIET2)$$(if $$(_TARGET_$(target)_DIGEST),$$(APPEND) $$@ "_TARGET_$(target)_DIGEST_PREV := $(_TARGET_$(target)_DIGEST)")
[760]366
[689]367.NOTPARALLEL: $(out).lst $(out)_unfetched $(out)
[688]368
[640]369endef
370
371
372##
373# Deal with one fetch target.
374# @param target
375# @param bld_trg
376# @param bld_trg_arch
377define def_fetch
378# common
[2475]379## @todo Install-revamp: FIXME
[640]380INSTARGET_$(target) := $($(target)_INST)
381ifneq ($(patsubst %/,ok,$(INSTARGET_$(target))),ok)
[679]382$(error kBuild: Bad or missing INST property for target '$(target)'. \
383 $(target)_INST='$($(target)_INST)' ($(origin $(target)_INST)))
[640]384endif
385_TARGET_$(target)_FETCHED :=
386_TARGET_$(target)_UNPACKED :=
387_TARGET_$(target)_DIGEST :=
[1614]388local clean_files := $($(target)_CLEAN) $($(target)_CLEAN.$(bld_trg)) $($(target)_CLEAN.$(bld_trg).$(bld_trg_arch)) $($(target)_CLEAN.$(bld_trg_arch)) $($(target)_CLEAN.$(KBUILD_TYPE))
[640]389
390# The 'sources'.
[1614]391#$ (warning dbg fetch: target=$(target) sources=$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(KBUILD_TARGET)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(KBUILD_TARGET).$(bld_trg_arch)))
392$(foreach source,$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(bld_trg)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(bld_trg).$(bld_trg_arch)),\
[1496]393 $(evalval def_fetch_src))
[640]394
395# The target.
[1496]396local inst := $(INSTARGET_$(target))
397local out := $(inst)_kBuild_fetch_$(target)
[640]398
[689]399$(eval includedep $(out))
[690]400
[640]401$(eval $(def_fetch_rules))
402
[690]403# Define the aliases here (doesn't work if defined in def_fetch_rules, just like includedep).
[749]404$(target): $(out)
405$(target)_unfetch: $(out)_unfetched
[690]406
[689]407_FETCHES += $(out)
[640]408_DOWNLOADS += $(_TARGET_$(target)_FETCHED)
409_UNPACKS += $(_TARGET_$(target)_UNPACKED)
410_UNFETCHES += $(out)_unfetched
411_DIRS += $(inst)
[1613]412_CLEAN_FILES += $(clean_files)
[640]413
414endef
[1936]415$(eval-opt-var def_fetch)
[640]416
417# Walk the FETCH target lists.
[1504]418bld_trg := $(KBUILD_TARGET)
419bld_trg_arch := $(KBUILD_TARGET_ARCH)
[1648]420$(foreach target, $(_ALL_FETCHES), \
[1613]421 $(evalvalctx def_fetch))
[640]422
423# some aliases.
424download: $(_DOWNLOADS)
[688]425unpack: $(_UNPACKS)
426fetch: $(_FETCHES)
427unfetch: $(_UNFETCHES)
[640]428
[988]429ifdef KBUILD_PROFILE_SELF
[2008]430 $(evalcall def_profile_self, done fetching targets)
[988]431endif
Note: See TracBrowser for help on using the repository browser.