source: trunk/kBuild/footer-pass2-fetches.kmk

Last change on this file was 3608, checked in by bird, 10 months ago

footer-pass2-fetches.kmk,header.kmk: Added a NAME property to the fetch targets to facilitate cleanup (nsis mess); fixed cleanup (unfetch) issues when files or directories contains spaces.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 14.8 KB
Line 
1# $Id: footer-pass2-fetches.kmk 3608 2024-09-09 14:53:00Z bird $
2## @file
3# kBuild - Footer - Target lists - Pass 2 - Fetches.
4#
5
6#
7# Copyright (c) 2004-2017 knut st. osmundsen <bird-kBuild-spam-xviiv@anduin.net>
8#
9# This file is part of kBuild.
10#
11# kBuild is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version source of the License, or
14# (at your option) any later version.
15#
16# kBuild is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with kBuild; if not, write to the Free Software
23# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24#
25#
26# As a special exception you are granted permission to include this file, via
27# the kmk include directive, as you wish without this in itself causing the
28# resulting makefile, program or whatever to be covered by the GPL license.
29# This exception does not however invalidate any other reasons why the makefile,
30# program, whatever should not be covered the GPL.
31#
32#
33
34
35## @page pg_fetches Fetching Tools, Sources and Similar.
36#
37# The targets listed in the the FETCHES target list have the following attributes:
38# NAME
39# SOURCES
40# INST
41# FETCHTOOL
42# FETCHFLAGS
43# FETCHDIR
44# UNPACKTOOL
45# UNPACKFLAGS
46#
47# As usual the target name is an alias for 'creating' the target. Other
48# aliases are:
49# pass_fetches
50# fetch
51# unfetch
52# download
53# unpack
54#
55# @remark
56#
57# This is a little bit complex because we must guarantee that if a source file
58# changes only sligtly we must refetch it and to a proper unpacking of it. It
59# is also a desire that fetched archives and unpacked files can be deleted to
60# save space.
61#
62# Thus, we must be able to cleanup what we've unpacked should any of the
63# sources be removed. We do this by maintaining a file listing the files
64# and directories that was unpacked. This operation is named 'unfetch'.
65#
66# We make use of the SIZE and MD5 attributes for each of the sources to
67# create a digest that is stored in the primary target file. Subsequent
68# runswill compare their digest with it to decide if a refetch is required.
69# When a refetch is found necessary, an 'unfetch' is performed first to
70# clean out old files and directores. Note even changes in source order
71# will cause a refetch due to the way the digest is constructed and
72# evaluated.
73#
74# By not depending directly on the archives (nor on any unpacked files)
75# but on a goal made up from the archive name, size and md5, we allow
76# the user to delete the archives. Naturally, this means we'll have to
77# check and fetch missing archives before attempting to unpack them.
78#
79# @remark
80#
81# This feature will *NOT* work correctly with vanilla GNU make becuase
82# it makes use of includedep to avoid too many unnecessary files.
83#
84# @todo
85# 0. Move the fetches out into a unit.
86# 1. Download corruption / continuation.
87# 2. It's quite possible that there is one too many indirect dependency now...
88#
89
90## generates the fetch rule
91define def_fetch_src_fetch_rule
92# Indirect goal for downloading something.
93.PRECIOUS: $(out)
94$(out) + $($(target)_$(srcname)_FETCH_2_OUTPUT) +| $($(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE) : \
95 | $($(target)_$(srcname)_FETCH_2_DEPORD)
96 %$$(call MSG_FETCH_DL,$(target),$(source),$(out))
97 @## @todo do fancy stuff like download continuation.
98 $$(QUIET)$$(RM) -f -- $(out)
99 $(cmds)
100 $$(QUIET)$(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out))
101
102
103
104# Intermediate goal for making sure the md5 and size matches. it will (re) fetch the archive if necessary.
105$(out).checked_$(md5)_$(size): $($(target)_$(srcname)_FETCH_2_DEPEND) | $($(target)_$(srcname)_FETCH_2_DEPORD)
106 %$$(call MSG_FETCH_CHK,$(target),$(source),$(out))
107 $$(QUIET)$$(RM) -f -- $$@
108 @# (re)fetch the file if it doesn't exist or if it doesn't matches the md5.
109 @## @todo do fancy stuff like download continuation.
110 $$(QUIET)( test -f $(out) && $(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out), true) ) \
111 || ( $$(RM_EXT) -f $(out) \
112 && $$(MAKE) $(out) -f $(MAKEFILE) --no-print-directory )
113 $$(QUIET2)$$(APPEND) $$@
114
115_TARGET_$(target)_FETCHED += $(out) $(out).checked_$(md5)_$(size)
116
117# Just a little precaution.
118.NOTPARALLEL: $(out) $(out).checked_$(md5)_$(size)
119
120endef # def_fetch_src_fetch_rule
121$(eval-opt-var def_fetch_src_fetch_rule)
122
123## generates the unpack rule
124define def_fetch_src_unpack_rule
125# This is the unpack rule. it has an order-only dependency on the download check.
126$(out) + $($(target)_$(srcname)_UNPACK_2_OUTPUT) +| $($(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE) : \
127 $($(target)_$(srcname)_UNPACK_2_DEPEND) \
128 | $($(target)_$(srcname)_UNPACK_2_DEPORD) \
129 $(archive).checked_$(md5)_$(size) \
130 $(dir $(out))
131 %$$(call MSG_FETCH_UP,$(target),$(archive),$(inst))
132 $$(QUIET)$$(RM) -f -- $(out)
133 $$(QUIET)$$(MKDIR) -p -- $(dir $(out))
134 @# if the source archive doesn't exist fetch it (may have been deleted to save space).
135 $$(QUIET)test -f $(archive) \
136 || ( $$(RM_EXT) -f $(archive).checked_$(md5)_$(size) \
137 && $$(MAKE) $(archive).checked_$(md5)_$(size) -f $(MAKEFILE) --no-print-directory )
138 $(cmds)
139 $$(QUIET2)$$(APPEND) $(out) $(notdir $(archive).checked_$(md5)_$(size))
140 $$(QUIET2)$$(APPEND) $(out) $(notdir $(out))
141
142$(eval _TARGET_$(target)_UNPACKED += $(out))
143_TARGET_$(target)_DIGEST := $(_TARGET_$(target)_DIGEST)-$(srcname)_$(md5)_$(size)
144
145.NOTPARALLEL: $(out)
146
147endef # def_fetch_src_unpack_rule
148$(eval-opt-var def_fetch_src_unpack_rule)
149
150## Processes a fetch source
151#
152define def_fetch_src
153#$ (warning dbg: def_fetch_src: source='$(source)' target='$(target)')
154
155# common
156local srcname := $(notdir $(source))
157local inst := $(firstword \
158 $($(target)_$(source)_INST)\
159 $($(target)_$(srcname)_INST)\
160 $($(source)_INST)\
161 $($(srcname)_INST)\
162 $($(target)_INST)\
163)
164ifneq ($(patsubst %/,ok,$(inst)),ok)
165$(error kBuild: Bad or missing INST property for source '$(source)' in target '$(target)': $(inst))
166endif
167## @todo Install-revamp: FIXME
168INSTARGET_$(target)_$(srcname) := $(inst)
169local fetchdir := $(firstword \
170 $($(target)_$(source)_FETCHDIR)\
171 $($(target)_$(srcname)_FETCHDIR)\
172 $($(source)_FETCHDIR)\
173 $($(srcname)_FETCHDIR)\
174 $($(target)_FETCHDIR)\
175 $(FETCHDIR)\
176 $(PATH_TARGET)\
177)
178local deps := \
179 $($(target)_$(source)_DEPS)\
180 $($(target)_$(srcname)_DEPS)\
181 $($(source)_DEPS)\
182 $($(srcname)_DEPS)\
183 $($(target)_DEPS)
184local orderdeps := \
185 $($(target)_$(source)_ORDERDEPS)\
186 $($(target)_$(srcname)_ORDERDEPS)\
187 $($(source)_ORDERDEPS)\
188 $($(srcname)_ORDERDEPS)\
189 $($(target)_ORDERDEPS)
190local md5 := $(firstword \
191 $($(target)_$(source)_MD5)\
192 $($(target)_$(srcname)_MD5)\
193 $($(source)_MD5)\
194 $($(srcname)_MD5)\
195 $($(target)_MD5)\
196)
197local size := $(firstword \
198 $($(target)_$(source)_SIZE)\
199 $($(target)_$(srcname)_SIZE)\
200 $($(source)_SIZE)\
201 $($(srcname)_SIZE)\
202 $($(target)_SIZE)\
203)
204clean_files += \
205 $($(target)_$(source)_CLEAN)\
206 $($(target)_$(srcname)_CLEAN)\
207 $($(source)_CLEAN)\
208 $($(srcname)_CLEAN)
209local dep := # not legal for fetch and unpack tools
210
211
212#
213# The fetching.
214#
215local out := $(fetchdir)/$(srcname)
216local archive := $(out)
217$(target)_$(srcname)_1_TARGET = $(TARGET_$(target)_$(srcname))
218$(call KB_FN_ASSIGN_DEPRECATED,TARGET_$(target)_$(srcname),$(TARGET_$(target)_$(srcname)),TARGET_$(target)_$(srcname))
219
220local dirdep := $(call DIRDEP,$(fetchdir))
221local tool := $(firstword \
222 $($(target)_$(source)_FETCHTOOL)\
223 $($(target)_$(srcname)_FETCHTOOL)\
224 $($(target)_$(source)_TOOL)\
225 $($(target)_$(srcname)_TOOL)\
226 $($(source)_FETCHTOOL)\
227 $($(srcname)_FETCHTOOL)\
228 $($(source)_TOOL)\
229 $($(srcname)_TOOL)\
230 $($(target)_FETCHTOOL)\
231 $($(target)_TOOL)\
232 )
233local flags :=\
234 $(TOOL_$(tool)_FETCHFLAGS)\
235 $(FETCHFLAGS)\
236 $($(target)_FETCHFLAGS)\
237 $($(srcname)_FETCHFLAGS)\
238 $($(source)_FETCHFLAGS)\
239 $($(target)_$(srcname)_FETCHFLAGS)\
240 $($(target)_$(source)_FETCHFLAGS)
241
242#$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) fetchdir=$(fetchdir) md5=$(md5) size=$(size))
243
244ifndef TOOL_$(tool)_FETCH_CMDS
245$(warning kBuild: tools: \
246 1 $($(target)_$(source)_FETCHTOOL)\
247 2 $($(target)_$(srcname)_FETCHTOOL)\
248 3 $($(target)_$(source)_TOOL)\
249 4 $($(target)_$(srcname)_TOOL)\
250 5 $($(source)_FETCHTOOL)\
251 6 $($(srcname)_FETCHTOOL)\
252 7 $($(source)_TOOL)\
253 8 $($(srcname)_TOOL)\
254 9 $($(target)_FETCHTOOL)\
255 10 $($(target)_TOOL) )
256$(error kBuild: TOOL_$(tool)_FETCH_CMDS is not defined. source=$(source) target=$(target) )
257endif
258
259# call the tool
260local cmds := $(TOOL_$(tool)_FETCH_CMDS)
261$(target)_$(srcname)_FETCH_2_OUTPUT := $(TOOL_$(tool)_FETCH_OUTPUT)
262$(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE := $(TOOL_$(tool)_FETCH_OUTPUT_MAYBE)
263$(target)_$(srcname)_FETCH_2_DEPEND := $(TOOL_$(tool)_FETCH_DEPEND) $(deps)
264$(target)_$(srcname)_FETCH_2_DEPORD := $(TOOL_$(tool)_FETCH_DEPORD) $(dirdep) $(orderdeps)
265
266# generate the fetch rule.
267$(eval $(def_fetch_src_fetch_rule))
268
269
270#
271# The unpacking / installing.
272#
273local out := $(inst)_kBuild_$(name)_$(srcname)_unpacked.lst
274local dirdep := $(call DIRDEP,$(inst))
275local tool := $(firstword \
276 $($(target)_$(source)_UNPACKTOOL)\
277 $($(target)_$(srcname)_UNPACKTOOL)\
278 $($(target)_$(source)_TOOL)\
279 $($(target)_$(srcname)_TOOL)\
280 $($(source)_UNPACKTOOL)\
281 $($(srcname)_UNPACKTOOL)\
282 $($(source)_TOOL)\
283 $($(srcname)_TOOL)\
284 $($(target)_UNPACKTOOL)\
285 $($(target)_TOOL) \
286 )
287ifeq ($(tool),)
288local tool := $(toupper $(subst .,,$(suffix $(subst tar.,TAR,$(srcname)))))
289$(evalval def_tools_include)
290endif
291local flags :=\
292 $(TOOL_$(tool)_UNPACKFLAGS)\
293 $(UNPACKFLAGS)\
294 $($(target)_UNPACKFLAGS)\
295 $($(srcname)_UNPACKFLAGS)\
296 $($(source)_UNPACKFLAGS)\
297 $($(target)_$(srcname)_UNPACKFLAGS)\
298 $($(target)_$(source)_UNPACKFLAGS)
299
300#$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) inst=$(inst) md5=$(md5) size=$(size))
301ifndef TOOL_$(tool)_UNPACK_CMDS
302$(warning kBuild: tools: \
303 1 $($(target)_$(source)_UNPACKTOOL)\
304 2 $($(target)_$(srcname)_UNPACKTOOL)\
305 3 $($(target)_$(source)_TOOL)\
306 4 $($(target)_$(srcname)_TOOL)\
307 5 $($(source)_UNPACKTOOL)\
308 6 $($(srcname)_UNPACKTOOL)\
309 7 $($(source)_TOOL)\
310 8 $($(srcname)_TOOL)\
311 9 $($(target)_UNPACKTOOL)\
312 10 $($(target)_TOOL) \
313 11 $(toupper $(subst tar.,TAR,$(ext $(srcname)))) \
314 )
315$(error kBuild: TOOL_$(tool)_UNPACK_CMDS is not defined. source=$(source) target=$(target) )
316endif
317
318# call the tool
319local cmds := $(TOOL_$(tool)_UNPACK_CMDS)
320$(target)_$(srcname)_UNPACK_2_OUTPUT := $(TOOL_$(tool)_UNPACK_OUTPUT)
321$(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE := $(TOOL_$(tool)_UNPACK_OUTPUT_MAYBE)
322$(target)_$(srcname)_UNPACK_2_DEPEND := $(TOOL_$(tool)_UNPACK_DEPEND) $(deps)
323$(target)_$(srcname)_UNPACK_2_DEPORD := $(TOOL_$(tool)_UNPACK_DEPORD) $(dirdep) $(orderdeps)
324
325# generate the fetch rule.
326$(eval $(def_fetch_src_unpack_rule))
327
328_DIRS += $(inst) $(fetchdir)
329
330endef # def_fetch_src
331$(eval-opt-var def_fetch_src)
332
333
334##
335# Define the target level rules for a fetch.
336# @param target
337# @param out
338# @param inst
339# @param _TARGET_$(target)_UNPACKED
340# @param _TARGET_$(target)_DIGEST
341# @param bld_trg
342# @param bld_trg_arch
343define def_fetch_rules
344
345$(out).lst: $(_TARGET_$(target)_UNPACKED) | $(call DIRDEP,$(inst))
346 %$$(call MSG_FETCH_OK,$(target))
347 $$(QUIET)$$(RM) -f -- $$@ $$@.tmp
348 $$(QUIET2)$$(APPEND) $$@.tmp '$(notdir $(out))'
349 $$(QUIET)$(if $(_TARGET_$(target)_UNPACKED),$$(CAT_EXT) $(_TARGET_$(target)_UNPACKED) >> $$@.tmp)
350 $$(QUIET)$$(MV) -f -- $$@.tmp $$@
351
352$(out)_unfetched:
353 %$$(call MSG_UNFETCH,$(target))
354 $$(QUIET)$$(RM) -f -- $$(qaddprefix sh,$(inst),$$(shell $$(CAT_EXT) $(out).lst 2> /dev/null | $$(SED) -e '/\/$$$$/d' -e 's/ /\\ /g'))
355 $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@) \
356 $$(qaddprefix sh,$(inst),$$(qsortfiles ,$$(qdir ,$$(shell $$(CAT_EXT) $(out).lst 2> /dev/null | $$(SED) -e '/\/$$$$/d' -e 's/ /\\ /g'))))
357 $$(QUIET)$$(RM) -f -- $(out).lst $(out)
358 $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@)
359
360$(out): $(comp-vars _TARGET_$(target)_DIGEST_PREV,_TARGET_$(target)_DIGEST,FORCE) | $(call DIRDEP,$(inst))
361 $$(QUIET)$$(RM) -f -- $$@
362 %$$(if $$(_TARGET_$(target)_DIGEST),$$(if $$(eq $$(file-size $(out).lst),-1)\
363 ,$$(call MSG_REFETCH,$(target)),$$(call MSG_FETCH,$(target))),$$(call MSG_UNFETCH,$(target)))
364 $$(QUIET)$(TEST_EXT) -f $(out).lst -- $$(MAKE) -f $(MAKEFILE) --no-print-directory $(out)_unfetched
365if $(KBUILD_KMK_REVISION) > 2911
366 $$(QUIET)kmk_builtin_dircache deleted "$(dir $(out))"
367endif
368 $$(QUIET)$$(if $$(_TARGET_$(target)_DIGEST),$$(MAKE) -f $(MAKEFILE) --no-print-directory $(out).lst,$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@))
369 $$(QUIET2)$$(if $$(_TARGET_$(target)_DIGEST),$$(APPEND) $$@ "_TARGET_$(target)_DIGEST_PREV := $(_TARGET_$(target)_DIGEST)")
370
371.NOTPARALLEL: $(out).lst $(out)_unfetched $(out)
372
373endef
374
375
376##
377# Deal with one fetch target.
378# @param target
379# @param bld_trg
380# @param bld_trg_arch
381define def_fetch
382# common
383local name := $(firstword $($(target)_NAME) $(target))
384## @todo Install-revamp: FIXME
385INSTARGET_$(target) := $($(target)_INST)
386ifneq ($(patsubst %/,ok,$(INSTARGET_$(target))),ok)
387$(error kBuild: Bad or missing INST property for target '$(target)'. \
388 $(target)_INST='$($(target)_INST)' ($(origin $(target)_INST)))
389endif
390_TARGET_$(target)_FETCHED :=
391_TARGET_$(target)_UNPACKED :=
392_TARGET_$(target)_DIGEST :=
393local clean_files := $($(target)_CLEAN) $($(target)_CLEAN.$(bld_trg)) $($(target)_CLEAN.$(bld_trg).$(bld_trg_arch)) $($(target)_CLEAN.$(bld_trg_arch)) $($(target)_CLEAN.$(KBUILD_TYPE))
394
395# The 'sources'.
396#$ (warning dbg fetch: target=$(target) name=$(name) sources=$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(KBUILD_TARGET)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(KBUILD_TARGET).$(bld_trg_arch)))
397$(foreach source,$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(bld_trg)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(bld_trg).$(bld_trg_arch)),\
398 $(evalval def_fetch_src))
399
400# The target.
401local inst := $(INSTARGET_$(target))
402local out := $(inst)_kBuild_fetch_$(name)
403
404$(eval includedep $(out))
405
406$(eval $(def_fetch_rules))
407
408# Define the aliases here (doesn't work if defined in def_fetch_rules, just like includedep).
409$(target): $(out)
410$(target)_unfetch: $(out)_unfetched
411
412_FETCHES += $(out)
413_DOWNLOADS += $(_TARGET_$(target)_FETCHED)
414_UNPACKS += $(_TARGET_$(target)_UNPACKED)
415_UNFETCHES += $(out)_unfetched
416_DIRS += $(inst)
417_CLEAN_FILES += $(clean_files)
418
419endef
420$(eval-opt-var def_fetch)
421
422# Walk the FETCH target lists.
423bld_trg := $(KBUILD_TARGET)
424bld_trg_arch := $(KBUILD_TARGET_ARCH)
425$(foreach target, $(_ALL_FETCHES), \
426 $(evalvalctx def_fetch))
427
428# some aliases.
429download: $(_DOWNLOADS)
430unpack: $(_UNPACKS)
431fetch: $(_FETCHES)
432unfetch: $(_UNFETCHES)
433
434ifdef KBUILD_PROFILE_SELF
435 $(evalcall def_profile_self, done fetching targets)
436endif
Note: See TracBrowser for help on using the repository browser.