source: trunk/kBuild/footer-pass2-fetches.kmk@ 3675

# $Id: footer-pass2-fetches.kmk 3608 2024-09-09 14:53:00Z bird $
## @file
# kBuild - Footer - Target lists - Pass 2 - Fetches.
#

#
# Copyright (c) 2004-2017 knut st. osmundsen <bird-kBuild-spam-xviiv@anduin.net>
#
# This file is part of kBuild.
#
# kBuild is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version source of the License, or
# (at your option) any later version.
#
# kBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with kBuild; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#
# As a special exception you are granted permission to include this file, via
# the kmk include directive, as you wish without this in itself causing the
# resulting makefile, program or whatever to be covered by the GPL license.
# This exception does not however invalidate any other reasons why the makefile,
# program, whatever should not be covered the GPL.
#
#


## @page pg_fetches     Fetching Tools, Sources and Similar.
#
# The targets listed in the the FETCHES target list have the following attributes:
#       NAME
#       SOURCES
#       INST
#       FETCHTOOL
#       FETCHFLAGS
#       FETCHDIR
#       UNPACKTOOL
#       UNPACKFLAGS
#
# As usual the target name is an alias for 'creating' the target. Other
# aliases are:
#       pass_fetches
#       fetch
#       unfetch
#       download
#       unpack
#
# @remark
#
#   This is a little bit complex because we must guarantee that if a source file
#   changes only sligtly we must refetch it and to a proper unpacking of it. It
#   is also a desire that fetched archives and unpacked files can be deleted to
#   save space.
#
#   Thus, we must be able to cleanup what we've unpacked should any of the
#   sources be removed. We do this by maintaining a file listing the files
#   and directories that was unpacked. This operation is named 'unfetch'.
#
#   We make use of the SIZE and MD5 attributes for each of the sources to
#   create a digest that is stored in the primary target file. Subsequent
#   runswill compare their digest with it to decide if a refetch is required.
#   When a refetch is found necessary, an 'unfetch' is performed first to
#       clean out old files and directores. Note even changes in source order
#   will cause a refetch due to the way the digest is constructed and
#   evaluated.
#
#   By not depending directly on the archives (nor on any unpacked files)
#       but on a goal made up from the archive name, size and md5, we allow
#   the user to delete the archives. Naturally, this means we'll have to
#   check and fetch missing archives before attempting to unpack them.
#
# @remark
#
#       This feature will *NOT* work correctly with vanilla GNU make becuase
#   it makes use of includedep to avoid too many unnecessary files.
#
# @todo
#   0. Move the fetches out into a unit.
#   1. Download corruption / continuation.
#   2. It's quite possible that there is one too many indirect dependency now...
#

## generates the fetch rule
define def_fetch_src_fetch_rule
# Indirect goal for downloading something.
.PRECIOUS: $(out)
$(out) + $($(target)_$(srcname)_FETCH_2_OUTPUT) +| $($(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE)  : \
                | $($(target)_$(srcname)_FETCH_2_DEPORD)
        %$$(call MSG_FETCH_DL,$(target),$(source),$(out))
        @## @todo do fancy stuff like download continuation.
        $$(QUIET)$$(RM) -f -- $(out)
        $(cmds)
        $$(QUIET)$(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out))



# Intermediate goal for making sure the md5 and size matches. it will (re) fetch the archive if necessary.
$(out).checked_$(md5)_$(size): $($(target)_$(srcname)_FETCH_2_DEPEND) | $($(target)_$(srcname)_FETCH_2_DEPORD)
        %$$(call MSG_FETCH_CHK,$(target),$(source),$(out))
        $$(QUIET)$$(RM) -f -- $$@
        @# (re)fetch the file if it doesn't exist or if it doesn't matches the md5.
        @## @todo do fancy stuff like download continuation.
        $$(QUIET)( test -f $(out) && $(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out), true) ) \
                  || (   $$(RM_EXT) -f $(out) \
                      && $$(MAKE)      $(out) -f $(MAKEFILE) --no-print-directory )
        $$(QUIET2)$$(APPEND) $$@

_TARGET_$(target)_FETCHED += $(out) $(out).checked_$(md5)_$(size)

# Just a little precaution.
.NOTPARALLEL: $(out) $(out).checked_$(md5)_$(size)

endef # def_fetch_src_fetch_rule
$(eval-opt-var def_fetch_src_fetch_rule)

## generates the unpack rule
define def_fetch_src_unpack_rule
# This is the unpack rule. it has an order-only dependency on the download check.
$(out) + $($(target)_$(srcname)_UNPACK_2_OUTPUT) +| $($(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE) : \
                $($(target)_$(srcname)_UNPACK_2_DEPEND) \
                | $($(target)_$(srcname)_UNPACK_2_DEPORD) \
                $(archive).checked_$(md5)_$(size) \
                $(dir $(out))
        %$$(call MSG_FETCH_UP,$(target),$(archive),$(inst))
        $$(QUIET)$$(RM) -f -- $(out)
        $$(QUIET)$$(MKDIR) -p -- $(dir $(out))
        @# if the source archive doesn't exist fetch it (may have been deleted to save space).
        $$(QUIET)test -f $(archive) \
                 || (   $$(RM_EXT) -f $(archive).checked_$(md5)_$(size) \
                     && $$(MAKE)      $(archive).checked_$(md5)_$(size) -f $(MAKEFILE) --no-print-directory )
        $(cmds)
        $$(QUIET2)$$(APPEND) $(out) $(notdir $(archive).checked_$(md5)_$(size))
        $$(QUIET2)$$(APPEND) $(out) $(notdir $(out))

$(eval _TARGET_$(target)_UNPACKED += $(out))
_TARGET_$(target)_DIGEST := $(_TARGET_$(target)_DIGEST)-$(srcname)_$(md5)_$(size)

.NOTPARALLEL: $(out)

endef # def_fetch_src_unpack_rule
$(eval-opt-var def_fetch_src_unpack_rule)

## Processes a fetch source
#
define def_fetch_src
#$ (warning dbg: def_fetch_src: source='$(source)' target='$(target)')

# common
local srcname := $(notdir $(source))
local inst := $(firstword \
        $($(target)_$(source)_INST)\
        $($(target)_$(srcname)_INST)\
        $($(source)_INST)\
        $($(srcname)_INST)\
        $($(target)_INST)\
)
ifneq ($(patsubst %/,ok,$(inst)),ok)
$(error kBuild: Bad or missing INST property for source '$(source)' in target '$(target)': $(inst))
endif
## @todo Install-revamp: FIXME
INSTARGET_$(target)_$(srcname) := $(inst)
local fetchdir := $(firstword \
        $($(target)_$(source)_FETCHDIR)\
        $($(target)_$(srcname)_FETCHDIR)\
        $($(source)_FETCHDIR)\
        $($(srcname)_FETCHDIR)\
        $($(target)_FETCHDIR)\
        $(FETCHDIR)\
        $(PATH_TARGET)\
)
local deps := \
        $($(target)_$(source)_DEPS)\
        $($(target)_$(srcname)_DEPS)\
        $($(source)_DEPS)\
        $($(srcname)_DEPS)\
        $($(target)_DEPS)
local orderdeps := \
        $($(target)_$(source)_ORDERDEPS)\
        $($(target)_$(srcname)_ORDERDEPS)\
        $($(source)_ORDERDEPS)\
        $($(srcname)_ORDERDEPS)\
        $($(target)_ORDERDEPS)
local md5 := $(firstword \
        $($(target)_$(source)_MD5)\
        $($(target)_$(srcname)_MD5)\
        $($(source)_MD5)\
        $($(srcname)_MD5)\
        $($(target)_MD5)\
)
local size := $(firstword \
        $($(target)_$(source)_SIZE)\
        $($(target)_$(srcname)_SIZE)\
        $($(source)_SIZE)\
        $($(srcname)_SIZE)\
        $($(target)_SIZE)\
)
clean_files += \
        $($(target)_$(source)_CLEAN)\
        $($(target)_$(srcname)_CLEAN)\
        $($(source)_CLEAN)\
        $($(srcname)_CLEAN)
local dep := # not legal for fetch and unpack tools


#
# The fetching.
#
local out := $(fetchdir)/$(srcname)
local archive := $(out)
$(target)_$(srcname)_1_TARGET = $(TARGET_$(target)_$(srcname))
$(call KB_FN_ASSIGN_DEPRECATED,TARGET_$(target)_$(srcname),$(TARGET_$(target)_$(srcname)),TARGET_$(target)_$(srcname))

local dirdep := $(call DIRDEP,$(fetchdir))
local tool := $(firstword \
        $($(target)_$(source)_FETCHTOOL)\
        $($(target)_$(srcname)_FETCHTOOL)\
        $($(target)_$(source)_TOOL)\
        $($(target)_$(srcname)_TOOL)\
        $($(source)_FETCHTOOL)\
        $($(srcname)_FETCHTOOL)\
        $($(source)_TOOL)\
        $($(srcname)_TOOL)\
        $($(target)_FETCHTOOL)\
        $($(target)_TOOL)\
        )
local flags :=\
        $(TOOL_$(tool)_FETCHFLAGS)\
        $(FETCHFLAGS)\
        $($(target)_FETCHFLAGS)\
        $($(srcname)_FETCHFLAGS)\
        $($(source)_FETCHFLAGS)\
        $($(target)_$(srcname)_FETCHFLAGS)\
        $($(target)_$(source)_FETCHFLAGS)

#$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) fetchdir=$(fetchdir) md5=$(md5) size=$(size))

ifndef TOOL_$(tool)_FETCH_CMDS
$(warning kBuild: tools: \
        1 $($(target)_$(source)_FETCHTOOL)\
        2 $($(target)_$(srcname)_FETCHTOOL)\
        3 $($(target)_$(source)_TOOL)\
        4 $($(target)_$(srcname)_TOOL)\
        5 $($(source)_FETCHTOOL)\
        6 $($(srcname)_FETCHTOOL)\
        7 $($(source)_TOOL)\
        8 $($(srcname)_TOOL)\
        9 $($(target)_FETCHTOOL)\
        10 $($(target)_TOOL) )
$(error kBuild: TOOL_$(tool)_FETCH_CMDS is not defined. source=$(source) target=$(target) )
endif

# call the tool
local cmds := $(TOOL_$(tool)_FETCH_CMDS)
$(target)_$(srcname)_FETCH_2_OUTPUT := $(TOOL_$(tool)_FETCH_OUTPUT)
$(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE := $(TOOL_$(tool)_FETCH_OUTPUT_MAYBE)
$(target)_$(srcname)_FETCH_2_DEPEND := $(TOOL_$(tool)_FETCH_DEPEND) $(deps)
$(target)_$(srcname)_FETCH_2_DEPORD := $(TOOL_$(tool)_FETCH_DEPORD) $(dirdep) $(orderdeps)

# generate the fetch rule.
$(eval $(def_fetch_src_fetch_rule))


#
# The unpacking / installing.
#
local out := $(inst)_kBuild_$(name)_$(srcname)_unpacked.lst
local dirdep := $(call DIRDEP,$(inst))
local tool := $(firstword \
        $($(target)_$(source)_UNPACKTOOL)\
        $($(target)_$(srcname)_UNPACKTOOL)\
        $($(target)_$(source)_TOOL)\
        $($(target)_$(srcname)_TOOL)\
        $($(source)_UNPACKTOOL)\
        $($(srcname)_UNPACKTOOL)\
        $($(source)_TOOL)\
        $($(srcname)_TOOL)\
        $($(target)_UNPACKTOOL)\
        $($(target)_TOOL) \
        )
ifeq ($(tool),)
local tool := $(toupper $(subst .,,$(suffix $(subst tar.,TAR,$(srcname)))))
$(evalval def_tools_include)
endif
local flags :=\
        $(TOOL_$(tool)_UNPACKFLAGS)\
        $(UNPACKFLAGS)\
        $($(target)_UNPACKFLAGS)\
        $($(srcname)_UNPACKFLAGS)\
        $($(source)_UNPACKFLAGS)\
        $($(target)_$(srcname)_UNPACKFLAGS)\
        $($(target)_$(source)_UNPACKFLAGS)

#$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) inst=$(inst) md5=$(md5) size=$(size))
ifndef TOOL_$(tool)_UNPACK_CMDS
$(warning kBuild: tools: \
        1 $($(target)_$(source)_UNPACKTOOL)\
        2 $($(target)_$(srcname)_UNPACKTOOL)\
        3 $($(target)_$(source)_TOOL)\
        4 $($(target)_$(srcname)_TOOL)\
        5 $($(source)_UNPACKTOOL)\
        6 $($(srcname)_UNPACKTOOL)\
        7 $($(source)_TOOL)\
        8 $($(srcname)_TOOL)\
        9 $($(target)_UNPACKTOOL)\
        10 $($(target)_TOOL) \
        11 $(toupper $(subst tar.,TAR,$(ext $(srcname)))) \
        )
$(error kBuild: TOOL_$(tool)_UNPACK_CMDS is not defined. source=$(source) target=$(target) )
endif

# call the tool
local cmds := $(TOOL_$(tool)_UNPACK_CMDS)
$(target)_$(srcname)_UNPACK_2_OUTPUT := $(TOOL_$(tool)_UNPACK_OUTPUT)
$(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE := $(TOOL_$(tool)_UNPACK_OUTPUT_MAYBE)
$(target)_$(srcname)_UNPACK_2_DEPEND := $(TOOL_$(tool)_UNPACK_DEPEND) $(deps)
$(target)_$(srcname)_UNPACK_2_DEPORD := $(TOOL_$(tool)_UNPACK_DEPORD) $(dirdep) $(orderdeps)

# generate the fetch rule.
$(eval $(def_fetch_src_unpack_rule))

_DIRS += $(inst) $(fetchdir)

endef # def_fetch_src
$(eval-opt-var def_fetch_src)


##
# Define the target level rules for a fetch.
# @param        target
# @param        out
# @param        inst
# @param        _TARGET_$(target)_UNPACKED
# @param        _TARGET_$(target)_DIGEST
# @param        bld_trg
# @param        bld_trg_arch
define def_fetch_rules

$(out).lst: $(_TARGET_$(target)_UNPACKED) | $(call DIRDEP,$(inst))
        %$$(call MSG_FETCH_OK,$(target))
        $$(QUIET)$$(RM) -f -- $$@ $$@.tmp
        $$(QUIET2)$$(APPEND) $$@.tmp '$(notdir $(out))'
        $$(QUIET)$(if $(_TARGET_$(target)_UNPACKED),$$(CAT_EXT) $(_TARGET_$(target)_UNPACKED) >> $$@.tmp)
        $$(QUIET)$$(MV) -f -- $$@.tmp $$@

$(out)_unfetched:
        %$$(call MSG_UNFETCH,$(target))
        $$(QUIET)$$(RM) -f -- $$(qaddprefix sh,$(inst),$$(shell $$(CAT_EXT) $(out).lst 2> /dev/null | $$(SED) -e '/\/$$$$/d' -e 's/ /\\ /g'))
        $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@) \
                $$(qaddprefix sh,$(inst),$$(qsortfiles ,$$(qdir ,$$(shell $$(CAT_EXT) $(out).lst 2> /dev/null | $$(SED) -e '/\/$$$$/d' -e 's/ /\\ /g'))))
        $$(QUIET)$$(RM) -f -- $(out).lst $(out)
        $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@)

$(out): $(comp-vars _TARGET_$(target)_DIGEST_PREV,_TARGET_$(target)_DIGEST,FORCE) | $(call DIRDEP,$(inst))
        $$(QUIET)$$(RM) -f -- $$@
        %$$(if $$(_TARGET_$(target)_DIGEST),$$(if $$(eq $$(file-size $(out).lst),-1)\
                ,$$(call MSG_REFETCH,$(target)),$$(call MSG_FETCH,$(target))),$$(call MSG_UNFETCH,$(target)))
        $$(QUIET)$(TEST_EXT) -f $(out).lst -- $$(MAKE) -f $(MAKEFILE) --no-print-directory $(out)_unfetched
if $(KBUILD_KMK_REVISION) > 2911
        $$(QUIET)kmk_builtin_dircache deleted "$(dir $(out))"
endif
        $$(QUIET)$$(if  $$(_TARGET_$(target)_DIGEST),$$(MAKE) -f $(MAKEFILE) --no-print-directory $(out).lst,$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@))
        $$(QUIET2)$$(if $$(_TARGET_$(target)_DIGEST),$$(APPEND) $$@ "_TARGET_$(target)_DIGEST_PREV := $(_TARGET_$(target)_DIGEST)")

.NOTPARALLEL: $(out).lst $(out)_unfetched $(out)

endef


##
# Deal with one fetch target.
# @param        target
# @param        bld_trg
# @param        bld_trg_arch
define def_fetch
# common
local name := $(firstword $($(target)_NAME) $(target))
## @todo Install-revamp: FIXME
INSTARGET_$(target) := $($(target)_INST)
ifneq ($(patsubst %/,ok,$(INSTARGET_$(target))),ok)
$(error kBuild: Bad or missing INST property for target '$(target)'. \
        $(target)_INST='$($(target)_INST)' ($(origin $(target)_INST)))
endif
_TARGET_$(target)_FETCHED :=
_TARGET_$(target)_UNPACKED :=
_TARGET_$(target)_DIGEST :=
local clean_files := $($(target)_CLEAN) $($(target)_CLEAN.$(bld_trg)) $($(target)_CLEAN.$(bld_trg).$(bld_trg_arch)) $($(target)_CLEAN.$(bld_trg_arch)) $($(target)_CLEAN.$(KBUILD_TYPE))

# The 'sources'.
#$ (warning dbg fetch: target=$(target) name=$(name) sources=$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(KBUILD_TARGET)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(KBUILD_TARGET).$(bld_trg_arch)))
$(foreach source,$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(bld_trg)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(bld_trg).$(bld_trg_arch)),\
        $(evalval def_fetch_src))

# The target.
local inst := $(INSTARGET_$(target))
local out := $(inst)_kBuild_fetch_$(name)

$(eval includedep $(out))

$(eval $(def_fetch_rules))

# Define the aliases here (doesn't work if defined in def_fetch_rules, just like includedep).
$(target): $(out)
$(target)_unfetch: $(out)_unfetched

_FETCHES += $(out)
_DOWNLOADS += $(_TARGET_$(target)_FETCHED)
_UNPACKS += $(_TARGET_$(target)_UNPACKED)
_UNFETCHES += $(out)_unfetched
_DIRS += $(inst)
_CLEAN_FILES += $(clean_files)

endef
$(eval-opt-var def_fetch)

# Walk the FETCH target lists.
bld_trg := $(KBUILD_TARGET)
bld_trg_arch := $(KBUILD_TARGET_ARCH)
$(foreach target, $(_ALL_FETCHES), \
        $(evalvalctx def_fetch))

# some aliases.
download: $(_DOWNLOADS)
unpack: $(_UNPACKS)
fetch: $(_FETCHES)
unfetch: $(_UNFETCHES)

ifdef KBUILD_PROFILE_SELF
 $(evalcall def_profile_self, done fetching targets)
endif