Changeset 988 for vendor/current/buildtools/wafsamba/symbols.py

Timestamp:

Nov 24, 2016, 1:14:11 PM (9 years ago)

Author:

Silvan Scherrer

Message:

Samba Server: update vendor to version 4.4.3

File:

• vendor/current/buildtools/wafsamba/symbols.py (modified) (20 diffs)

vendor/current/buildtools/wafsamba/symbols.py

@@ -2 +2 @@
 # using nm, producing a set of exposed defined/undefined symbols
 
-import Utils, Build, subprocess, Logs
-from samba_wildcard import fake_build_environment
-from samba_utils import *
+import os, re, subprocess
+import Utils, Build, Options, Logs
+from Logs import debug
+from samba_utils import TO_LIST, LOCAL_CACHE, get_tgt_list, os_path_relpath
 
 # these are the data structures used in symbols.py:
@@ -18 +19 @@
 # bld.env.syslib_symbols: dictionary mapping system library name to set of symbols
 #                         for that library
+# bld.env.library_dict  : dictionary mapping built library paths to subsystem names
 #
 # LOCAL_CACHE(bld, 'TARGET_TYPE') : dictionary mapping subsystem name to target type
 
-def symbols_extract(objfiles, dynamic=False):
+
+def symbols_extract(bld, objfiles, dynamic=False):
     '''extract symbols from objfile, returning a dictionary containing
        the set of undefined and public symbols for each file'''
 
     ret = {}
+
+    # see if we can get some results from the nm cache
+    if not bld.env.nm_cache:
+        bld.env.nm_cache = {}
+
+    objfiles = set(objfiles).copy()
+
+    remaining = set()
+    for obj in objfiles:
+        if obj in bld.env.nm_cache:
+            ret[obj] = bld.env.nm_cache[obj].copy()
+        else:
+            remaining.add(obj)
+    objfiles = remaining
+
+    if len(objfiles) == 0:
+        return ret
 
     cmd = ["nm"]
@@ -31 +51 @@
         # needed for some .so files
         cmd.append("-D")
-    cmd.extend(objfiles)
+    cmd.extend(list(objfiles))
 
     nmpipe = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout
     if len(objfiles) == 1:
-        filename = objfiles[0]
+        filename = list(objfiles)[0]
         ret[filename] = { "PUBLIC": set(), "UNDEFINED" : set()}
 
@@ -60 +80 @@
             ret[filename]["UNDEFINED"].add(symbol)
 
+    # add to the cache
+    for obj in objfiles:
+        if obj in ret:
+            bld.env.nm_cache[obj] = ret[obj].copy()
+        else:
+            bld.env.nm_cache[obj] = { "PUBLIC": set(), "UNDEFINED" : set() }
+
     return ret
 
@@ -69 +96 @@
 
 
-def find_syslib_path(bld, libname, deps):
+def find_ldd_path(bld, libname, binary):
     '''find the path to the syslib we will link against'''
-    # the strategy is to use the targets that depend on the library, and run ldd
-    # on it to find the real location of the library that is used
-
-    linkpath = deps[0].link_task.outputs[0].abspath(bld.env)
-
-    if libname == "python":
-        libname += bld.env.PYTHON_VERSION
-
     ret = None
-
-    lddpipe = subprocess.Popen(['ldd', linkpath], stdout=subprocess.PIPE).stdout
+    if not bld.env.syslib_paths:
+        bld.env.syslib_paths = {}
+    if libname in bld.env.syslib_paths:
+        return bld.env.syslib_paths[libname]
+
+    lddpipe = subprocess.Popen(['ldd', binary], stdout=subprocess.PIPE).stdout
     for line in lddpipe:
         line = line.strip()
@@ -87 +110 @@
         if len(cols) < 3 or cols[1] != "=>":
             continue
-        if cols[0].startswith("lib%s." % libname.lower()):
-            ret = cols[2]
         if cols[0].startswith("libc."):
             # save this one too
             bld.env.libc_path = cols[2]
+        if cols[0].startswith(libname):
+            ret = cols[2]
+    bld.env.syslib_paths[libname] = ret
     return ret
+
+
+# some regular expressions for parsing readelf output
+re_sharedlib = re.compile('Shared library: \[(.*)\]')
+re_rpath     = re.compile('Library rpath: \[(.*)\]')
+
+def get_libs(bld, binname):
+    '''find the list of linked libraries for any binary or library
+    binname is the path to the binary/library on disk
+
+    We do this using readelf instead of ldd as we need to avoid recursing
+    into system libraries
+    '''
+
+    # see if we can get the result from the ldd cache
+    if not bld.env.lib_cache:
+        bld.env.lib_cache = {}
+    if binname in bld.env.lib_cache:
+        return bld.env.lib_cache[binname].copy()
+
+    rpath = []
+    libs = set()
+
+    elfpipe = subprocess.Popen(['readelf', '--dynamic', binname], stdout=subprocess.PIPE).stdout
+    for line in elfpipe:
+        m = re_sharedlib.search(line)
+        if m:
+            libs.add(m.group(1))
+        m = re_rpath.search(line)
+        if m:
+            rpath.extend(m.group(1).split(":"))
+
+    ret = set()
+    for lib in libs:
+        found = False
+        for r in rpath:
+            path = os.path.join(r, lib)
+            if os.path.exists(path):
+                ret.add(os.path.realpath(path))
+                found = True
+                break
+        if not found:
+            # we didn't find this lib using rpath. It is probably a system
+            # library, so to find the path to it we either need to use ldd
+            # or we need to start parsing /etc/ld.so.conf* ourselves. We'll
+            # use ldd for now, even though it is slow
+            path = find_ldd_path(bld, lib, binname)
+            if path:
+                ret.add(os.path.realpath(path))
+
+    bld.env.lib_cache[binname] = ret.copy()
+
+    return ret
+
+
+def get_libs_recursive(bld, binname, seen):
+    '''find the recursive list of linked libraries for any binary or library
+    binname is the path to the binary/library on disk. seen is a set used
+    to prevent loops
+    '''
+    if binname in seen:
+        return set()
+    ret = get_libs(bld, binname)
+    seen.add(binname)
+    for lib in ret:
+        # we don't want to recurse into system libraries. If a system
+        # library that we use (eg. libcups) happens to use another library
+        # (such as libkrb5) which contains common symbols with our own
+        # libraries, then that is not an error
+        if lib in bld.env.library_dict:
+            ret = ret.union(get_libs_recursive(bld, lib, seen))
+    return ret
+
+
+
+def find_syslib_path(bld, libname, deps):
+    '''find the path to the syslib we will link against'''
+    # the strategy is to use the targets that depend on the library, and run ldd
+    # on it to find the real location of the library that is used
+
+    linkpath = deps[0].link_task.outputs[0].abspath(bld.env)
+
+    if libname == "python":
+        libname += bld.env.PYTHON_VERSION
+
+    return find_ldd_path(bld, "lib%s" % libname.lower(), linkpath)
 
 
@@ -114 +224 @@
                 objmap[objpath] = t
 
-    symbols = symbols_extract(objlist)
+    symbols = symbols_extract(bld, objlist)
     for obj in objlist:
         t = objmap[obj]
@@ -143 +253 @@
         if t.samba_type == 'LIBRARY':
             for dep in t.add_objects:
-                t2 = bld.name_to_obj(dep, bld.env)
+                t2 = bld.get_tgen_by_name(dep)
                 bld.ASSERT(t2 is not None, "Library '%s' has unknown dependency '%s'" % (name, dep))
                 bld.env.public_symbols[name] = bld.env.public_symbols[name].union(t2.public_symbols)
@@ -156 +266 @@
         if t.samba_type == 'LIBRARY':
             for dep in t.add_objects:
-                t2 = bld.name_to_obj(dep, bld.env)
+                t2 = bld.get_tgen_by_name(dep)
                 bld.ASSERT(t2 is not None, "Library '%s' has unknown dependency '%s'" % (name, dep))
                 bld.env.used_symbols[name] = bld.env.used_symbols[name].union(t2.used_symbols)
+
+
+def build_library_dict(bld, tgt_list):
+    '''build the library_dict dictionary'''
+
+    if bld.env.library_dict:
+        return
+
+    bld.env.library_dict = {}
+
+    for t in tgt_list:
+        if t.samba_type in [ 'LIBRARY', 'PYTHON' ]:
+            linkpath = os.path.realpath(t.link_task.outputs[0].abspath(bld.env))
+            bld.env.library_dict[linkpath] = t.sname
 
 
@@ -193 +317 @@
     objmap[bld.env.libc_path] = 'c'
 
-    symbols = symbols_extract(syslib_paths, dynamic=True)
+    symbols = symbols_extract(bld, syslib_paths, dynamic=True)
 
     # keep a map of syslib names to public symbols
@@ -240 +364 @@
                 deps.add(depname[0])
                 continue
-            t2 = bld.name_to_obj(depname[0], bld.env)
+            t2 = bld.get_tgen_by_name(depname[0])
             if len(t2.in_library) != 1:
                 deps.add(depname[0])
@@ -263 +387 @@
         if t.samba_type in [ 'LIBRARY' ]:
             for obj in t.samba_deps_extended:
-                t2 = bld.name_to_obj(obj, bld.env)
+                t2 = bld.get_tgen_by_name(obj)
                 if t2 and t2.samba_type in [ 'SUBSYSTEM', 'ASN1' ]:
                     if not t.sname in t2.in_library:
@@ -280 +404 @@
 
     for dep in t.autodeps:
-        t2 = bld.name_to_obj(dep, bld.env)
+        t2 = bld.get_tgen_by_name(dep)
         if t2 is None:
             continue
@@ -313 +437 @@
     '''check for depenencies that should be changed'''
 
-    if bld.name_to_obj(t.sname + ".objlist", bld.env):
+    if bld.get_tgen_by_name(t.sname + ".objlist"):
         return
 
@@ -325 +449 @@
     deps = set(t.samba_deps)
     for d in t.samba_deps:
-        if targets[d] in [ 'EMPTY', 'DISABLED', 'SYSLIB' ]:
+        if targets[d] in [ 'EMPTY', 'DISABLED', 'SYSLIB', 'GENERATOR' ]:
             continue
         bld.ASSERT(d in bld.env.public_symbols, "Failed to find symbol list for dependency '%s'" % d)
@@ -353 +477 @@
     '''check for syslib depenencies'''
 
-    if bld.name_to_obj(t.sname + ".objlist", bld.env):
+    if bld.get_tgen_by_name(t.sname + ".objlist"):
         return
 
@@ -362 +486 @@
     features = TO_LIST(t.features)
     if 'pyembed' in features or 'pyext' in features:
-        t.unsatisfied_symbols = t.unsatisfied_symbols.difference(bld.env.public_symbols['python'])
+        if 'python' in bld.env.public_symbols:
+            t.unsatisfied_symbols = t.unsatisfied_symbols.difference(bld.env.public_symbols['python'])
 
     needed = {}
@@ -444 +569 @@
 
 
-
-def symbols_dupcheck(task):
+def report_duplicate(bld, binname, sym, libs, fail_on_error):
+    '''report duplicated symbols'''
+    if sym in ['_init', '_fini', '_edata', '_end', '__bss_start']:
+        return
+    libnames = []
+    for lib in libs:
+        if lib in bld.env.library_dict:
+            libnames.append(bld.env.library_dict[lib])
+        else:
+            libnames.append(lib)
+    if fail_on_error:
+        raise Utils.WafError("%s: Symbol %s linked in multiple libraries %s" % (binname, sym, libnames))
+    else:
+        print("%s: Symbol %s linked in multiple libraries %s" % (binname, sym, libnames))
+
+
+def symbols_dupcheck_binary(bld, binname, fail_on_error):
+    '''check for duplicated symbols in one binary'''
+
+    libs = get_libs_recursive(bld, binname, set())
+    symlist = symbols_extract(bld, libs, dynamic=True)
+
+    symmap = {}
+    for libpath in symlist:
+        for sym in symlist[libpath]['PUBLIC']:
+            if sym == '_GLOBAL_OFFSET_TABLE_':
+                continue
+            if not sym in symmap:
+                symmap[sym] = set()
+            symmap[sym].add(libpath)
+    for sym in symmap:
+        if len(symmap[sym]) > 1:
+            for libpath in symmap[sym]:
+                if libpath in bld.env.library_dict:
+                    report_duplicate(bld, binname, sym, symmap[sym], fail_on_error)
+                    break
+
+def symbols_dupcheck(task, fail_on_error=False):
     '''check for symbols defined in two different subsystems'''
     bld = task.env.bld
@@ -452 +613 @@
     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 
-    Logs.info("Checking for duplicate symbols")
-    for sym in bld.env.symbol_map:
-        subsystems = set(bld.env.symbol_map[sym])
-        if len(subsystems) == 1:
-            continue
-
-        if sym in ['main', '_init', '_fini', 'init_samba_module', 'samba_init_module', 'ldb_init_module' ]:
-            # these are expected to be in many subsystems
-            continue
-
-        # if all of them are in system libraries, we can ignore them. This copes
-        # with the duplication between libc, libpthread and libattr
-        all_syslib = True
-        for s in subsystems:
-            if s != 'c' and (not s in targets or targets[s] != 'SYSLIB'):
-                all_syslib = False
-        if all_syslib:
-            continue
-        Logs.info("symbol %s appears in %s" % (sym, subsystems))
+    build_library_dict(bld, tgt_list)
+    for t in tgt_list:
+        if t.samba_type == 'BINARY':
+            binname = os_path_relpath(t.link_task.outputs[0].abspath(bld.env), os.getcwd())
+            symbols_dupcheck_binary(bld, binname, fail_on_error)
+
+
+def symbols_dupcheck_fatal(task):
+    '''check for symbols defined in two different subsystems (and fail if duplicates are found)'''
+    symbols_dupcheck(task, fail_on_error=True)
 
 
@@ -495 +647 @@
 
 Build.BuildContext.SYMBOL_CHECK = SYMBOL_CHECK
+
+def DUP_SYMBOL_CHECK(bld):
+    if Options.options.DUP_SYMBOLCHECK and bld.env.DEVELOPER:
+        '''check for duplicate symbols'''
+        bld.SET_BUILD_GROUP('syslibcheck')
+        task = bld(rule=symbols_dupcheck_fatal, always=True, name='symbol duplicate checking')
+        task.env.bld = bld
+
+Build.BuildContext.DUP_SYMBOL_CHECK = DUP_SYMBOL_CHECK