| 1 | """distutils.filelist
|
|---|
| 2 |
|
|---|
| 3 | Provides the FileList class, used for poking about the filesystem
|
|---|
| 4 | and building lists of files.
|
|---|
| 5 | """
|
|---|
| 6 |
|
|---|
| 7 | __revision__ = "$Id$"
|
|---|
| 8 |
|
|---|
| 9 | import os, re
|
|---|
| 10 | import fnmatch
|
|---|
| 11 | from distutils.util import convert_path
|
|---|
| 12 | from distutils.errors import DistutilsTemplateError, DistutilsInternalError
|
|---|
| 13 | from distutils import log
|
|---|
| 14 |
|
|---|
| 15 | class FileList:
|
|---|
| 16 | """A list of files built by on exploring the filesystem and filtered by
|
|---|
| 17 | applying various patterns to what we find there.
|
|---|
| 18 |
|
|---|
| 19 | Instance attributes:
|
|---|
| 20 | dir
|
|---|
| 21 | directory from which files will be taken -- only used if
|
|---|
| 22 | 'allfiles' not supplied to constructor
|
|---|
| 23 | files
|
|---|
| 24 | list of filenames currently being built/filtered/manipulated
|
|---|
| 25 | allfiles
|
|---|
| 26 | complete list of files under consideration (ie. without any
|
|---|
| 27 | filtering applied)
|
|---|
| 28 | """
|
|---|
| 29 |
|
|---|
| 30 | def __init__(self, warn=None, debug_print=None):
|
|---|
| 31 | # ignore argument to FileList, but keep them for backwards
|
|---|
| 32 | # compatibility
|
|---|
| 33 | self.allfiles = None
|
|---|
| 34 | self.files = []
|
|---|
| 35 |
|
|---|
| 36 | def set_allfiles(self, allfiles):
|
|---|
| 37 | self.allfiles = allfiles
|
|---|
| 38 |
|
|---|
| 39 | def findall(self, dir=os.curdir):
|
|---|
| 40 | self.allfiles = findall(dir)
|
|---|
| 41 |
|
|---|
| 42 | def debug_print(self, msg):
|
|---|
| 43 | """Print 'msg' to stdout if the global DEBUG (taken from the
|
|---|
| 44 | DISTUTILS_DEBUG environment variable) flag is true.
|
|---|
| 45 | """
|
|---|
| 46 | from distutils.debug import DEBUG
|
|---|
| 47 | if DEBUG:
|
|---|
| 48 | print msg
|
|---|
| 49 |
|
|---|
| 50 | # -- List-like methods ---------------------------------------------
|
|---|
| 51 |
|
|---|
| 52 | def append(self, item):
|
|---|
| 53 | self.files.append(item)
|
|---|
| 54 |
|
|---|
| 55 | def extend(self, items):
|
|---|
| 56 | self.files.extend(items)
|
|---|
| 57 |
|
|---|
| 58 | def sort(self):
|
|---|
| 59 | # Not a strict lexical sort!
|
|---|
| 60 | sortable_files = map(os.path.split, self.files)
|
|---|
| 61 | sortable_files.sort()
|
|---|
| 62 | self.files = []
|
|---|
| 63 | for sort_tuple in sortable_files:
|
|---|
| 64 | self.files.append(os.path.join(*sort_tuple))
|
|---|
| 65 |
|
|---|
| 66 |
|
|---|
| 67 | # -- Other miscellaneous utility methods ---------------------------
|
|---|
| 68 |
|
|---|
| 69 | def remove_duplicates(self):
|
|---|
| 70 | # Assumes list has been sorted!
|
|---|
| 71 | for i in range(len(self.files) - 1, 0, -1):
|
|---|
| 72 | if self.files[i] == self.files[i - 1]:
|
|---|
| 73 | del self.files[i]
|
|---|
| 74 |
|
|---|
| 75 |
|
|---|
| 76 | # -- "File template" methods ---------------------------------------
|
|---|
| 77 |
|
|---|
| 78 | def _parse_template_line(self, line):
|
|---|
| 79 | words = line.split()
|
|---|
| 80 | action = words[0]
|
|---|
| 81 |
|
|---|
| 82 | patterns = dir = dir_pattern = None
|
|---|
| 83 |
|
|---|
| 84 | if action in ('include', 'exclude',
|
|---|
| 85 | 'global-include', 'global-exclude'):
|
|---|
| 86 | if len(words) < 2:
|
|---|
| 87 | raise DistutilsTemplateError, \
|
|---|
| 88 | "'%s' expects <pattern1> <pattern2> ..." % action
|
|---|
| 89 |
|
|---|
| 90 | patterns = map(convert_path, words[1:])
|
|---|
| 91 |
|
|---|
| 92 | elif action in ('recursive-include', 'recursive-exclude'):
|
|---|
| 93 | if len(words) < 3:
|
|---|
| 94 | raise DistutilsTemplateError, \
|
|---|
| 95 | "'%s' expects <dir> <pattern1> <pattern2> ..." % action
|
|---|
| 96 |
|
|---|
| 97 | dir = convert_path(words[1])
|
|---|
| 98 | patterns = map(convert_path, words[2:])
|
|---|
| 99 |
|
|---|
| 100 | elif action in ('graft', 'prune'):
|
|---|
| 101 | if len(words) != 2:
|
|---|
| 102 | raise DistutilsTemplateError, \
|
|---|
| 103 | "'%s' expects a single <dir_pattern>" % action
|
|---|
| 104 |
|
|---|
| 105 | dir_pattern = convert_path(words[1])
|
|---|
| 106 |
|
|---|
| 107 | else:
|
|---|
| 108 | raise DistutilsTemplateError, "unknown action '%s'" % action
|
|---|
| 109 |
|
|---|
| 110 | return (action, patterns, dir, dir_pattern)
|
|---|
| 111 |
|
|---|
| 112 | def process_template_line(self, line):
|
|---|
| 113 | # Parse the line: split it up, make sure the right number of words
|
|---|
| 114 | # is there, and return the relevant words. 'action' is always
|
|---|
| 115 | # defined: it's the first word of the line. Which of the other
|
|---|
| 116 | # three are defined depends on the action; it'll be either
|
|---|
| 117 | # patterns, (dir and patterns), or (dir_pattern).
|
|---|
| 118 | action, patterns, dir, dir_pattern = self._parse_template_line(line)
|
|---|
| 119 |
|
|---|
| 120 | # OK, now we know that the action is valid and we have the
|
|---|
| 121 | # right number of words on the line for that action -- so we
|
|---|
| 122 | # can proceed with minimal error-checking.
|
|---|
| 123 | if action == 'include':
|
|---|
| 124 | self.debug_print("include " + ' '.join(patterns))
|
|---|
| 125 | for pattern in patterns:
|
|---|
| 126 | if not self.include_pattern(pattern, anchor=1):
|
|---|
| 127 | log.warn("warning: no files found matching '%s'",
|
|---|
| 128 | pattern)
|
|---|
| 129 |
|
|---|
| 130 | elif action == 'exclude':
|
|---|
| 131 | self.debug_print("exclude " + ' '.join(patterns))
|
|---|
| 132 | for pattern in patterns:
|
|---|
| 133 | if not self.exclude_pattern(pattern, anchor=1):
|
|---|
| 134 | log.warn(("warning: no previously-included files "
|
|---|
| 135 | "found matching '%s'"), pattern)
|
|---|
| 136 |
|
|---|
| 137 | elif action == 'global-include':
|
|---|
| 138 | self.debug_print("global-include " + ' '.join(patterns))
|
|---|
| 139 | for pattern in patterns:
|
|---|
| 140 | if not self.include_pattern(pattern, anchor=0):
|
|---|
| 141 | log.warn(("warning: no files found matching '%s' " +
|
|---|
| 142 | "anywhere in distribution"), pattern)
|
|---|
| 143 |
|
|---|
| 144 | elif action == 'global-exclude':
|
|---|
| 145 | self.debug_print("global-exclude " + ' '.join(patterns))
|
|---|
| 146 | for pattern in patterns:
|
|---|
| 147 | if not self.exclude_pattern(pattern, anchor=0):
|
|---|
| 148 | log.warn(("warning: no previously-included files matching "
|
|---|
| 149 | "'%s' found anywhere in distribution"),
|
|---|
| 150 | pattern)
|
|---|
| 151 |
|
|---|
| 152 | elif action == 'recursive-include':
|
|---|
| 153 | self.debug_print("recursive-include %s %s" %
|
|---|
| 154 | (dir, ' '.join(patterns)))
|
|---|
| 155 | for pattern in patterns:
|
|---|
| 156 | if not self.include_pattern(pattern, prefix=dir):
|
|---|
| 157 | log.warn(("warning: no files found matching '%s' " +
|
|---|
| 158 | "under directory '%s'"),
|
|---|
| 159 | pattern, dir)
|
|---|
| 160 |
|
|---|
| 161 | elif action == 'recursive-exclude':
|
|---|
| 162 | self.debug_print("recursive-exclude %s %s" %
|
|---|
| 163 | (dir, ' '.join(patterns)))
|
|---|
| 164 | for pattern in patterns:
|
|---|
| 165 | if not self.exclude_pattern(pattern, prefix=dir):
|
|---|
| 166 | log.warn(("warning: no previously-included files matching "
|
|---|
| 167 | "'%s' found under directory '%s'"),
|
|---|
| 168 | pattern, dir)
|
|---|
| 169 |
|
|---|
| 170 | elif action == 'graft':
|
|---|
| 171 | self.debug_print("graft " + dir_pattern)
|
|---|
| 172 | if not self.include_pattern(None, prefix=dir_pattern):
|
|---|
| 173 | log.warn("warning: no directories found matching '%s'",
|
|---|
| 174 | dir_pattern)
|
|---|
| 175 |
|
|---|
| 176 | elif action == 'prune':
|
|---|
| 177 | self.debug_print("prune " + dir_pattern)
|
|---|
| 178 | if not self.exclude_pattern(None, prefix=dir_pattern):
|
|---|
| 179 | log.warn(("no previously-included directories found " +
|
|---|
| 180 | "matching '%s'"), dir_pattern)
|
|---|
| 181 | else:
|
|---|
| 182 | raise DistutilsInternalError, \
|
|---|
| 183 | "this cannot happen: invalid action '%s'" % action
|
|---|
| 184 |
|
|---|
| 185 | # -- Filtering/selection methods -----------------------------------
|
|---|
| 186 |
|
|---|
| 187 | def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
|
|---|
| 188 | """Select strings (presumably filenames) from 'self.files' that
|
|---|
| 189 | match 'pattern', a Unix-style wildcard (glob) pattern.
|
|---|
| 190 |
|
|---|
| 191 | Patterns are not quite the same as implemented by the 'fnmatch'
|
|---|
| 192 | module: '*' and '?' match non-special characters, where "special"
|
|---|
| 193 | is platform-dependent: slash on Unix; colon, slash, and backslash on
|
|---|
| 194 | DOS/Windows; and colon on Mac OS.
|
|---|
| 195 |
|
|---|
| 196 | If 'anchor' is true (the default), then the pattern match is more
|
|---|
| 197 | stringent: "*.py" will match "foo.py" but not "foo/bar.py". If
|
|---|
| 198 | 'anchor' is false, both of these will match.
|
|---|
| 199 |
|
|---|
| 200 | If 'prefix' is supplied, then only filenames starting with 'prefix'
|
|---|
| 201 | (itself a pattern) and ending with 'pattern', with anything in between
|
|---|
| 202 | them, will match. 'anchor' is ignored in this case.
|
|---|
| 203 |
|
|---|
| 204 | If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and
|
|---|
| 205 | 'pattern' is assumed to be either a string containing a regex or a
|
|---|
| 206 | regex object -- no translation is done, the regex is just compiled
|
|---|
| 207 | and used as-is.
|
|---|
| 208 |
|
|---|
| 209 | Selected strings will be added to self.files.
|
|---|
| 210 |
|
|---|
| 211 | Return 1 if files are found.
|
|---|
| 212 | """
|
|---|
| 213 | # XXX docstring lying about what the special chars are?
|
|---|
| 214 | files_found = 0
|
|---|
| 215 | pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
|
|---|
| 216 | self.debug_print("include_pattern: applying regex r'%s'" %
|
|---|
| 217 | pattern_re.pattern)
|
|---|
| 218 |
|
|---|
| 219 | # delayed loading of allfiles list
|
|---|
| 220 | if self.allfiles is None:
|
|---|
| 221 | self.findall()
|
|---|
| 222 |
|
|---|
| 223 | for name in self.allfiles:
|
|---|
| 224 | if pattern_re.search(name):
|
|---|
| 225 | self.debug_print(" adding " + name)
|
|---|
| 226 | self.files.append(name)
|
|---|
| 227 | files_found = 1
|
|---|
| 228 |
|
|---|
| 229 | return files_found
|
|---|
| 230 |
|
|---|
| 231 |
|
|---|
| 232 | def exclude_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
|
|---|
| 233 | """Remove strings (presumably filenames) from 'files' that match
|
|---|
| 234 | 'pattern'.
|
|---|
| 235 |
|
|---|
| 236 | Other parameters are the same as for 'include_pattern()', above.
|
|---|
| 237 | The list 'self.files' is modified in place. Return 1 if files are
|
|---|
| 238 | found.
|
|---|
| 239 | """
|
|---|
| 240 | files_found = 0
|
|---|
| 241 | pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
|
|---|
| 242 | self.debug_print("exclude_pattern: applying regex r'%s'" %
|
|---|
| 243 | pattern_re.pattern)
|
|---|
| 244 | for i in range(len(self.files)-1, -1, -1):
|
|---|
| 245 | if pattern_re.search(self.files[i]):
|
|---|
| 246 | self.debug_print(" removing " + self.files[i])
|
|---|
| 247 | del self.files[i]
|
|---|
| 248 | files_found = 1
|
|---|
| 249 |
|
|---|
| 250 | return files_found
|
|---|
| 251 |
|
|---|
| 252 |
|
|---|
| 253 | # ----------------------------------------------------------------------
|
|---|
| 254 | # Utility functions
|
|---|
| 255 |
|
|---|
| 256 | def findall(dir = os.curdir):
|
|---|
| 257 | """Find all files under 'dir' and return the list of full filenames
|
|---|
| 258 | (relative to 'dir').
|
|---|
| 259 | """
|
|---|
| 260 | from stat import ST_MODE, S_ISREG, S_ISDIR, S_ISLNK
|
|---|
| 261 |
|
|---|
| 262 | list = []
|
|---|
| 263 | stack = [dir]
|
|---|
| 264 | pop = stack.pop
|
|---|
| 265 | push = stack.append
|
|---|
| 266 |
|
|---|
| 267 | while stack:
|
|---|
| 268 | dir = pop()
|
|---|
| 269 | names = os.listdir(dir)
|
|---|
| 270 |
|
|---|
| 271 | for name in names:
|
|---|
| 272 | if dir != os.curdir: # avoid the dreaded "./" syndrome
|
|---|
| 273 | fullname = os.path.join(dir, name)
|
|---|
| 274 | else:
|
|---|
| 275 | fullname = name
|
|---|
| 276 |
|
|---|
| 277 | # Avoid excess stat calls -- just one will do, thank you!
|
|---|
| 278 | stat = os.stat(fullname)
|
|---|
| 279 | mode = stat[ST_MODE]
|
|---|
| 280 | if S_ISREG(mode):
|
|---|
| 281 | list.append(fullname)
|
|---|
| 282 | elif S_ISDIR(mode) and not S_ISLNK(mode):
|
|---|
| 283 | push(fullname)
|
|---|
| 284 |
|
|---|
| 285 | return list
|
|---|
| 286 |
|
|---|
| 287 |
|
|---|
| 288 | def glob_to_re(pattern):
|
|---|
| 289 | """Translate a shell-like glob pattern to a regular expression.
|
|---|
| 290 |
|
|---|
| 291 | Return a string containing the regex. Differs from
|
|---|
| 292 | 'fnmatch.translate()' in that '*' does not match "special characters"
|
|---|
| 293 | (which are platform-specific).
|
|---|
| 294 | """
|
|---|
| 295 | pattern_re = fnmatch.translate(pattern)
|
|---|
| 296 |
|
|---|
| 297 | # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which
|
|---|
| 298 | # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
|
|---|
| 299 | # and by extension they shouldn't match such "special characters" under
|
|---|
| 300 | # any OS. So change all non-escaped dots in the RE to match any
|
|---|
| 301 | # character except the special characters (currently: just os.sep).
|
|---|
| 302 | sep = os.sep
|
|---|
| 303 | if os.sep == '\\':
|
|---|
| 304 | # we're using a regex to manipulate a regex, so we need
|
|---|
| 305 | # to escape the backslash twice
|
|---|
| 306 | sep = r'\\\\'
|
|---|
| 307 | escaped = r'\1[^%s]' % sep
|
|---|
| 308 | pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)
|
|---|
| 309 | return pattern_re
|
|---|
| 310 |
|
|---|
| 311 |
|
|---|
| 312 | def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0):
|
|---|
| 313 | """Translate a shell-like wildcard pattern to a compiled regular
|
|---|
| 314 | expression.
|
|---|
| 315 |
|
|---|
| 316 | Return the compiled regex. If 'is_regex' true,
|
|---|
| 317 | then 'pattern' is directly compiled to a regex (if it's a string)
|
|---|
| 318 | or just returned as-is (assumes it's a regex object).
|
|---|
| 319 | """
|
|---|
| 320 | if is_regex:
|
|---|
| 321 | if isinstance(pattern, str):
|
|---|
| 322 | return re.compile(pattern)
|
|---|
| 323 | else:
|
|---|
| 324 | return pattern
|
|---|
| 325 |
|
|---|
| 326 | if pattern:
|
|---|
| 327 | pattern_re = glob_to_re(pattern)
|
|---|
| 328 | else:
|
|---|
| 329 | pattern_re = ''
|
|---|
| 330 |
|
|---|
| 331 | if prefix is not None:
|
|---|
| 332 | # ditch end of pattern character
|
|---|
| 333 | empty_pattern = glob_to_re('')
|
|---|
| 334 | prefix_re = glob_to_re(prefix)[:-len(empty_pattern)]
|
|---|
| 335 | sep = os.sep
|
|---|
| 336 | if os.sep == '\\':
|
|---|
| 337 | sep = r'\\'
|
|---|
| 338 | pattern_re = "^" + sep.join((prefix_re, ".*" + pattern_re))
|
|---|
| 339 | else: # no prefix -- respect anchor flag
|
|---|
| 340 | if anchor:
|
|---|
| 341 | pattern_re = "^" + pattern_re
|
|---|
| 342 |
|
|---|
| 343 | return re.compile(pattern_re)
|
|---|
