| 1 | # -*- coding: iso-8859-1 -*-
|
|---|
| 2 | """A lexical analyzer class for simple shell-like syntaxes."""
|
|---|
| 3 |
|
|---|
| 4 | # Module and documentation by Eric S. Raymond, 21 Dec 1998
|
|---|
| 5 | # Input stacking and error message cleanup added by ESR, March 2000
|
|---|
| 6 | # push_source() and pop_source() made explicit by ESR, January 2001.
|
|---|
| 7 | # Posix compliance, split(), string arguments, and
|
|---|
| 8 | # iterator interface by Gustavo Niemeyer, April 2003.
|
|---|
| 9 |
|
|---|
| 10 | import os.path
|
|---|
| 11 | import sys
|
|---|
| 12 | from collections import deque
|
|---|
| 13 |
|
|---|
| 14 | try:
|
|---|
| 15 | from cStringIO import StringIO
|
|---|
| 16 | except ImportError:
|
|---|
| 17 | from StringIO import StringIO
|
|---|
| 18 |
|
|---|
| 19 | __all__ = ["shlex", "split"]
|
|---|
| 20 |
|
|---|
| 21 | class shlex:
|
|---|
| 22 | "A lexical analyzer class for simple shell-like syntaxes."
|
|---|
| 23 | def __init__(self, instream=None, infile=None, posix=False):
|
|---|
| 24 | if isinstance(instream, basestring):
|
|---|
| 25 | instream = StringIO(instream)
|
|---|
| 26 | if instream is not None:
|
|---|
| 27 | self.instream = instream
|
|---|
| 28 | self.infile = infile
|
|---|
| 29 | else:
|
|---|
| 30 | self.instream = sys.stdin
|
|---|
| 31 | self.infile = None
|
|---|
| 32 | self.posix = posix
|
|---|
| 33 | if posix:
|
|---|
| 34 | self.eof = None
|
|---|
| 35 | else:
|
|---|
| 36 | self.eof = ''
|
|---|
| 37 | self.commenters = '#'
|
|---|
| 38 | self.wordchars = ('abcdfeghijklmnopqrstuvwxyz'
|
|---|
| 39 | 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')
|
|---|
| 40 | if self.posix:
|
|---|
| 41 | self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ'
|
|---|
| 42 | 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')
|
|---|
| 43 | self.whitespace = ' \t\r\n'
|
|---|
| 44 | self.whitespace_split = False
|
|---|
| 45 | self.quotes = '\'"'
|
|---|
| 46 | self.escape = '\\'
|
|---|
| 47 | self.escapedquotes = '"'
|
|---|
| 48 | self.state = ' '
|
|---|
| 49 | self.pushback = deque()
|
|---|
| 50 | self.lineno = 1
|
|---|
| 51 | self.debug = 0
|
|---|
| 52 | self.token = ''
|
|---|
| 53 | self.filestack = deque()
|
|---|
| 54 | self.source = None
|
|---|
| 55 | if self.debug:
|
|---|
| 56 | print 'shlex: reading from %s, line %d' \
|
|---|
| 57 | % (self.instream, self.lineno)
|
|---|
| 58 |
|
|---|
| 59 | def push_token(self, tok):
|
|---|
| 60 | "Push a token onto the stack popped by the get_token method"
|
|---|
| 61 | if self.debug >= 1:
|
|---|
| 62 | print "shlex: pushing token " + repr(tok)
|
|---|
| 63 | self.pushback.appendleft(tok)
|
|---|
| 64 |
|
|---|
| 65 | def push_source(self, newstream, newfile=None):
|
|---|
| 66 | "Push an input source onto the lexer's input source stack."
|
|---|
| 67 | if isinstance(newstream, basestring):
|
|---|
| 68 | newstream = StringIO(newstream)
|
|---|
| 69 | self.filestack.appendleft((self.infile, self.instream, self.lineno))
|
|---|
| 70 | self.infile = newfile
|
|---|
| 71 | self.instream = newstream
|
|---|
| 72 | self.lineno = 1
|
|---|
| 73 | if self.debug:
|
|---|
| 74 | if newfile is not None:
|
|---|
| 75 | print 'shlex: pushing to file %s' % (self.infile,)
|
|---|
| 76 | else:
|
|---|
| 77 | print 'shlex: pushing to stream %s' % (self.instream,)
|
|---|
| 78 |
|
|---|
| 79 | def pop_source(self):
|
|---|
| 80 | "Pop the input source stack."
|
|---|
| 81 | self.instream.close()
|
|---|
| 82 | (self.infile, self.instream, self.lineno) = self.filestack.popleft()
|
|---|
| 83 | if self.debug:
|
|---|
| 84 | print 'shlex: popping to %s, line %d' \
|
|---|
| 85 | % (self.instream, self.lineno)
|
|---|
| 86 | self.state = ' '
|
|---|
| 87 |
|
|---|
| 88 | def get_token(self):
|
|---|
| 89 | "Get a token from the input stream (or from stack if it's nonempty)"
|
|---|
| 90 | if self.pushback:
|
|---|
| 91 | tok = self.pushback.popleft()
|
|---|
| 92 | if self.debug >= 1:
|
|---|
| 93 | print "shlex: popping token " + repr(tok)
|
|---|
| 94 | return tok
|
|---|
| 95 | # No pushback. Get a token.
|
|---|
| 96 | raw = self.read_token()
|
|---|
| 97 | # Handle inclusions
|
|---|
| 98 | if self.source is not None:
|
|---|
| 99 | while raw == self.source:
|
|---|
| 100 | spec = self.sourcehook(self.read_token())
|
|---|
| 101 | if spec:
|
|---|
| 102 | (newfile, newstream) = spec
|
|---|
| 103 | self.push_source(newstream, newfile)
|
|---|
| 104 | raw = self.get_token()
|
|---|
| 105 | # Maybe we got EOF instead?
|
|---|
| 106 | while raw == self.eof:
|
|---|
| 107 | if not self.filestack:
|
|---|
| 108 | return self.eof
|
|---|
| 109 | else:
|
|---|
| 110 | self.pop_source()
|
|---|
| 111 | raw = self.get_token()
|
|---|
| 112 | # Neither inclusion nor EOF
|
|---|
| 113 | if self.debug >= 1:
|
|---|
| 114 | if raw != self.eof:
|
|---|
| 115 | print "shlex: token=" + repr(raw)
|
|---|
| 116 | else:
|
|---|
| 117 | print "shlex: token=EOF"
|
|---|
| 118 | return raw
|
|---|
| 119 |
|
|---|
| 120 | def read_token(self):
|
|---|
| 121 | quoted = False
|
|---|
| 122 | escapedstate = ' '
|
|---|
| 123 | while True:
|
|---|
| 124 | nextchar = self.instream.read(1)
|
|---|
| 125 | if nextchar == '\n':
|
|---|
| 126 | self.lineno = self.lineno + 1
|
|---|
| 127 | if self.debug >= 3:
|
|---|
| 128 | print "shlex: in state", repr(self.state), \
|
|---|
| 129 | "I see character:", repr(nextchar)
|
|---|
| 130 | if self.state is None:
|
|---|
| 131 | self.token = '' # past end of file
|
|---|
| 132 | break
|
|---|
| 133 | elif self.state == ' ':
|
|---|
| 134 | if not nextchar:
|
|---|
| 135 | self.state = None # end of file
|
|---|
| 136 | break
|
|---|
| 137 | elif nextchar in self.whitespace:
|
|---|
| 138 | if self.debug >= 2:
|
|---|
| 139 | print "shlex: I see whitespace in whitespace state"
|
|---|
| 140 | if self.token or (self.posix and quoted):
|
|---|
| 141 | break # emit current token
|
|---|
| 142 | else:
|
|---|
| 143 | continue
|
|---|
| 144 | elif nextchar in self.commenters:
|
|---|
| 145 | self.instream.readline()
|
|---|
| 146 | self.lineno = self.lineno + 1
|
|---|
| 147 | elif self.posix and nextchar in self.escape:
|
|---|
| 148 | escapedstate = 'a'
|
|---|
| 149 | self.state = nextchar
|
|---|
| 150 | elif nextchar in self.wordchars:
|
|---|
| 151 | self.token = nextchar
|
|---|
| 152 | self.state = 'a'
|
|---|
| 153 | elif nextchar in self.quotes:
|
|---|
| 154 | if not self.posix:
|
|---|
| 155 | self.token = nextchar
|
|---|
| 156 | self.state = nextchar
|
|---|
| 157 | elif self.whitespace_split:
|
|---|
| 158 | self.token = nextchar
|
|---|
| 159 | self.state = 'a'
|
|---|
| 160 | else:
|
|---|
| 161 | self.token = nextchar
|
|---|
| 162 | if self.token or (self.posix and quoted):
|
|---|
| 163 | break # emit current token
|
|---|
| 164 | else:
|
|---|
| 165 | continue
|
|---|
| 166 | elif self.state in self.quotes:
|
|---|
| 167 | quoted = True
|
|---|
| 168 | if not nextchar: # end of file
|
|---|
| 169 | if self.debug >= 2:
|
|---|
| 170 | print "shlex: I see EOF in quotes state"
|
|---|
| 171 | # XXX what error should be raised here?
|
|---|
| 172 | raise ValueError, "No closing quotation"
|
|---|
| 173 | if nextchar == self.state:
|
|---|
| 174 | if not self.posix:
|
|---|
| 175 | self.token = self.token + nextchar
|
|---|
| 176 | self.state = ' '
|
|---|
| 177 | break
|
|---|
| 178 | else:
|
|---|
| 179 | self.state = 'a'
|
|---|
| 180 | elif self.posix and nextchar in self.escape and \
|
|---|
| 181 | self.state in self.escapedquotes:
|
|---|
| 182 | escapedstate = self.state
|
|---|
| 183 | self.state = nextchar
|
|---|
| 184 | else:
|
|---|
| 185 | self.token = self.token + nextchar
|
|---|
| 186 | elif self.state in self.escape:
|
|---|
| 187 | if not nextchar: # end of file
|
|---|
| 188 | if self.debug >= 2:
|
|---|
| 189 | print "shlex: I see EOF in escape state"
|
|---|
| 190 | # XXX what error should be raised here?
|
|---|
| 191 | raise ValueError, "No escaped character"
|
|---|
| 192 | # In posix shells, only the quote itself or the escape
|
|---|
| 193 | # character may be escaped within quotes.
|
|---|
| 194 | if escapedstate in self.quotes and \
|
|---|
| 195 | nextchar != self.state and nextchar != escapedstate:
|
|---|
| 196 | self.token = self.token + self.state
|
|---|
| 197 | self.token = self.token + nextchar
|
|---|
| 198 | self.state = escapedstate
|
|---|
| 199 | elif self.state == 'a':
|
|---|
| 200 | if not nextchar:
|
|---|
| 201 | self.state = None # end of file
|
|---|
| 202 | break
|
|---|
| 203 | elif nextchar in self.whitespace:
|
|---|
| 204 | if self.debug >= 2:
|
|---|
| 205 | print "shlex: I see whitespace in word state"
|
|---|
| 206 | self.state = ' '
|
|---|
| 207 | if self.token or (self.posix and quoted):
|
|---|
| 208 | break # emit current token
|
|---|
| 209 | else:
|
|---|
| 210 | continue
|
|---|
| 211 | elif nextchar in self.commenters:
|
|---|
| 212 | self.instream.readline()
|
|---|
| 213 | self.lineno = self.lineno + 1
|
|---|
| 214 | if self.posix:
|
|---|
| 215 | self.state = ' '
|
|---|
| 216 | if self.token or (self.posix and quoted):
|
|---|
| 217 | break # emit current token
|
|---|
| 218 | else:
|
|---|
| 219 | continue
|
|---|
| 220 | elif self.posix and nextchar in self.quotes:
|
|---|
| 221 | self.state = nextchar
|
|---|
| 222 | elif self.posix and nextchar in self.escape:
|
|---|
| 223 | escapedstate = 'a'
|
|---|
| 224 | self.state = nextchar
|
|---|
| 225 | elif nextchar in self.wordchars or nextchar in self.quotes \
|
|---|
| 226 | or self.whitespace_split:
|
|---|
| 227 | self.token = self.token + nextchar
|
|---|
| 228 | else:
|
|---|
| 229 | self.pushback.appendleft(nextchar)
|
|---|
| 230 | if self.debug >= 2:
|
|---|
| 231 | print "shlex: I see punctuation in word state"
|
|---|
| 232 | self.state = ' '
|
|---|
| 233 | if self.token:
|
|---|
| 234 | break # emit current token
|
|---|
| 235 | else:
|
|---|
| 236 | continue
|
|---|
| 237 | result = self.token
|
|---|
| 238 | self.token = ''
|
|---|
| 239 | if self.posix and not quoted and result == '':
|
|---|
| 240 | result = None
|
|---|
| 241 | if self.debug > 1:
|
|---|
| 242 | if result:
|
|---|
| 243 | print "shlex: raw token=" + repr(result)
|
|---|
| 244 | else:
|
|---|
| 245 | print "shlex: raw token=EOF"
|
|---|
| 246 | return result
|
|---|
| 247 |
|
|---|
| 248 | def sourcehook(self, newfile):
|
|---|
| 249 | "Hook called on a filename to be sourced."
|
|---|
| 250 | if newfile[0] == '"':
|
|---|
| 251 | newfile = newfile[1:-1]
|
|---|
| 252 | # This implements cpp-like semantics for relative-path inclusion.
|
|---|
| 253 | if isinstance(self.infile, basestring) and not os.path.isabs(newfile):
|
|---|
| 254 | newfile = os.path.join(os.path.dirname(self.infile), newfile)
|
|---|
| 255 | return (newfile, open(newfile, "r"))
|
|---|
| 256 |
|
|---|
| 257 | def error_leader(self, infile=None, lineno=None):
|
|---|
| 258 | "Emit a C-compiler-like, Emacs-friendly error-message leader."
|
|---|
| 259 | if infile is None:
|
|---|
| 260 | infile = self.infile
|
|---|
| 261 | if lineno is None:
|
|---|
| 262 | lineno = self.lineno
|
|---|
| 263 | return "\"%s\", line %d: " % (infile, lineno)
|
|---|
| 264 |
|
|---|
| 265 | def __iter__(self):
|
|---|
| 266 | return self
|
|---|
| 267 |
|
|---|
| 268 | def next(self):
|
|---|
| 269 | token = self.get_token()
|
|---|
| 270 | if token == self.eof:
|
|---|
| 271 | raise StopIteration
|
|---|
| 272 | return token
|
|---|
| 273 |
|
|---|
| 274 | def split(s, comments=False, posix=True):
|
|---|
| 275 | lex = shlex(s, posix=posix)
|
|---|
| 276 | lex.whitespace_split = True
|
|---|
| 277 | if not comments:
|
|---|
| 278 | lex.commenters = ''
|
|---|
| 279 | return list(lex)
|
|---|
| 280 |
|
|---|
| 281 | if __name__ == '__main__':
|
|---|
| 282 | if len(sys.argv) == 1:
|
|---|
| 283 | lexer = shlex()
|
|---|
| 284 | else:
|
|---|
| 285 | file = sys.argv[1]
|
|---|
| 286 | lexer = shlex(open(file), file)
|
|---|
| 287 | while 1:
|
|---|
| 288 | tt = lexer.get_token()
|
|---|
| 289 | if tt:
|
|---|
| 290 | print "Token: " + repr(tt)
|
|---|
| 291 | else:
|
|---|
| 292 | break
|
|---|
