Coverage for /home/anselor/src/cmd2/cmd2/parsing.py : 98%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
# # -*- coding: utf-8 -*-
"""String subclass with additional attributes to store the results of parsing.
The cmd module in the standard library passes commands around as a string. To retain backwards compatibility, cmd2 does the same. However, we need a place to capture the additional output of the command parsing, so we add our own attributes to this subclass.
The string portion of the class contains the arguments, but not the command, nor the output redirection clauses.
:var raw: string containing exactly what we input by the user :type raw: str :var command: the command, i.e. the first whitespace delimited word :type command: str or None :var multiline_command: if the command is a multiline command, the name of the command, otherwise None :type command: str or None :var args: the arguments to the command, not including any output redirection or terminators. quoted arguments remain quoted. :type args: str or None :var: argv: a list of arguments a la sys.argv. Quotes, if any, are removed from the elements of the list, and aliases and shortcuts are expanded :type argv: list :var terminator: the charater which terminated the multiline command, if there was one :type terminator: str or None :var suffix: characters appearing after the terminator but before output redirection, if any :type suffix: str or None :var pipe_to: if output was piped to a shell command, the shell command as a list of tokens :type pipe_to: list :var output: if output was redirected, the redirection token, i.e. '>>' :type output: str or None :var output_to: if output was redirected, the destination, usually a filename :type output_to: str or None
"""
def command_and_args(self): """Combine command and args with a space separating them.
Quoted arguments remain quoted. """ # we are trusting that if we get here that self.args is None else:
"""Parse raw text into command components.
Shortcuts is a list of tuples with each tuple containing the shortcut and the expansion. """ self, allow_redirection=True, terminators=None, multiline_commands=None, aliases=None, shortcuts=None, ): self.terminators = [';'] else: self.multiline_commands = [] else: self.aliases = {} else: self.shortcuts = [] else:
# this regular expression matches C-style comments and quoted # strings, i.e. stuff between single or double quote marks # it's used with _comment_replacer() to strip out the C-style # comments, while leaving C-style comments that are inside either # double or single quotes. # # this big regular expression can be broken down into 3 regular # expressions that are OR'ed together. # # /\*.*?(\*/|$) matches C-style comments, with an optional # closing '*/'. The optional closing '*/' is # there to retain backward compatibility with # the pyparsing implementation of cmd2 < 0.9.0 # \'(?:\\.|[^\\\'])*\' matches a single quoted string, allowing # for embedded backslash escaped single quote # marks # "(?:\\.|[^\\"])*" matches a double quoted string, allowing # for embedded backslash escaped double quote # marks # # by way of reminder the (?:...) regular expression syntax is just # a non-capturing version of regular parenthesis. We need the non- # capturing syntax because _comment_replacer() looks at match # groups r'/\*.*?(\*/|$)|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE )
# commands have to be a word, so make a regular expression # that matches the first word in the line. This regex has three # parts: # - the '\A\s*' matches the beginning of the string (even # if contains multiple lines) and gobbles up any leading # whitespace # - the first parenthesis enclosed group matches one # or more non-whitespace characters with a non-greedy match # (that's what the '+?' part does). The non-greedy match # ensures that this first group doesn't include anything # matched by the second group # - the second parenthesis group must be dynamically created # because it needs to match either whitespace, something in # REDIRECTION_CHARS, one of the terminators, or the end of # the string (\Z matches the end of the string even if it # contains multiple lines) # # escape each item so it will for sure get treated as a literal # add the whitespace and end of string, not escaped because they # are not literals # join them up with a pipe # build the regular expression
"""Determine whether a word is a valid alias.
Aliases can not include redirection characters, whitespace, or termination characters.
If word is not a valid command, return False and a comma separated string of characters that can not appear in a command. This string is suitable for inclusion in an error message of your choice:
valid, invalidchars = statement_parser.is_valid_command('>') if not valid: errmsg = "Aliases can not contain: {}".format(invalidchars) """
"""Lex a string into a list of tokens.
Comments are removed, and shortcuts and aliases are expanded.
Raises ValueError if there are unclosed quotation marks. """
# strip C-style comments # shlex will handle the python/shell style comments for us
# expand shortcuts and aliases
# split on whitespace
# custom lexing
"""Tokenize the input and parse it into a Statement object, stripping comments, expanding aliases and shortcuts, and extracting output redirection directives.
Raises ValueError if there are unclosed quotation marks. """
# handle the special case/hardcoded terminator of a blank line # we have to do this before we tokenize because tokenizing # destroys all unquoted whitespace in the input
# lex the input into a list of tokens
# of the valid terminators, find the first one to occur in the input # break the inner loop, and we want to break the # outer loop too else: # this else clause is only run if the inner loop # didn't execute a break. If it didn't, then # continue to the next iteration of the outer loop # inner loop was broken, break the outer
# everything before the first terminator is the command and the args # we will set the suffix later # remove all the tokens before and including the terminator else: # no terminator on this line but we have a multiline command # everything else on the line is part of the args # because redirectors can only be after a terminator
# check for a pipe to a shell process # if there is a pipe, everything after the pipe needs to be passed # to the shell, even redirected output # this allows '(Cmd) say hello | wc > countit.txt' # find the first pipe if it exists # save everything after the first pipe as tokens # remove all the tokens after the pipe # no pipe in the tokens
# check for output redirect # remove all the tokens after the output redirect
# remove all tokens after the output redirect
# whatever is left is the suffix else: # no terminator, so whatever is left is the command and the args # command could already have been set, if so, don't set it again
# set multiline else:
# build the statement # string representation of args must be an empty string instead of # None for compatibility with standard library cmd # if there are no args we will use None since we don't have to worry # about compatibility with standard library cmd
"""Partially parse input into a Statement object.
The command is identified, and shortcuts and aliases are expanded. Terminators, multiline commands, and output redirection are not parsed.
This method is used by tab completion code and therefore must not generate an exception if there are unclosed quotes.
The Statement object returned by this method can at most contained values in the following attributes: - raw - command - args
Different from parse(), this method does not remove redundant whitespace within statement.args. It does however, ensure args does not have leading or trailing whitespace. """ # expand shortcuts and aliases
# we got a match, extract the command # the match could be an empty string, if so, turn it into none # the _command_pattern regex is designed to match the spaces # between command and args with a second match group. Using # the end of the second match group ensures that args has # no leading whitespace. The rstrip() makes sure there is # no trailing whitespace # if the command is none that means the input was either empty # or something wierd like '>'. args should be None if we couldn't # parse a command
# build the statement # string representation of args must be an empty string instead of # None for compatibility with standard library cmd
"""Expand shortcuts and aliases"""
# expand aliases # make a copy of aliases so we can edit it # apply our regex to line # we got a match, extract the command # rebuild line with the expanded alias
# expand shortcuts # If the next character after the shortcut isn't a space, then insert one
# Expand the shortcut
"""Given a list of tokens, return a tuple of the command and the args as a string.
The args string will be '' instead of None to retain backwards compatibility with cmd in the standard library. """
def _comment_replacer(match): # the matched string was a comment, so remove it # the matched string was a quoted string, return the match
""" # Further splits tokens from a command line using punctuation characters # as word breaks when they are in unquoted strings. Each run of punctuation # characters is treated as a single token.
:param tokens: the tokens as parsed by shlex :return: the punctuated tokens """
# Save tokens up to 1 character in length or quoted tokens. No need to parse these.
# Iterate over each character in this token
# Keep track of the token we are building
# Keep appending to new_token until we hit a punctuation char else:
else:
# Keep appending to new_token until we hit something other than cur_punc else:
# Save the new token
# Check if we've viewed all characters
|