diff --git a/pkg/shparse/shparse.go b/pkg/shparse/shparse.go index ec4e966e..acb21998 100644 --- a/pkg/shparse/shparse.go +++ b/pkg/shparse/shparse.go @@ -81,6 +81,8 @@ const ( WordTypeOp = "op" // single: & ; | ( ) < > \n multi(2): && || ;; << >> <& >& <> >| (( multi(3): <<- ('((' requires special processing) WordTypeKey = "key" // if then else elif fi do done case esac while until for in { } ! (( [[ WordTypeSimpleVar = "svar" // simplevar $ + WordTypeGroup = "grp" // contains other words e.g. "hello"foo'bar'$x + WordTypeArith = "ath" // each of these can also be used as an entry in quoteContext WordTypeDQ = "dq" // " @@ -129,11 +131,10 @@ type wordType struct { } func (c *parseContext) clone(pos int, newQuote string) *parseContext { - rtn := *c + rtn := parseContext{Input: c.Input[pos:], QC: c.QC} if newQuote != "" { - rtn.QC = append(rtn.QC, newQuote) + rtn.QC = rtn.QC.push(newQuote) } - rtn.Input = rtn.Input[pos:] return &rtn } @@ -264,12 +265,15 @@ func (c *parseContext) parseStrDQ() *wordType { if !c.match('"') { return nil } - newOffset, complete := c.skipToChar(1, '"', false) + newContext := c.clone(c.Pos+1, WordTypeDQ) + subWords, eofExit := newContext.tokenizeDQ() + newOffset := newContext.Pos + 1 w := &wordType{ Type: WordTypeDQ, Offset: c.Pos, Raw: c.Input[c.Pos : c.Pos+newOffset], - Complete: complete, + Complete: !eofExit, + Subs: subWords, } c.Pos = c.Pos + newOffset return w @@ -320,6 +324,19 @@ func (c *parseContext) parseStrDDQ() *wordType { return w } +func (c *parseContext) parseArith(mustComplete bool) *wordType { + if !c.match2('(', '(') { + return nil + } + newOffset, complete := c.skipToChar2(2, ')', ')', false) + if mustComplete && !complete { + return nil + } + w := &wordType{Type: WordTypeArith, Offset: c.Pos, Raw: c.Input[c.Pos : c.Pos+newOffset], Complete: complete} + c.Pos = c.Pos + newOffset + return w +} + func (c *parseContext) parseExpansion() *wordType { if !c.match('$') { return nil @@ -332,8 +349,12 @@ func (c *parseContext) parseExpansion() *wordType { } if c.match2('$', '(') { // subshell - newOffset, complete := c.skipToChar(2, ')', false) - w := &wordType{Type: WordTypeDP, Offset: c.Pos, Raw: c.Input[c.Pos : c.Pos+newOffset], Complete: complete} + newContext := c.clone(c.Pos+2, WordTypeDP) + subWords, eofExit := newContext.tokenizeRaw() + newOffset := newContext.Pos + 2 + // newOffset, complete := c.skipToChar(2, ')', false) + w := &wordType{Type: WordTypeDP, Offset: c.Pos, Raw: c.Input[c.Pos : c.Pos+newOffset], Complete: !eofExit} + w.Subs = subWords c.Pos = c.Pos + newOffset return w } @@ -346,8 +367,10 @@ func (c *parseContext) parseExpansion() *wordType { } if c.match2('$', '{') { // variable expansion - newOffset, complete := c.skipToChar(2, '}', false) - w := &wordType{Type: WordTypeVarBrace, Offset: c.Pos, Raw: c.Input[c.Pos : c.Pos+newOffset], Complete: complete} + newContext := c.clone(c.Pos+2, WordTypeVarBrace) + _, eofExit := newContext.tokenizeVarBrace() + newOffset := newContext.Pos + 2 + w := &wordType{Type: WordTypeVarBrace, Offset: c.Pos, Raw: c.Input[c.Pos : c.Pos+newOffset], Complete: !eofExit} c.Pos = c.Pos + newOffset return w } @@ -520,7 +543,7 @@ func (w *wordType) String() string { if !w.Complete { notCompleteFlag = "*" } - return fmt.Sprintf("%4s[%3d]%s %s%q", w.Type, w.Offset, notCompleteFlag, makeSpaceStr(len(w.Prefix)), string(w.Raw)) + return fmt.Sprintf("%4s[%3d]%s %s%q", w.Type, w.Offset, notCompleteFlag, makeSpaceStr(len(w.Prefix)), string(w.FullRawString())) } func dumpWords(words []*wordType, indentStr string) { diff --git a/pkg/shparse/shparse_test.go b/pkg/shparse/shparse_test.go index afed4770..187c50cf 100644 --- a/pkg/shparse/shparse_test.go +++ b/pkg/shparse/shparse_test.go @@ -35,4 +35,12 @@ func Test1(t *testing.T) { testParse(t, `ls ${x:"hello"} $[2+2] $((5 * 10)) $(ls; ls&)`) testParse(t, `ls;ls&./foo > out 2> "out2"`) testParse(t, `(( x = 5)); ls& cd ~/work/"hello again"`) + testParse(t, `echo "hello"abc$(ls)$x${y:foo}`) + testParse(t, `echo $(ls; ./x "foo")`) + testParse(t, `echo $(ls; (cd foo; ls); (cd bar; ls))xyz`) + testParse(t, `echo "$x ${y:-foo}"`) + testParse(t, `command="$(echo "$input" | sed -e "s/^[ \t]*\([^ \t]*\)[ \t]*.*$/\1/g")"`) + testParse(t, `echo $(ls $)`) + testParse(t, `echo ${x:-hello\}"}"} 2nd`) + testParse(t, `echo "$(ls "foo") more $x"`) } diff --git a/pkg/shparse/tokenize.go b/pkg/shparse/tokenize.go index 9d447b89..d2b45b68 100644 --- a/pkg/shparse/tokenize.go +++ b/pkg/shparse/tokenize.go @@ -2,6 +2,7 @@ package shparse import ( "bytes" + "fmt" "unicode" ) @@ -16,7 +17,8 @@ type tokenizeOutputState struct { SavedPrefix []rune } -func (state *tokenizeOutputState) appendWord(word *wordType) { +// does not set CurWord +func (state *tokenizeOutputState) appendStandaloneWord(word *wordType) { state.delimitCurWord() if len(state.SavedPrefix) > 0 { word.Prefix = state.SavedPrefix @@ -25,12 +27,72 @@ func (state *tokenizeOutputState) appendWord(word *wordType) { state.Rtn = append(state.Rtn, word) } -func (state *tokenizeOutputState) ensureCurWord(pc *parseContext) { - if state.CurWord != nil { +func (state *tokenizeOutputState) appendWord(word *wordType) { + if len(state.SavedPrefix) > 0 { + word.Prefix = state.SavedPrefix + state.SavedPrefix = nil + } + if state.CurWord == nil { + state.CurWord = word return } - state.CurWord = &wordType{Type: WordTypeLit, Offset: pc.Pos, Complete: true, Prefix: state.SavedPrefix} - state.SavedPrefix = nil + state.ensureGroupWord() + state.CurWord.Subs = append(state.CurWord.Subs, word) +} + +func (state *tokenizeOutputState) ensureGroupWord() { + if state.CurWord == nil { + panic("invalid state, cannot make group word when CurWord is nil") + } + if state.CurWord.Type == WordTypeGroup { + return + } + // moves the prefix from CurWord to the new group word + groupWord := &wordType{ + Type: WordTypeGroup, + Offset: state.CurWord.Offset, + Complete: true, + Prefix: state.CurWord.Prefix, + } + state.CurWord.Prefix = nil + groupWord.Subs = []*wordType{state.CurWord} + state.CurWord = groupWord +} + +func ungroupWord(w *wordType) []*wordType { + if w.Type != WordTypeGroup { + return []*wordType{w} + } + rtn := w.Subs + if len(w.Prefix) > 0 && len(rtn) > 0 { + newPrefix := append([]rune{}, w.Prefix...) + newPrefix = append(newPrefix, rtn[0].Prefix...) + rtn[0].Prefix = newPrefix + } + return rtn +} + +func (state *tokenizeOutputState) ensureLitCurWord(pc *parseContext) { + if state.CurWord == nil { + state.CurWord = &wordType{Type: WordTypeLit, Offset: pc.Pos, Complete: true, Prefix: state.SavedPrefix} + state.SavedPrefix = nil + return + } + if state.CurWord.Type == WordTypeLit { + return + } + state.ensureGroupWord() + lastWord := state.CurWord.Subs[len(state.CurWord.Subs)-1] + if lastWord.Type != WordTypeLit { + if len(state.SavedPrefix) > 0 { + dumpWords(state.Rtn, "**") + dumpWords([]*wordType{state.CurWord}, ">>") + fmt.Printf("sp: %q\n", state.SavedPrefix) + panic("invalid state, there can be no saved prefix") + } + litWord := &wordType{Type: WordTypeLit, Offset: pc.Pos, Complete: true} + state.CurWord.Subs = append(state.CurWord.Subs, litWord) + } } func (state *tokenizeOutputState) delimitCurWord() { @@ -45,40 +107,51 @@ func (state *tokenizeOutputState) delimitWithSpace(spaceCh rune) { state.SavedPrefix = append(state.SavedPrefix, spaceCh) } +func (state *tokenizeOutputState) appendLiteral(pc *parseContext, ch rune) { + state.ensureLitCurWord(pc) + if state.CurWord.Type == WordTypeLit { + state.CurWord.Raw = append(state.CurWord.Raw, ch) + } else if state.CurWord.Type == WordTypeGroup { + lastWord := state.CurWord.Subs[len(state.CurWord.Subs)-1] + if lastWord.Type != WordTypeLit { + panic(fmt.Sprintf("invalid curword type (group) %q", state.CurWord.Type)) + } + lastWord.Raw = append(lastWord.Raw, ch) + } else { + panic(fmt.Sprintf("invalid curword type %q", state.CurWord.Type)) + } +} + func (state *tokenizeOutputState) finish(pc *parseContext) { state.delimitCurWord() if len(state.SavedPrefix) > 0 { - state.ensureCurWord(pc) + state.ensureLitCurWord(pc) state.delimitCurWord() } } -func Tokenize(cmd string) []*wordType { - c := &parseContext{Input: []rune(cmd)} +func (c *parseContext) tokenizeVarBrace() ([]*wordType, bool) { state := &tokenizeOutputState{} + eofExit := false for { ch := c.cur() if ch == 0 { + eofExit = true break } - // fmt.Printf("ch %d %q\n", c.Pos, string([]rune{ch})) - foundOp, newOffset := c.parseOp(0) - if foundOp { - opWord := &wordType{Type: WordTypeOp, Offset: c.Pos, Raw: c.Input[c.Pos : c.Pos+newOffset], Complete: true} - opWord.Val = string(opWord.Raw) - c.Pos = c.Pos + newOffset - state.appendWord(opWord) - continue + if ch == '}' { + c.Pos++ + break } var quoteWord *wordType - switch ch { - case '\'': + if ch == '\'' { quoteWord = c.parseStrSQ() - - case '"': + } + if quoteWord == nil && ch == '"' { quoteWord = c.parseStrDQ() - - case '$': + } + isNextBrace := c.at(1) == '}' + if quoteWord == nil && ch == '$' && !isNextBrace { quoteWord = c.parseStrANSI() if quoteWord == nil { quoteWord = c.parseStrDDQ() @@ -92,8 +165,124 @@ func Tokenize(cmd string) []*wordType { continue } if ch == '\\' && c.at(1) != 0 { - state.ensureCurWord(c) - state.CurWord.Raw = append(state.CurWord.Raw, ch, c.at(1)) + state.appendLiteral(c, ch) + state.appendLiteral(c, c.at(1)) + c.Pos += 2 + continue + } + state.appendLiteral(c, ch) + c.Pos++ + } + return state.Rtn, eofExit +} + +func (c *parseContext) tokenizeDQ() ([]*wordType, bool) { + state := &tokenizeOutputState{} + eofExit := false + for { + ch := c.cur() + if ch == 0 { + eofExit = true + break + } + if ch == '"' { + c.Pos++ + break + } + if ch == '$' && c.at(1) != 0 { + quoteWord := c.parseStrANSI() + if quoteWord == nil { + quoteWord = c.parseStrDDQ() + } + if quoteWord == nil { + quoteWord = c.parseExpansion() + } + if quoteWord != nil { + state.appendWord(quoteWord) + continue + } + } + if ch == '\\' && c.at(1) != 0 { + state.appendLiteral(c, ch) + state.appendLiteral(c, c.at(1)) + c.Pos += 2 + continue + } + state.appendLiteral(c, ch) + c.Pos++ + } + state.finish(c) + if len(state.Rtn) == 0 { + return nil, eofExit + } + if len(state.Rtn) == 1 && state.Rtn[0].Type == WordTypeGroup { + return ungroupWord(state.Rtn[0]), eofExit + } + return state.Rtn, eofExit +} + +// returns (words, eofexit) +func (c *parseContext) tokenizeRaw() ([]*wordType, bool) { + state := &tokenizeOutputState{} + isExpSubShell := c.QC.cur() == WordTypeDP + parenLevel := 0 + eofExit := false + for { + ch := c.cur() + if ch == 0 { + eofExit = true + break + } + if isExpSubShell && ch == ')' && parenLevel == 0 { + c.Pos++ + break + } + // fmt.Printf("ch %d %q\n", c.Pos, string([]rune{ch})) + foundOp, newOffset := c.parseOp(0) + if foundOp { + rawOp := c.Input[c.Pos : c.Pos+newOffset] + opVal := string(rawOp) + opWord := &wordType{Type: WordTypeOp, Offset: c.Pos, Raw: rawOp, Val: opVal, Complete: true} + if opWord.Val == "(" { + arithWord := c.parseArith(true) + if arithWord != nil { + state.appendStandaloneWord(arithWord) + continue + } else { + parenLevel++ + } + } + if opWord.Val == ")" { + parenLevel-- + } + c.Pos = c.Pos + newOffset + state.appendStandaloneWord(opWord) + continue + } + var quoteWord *wordType + if ch == '\'' { + quoteWord = c.parseStrSQ() + } + if quoteWord == nil && ch == '"' { + quoteWord = c.parseStrDQ() + } + isNextParen := isExpSubShell && c.at(1) == ')' + if quoteWord == nil && ch == '$' && !isNextParen { + quoteWord = c.parseStrANSI() + if quoteWord == nil { + quoteWord = c.parseStrDDQ() + } + if quoteWord == nil { + quoteWord = c.parseExpansion() + } + } + if quoteWord != nil { + state.appendWord(quoteWord) + continue + } + if ch == '\\' && c.at(1) != 0 { + state.appendLiteral(c, ch) + state.appendLiteral(c, c.at(1)) c.Pos += 2 continue } @@ -102,12 +291,28 @@ func Tokenize(cmd string) []*wordType { c.Pos++ continue } - state.ensureCurWord(c) - state.CurWord.Raw = append(state.CurWord.Raw, ch) + state.appendLiteral(c, ch) c.Pos++ } state.finish(c) - return state.Rtn + return state.Rtn, eofExit +} + +func Tokenize(cmd string) []*wordType { + c := &parseContext{Input: []rune(cmd)} + rtn, _ := c.tokenizeRaw() + return rtn +} + +func (w *wordType) FullRawString() []rune { + if w.Type == WordTypeGroup { + var rtn []rune + for _, sw := range w.Subs { + rtn = append(rtn, sw.FullRawString()...) + } + return rtn + } + return w.Raw } func wordsToStr(words []*wordType) string { @@ -116,7 +321,7 @@ func wordsToStr(words []*wordType) string { if len(word.Prefix) > 0 { buf.WriteString(string(word.Prefix)) } - buf.WriteString(string(word.Raw)) + buf.WriteString(string(word.FullRawString())) } return buf.String() }