diff --git a/pkg/shparse/extend.go b/pkg/shparse/extend.go index 457a454d..a616a985 100644 --- a/pkg/shparse/extend.go +++ b/pkg/shparse/extend.go @@ -2,7 +2,6 @@ package shparse import ( "bytes" - "fmt" "unicode" "unicode/utf8" @@ -51,15 +50,18 @@ func (w *wordType) writeString(s string) { } func (w *wordType) writeRune(ch rune) { - if w.Type == WordTypeLit { - w.Raw = append(w.Raw, ch) - return - } - if w.Type == WordTypeDQ || w.Type == WordTypeDDQ || w.Type == WordTypeSimpleVar || w.Type == WordTypeVarBrace || w.Type == WordTypeSQ || w.Type == WordTypeDSQ { + wmeta := wordMetaMap[w.Type] + if w.Complete && wmeta.SuffixLen == 1 { w.Raw = append(w.Raw[0:len(w.Raw)-1], ch, w.Raw[len(w.Raw)-1]) return } - panic(fmt.Sprintf("cannot extend type %q", w.Type)) + if w.Complete && wmeta.SuffixLen == 2 { + w.Raw = append(w.Raw[0:len(w.Raw)-2], ch, w.Raw[len(w.Raw)-2], w.Raw[len(w.Raw)-1]) + return + } + // not complete or SuffixLen == 0 (2+ is not supported) + w.Raw = append(w.Raw, ch) + return } func (w *wordType) cloneRaw() { @@ -77,12 +79,13 @@ type extendContext struct { Intention string } -func makeExtendContext(qc quoteContext, w *wordType, intention string) *extendContext { - rtn := &extendContext{QC: qc, Intention: intention} +func makeExtendContext(qc quoteContext, w *wordType) *extendContext { + rtn := &extendContext{QC: qc, Intention: WordTypeLit} if w != nil { w.cloneRaw() rtn.Rtn = []*wordType{w} rtn.CurWord = w + rtn.Intention = w.Type } return rtn } @@ -92,30 +95,36 @@ func (ec *extendContext) appendWord(w *wordType) { ec.CurWord = w } +func (ec *extendContext) ensureCurWord() { + if ec.CurWord == nil || ec.CurWord.Type != ec.Intention { + ec.CurWord = makeEmptyWord(ec.Intention, ec.QC, 0) + ec.Rtn = append(ec.Rtn, ec.CurWord) + } +} + func (ec *extendContext) extend(ch rune) { if ch == 0 { return } - if ec.CurWord == nil { - ec.CurWord = &wordType{Type: WordTypeLit, QC: ec.QC} - ec.Rtn = append(ec.Rtn, ec.CurWord) - } - switch ec.CurWord.Type { - case WordTypeSimpleVar: + switch ec.Intention { + + case WordTypeSimpleVar, WordTypeVarBrace: + ec.extendVar(ch) case WordTypeDQ, WordTypeDDQ: - - case WordTypeVarBrace: + ec.extendDQ(ch) case WordTypeSQ: ec.extendSQ(ch) case WordTypeDSQ: + ec.extendDSQ(ch) case WordTypeLit: + ec.extendLit(ch) default: - + return } } @@ -126,18 +135,27 @@ func getSpecialEscape(ch rune) string { return specialEsc[byte(ch)] } -func (ec *extendContext) extendSQ(ch rune) { +func isVarNameChar(ch rune) bool { + return ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') +} + +func (ec *extendContext) extendVar(ch rune) { if ch == 0 { return } - if ch == '\'' { - litWord := &wordType{Type: WordTypeLit, QC: ec.QC} - litWord.Raw = []rune{'\\', '\''} - ec.appendWord(litWord) + if !isVarNameChar(ch) { + return + } + ec.ensureCurWord() + ec.CurWord.writeRune(ch) +} + +func (ec *extendContext) extendLit(ch rune) { + if ch == 0 { + return } if ch > unicode.MaxASCII || !unicode.IsPrint(ch) { - dsqWord := &wordType{Type: WordTypeDSQ, QC: ec.QC} - dsqWord.Raw = []rune{'$', '\'', '\''} + dsqWord := makeEmptyWord(WordTypeDSQ, ec.QC, 0) ec.appendWord(dsqWord) sesc := getSpecialEscape(ch) if sesc != "" { @@ -147,6 +165,92 @@ func (ec *extendContext) extendSQ(ch rune) { utf8Lit := getUtf8Literal(ch) dsqWord.writeString(utf8Lit) } + return + } + var bch = byte(ch) + ec.ensureCurWord() + if noEscChars[bch] { + ec.CurWord.writeRune(ch) + return + } + ec.CurWord.writeRune('\\') + ec.CurWord.writeRune(ch) + return +} + +func (ec *extendContext) extendDSQ(ch rune) { + if ch == 0 { + return + } + ec.ensureCurWord() + if ch == '\'' { + ec.CurWord.writeRune('\\') + ec.CurWord.writeRune(ch) + return + } + if ch > unicode.MaxASCII || !unicode.IsPrint(ch) { + sesc := getSpecialEscape(ch) + if sesc != "" { + ec.CurWord.writeString(sesc) + } else { + utf8Lit := getUtf8Literal(ch) + ec.CurWord.writeString(utf8Lit) + } + return } ec.CurWord.writeRune(ch) + return +} + +func (ec *extendContext) extendSQ(ch rune) { + if ch == 0 { + return + } + if ch == '\'' { + litWord := &wordType{Type: WordTypeLit, QC: ec.QC} + litWord.Raw = []rune{'\\', '\''} + ec.appendWord(litWord) + return + } + if ch > unicode.MaxASCII || !unicode.IsPrint(ch) { + dsqWord := makeEmptyWord(WordTypeDSQ, ec.QC, 0) + ec.appendWord(dsqWord) + sesc := getSpecialEscape(ch) + if sesc != "" { + dsqWord.writeString(sesc) + } else { + utf8Lit := getUtf8Literal(ch) + dsqWord.writeString(utf8Lit) + } + return + } + ec.ensureCurWord() + ec.CurWord.writeRune(ch) + return +} + +func (ec *extendContext) extendDQ(ch rune) { + if ch == 0 { + return + } + if ch == '"' || ch == '\\' || ch == '$' || ch == '`' { + ec.ensureCurWord() + ec.CurWord.writeRune('\\') + ec.CurWord.writeRune(ch) + return + } + if ch > unicode.MaxASCII || !unicode.IsPrint(ch) { + dsqWord := makeEmptyWord(WordTypeDSQ, ec.QC, 0) + ec.appendWord(dsqWord) + sesc := getSpecialEscape(ch) + if sesc != "" { + dsqWord.writeString(sesc) + } else { + utf8Lit := getUtf8Literal(ch) + dsqWord.writeString(utf8Lit) + } + return + } + ec.CurWord.writeRune(ch) + return } diff --git a/pkg/shparse/shparse.go b/pkg/shparse/shparse.go index 2e71fc07..13030b34 100644 --- a/pkg/shparse/shparse.go +++ b/pkg/shparse/shparse.go @@ -75,6 +75,8 @@ import ( // tokenization https://pubs.opengroup.org/onlinepubs/7908799/xcu/chap2.html#tag_001_003 +// can-extend: WordTypeLit, WordTypeSimpleVar, WordTypeVarBrace, WordTypeDQ, WordTypeDDQ, WordTypeSQ, WordTypeDSQ + const ( WordTypeRaw = "raw" WordTypeLit = "lit" // (can-extend) @@ -96,6 +98,52 @@ const ( WordTypeDB = "db" // $[ (internals not parsed) ) +var wordMetaMap map[string]wordMeta + +type wordMeta struct { + Type string + EmptyWord []rune + SuffixLen int + CanExtend bool + QuoteContext bool +} + +func makeWordMeta(wtype string, emptyWord string, suffixLen int, canExtend bool, quoteContext bool) { + wordMetaMap[wtype] = wordMeta{wtype, []rune(emptyWord), suffixLen, canExtend, quoteContext} +} + +func init() { + wordMetaMap = make(map[string]wordMeta) + makeWordMeta(WordTypeRaw, "", 0, false, false) + makeWordMeta(WordTypeLit, "", 0, true, false) + makeWordMeta(WordTypeOp, "", 0, false, false) + makeWordMeta(WordTypeKey, "", 0, false, false) + makeWordMeta(WordTypeGroup, "", 0, false, false) + makeWordMeta(WordTypeSimpleVar, "$", 0, true, false) + makeWordMeta(WordTypeVarBrace, "${}", 1, true, true) + makeWordMeta(WordTypeDQ, `""`, 1, true, true) + makeWordMeta(WordTypeDDQ, `$""`, 1, true, true) + makeWordMeta(WordTypeDP, "$()", 1, false, false) + makeWordMeta(WordTypeBQ, "``", 1, false, false) + makeWordMeta(WordTypeSQ, "''", 1, true, false) + makeWordMeta(WordTypeDSQ, "$''", 1, true, false) + makeWordMeta(WordTypeDPP, "$(())", 2, false, false) + makeWordMeta(WordTypePP, "(())", 2, false, false) + makeWordMeta(WordTypeDB, "$[]", 1, false, false) +} + +func makeEmptyWord(wtype string, qc quoteContext, offset int) *wordType { + meta := wordMetaMap[wtype] + if meta.Type == "" { + meta = wordMetaMap[WordTypeRaw] + } + rtn := &wordType{Type: meta.Type, QC: qc, Offset: offset, Complete: true} + if len(meta.EmptyWord) > 0 { + rtn.Raw = append([]rune(nil), meta.EmptyWord...) + } + return rtn +} + type quoteContext []string func (qc quoteContext) push(q string) quoteContext { diff --git a/pkg/shparse/shparse_test.go b/pkg/shparse/shparse_test.go index 97031882..ae2074c9 100644 --- a/pkg/shparse/shparse_test.go +++ b/pkg/shparse/shparse_test.go @@ -46,3 +46,39 @@ func Test1(t *testing.T) { testParse(t, "echo `ls $x \"hello $x\" \\`ls\\`; ./foo`") testParse(t, `echo $"hello $x $(ls)"`) } + +func lastWord(words []*wordType) *wordType { + if len(words) == 0 { + return nil + } + return words[len(words)-1] +} + +func testExtend(t *testing.T, startStr string, extendStr string, expectedStr string) { + words := Tokenize(startStr) + ec := makeExtendContext(nil, lastWord(words)) + for _, ch := range extendStr { + ec.extend(ch) + } + ec.ensureCurWord() + output := wordsToStr(ec.Rtn) + fmt.Printf("[%s] + [%s] => [%s]\n", startStr, extendStr, output) + if output != expectedStr { + t.Errorf("extension does not match: [%s] + [%s] => [%s] expected [%s]\n", startStr, extendStr, output, expectedStr) + } +} + +func Test2(t *testing.T) { + testExtend(t, `'he'`, "llo", `'hello'`) + testExtend(t, `'he'`, "'", `'he'\'''`) + testExtend(t, `'he'`, "'\x01", `'he'\'$'\x01'''`) + testExtend(t, `he`, "llo", `hello`) + testExtend(t, `he`, "l*l'\x01\x07o", `hel\*l\'$'\x01'$'\a'o`) + testExtend(t, `$x`, "fo|o", `$xfoo`) + testExtend(t, `${x`, "fo|o", `${xfoo`) + testExtend(t, `$'f`, "oo", `$'foo`) + testExtend(t, `$'f`, "'\x01\x07o", `$'f\'\x01\ao`) + testExtend(t, `"f"`, "oo", `"foo"`) + testExtend(t, `"mi"`, "ke's \"hello\"", `"mike's \"hello\""`) + testExtend(t, `"t"`, "t\x01\x07", `"tt"$'\x01'$'\x07'""`) +}