waveterm/backup
0
0
Fork 0
mirror of https://github.com/wavetermdev/backup.git synced 2026-08-05 13:57:07 -07:00
Code Issues Packages Projects Releases Wiki Activity

split words into simple commands. identify bash keywords. light parsing of bash language to get command separation

Browse Source
This commit is contained in:
sawka
2022-11-18 00:09:18 -08:00
parent 082b146409
commit a599dc473a
4 changed files with 646 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/shparse/extend.go
+14 -14
View File
@@ -43,13 +43,13 @@ func getUtf8Literal(ch rune) string {
return buf.String()
}
func (w *wordType) writeString(s string) {
func (w *WordType) writeString(s string) {
for _, ch := range s {
w.writeRune(ch)
}
}
func (w *wordType) writeRune(ch rune) {
func (w *WordType) writeRune(ch rune) {
wmeta := wordMetaMap[w.Type]
if w.Complete && wmeta.SuffixLen == 1 {
w.Raw = append(w.Raw[0:len(w.Raw)-1], ch, w.Raw[len(w.Raw)-1])
@@ -64,7 +64,7 @@ func (w *wordType) writeRune(ch rune) {
return
}
func (w *wordType) cloneRaw() {
func (w *WordType) cloneRaw() {
if len(w.Raw) == 0 {
return
}
@@ -73,31 +73,31 @@ func (w *wordType) cloneRaw() {
}
type extendContext struct {
QC quoteContext
Rtn []*wordType
CurWord *wordType
QC QuoteContext
Rtn []*WordType
CurWord *WordType
Intention string
}
func makeExtendContext(qc quoteContext, w *wordType) *extendContext {
func makeExtendContext(qc QuoteContext, w *WordType) *extendContext {
rtn := &extendContext{QC: qc, Intention: WordTypeLit}
if w != nil {
w.cloneRaw()
rtn.Rtn = []*wordType{w}
rtn.Rtn = []*WordType{w}
rtn.CurWord = w
rtn.Intention = w.Type
}
return rtn
}
func (ec *extendContext) appendWord(w *wordType) {
func (ec *extendContext) appendWord(w *WordType) {
ec.Rtn = append(ec.Rtn, w)
ec.CurWord = w
}
func (ec *extendContext) ensureCurWord() {
if ec.CurWord == nil || ec.CurWord.Type != ec.Intention {
ec.CurWord = makeEmptyWord(ec.Intention, ec.QC, 0)
ec.CurWord = MakeEmptyWord(ec.Intention, ec.QC, 0)
ec.Rtn = append(ec.Rtn, ec.CurWord)
}
}
@@ -155,7 +155,7 @@ func (ec *extendContext) extendLit(ch rune) {
return
}
if ch > unicode.MaxASCII || !unicode.IsPrint(ch) {
dsqWord := makeEmptyWord(WordTypeDSQ, ec.QC, 0)
dsqWord := MakeEmptyWord(WordTypeDSQ, ec.QC, 0)
ec.appendWord(dsqWord)
sesc := getSpecialEscape(ch)
if sesc != "" {
@@ -207,13 +207,13 @@ func (ec *extendContext) extendSQ(ch rune) {
return
}
if ch == '\'' {
litWord := &wordType{Type: WordTypeLit, QC: ec.QC}
litWord := &WordType{Type: WordTypeLit, QC: ec.QC}
litWord.Raw = []rune{'\\', '\''}
ec.appendWord(litWord)
return
}
if ch > unicode.MaxASCII || !unicode.IsPrint(ch) {
dsqWord := makeEmptyWord(WordTypeDSQ, ec.QC, 0)
dsqWord := MakeEmptyWord(WordTypeDSQ, ec.QC, 0)
ec.appendWord(dsqWord)
sesc := getSpecialEscape(ch)
if sesc != "" {
@@ -240,7 +240,7 @@ func (ec *extendContext) extendDQ(ch rune) {
return
}
if ch > unicode.MaxASCII || !unicode.IsPrint(ch) {
dsqWord := makeEmptyWord(WordTypeDSQ, ec.QC, 0)
dsqWord := MakeEmptyWord(WordTypeDSQ, ec.QC, 0)
ec.appendWord(dsqWord)
sesc := getSpecialEscape(ch)
if sesc != "" {
pkg/shparse/shparse.go
+337 -323
View File
File diff suppressed because it is too large Load Diff
pkg/shparse/shparse_test.go
+20 -2
View File
@@ -45,9 +45,10 @@ func Test1(t *testing.T) {
testParse(t, `echo "$(ls "foo") more $x"`)
testParse(t, "echo `ls $x \"hello $x\" \\`ls\\`; ./foo`")
testParse(t, `echo $"hello $x $(ls)"`)
testParse(t, "echo 'hello'\nls\n")
}
func lastWord(words []*wordType) *wordType {
func lastWord(words []*WordType) *WordType {
if len(words) == 0 {
return nil
}
@@ -80,5 +81,22 @@ func Test2(t *testing.T) {
testExtend(t, `$'f`, "'\x01\x07o", `$'f\'\x01\ao`)
testExtend(t, `"f"`, "oo", `"foo"`)
testExtend(t, `"mi"`, "ke's \"hello\"", `"mike's \"hello\""`)
testExtend(t, `"t"`, "t\x01\x07", `"tt"$'\x01'$'\x07'""`)
testExtend(t, `"t"`, "t\x01\x07", `"tt"$'\x01'$'\a'""`)
}
func testParseCommands(t *testing.T, str string) {
fmt.Printf("parse: %q\n", str)
words := Tokenize(str)
cmds := ParseCommands(words)
dumpCommands(cmds, " ")
fmt.Printf("\n")
}
func TestCmd(t *testing.T) {
testParseCommands(t, "ls foo")
testParseCommands(t, "ls foo && ls bar; ./run $x hello | xargs foo; ")
testParseCommands(t, "if [[ 2 > 1 ]]; then echo hello\nelse echo world; echo next; done")
testParseCommands(t, "case lots of stuff; i don\\'t know how to parse; esac; ls foo")
testParseCommands(t, "(ls & ./x); for x in $vars 3; do { echo $x; ls foo; } done")
testParseCommands(t, "function foo () { echo hello; }")
}
pkg/shparse/tokenize.go
+275 -38
View File
@@ -1,7 +1,6 @@
package shparse
import (
"bytes"
"fmt"
"unicode"
)
@@ -12,13 +11,13 @@ import (
//
type tokenizeOutputState struct {
Rtn []*wordType
CurWord *wordType
Rtn []*WordType
CurWord *WordType
SavedPrefix []rune
}
// does not set CurWord
func (state *tokenizeOutputState) appendStandaloneWord(word *wordType) {
func (state *tokenizeOutputState) appendStandaloneWord(word *WordType) {
state.delimitCurWord()
if len(state.SavedPrefix) > 0 {
word.Prefix = state.SavedPrefix
@@ -27,7 +26,7 @@ func (state *tokenizeOutputState) appendStandaloneWord(word *wordType) {
state.Rtn = append(state.Rtn, word)
}
func (state *tokenizeOutputState) appendWord(word *wordType) {
func (state *tokenizeOutputState) appendWord(word *WordType) {
if len(state.SavedPrefix) > 0 {
word.Prefix = state.SavedPrefix
state.SavedPrefix = nil
@@ -48,7 +47,7 @@ func (state *tokenizeOutputState) ensureGroupWord() {
return
}
// moves the prefix from CurWord to the new group word
groupWord := &wordType{
groupWord := &WordType{
Type: WordTypeGroup,
Offset: state.CurWord.Offset,
QC: state.CurWord.QC,
@@ -56,13 +55,13 @@ func (state *tokenizeOutputState) ensureGroupWord() {
Prefix: state.CurWord.Prefix,
}
state.CurWord.Prefix = nil
groupWord.Subs = []*wordType{state.CurWord}
groupWord.Subs = []*WordType{state.CurWord}
state.CurWord = groupWord
}
func ungroupWord(w *wordType) []*wordType {
func ungroupWord(w *WordType) []*WordType {
if w.Type != WordTypeGroup {
return []*wordType{w}
return []*WordType{w}
}
rtn := w.Subs
if len(w.Prefix) > 0 && len(rtn) > 0 {
@@ -129,7 +128,7 @@ func (state *tokenizeOutputState) finish(pc *parseContext) {
}
}
func (c *parseContext) tokenizeVarBrace() ([]*wordType, bool) {
func (c *parseContext) tokenizeVarBrace() ([]*WordType, bool) {
state := &tokenizeOutputState{}
eofExit := false
for {
@@ -142,7 +141,7 @@ func (c *parseContext) tokenizeVarBrace() ([]*wordType, bool) {
c.Pos++
break
}
var quoteWord *wordType
var quoteWord *WordType
if ch == '\'' {
quoteWord = c.parseStrSQ()
}
@@ -175,7 +174,7 @@ func (c *parseContext) tokenizeVarBrace() ([]*wordType, bool) {
return state.Rtn, eofExit
}
func (c *parseContext) tokenizeDQ() ([]*wordType, bool) {
func (c *parseContext) tokenizeDQ() ([]*WordType, bool) {
state := &tokenizeOutputState{}
eofExit := false
for {
@@ -223,7 +222,7 @@ func (c *parseContext) tokenizeDQ() ([]*wordType, bool) {
// returns (words, eofexit)
// backticks (WordTypeBQ) handle backslash in a special way, but that seems to mainly effect execution (not completion)
// de_backslash => removes initial backslash in \`, \\, and \$ before execution
func (c *parseContext) tokenizeRaw() ([]*wordType, bool) {
func (c *parseContext) tokenizeRaw() ([]*WordType, bool) {
state := &tokenizeOutputState{}
isExpSubShell := c.QC.cur() == WordTypeDP
isInBQ := c.QC.cur() == WordTypeBQ
@@ -260,11 +259,10 @@ func (c *parseContext) tokenizeRaw() ([]*wordType, bool) {
parenLevel--
}
opWord := c.makeWord(WordTypeOp, newOffset, true)
opWord.Val = opVal
state.appendStandaloneWord(opWord)
continue
}
var quoteWord *wordType
var quoteWord *WordType
if ch == '\'' {
quoteWord = c.parseStrSQ()
}
@@ -294,6 +292,11 @@ func (c *parseContext) tokenizeRaw() ([]*wordType, bool) {
c.Pos += 2
continue
}
if ch == '\n' {
newlineWord := c.makeWord(WordTypeOp, 1, true)
state.appendStandaloneWord(newlineWord)
continue
}
if unicode.IsSpace(ch) {
state.delimitWithSpace(ch)
c.Pos++
@@ -306,30 +309,264 @@ func (c *parseContext) tokenizeRaw() ([]*wordType, bool) {
return state.Rtn, eofExit
}
func Tokenize(cmd string) []*wordType {
type parseContext struct {
Input []rune
Pos int
QC QuoteContext
}
func (c *parseContext) clone(pos int, newQuote string) *parseContext {
rtn := parseContext{Input: c.Input[pos:], QC: c.QC}
if newQuote != "" {
rtn.QC = rtn.QC.push(newQuote)
}
return &rtn
}
func (c *parseContext) at(offset int) rune {
pos := c.Pos + offset
if pos < 0 || pos >= len(c.Input) {
return 0
}
return c.Input[pos]
}
func (c *parseContext) eof() bool {
return c.Pos >= len(c.Input)
}
func (c *parseContext) cur() rune {
return c.at(0)
}
func (c *parseContext) match(ch rune) bool {
return c.at(0) == ch
}
func (c *parseContext) match2(ch rune, ch2 rune) bool {
return c.at(0) == ch && c.at(1) == ch2
}
func (c *parseContext) match3(ch rune, ch2 rune, ch3 rune) bool {
return c.at(0) == ch && c.at(1) == ch2 && c.at(2) == ch3
}
func (c *parseContext) makeWord(t string, length int, complete bool) *WordType {
rtn := &WordType{Type: t}
rtn.Offset = c.Pos
rtn.QC = c.QC
rtn.Raw = c.Input[c.Pos : c.Pos+length]
rtn.Complete = complete
c.Pos += length
return rtn
}
// returns (found, newOffset)
// shell_meta_chars "()<>;&|"
// possible to maybe add ;;& &>> &> |& ;&
func (c *parseContext) parseOp(offset int) (bool, int) {
ch := c.at(offset)
if ch == '(' || ch == ')' || ch == '<' || ch == '>' || ch == ';' || ch == '&' || ch == '|' {
ch2 := c.at(offset + 1)
if ch2 == 0 {
return true, offset + 1
}
r2 := string([]rune{ch, ch2})
if r2 == "<<" {
ch3 := c.at(offset + 2)
if ch3 == '-' || ch3 == '<' {
return true, offset + 3 // "<<-" or "<<<"
}
return true, offset + 2 // "<<"
}
if r2 == ">>" || r2 == "&&" || r2 == "||" || r2 == ";;" || r2 == "<<" || r2 == "<&" || r2 == ">&" || r2 == "<>" || r2 == ">|" {
// we don't return '((' here (requires special processing)
return true, offset + 2
}
return true, offset + 1
}
return false, 0
}
// returns (new-offset, complete)
func (c *parseContext) skipToChar(offset int, endCh rune, allowEsc bool) (int, bool) {
for {
ch := c.at(offset)
if ch == 0 {
return offset, false
}
if allowEsc && ch == '\\' {
if c.at(offset+1) == 0 {
return offset + 1, false
}
offset += 2
continue
}
if ch == endCh {
return offset + 1, true
}
offset++
}
}
// returns (new-offset, complete)
func (c *parseContext) skipToChar2(offset int, endCh rune, endCh2 rune, allowEsc bool) (int, bool) {
for {
ch := c.at(offset)
ch2 := c.at(offset + 1)
if ch == 0 {
return offset, false
}
if ch2 == 0 {
return offset + 1, false
}
if allowEsc && ch == '\\' {
offset += 2
continue
}
if ch == endCh && ch2 == endCh2 {
return offset + 2, true
}
offset++
}
}
func (c *parseContext) parseStrSQ() *WordType {
if !c.match('\'') {
return nil
}
newOffset, complete := c.skipToChar(1, '\'', false)
w := c.makeWord(WordTypeSQ, newOffset, complete)
return w
}
func (c *parseContext) parseStrDQ() *WordType {
if !c.match('"') {
return nil
}
newContext := c.clone(c.Pos+1, WordTypeDQ)
subWords, eofExit := newContext.tokenizeDQ()
newOffset := newContext.Pos + 1
w := c.makeWord(WordTypeDQ, newOffset, !eofExit)
w.Subs = subWords
return w
}
func (c *parseContext) parseStrDDQ() *WordType {
if !c.match2('$', '"') {
return nil
}
newContext := c.clone(c.Pos+2, WordTypeDDQ)
subWords, eofExit := newContext.tokenizeDQ()
newOffset := newContext.Pos + 2
w := c.makeWord(WordTypeDDQ, newOffset, !eofExit)
w.Subs = subWords
return w
}
func (c *parseContext) parseStrBQ() *WordType {
if !c.match('`') {
return nil
}
newContext := c.clone(c.Pos+1, WordTypeBQ)
subWords, eofExit := newContext.tokenizeRaw()
newOffset := newContext.Pos + 1
w := c.makeWord(WordTypeBQ, newOffset, !eofExit)
w.Subs = subWords
return w
}
func (c *parseContext) parseStrANSI() *WordType {
if !c.match2('$', '\'') {
return nil
}
newOffset, complete := c.skipToChar(2, '\'', true)
w := c.makeWord(WordTypeDSQ, newOffset, complete)
return w
}
func (c *parseContext) parseArith(mustComplete bool) *WordType {
if !c.match2('(', '(') {
return nil
}
newOffset, complete := c.skipToChar2(2, ')', ')', false)
if mustComplete && !complete {
return nil
}
w := c.makeWord(WordTypePP, newOffset, complete)
return w
}
func (c *parseContext) parseExpansion() *WordType {
if !c.match('$') {
return nil
}
if c.match3('$', '(', '(') {
newOffset, complete := c.skipToChar2(3, ')', ')', false)
w := c.makeWord(WordTypeDPP, newOffset, complete)
return w
}
if c.match2('$', '(') {
// subshell
newContext := c.clone(c.Pos+2, WordTypeDP)
subWords, eofExit := newContext.tokenizeRaw()
newOffset := newContext.Pos + 2
w := c.makeWord(WordTypeDP, newOffset, !eofExit)
w.Subs = subWords
return w
}
if c.match2('$', '[') {
// deprecated arith expansion
newOffset, complete := c.skipToChar(2, ']', false)
w := c.makeWord(WordTypeDB, newOffset, complete)
return w
}
if c.match2('$', '{') {
// variable expansion
newContext := c.clone(c.Pos+2, WordTypeVarBrace)
_, eofExit := newContext.tokenizeVarBrace()
newOffset := newContext.Pos + 2
w := c.makeWord(WordTypeVarBrace, newOffset, !eofExit)
return w
}
ch2 := c.at(1)
if ch2 == 0 || unicode.IsSpace(ch2) {
// no expansion
return nil
}
newOffset := c.parseSimpleVarName(1)
if newOffset > 1 {
// simple variable name
w := c.makeWord(WordTypeSimpleVar, newOffset, true)
return w
}
if ch2 == '*' || ch2 == '@' || ch2 == '#' || ch2 == '?' || ch2 == '-' || ch2 == '$' || ch2 == '!' || (ch2 >= '0' && ch2 <= '9') {
// single character variable name, e.g. $@, $_, $1, etc.
w := c.makeWord(WordTypeSimpleVar, 2, true)
return w
}
return nil
}
// returns newOffset
func (c *parseContext) parseSimpleVarName(offset int) int {
first := true
for {
ch := c.at(offset)
if ch == 0 {
return offset
}
if (ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')) || (!first && ch >= '0' && ch <= '9') {
first = false
offset++
continue
}
return offset
}
}
func Tokenize(cmd string) []*WordType {
c := &parseContext{Input: []rune(cmd)}
rtn, _ := c.tokenizeRaw()
return rtn
}
func (w *wordType) FullRawString() []rune {
if w.Type == WordTypeGroup {
var rtn []rune
for _, sw := range w.Subs {
rtn = append(rtn, sw.FullRawString()...)
}
return rtn
}
return w.Raw
}
func wordsToStr(words []*wordType) string {
var buf bytes.Buffer
for _, word := range words {
if len(word.Prefix) > 0 {
buf.WriteString(string(word.Prefix))
}
buf.WriteString(string(word.FullRawString()))
}
return buf.String()
}