diff --git a/pkg/binpack/binpack.go b/pkg/binpack/binpack.go index a1145542..67e25337 100644 --- a/pkg/binpack/binpack.go +++ b/pkg/binpack/binpack.go @@ -2,6 +2,7 @@ package binpack import ( "encoding/binary" + "encoding/json" "fmt" "io" ) @@ -32,6 +33,14 @@ func PackValue(w io.Writer, barr []byte) error { return nil } +func PackStrArr(w io.Writer, strs []string) error { + barr, err := json.Marshal(strs) + if err != nil { + return err + } + return PackValue(w, barr) +} + func PackInt(w io.Writer, ival int) error { viBuf := make([]byte, binary.MaxVarintLen64) l := binary.PutUvarint(viBuf, uint64(ival)) @@ -55,6 +64,19 @@ func UnpackValue(r FullByteReader) ([]byte, error) { return rtnBuf, nil } +func UnpackStrArr(r FullByteReader) ([]string, error) { + barr, err := UnpackValue(r) + if err != nil { + return nil, err + } + var strs []string + err = json.Unmarshal(barr, &strs) + if err != nil { + return nil, err + } + return strs, nil +} + func UnpackInt(r io.ByteReader) (int, error) { ival64, err := binary.ReadVarint(r) if err != nil { @@ -85,6 +107,17 @@ func (u *Unpacker) UnpackInt(name string) int { return rtn } +func (u *Unpacker) UnpackStrArr(name string) []string { + if u.Err != nil { + return nil + } + rtn, err := UnpackStrArr(u.R) + if err != nil { + u.Err = fmt.Errorf("cannot unpack %s: %v", name, err) + } + return rtn +} + func (u *Unpacker) Error() error { return u.Err } diff --git a/pkg/packet/shellstate.go b/pkg/packet/shellstate.go index f4187681..fce4026c 100644 --- a/pkg/packet/shellstate.go +++ b/pkg/packet/shellstate.go @@ -21,22 +21,32 @@ type ShellState struct { Aliases string `json:"aliases,omitempty"` Funcs string `json:"funcs,omitempty"` Error string `json:"error,omitempty"` + HashVal string `json:"-"` } type ShellStateDiff struct { - Version string `json:"version"` // [type] [semver] - BaseHash string `json:"basehash"` - Cwd string `json:"cwd,omitempty"` - VarsDiff []byte `json:"shellvarsdiff,omitempty"` // vardiff - AliasesDiff []byte `json:"aliasesdiff,omitempty"` // linediff - FuncsDiff []byte `json:"funcsdiff,omitempty"` // linediff - Error string `json:"error,omitempty"` + Version string `json:"version"` // [type] [semver] + BaseHash string `json:"basehash"` + DiffHashArr []string `json:"diffhasharr,omitempty"` + Cwd string `json:"cwd,omitempty"` + VarsDiff []byte `json:"shellvarsdiff,omitempty"` // vardiff + AliasesDiff []byte `json:"aliasesdiff,omitempty"` // linediff + FuncsDiff []byte `json:"funcsdiff,omitempty"` // linediff + Error string `json:"error,omitempty"` + HashVal string `json:"-"` } func (state ShellState) IsEmpty() bool { return state.Version == "" && state.Cwd == "" && len(state.ShellVars) == 0 && state.Aliases == "" && state.Funcs == "" && state.Error == "" } +// returns base64 hash of data +func sha1Hash(data []byte) string { + hvalRaw := sha1.Sum(data) + hval := base64.StdEncoding.EncodeToString(hvalRaw[:]) + return hval +} + // returns (SHA1, encoded-state) func (state ShellState) EncodeAndHash() (string, []byte) { var buf bytes.Buffer @@ -47,22 +57,24 @@ func (state ShellState) EncodeAndHash() (string, []byte) { binpack.PackValue(&buf, []byte(state.Aliases)) binpack.PackValue(&buf, []byte(state.Funcs)) binpack.PackValue(&buf, []byte(state.Error)) - hvalRaw := sha1.Sum(buf.Bytes()) - hval := base64.StdEncoding.EncodeToString(hvalRaw[:]) - return hval, buf.Bytes() + return sha1Hash(buf.Bytes()), buf.Bytes() } func (state ShellState) MarshalJSON() ([]byte, error) { - _, encodedState := state.EncodeAndHash() - return json.Marshal(encodedState) + _, encodedBytes := state.EncodeAndHash() + return json.Marshal(encodedBytes) } -func (state *ShellState) UnmarshalJSON(jsonBytes []byte) error { - var barr []byte - err := json.Unmarshal(jsonBytes, &barr) - if err != nil { - return err +// caches HashVal in struct +func (state *ShellState) GetHashVal(force bool) string { + if state.HashVal == "" || force { + state.HashVal, _ = state.EncodeAndHash() } + return state.HashVal +} + +func (state *ShellState) DecodeShellState(barr []byte) error { + state.HashVal = sha1Hash(barr) buf := bytes.NewBuffer(barr) u := binpack.MakeUnpacker(buf) version := u.UnpackInt("ShellState pack version") @@ -78,25 +90,36 @@ func (state *ShellState) UnmarshalJSON(jsonBytes []byte) error { return u.Error() } -func (sdiff ShellStateDiff) MarshalJSON() ([]byte, error) { - var buf bytes.Buffer - binpack.PackInt(&buf, ShellStateDiffPackVersion) - binpack.PackValue(&buf, []byte(sdiff.Version)) - binpack.PackValue(&buf, []byte(sdiff.BaseHash)) - binpack.PackValue(&buf, []byte(sdiff.Cwd)) - binpack.PackValue(&buf, sdiff.VarsDiff) - binpack.PackValue(&buf, sdiff.AliasesDiff) - binpack.PackValue(&buf, sdiff.FuncsDiff) - binpack.PackValue(&buf, []byte(sdiff.Error)) - return json.Marshal(buf.Bytes()) -} - -func (sdiff *ShellStateDiff) UnmarshalJSON(jsonBytes []byte) error { +func (state *ShellState) UnmarshalJSON(jsonBytes []byte) error { var barr []byte err := json.Unmarshal(jsonBytes, &barr) if err != nil { return err } + return state.DecodeShellState(barr) +} + +func (sdiff ShellStateDiff) EncodeAndHash() (string, []byte) { + var buf bytes.Buffer + binpack.PackInt(&buf, ShellStateDiffPackVersion) + binpack.PackValue(&buf, []byte(sdiff.Version)) + binpack.PackValue(&buf, []byte(sdiff.BaseHash)) + binpack.PackStrArr(&buf, sdiff.DiffHashArr) + binpack.PackValue(&buf, []byte(sdiff.Cwd)) + binpack.PackValue(&buf, sdiff.VarsDiff) + binpack.PackValue(&buf, sdiff.AliasesDiff) + binpack.PackValue(&buf, sdiff.FuncsDiff) + binpack.PackValue(&buf, []byte(sdiff.Error)) + return sha1Hash(buf.Bytes()), buf.Bytes() +} + +func (sdiff ShellStateDiff) MarshalJSON() ([]byte, error) { + _, encodedBytes := sdiff.EncodeAndHash() + return json.Marshal(encodedBytes) +} + +func (sdiff *ShellStateDiff) DecodeShellStateDiff(barr []byte) error { + sdiff.HashVal = sha1Hash(barr) buf := bytes.NewBuffer(barr) u := binpack.MakeUnpacker(buf) version := u.UnpackInt("ShellState pack version") @@ -105,6 +128,7 @@ func (sdiff *ShellStateDiff) UnmarshalJSON(jsonBytes []byte) error { } sdiff.Version = string(u.UnpackValue("ShellStateDiff.Version")) sdiff.BaseHash = string(u.UnpackValue("ShellStateDiff.BaseHash")) + sdiff.DiffHashArr = u.UnpackStrArr("ShellStateDiff.DiffHashArr") sdiff.Cwd = string(u.UnpackValue("ShellStateDiff.Cwd")) sdiff.VarsDiff = u.UnpackValue("ShellStateDiff.VarsDiff") sdiff.AliasesDiff = u.UnpackValue("ShellStateDiff.AliasesDiff") @@ -113,6 +137,23 @@ func (sdiff *ShellStateDiff) UnmarshalJSON(jsonBytes []byte) error { return u.Error() } +func (sdiff *ShellStateDiff) UnmarshalJSON(jsonBytes []byte) error { + var barr []byte + err := json.Unmarshal(jsonBytes, &barr) + if err != nil { + return err + } + return sdiff.DecodeShellStateDiff(barr) +} + +// caches HashVal in struct +func (sdiff *ShellStateDiff) GetHashVal(force bool) string { + if sdiff.HashVal == "" || force { + sdiff.HashVal, _ = sdiff.EncodeAndHash() + } + return sdiff.HashVal +} + func (sdiff ShellStateDiff) Dump() { fmt.Printf("ShellStateDiff:\n") fmt.Printf(" version: %s\n", sdiff.Version) diff --git a/pkg/shexec/parser.go b/pkg/shexec/parser.go index 7b331927..e75cbfeb 100644 --- a/pkg/shexec/parser.go +++ b/pkg/shexec/parser.go @@ -190,14 +190,13 @@ func ParseDeclLine(envLine string) *DeclareDeclType { } } +// returns name => full-line func parseDeclLineToKV(envLine string) (string, string) { - eqIdx := strings.Index(envLine, "=") - if eqIdx == -1 { + decl := ParseDeclLine(envLine) + if decl == nil { return "", "" } - namePart := envLine[0:eqIdx] - valPart := envLine[eqIdx+1:] - return namePart, valPart + return decl.Name, envLine } func shellStateVarsToMap(shellVars []byte) map[string]string { @@ -216,6 +215,35 @@ func shellStateVarsToMap(shellVars []byte) map[string]string { return rtn } +func strMapToShellStateVars(varMap map[string]string) []byte { + var buf bytes.Buffer + orderedKeys := getOrderedKeysStrMap(varMap) + for _, key := range orderedKeys { + val := varMap[key] + buf.WriteString(val) + buf.WriteByte(0) + } + return buf.Bytes() +} + +func getOrderedKeysStrMap(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for key, _ := range m { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func getOrderedKeysDeclMap(m map[string]*DeclareDeclType) []string { + keys := make([]string, 0, len(m)) + for key, _ := range m { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + func DeclMapFromState(state *packet.ShellState) map[string]*DeclareDeclType { if state == nil { return nil @@ -233,7 +261,9 @@ func DeclMapFromState(state *packet.ShellState) map[string]*DeclareDeclType { func SerializeDeclMap(declMap map[string]*DeclareDeclType) []byte { var rtn bytes.Buffer - for _, decl := range declMap { + orderedKeys := getOrderedKeysDeclMap(declMap) + for _, key := range orderedKeys { + decl := declMap[key] rtn.WriteString(decl.Serialize()) } return rtn.Bytes() @@ -269,6 +299,18 @@ func ShellVarMapFromState(state *packet.ShellState) map[string]string { return rtn } +func DumpVarMapFromState(state *packet.ShellState) { + fmt.Printf("DUMP-STATE-VARS:\n") + if state == nil { + fmt.Printf(" nil\n") + return + } + vars := bytes.Split(state.ShellVars, []byte{0}) + for _, varLine := range vars { + fmt.Printf(" %s\n", varLine) + } +} + func VarDeclsFromState(state *packet.ShellState) []*DeclareDeclType { if state == nil { return nil @@ -365,8 +407,8 @@ func parseDeclareOutput(state *packet.ShellState, declareBytes []byte) error { if err != nil { return err } - var varsBuffer bytes.Buffer var firstParseErr error + declMap := make(map[string]*DeclareDeclType) for _, stmt := range file.Stmts { decl, err := parseDeclareStmt(stmt, declareStr) if err != nil { @@ -375,10 +417,10 @@ func parseDeclareOutput(state *packet.ShellState, declareBytes []byte) error { } } if decl != nil && !NoStoreVarNames[decl.Name] { - varsBuffer.WriteString(decl.Serialize()) + declMap[decl.Name] = decl } } - state.ShellVars = varsBuffer.Bytes() + state.ShellVars = SerializeDeclMap(declMap) // this writes out the decls in a canonical order if firstParseErr != nil { state.Error = firstParseErr.Error() } @@ -533,3 +575,26 @@ func MakeShellStateDiff(oldState packet.ShellState, oldStateHash string, newStat rtn.FuncsDiff = statediff.MakeLineDiff(oldState.Funcs, newState.Funcs) return rtn, nil } + +func ApplyShellStateDiff(oldState packet.ShellState, diff packet.ShellStateDiff) (packet.ShellState, error) { + var rtnState packet.ShellState + var err error + rtnState.Version = oldState.Version + rtnState.Cwd = diff.Cwd + rtnState.Error = diff.Error + oldVars := shellStateVarsToMap(oldState.ShellVars) + newVars, err := statediff.ApplyMapDiff(oldVars, diff.VarsDiff) + if err != nil { + return rtnState, fmt.Errorf("applying mapdiff 'vars': %v", err) + } + rtnState.ShellVars = strMapToShellStateVars(newVars) + rtnState.Aliases, err = statediff.ApplyLineDiff(oldState.Aliases, diff.AliasesDiff) + if err != nil { + return rtnState, fmt.Errorf("applying diff 'aliases': %v", err) + } + rtnState.Funcs, err = statediff.ApplyLineDiff(oldState.Funcs, diff.FuncsDiff) + if err != nil { + return rtnState, fmt.Errorf("applying diff 'funcs': %v", err) + } + return rtnState, nil +} diff --git a/pkg/statediff/linediff.go b/pkg/statediff/linediff.go index 347b2e37..fa7ce11b 100644 --- a/pkg/statediff/linediff.go +++ b/pkg/statediff/linediff.go @@ -161,6 +161,9 @@ func makeLineDiff(oldData []string, newData []string) LineDiffType { } func MakeLineDiff(str1 string, str2 string) []byte { + if str1 == str2 { + return nil + } str1Arr := strings.Split(str1, "\n") str2Arr := strings.Split(str2, "\n") diff := makeLineDiff(str1Arr, str2Arr) @@ -168,6 +171,9 @@ func MakeLineDiff(str1 string, str2 string) []byte { } func ApplyLineDiff(str1 string, diffBytes []byte) (string, error) { + if len(diffBytes) == 0 { + return str1, nil + } var diff LineDiffType err := diff.Decode(diffBytes) if err != nil { diff --git a/pkg/statediff/mapdiff.go b/pkg/statediff/mapdiff.go index da0d14da..8e810f7f 100644 --- a/pkg/statediff/mapdiff.go +++ b/pkg/statediff/mapdiff.go @@ -111,10 +111,16 @@ func (diff *MapDiffType) Decode(diffBytes []byte) error { func MakeMapDiff(m1 map[string]string, m2 map[string]string) []byte { diff := makeMapDiff(m1, m2) + if len(diff.ToAdd) == 0 && len(diff.ToRemove) == 0 { + return nil + } return diff.Encode() } func ApplyMapDiff(oldMap map[string]string, diffBytes []byte) (map[string]string, error) { + if len(diffBytes) == 0 { + return oldMap, nil + } var diff MapDiffType err := diff.Decode(diffBytes) if err != nil {