From 5a151369cb6b87f5427bdbe091b41647279fc477 Mon Sep 17 00:00:00 2001 From: sawka Date: Fri, 25 Nov 2022 15:52:29 -0800 Subject: [PATCH] line/mapdiff code --- pkg/statediff/linediff.go | 182 ++++++++++++++++++++++++++++++++ pkg/statediff/mapdiff.go | 124 ++++++++++++++++++++++ pkg/statediff/statediff_test.go | 99 +++++++++++++++++ 3 files changed, 405 insertions(+) create mode 100644 pkg/statediff/linediff.go create mode 100644 pkg/statediff/mapdiff.go create mode 100644 pkg/statediff/statediff_test.go diff --git a/pkg/statediff/linediff.go b/pkg/statediff/linediff.go new file mode 100644 index 00000000..a2e36491 --- /dev/null +++ b/pkg/statediff/linediff.go @@ -0,0 +1,182 @@ +package statediff + +import ( + "bytes" + "encoding/binary" + "fmt" + "strings" +) + +const LineDiffVersion = 0 + +type SingleLineEntry struct { + LineVal int + Run int +} + +type LineDiffType struct { + Lines []SingleLineEntry + NewData []string +} + +func (diff LineDiffType) dump() { + fmt.Printf("DIFF:\n") + pos := 1 + for _, entry := range diff.Lines { + fmt.Printf(" %d-%d: %d\n", pos, pos+entry.Run, entry.LineVal) + pos += entry.Run + } + for idx, str := range diff.NewData { + fmt.Printf(" n%d: %s\n", idx+1, str) + } +} + +// simple encoding +// a 0 means read a line from NewData +// a non-zero number means read the 1-indexed line from OldData +func (diff LineDiffType) applyDiff(oldData []string) ([]string, error) { + rtn := make([]string, 0, len(diff.Lines)) + newDataPos := 0 + for _, entry := range diff.Lines { + if entry.LineVal == 0 { + for i := 0; i < entry.Run; i++ { + if newDataPos >= len(diff.NewData) { + return nil, fmt.Errorf("not enough newdata for diff") + } + rtn = append(rtn, diff.NewData[newDataPos]) + newDataPos++ + } + } else { + oldDataPos := entry.LineVal - 1 // 1-indexed + for i := 0; i < entry.Run; i++ { + realPos := oldDataPos + i + if realPos < 0 || realPos >= len(oldData) { + return nil, fmt.Errorf("diff index out of bounds %d old-data-len:%d", realPos, len(oldData)) + } + rtn = append(rtn, oldData[realPos]) + } + } + } + return rtn, nil +} + +func putUVarint(buf *bytes.Buffer, viBuf []byte, ival int) { + l := binary.PutUvarint(viBuf, uint64(ival)) + buf.Write(viBuf[0:l]) +} + +// simple encoding +// write varints. first version, then len, then len-number-of-varints, then fill the rest with newdata +// [version] [len-varint] [varint]xlen... newdata (bytes) +func (diff LineDiffType) encode() []byte { + var buf bytes.Buffer + viBuf := make([]byte, binary.MaxVarintLen64) + putUVarint(&buf, viBuf, LineDiffVersion) + putUVarint(&buf, viBuf, len(diff.Lines)) + for _, entry := range diff.Lines { + putUVarint(&buf, viBuf, entry.LineVal) + putUVarint(&buf, viBuf, entry.Run) + } + for idx, str := range diff.NewData { + buf.WriteString(str) + if idx != len(diff.NewData)-1 { + buf.WriteByte('\n') + } + } + return buf.Bytes() +} + +func (rtn *LineDiffType) decode(diffBytes []byte) error { + r := bytes.NewBuffer(diffBytes) + version, err := binary.ReadUvarint(r) + if err != nil { + return fmt.Errorf("invalid diff, cannot read version: %v", err) + } + if version != LineDiffVersion { + return fmt.Errorf("invalid diff, bad version: %d", version) + } + linesLen64, err := binary.ReadUvarint(r) + if err != nil { + return fmt.Errorf("invalid diff, cannot read lines length: %v", err) + } + linesLen := int(linesLen64) + rtn.Lines = make([]SingleLineEntry, linesLen) + for idx := 0; idx < linesLen; idx++ { + lineVal, err := binary.ReadUvarint(r) + if err != nil { + return fmt.Errorf("invalid diff, cannot read line %d: %v", idx, err) + } + lineRun, err := binary.ReadUvarint(r) + if err != nil { + return fmt.Errorf("invalid diff, cannot read line-run %d: %v", idx, err) + } + rtn.Lines[idx] = SingleLineEntry{LineVal: int(lineVal), Run: int(lineRun)} + } + restOfInput := string(r.Bytes()) + if len(restOfInput) > 0 { + rtn.NewData = strings.Split(restOfInput, "\n") + } + return nil +} + +func makeLineDiff(oldData []string, newData []string) LineDiffType { + var rtn LineDiffType + oldDataMap := make(map[string]int) // 1-indexed + for idx, str := range oldData { + if _, found := oldDataMap[str]; found { + continue + } + oldDataMap[str] = idx + 1 + } + var cur *SingleLineEntry + rtn.Lines = make([]SingleLineEntry, 0) + for _, str := range newData { + oldIdx, found := oldDataMap[str] + if cur != nil && cur.LineVal != 0 { + checkLine := cur.LineVal + cur.Run - 1 + if checkLine < len(oldData) && oldData[checkLine] == str { + cur.Run++ + continue + } + } else if cur != nil && cur.LineVal == 0 && !found { + cur.Run++ + rtn.NewData = append(rtn.NewData, str) + continue + } + if cur != nil { + rtn.Lines = append(rtn.Lines, *cur) + } + cur = &SingleLineEntry{Run: 1} + if found { + cur.LineVal = oldIdx + } else { + cur.LineVal = 0 + rtn.NewData = append(rtn.NewData, str) + } + } + if cur != nil { + rtn.Lines = append(rtn.Lines, *cur) + } + return rtn +} + +func MakeLineDiff(str1 string, str2 string) []byte { + str1Arr := strings.Split(str1, "\n") + str2Arr := strings.Split(str2, "\n") + diff := makeLineDiff(str1Arr, str2Arr) + return diff.encode() +} + +func ApplyLineDiff(str1 string, diffBytes []byte) (string, error) { + var diff LineDiffType + err := diff.decode(diffBytes) + if err != nil { + return "", err + } + str1Arr := strings.Split(str1, "\n") + str2Arr, err := diff.applyDiff(str1Arr) + if err != nil { + return "", err + } + return strings.Join(str2Arr, "\n"), nil +} diff --git a/pkg/statediff/mapdiff.go b/pkg/statediff/mapdiff.go new file mode 100644 index 00000000..a06298b7 --- /dev/null +++ b/pkg/statediff/mapdiff.go @@ -0,0 +1,124 @@ +package statediff + +import ( + "bytes" + "encoding/binary" + "fmt" +) + +const MapDiffVersion = 0 + +// 0-bytes are not allowed in entries or keys (same as bash) + +type MapDiffType struct { + ToAdd map[string]string + ToRemove []string +} + +func (diff MapDiffType) dump() { + fmt.Printf("VAR-DIFF\n") + for name, val := range diff.ToAdd { + fmt.Printf(" add: %s=%s\n", name, val) + } + for _, name := range diff.ToRemove { + fmt.Printf(" rem: %s\n", name) + } +} + +func makeMapDiff(oldMap map[string]string, newMap map[string]string) MapDiffType { + var rtn MapDiffType + rtn.ToAdd = make(map[string]string) + for name, newVal := range newMap { + oldVal, found := oldMap[name] + if !found || oldVal != newVal { + rtn.ToAdd[name] = newVal + continue + } + } + for name, _ := range oldMap { + _, found := newMap[name] + if !found { + rtn.ToRemove = append(rtn.ToRemove, name) + } + } + return rtn +} + +func (diff MapDiffType) apply(oldMap map[string]string) map[string]string { + rtn := make(map[string]string) + for name, val := range oldMap { + rtn[name] = val + } + for name, val := range diff.ToAdd { + rtn[name] = val + } + for _, name := range diff.ToRemove { + delete(rtn, name) + } + return rtn +} + +func (diff MapDiffType) encode() []byte { + var buf bytes.Buffer + viBuf := make([]byte, binary.MaxVarintLen64) + putUVarint(&buf, viBuf, MapDiffVersion) + putUVarint(&buf, viBuf, len(diff.ToAdd)) + for key, val := range diff.ToAdd { + buf.WriteString(key) + buf.WriteByte(0) + buf.WriteString(val) + buf.WriteByte(0) + } + for _, val := range diff.ToRemove { + buf.WriteString(val) + buf.WriteByte(0) + } + return buf.Bytes() +} + +func (diff *MapDiffType) decode(diffBytes []byte) error { + r := bytes.NewBuffer(diffBytes) + version, err := binary.ReadUvarint(r) + if err != nil { + return fmt.Errorf("invalid diff, cannot read version: %v", err) + } + if version != MapDiffVersion { + return fmt.Errorf("invalid diff, bad version: %d", version) + } + mapLen64, err := binary.ReadUvarint(r) + if err != nil { + return fmt.Errorf("invalid diff, cannot map length: %v", err) + } + mapLen := int(mapLen64) + fields := bytes.Split(r.Bytes(), []byte{0}) + if len(fields) < 2*mapLen { + return fmt.Errorf("invalid diff, not enough fields, maplen:%d fields:%d", mapLen, len(fields)) + } + mapFields := fields[0 : 2*mapLen] + removeFields := fields[2*mapLen:] + diff.ToAdd = make(map[string]string) + for i := 0; i < len(mapFields); i += 2 { + diff.ToAdd[string(mapFields[i])] = string(mapFields[i+1]) + } + for _, removeVal := range removeFields { + if len(removeVal) == 0 { + continue + } + diff.ToRemove = append(diff.ToRemove, string(removeVal)) + } + return nil +} + +func MakeMapDiff(m1 map[string]string, m2 map[string]string) []byte { + diff := makeMapDiff(m1, m2) + return diff.encode() +} + +func ApplyMapDiff(oldMap map[string]string, diffBytes []byte) (map[string]string, error) { + var diff MapDiffType + err := diff.decode(diffBytes) + if err != nil { + return nil, err + } + return diff.apply(oldMap), nil +} diff --git a/pkg/statediff/statediff_test.go b/pkg/statediff/statediff_test.go new file mode 100644 index 00000000..3b231c34 --- /dev/null +++ b/pkg/statediff/statediff_test.go @@ -0,0 +1,99 @@ +package statediff + +import ( + "fmt" + "testing" +) + +const Str1 = ` +hello +line #2 +apple +grapes +banana +apple +` + +const Str2 = ` +line #2 +apple +grapes +banana +` + +const Str3 = ` +more +stuff +banana +coconut +` + +const Str4 = ` +more +stuff +banana2 +coconut +` + +func testLineDiff(t *testing.T, str1 string, str2 string) { + diffBytes := MakeLineDiff(str1, str2) + fmt.Printf("diff-len: %d\n", len(diffBytes)) + out, err := ApplyLineDiff(str1, diffBytes) + if err != nil { + t.Errorf("error in diff: %v", err) + return + } + if out != str2 { + t.Errorf("bad diff output") + } + var dt LineDiffType + err = dt.decode(diffBytes) + if err != nil { + t.Errorf("error decoding diff: %v\n", err) + } +} + +func TestLineDiff(t *testing.T) { + testLineDiff(t, Str1, Str2) + testLineDiff(t, Str2, Str3) + testLineDiff(t, Str1, Str3) + testLineDiff(t, Str3, Str1) + testLineDiff(t, Str3, Str4) +} + +func strMapsEqual(m1 map[string]string, m2 map[string]string) bool { + if len(m1) != len(m2) { + return false + } + for key, val := range m1 { + val2, ok := m2[key] + if !ok || val != val2 { + return false + } + } + for key, val := range m2 { + val2, ok := m1[key] + if !ok || val != val2 { + return false + } + } + return true +} + +func TestMapDiff(t *testing.T) { + m1 := map[string]string{"a": "5", "b": "hello", "c": "mike"} + m2 := map[string]string{"a": "5", "b": "goodbye", "d": "more"} + diffBytes := MakeMapDiff(m1, m2) + fmt.Printf("mapdifflen: %d\n", len(diffBytes)) + var diff MapDiffType + diff.decode(diffBytes) + diff.dump() + mcheck, err := ApplyMapDiff(m1, diffBytes) + if err != nil { + t.Fatalf("error applying map diff: %v", err) + } + if !strMapsEqual(m2, mcheck) { + t.Errorf("maps not equal") + } + fmt.Printf("%v\n", mcheck) +}