mirror of
https://github.com/wavetermdev/backup.git
synced 2026-08-05 13:57:07 -07:00
line/mapdiff code
This commit is contained in:
sawka
@@ -0,0 +1,182 @@
|
||||
package statediff
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const LineDiffVersion = 0
|
||||
|
||||
type SingleLineEntry struct {
|
||||
LineVal int
|
||||
Run int
|
||||
}
|
||||
|
||||
type LineDiffType struct {
|
||||
Lines []SingleLineEntry
|
||||
NewData []string
|
||||
}
|
||||
|
||||
func (diff LineDiffType) dump() {
|
||||
fmt.Printf("DIFF:\n")
|
||||
pos := 1
|
||||
for _, entry := range diff.Lines {
|
||||
fmt.Printf(" %d-%d: %d\n", pos, pos+entry.Run, entry.LineVal)
|
||||
pos += entry.Run
|
||||
}
|
||||
for idx, str := range diff.NewData {
|
||||
fmt.Printf(" n%d: %s\n", idx+1, str)
|
||||
}
|
||||
}
|
||||
|
||||
// simple encoding
|
||||
// a 0 means read a line from NewData
|
||||
// a non-zero number means read the 1-indexed line from OldData
|
||||
func (diff LineDiffType) applyDiff(oldData []string) ([]string, error) {
|
||||
rtn := make([]string, 0, len(diff.Lines))
|
||||
newDataPos := 0
|
||||
for _, entry := range diff.Lines {
|
||||
if entry.LineVal == 0 {
|
||||
for i := 0; i < entry.Run; i++ {
|
||||
if newDataPos >= len(diff.NewData) {
|
||||
return nil, fmt.Errorf("not enough newdata for diff")
|
||||
}
|
||||
rtn = append(rtn, diff.NewData[newDataPos])
|
||||
newDataPos++
|
||||
}
|
||||
} else {
|
||||
oldDataPos := entry.LineVal - 1 // 1-indexed
|
||||
for i := 0; i < entry.Run; i++ {
|
||||
realPos := oldDataPos + i
|
||||
if realPos < 0 || realPos >= len(oldData) {
|
||||
return nil, fmt.Errorf("diff index out of bounds %d old-data-len:%d", realPos, len(oldData))
|
||||
}
|
||||
rtn = append(rtn, oldData[realPos])
|
||||
}
|
||||
}
|
||||
}
|
||||
return rtn, nil
|
||||
}
|
||||
|
||||
func putUVarint(buf *bytes.Buffer, viBuf []byte, ival int) {
|
||||
l := binary.PutUvarint(viBuf, uint64(ival))
|
||||
buf.Write(viBuf[0:l])
|
||||
}
|
||||
|
||||
// simple encoding
|
||||
// write varints. first version, then len, then len-number-of-varints, then fill the rest with newdata
|
||||
// [version] [len-varint] [varint]xlen... newdata (bytes)
|
||||
func (diff LineDiffType) encode() []byte {
|
||||
var buf bytes.Buffer
|
||||
viBuf := make([]byte, binary.MaxVarintLen64)
|
||||
putUVarint(&buf, viBuf, LineDiffVersion)
|
||||
putUVarint(&buf, viBuf, len(diff.Lines))
|
||||
for _, entry := range diff.Lines {
|
||||
putUVarint(&buf, viBuf, entry.LineVal)
|
||||
putUVarint(&buf, viBuf, entry.Run)
|
||||
}
|
||||
for idx, str := range diff.NewData {
|
||||
buf.WriteString(str)
|
||||
if idx != len(diff.NewData)-1 {
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func (rtn *LineDiffType) decode(diffBytes []byte) error {
|
||||
r := bytes.NewBuffer(diffBytes)
|
||||
version, err := binary.ReadUvarint(r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid diff, cannot read version: %v", err)
|
||||
}
|
||||
if version != LineDiffVersion {
|
||||
return fmt.Errorf("invalid diff, bad version: %d", version)
|
||||
}
|
||||
linesLen64, err := binary.ReadUvarint(r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid diff, cannot read lines length: %v", err)
|
||||
}
|
||||
linesLen := int(linesLen64)
|
||||
rtn.Lines = make([]SingleLineEntry, linesLen)
|
||||
for idx := 0; idx < linesLen; idx++ {
|
||||
lineVal, err := binary.ReadUvarint(r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid diff, cannot read line %d: %v", idx, err)
|
||||
}
|
||||
lineRun, err := binary.ReadUvarint(r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid diff, cannot read line-run %d: %v", idx, err)
|
||||
}
|
||||
rtn.Lines[idx] = SingleLineEntry{LineVal: int(lineVal), Run: int(lineRun)}
|
||||
}
|
||||
restOfInput := string(r.Bytes())
|
||||
if len(restOfInput) > 0 {
|
||||
rtn.NewData = strings.Split(restOfInput, "\n")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeLineDiff(oldData []string, newData []string) LineDiffType {
|
||||
var rtn LineDiffType
|
||||
oldDataMap := make(map[string]int) // 1-indexed
|
||||
for idx, str := range oldData {
|
||||
if _, found := oldDataMap[str]; found {
|
||||
continue
|
||||
}
|
||||
oldDataMap[str] = idx + 1
|
||||
}
|
||||
var cur *SingleLineEntry
|
||||
rtn.Lines = make([]SingleLineEntry, 0)
|
||||
for _, str := range newData {
|
||||
oldIdx, found := oldDataMap[str]
|
||||
if cur != nil && cur.LineVal != 0 {
|
||||
checkLine := cur.LineVal + cur.Run - 1
|
||||
if checkLine < len(oldData) && oldData[checkLine] == str {
|
||||
cur.Run++
|
||||
continue
|
||||
}
|
||||
} else if cur != nil && cur.LineVal == 0 && !found {
|
||||
cur.Run++
|
||||
rtn.NewData = append(rtn.NewData, str)
|
||||
continue
|
||||
}
|
||||
if cur != nil {
|
||||
rtn.Lines = append(rtn.Lines, *cur)
|
||||
}
|
||||
cur = &SingleLineEntry{Run: 1}
|
||||
if found {
|
||||
cur.LineVal = oldIdx
|
||||
} else {
|
||||
cur.LineVal = 0
|
||||
rtn.NewData = append(rtn.NewData, str)
|
||||
}
|
||||
}
|
||||
if cur != nil {
|
||||
rtn.Lines = append(rtn.Lines, *cur)
|
||||
}
|
||||
return rtn
|
||||
}
|
||||
|
||||
func MakeLineDiff(str1 string, str2 string) []byte {
|
||||
str1Arr := strings.Split(str1, "\n")
|
||||
str2Arr := strings.Split(str2, "\n")
|
||||
diff := makeLineDiff(str1Arr, str2Arr)
|
||||
return diff.encode()
|
||||
}
|
||||
|
||||
func ApplyLineDiff(str1 string, diffBytes []byte) (string, error) {
|
||||
var diff LineDiffType
|
||||
err := diff.decode(diffBytes)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
str1Arr := strings.Split(str1, "\n")
|
||||
str2Arr, err := diff.applyDiff(str1Arr)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.Join(str2Arr, "\n"), nil
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package statediff
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const MapDiffVersion = 0
|
||||
|
||||
// 0-bytes are not allowed in entries or keys (same as bash)
|
||||
|
||||
type MapDiffType struct {
|
||||
ToAdd map[string]string
|
||||
ToRemove []string
|
||||
}
|
||||
|
||||
func (diff MapDiffType) dump() {
|
||||
fmt.Printf("VAR-DIFF\n")
|
||||
for name, val := range diff.ToAdd {
|
||||
fmt.Printf(" add: %s=%s\n", name, val)
|
||||
}
|
||||
for _, name := range diff.ToRemove {
|
||||
fmt.Printf(" rem: %s\n", name)
|
||||
}
|
||||
}
|
||||
|
||||
func makeMapDiff(oldMap map[string]string, newMap map[string]string) MapDiffType {
|
||||
var rtn MapDiffType
|
||||
rtn.ToAdd = make(map[string]string)
|
||||
for name, newVal := range newMap {
|
||||
oldVal, found := oldMap[name]
|
||||
if !found || oldVal != newVal {
|
||||
rtn.ToAdd[name] = newVal
|
||||
continue
|
||||
}
|
||||
}
|
||||
for name, _ := range oldMap {
|
||||
_, found := newMap[name]
|
||||
if !found {
|
||||
rtn.ToRemove = append(rtn.ToRemove, name)
|
||||
}
|
||||
}
|
||||
return rtn
|
||||
}
|
||||
|
||||
func (diff MapDiffType) apply(oldMap map[string]string) map[string]string {
|
||||
rtn := make(map[string]string)
|
||||
for name, val := range oldMap {
|
||||
rtn[name] = val
|
||||
}
|
||||
for name, val := range diff.ToAdd {
|
||||
rtn[name] = val
|
||||
}
|
||||
for _, name := range diff.ToRemove {
|
||||
delete(rtn, name)
|
||||
}
|
||||
return rtn
|
||||
}
|
||||
|
||||
func (diff MapDiffType) encode() []byte {
|
||||
var buf bytes.Buffer
|
||||
viBuf := make([]byte, binary.MaxVarintLen64)
|
||||
putUVarint(&buf, viBuf, MapDiffVersion)
|
||||
putUVarint(&buf, viBuf, len(diff.ToAdd))
|
||||
for key, val := range diff.ToAdd {
|
||||
buf.WriteString(key)
|
||||
buf.WriteByte(0)
|
||||
buf.WriteString(val)
|
||||
buf.WriteByte(0)
|
||||
}
|
||||
for _, val := range diff.ToRemove {
|
||||
buf.WriteString(val)
|
||||
buf.WriteByte(0)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func (diff *MapDiffType) decode(diffBytes []byte) error {
|
||||
r := bytes.NewBuffer(diffBytes)
|
||||
version, err := binary.ReadUvarint(r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid diff, cannot read version: %v", err)
|
||||
}
|
||||
if version != MapDiffVersion {
|
||||
return fmt.Errorf("invalid diff, bad version: %d", version)
|
||||
}
|
||||
mapLen64, err := binary.ReadUvarint(r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid diff, cannot map length: %v", err)
|
||||
}
|
||||
mapLen := int(mapLen64)
|
||||
fields := bytes.Split(r.Bytes(), []byte{0})
|
||||
if len(fields) < 2*mapLen {
|
||||
return fmt.Errorf("invalid diff, not enough fields, maplen:%d fields:%d", mapLen, len(fields))
|
||||
}
|
||||
mapFields := fields[0 : 2*mapLen]
|
||||
removeFields := fields[2*mapLen:]
|
||||
diff.ToAdd = make(map[string]string)
|
||||
for i := 0; i < len(mapFields); i += 2 {
|
||||
diff.ToAdd[string(mapFields[i])] = string(mapFields[i+1])
|
||||
}
|
||||
for _, removeVal := range removeFields {
|
||||
if len(removeVal) == 0 {
|
||||
continue
|
||||
}
|
||||
diff.ToRemove = append(diff.ToRemove, string(removeVal))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func MakeMapDiff(m1 map[string]string, m2 map[string]string) []byte {
|
||||
diff := makeMapDiff(m1, m2)
|
||||
return diff.encode()
|
||||
}
|
||||
|
||||
func ApplyMapDiff(oldMap map[string]string, diffBytes []byte) (map[string]string, error) {
|
||||
var diff MapDiffType
|
||||
err := diff.decode(diffBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return diff.apply(oldMap), nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package statediff
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const Str1 = `
|
||||
hello
|
||||
line #2
|
||||
apple
|
||||
grapes
|
||||
banana
|
||||
apple
|
||||
`
|
||||
|
||||
const Str2 = `
|
||||
line #2
|
||||
apple
|
||||
grapes
|
||||
banana
|
||||
`
|
||||
|
||||
const Str3 = `
|
||||
more
|
||||
stuff
|
||||
banana
|
||||
coconut
|
||||
`
|
||||
|
||||
const Str4 = `
|
||||
more
|
||||
stuff
|
||||
banana2
|
||||
coconut
|
||||
`
|
||||
|
||||
func testLineDiff(t *testing.T, str1 string, str2 string) {
|
||||
diffBytes := MakeLineDiff(str1, str2)
|
||||
fmt.Printf("diff-len: %d\n", len(diffBytes))
|
||||
out, err := ApplyLineDiff(str1, diffBytes)
|
||||
if err != nil {
|
||||
t.Errorf("error in diff: %v", err)
|
||||
return
|
||||
}
|
||||
if out != str2 {
|
||||
t.Errorf("bad diff output")
|
||||
}
|
||||
var dt LineDiffType
|
||||
err = dt.decode(diffBytes)
|
||||
if err != nil {
|
||||
t.Errorf("error decoding diff: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLineDiff(t *testing.T) {
|
||||
testLineDiff(t, Str1, Str2)
|
||||
testLineDiff(t, Str2, Str3)
|
||||
testLineDiff(t, Str1, Str3)
|
||||
testLineDiff(t, Str3, Str1)
|
||||
testLineDiff(t, Str3, Str4)
|
||||
}
|
||||
|
||||
func strMapsEqual(m1 map[string]string, m2 map[string]string) bool {
|
||||
if len(m1) != len(m2) {
|
||||
return false
|
||||
}
|
||||
for key, val := range m1 {
|
||||
val2, ok := m2[key]
|
||||
if !ok || val != val2 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for key, val := range m2 {
|
||||
val2, ok := m1[key]
|
||||
if !ok || val != val2 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestMapDiff(t *testing.T) {
|
||||
m1 := map[string]string{"a": "5", "b": "hello", "c": "mike"}
|
||||
m2 := map[string]string{"a": "5", "b": "goodbye", "d": "more"}
|
||||
diffBytes := MakeMapDiff(m1, m2)
|
||||
fmt.Printf("mapdifflen: %d\n", len(diffBytes))
|
||||
var diff MapDiffType
|
||||
diff.decode(diffBytes)
|
||||
diff.dump()
|
||||
mcheck, err := ApplyMapDiff(m1, diffBytes)
|
||||
if err != nil {
|
||||
t.Fatalf("error applying map diff: %v", err)
|
||||
}
|
||||
if !strMapsEqual(m2, mcheck) {
|
||||
t.Errorf("maps not equal")
|
||||
}
|
||||
fmt.Printf("%v\n", mcheck)
|
||||
}
|
||||
Reference in New Issue
Block a user