I18n fixes (#211)

* remove byte sanitization for user commands

When serializing jsonBytes in packet.go::MarshalPacket, a step existed
that attempted to manually sanitize the bytes before sending them. This
was initially done to avoid invalid characters in json; however, go
should handle this for us. But this sanitization broke
internationalization because it excluded characters required for unicode
in other languages. Because of that, it has been removed.

* properly decode non-ascii on frontend

The functions atob and btoa do not convert base 64 to strings in the
expected way. The base64ToArray function handles it properly but other
cases do not. These other cases have been replaced with a helper
function that makes use of the base64-js package. This package has
already been included as a dependency of another package we use, but it
was added to the package.json file to make the inclusion explicit.

* automatically set/share LANG var with waveshell

Waveterm previously did not set the LANG environment variable which
caused problems for international users. On Linux, this is done
automatically, but it needs to be done manually on macos.

Even on linux, the wavesrv LANG variable is shared with waveshell to
ensure the same one is used on remotes.

* only set the lang var if not previously set

In order to prevent waveterm from overriding the lang var entirely, this
ensures that it is only manually determined if it hasn't previously been
set.

* use envMap instead of os to determine var

This is slightly more performant and relies more directly on our code
instead of external code.

package.json

@@ -13,6 +13,7 @@
         "@tanstack/react-table": "^8.10.3",
         "@types/semver": "^7.5.6",
         "autobind-decorator": "^2.4.0",
+        "base64-js": "^1.5.1",
         "classnames": "^2.3.1",
         "dayjs": "^1.11.3",
         "dompurify": "^3.0.2",

src/model/model.ts

@@ -9,6 +9,8 @@ import { boundMethod } from "autobind-decorator";
 import { debounce } from "throttle-debounce";
 import {
     handleJsonFetchResponse,
+    base64ToString,
+    stringToBase64,
     base64ToArray,
     genMergeData,
     genMergeDataMap,
@@ -336,7 +338,7 @@ class Cmd {
             type: "feinput",
             ck: this.screenId + "/" + this.lineId,
             remote: this.remote,
-            inputdata64: btoa(data),
+            inputdata64: stringToBase64(data),
         };
         GlobalModel.sendInputPacket(inputPacket);
     }
@@ -2866,7 +2868,7 @@ class RemotesModalModel {
         let inputPacket: RemoteInputPacketType = {
             type: "remoteinput",
             remoteid: remoteId,
-            inputdata64: btoa(event.key),
+            inputdata64: stringToBase64(event.key),
         };
         GlobalModel.sendInputPacket(inputPacket);
     }
@@ -3045,7 +3047,7 @@ class RemotesModel {
         let inputPacket: RemoteInputPacketType = {
             type: "remoteinput",
             remoteid: remoteId,
-            inputdata64: btoa(event.key),
+            inputdata64: stringToBase64(event.key),
         };
         GlobalModel.sendInputPacket(inputPacket);
     }
@@ -4201,7 +4203,7 @@ class Model {
                     return resp.text() as any;
                 }
                 contentType = resp.headers.get("Content-Type");
-                fileInfo = JSON.parse(atob(resp.headers.get("X-FileInfo")));
+                fileInfo = JSON.parse(base64ToString(resp.headers.get("X-FileInfo")));
                 return resp.blob();
             })
             .then((blobOrText: any) => {

src/util/util.ts

@@ -6,6 +6,7 @@ import { sprintf } from "sprintf-js";
 import dayjs from "dayjs";
 import localizedFormat from "dayjs/plugin/localizedFormat";
 import type { RemoteType, CommandRtnType } from "../types/types";
+import base64 from "base64-js";
 
 type OV<V> = mobx.IObservableValue<V>;
 
@@ -70,6 +71,16 @@ function handleJsonFetchResponse(url: URL, resp: any): Promise<any> {
     return rtnData;
 }
 
+function base64ToString(b64: string): string {
+    let stringBytes = base64.toByteArray(b64)
+    return new TextDecoder().decode(stringBytes)
+}
+
+function stringToBase64(input: string): string {
+    let stringBytes = new TextEncoder().encode(input)
+    return base64.fromByteArray(stringBytes)
+}
+
 function base64ToArray(b64: string): Uint8Array {
     let rawStr = atob(b64);
     let rtnArr = new Uint8Array(new ArrayBuffer(rawStr.length));
@@ -229,26 +240,6 @@ function genMergeDataMap<ObjType extends IObjType<DataType>, DataType extends ID
     return rtn;
 }
 
-function parseEnv0(envStr64: string): Map<string, string> {
-    let envStr = atob(envStr64);
-    let parts = envStr.split("\x00");
-    let rtn: Map<string, string> = new Map();
-    for (let i = 0; i < parts.length; i++) {
-        let part = parts[i];
-        if (part == "") {
-            continue;
-        }
-        let eqIdx = part.indexOf("=");
-        if (eqIdx == -1) {
-            continue;
-        }
-        let varName = part.substr(0, eqIdx);
-        let varVal = part.substr(eqIdx + 1);
-        rtn.set(varName, varVal);
-    }
-    return rtn;
-}
-
 function boundInt(ival: number, minVal: number, maxVal: number): number {
     if (ival < minVal) {
         return minVal;
@@ -414,11 +405,12 @@ function getRemoteName(remote: RemoteType): string {
 
 export {
     handleJsonFetchResponse,
+    base64ToString,
+    stringToBase64,
     base64ToArray,
     genMergeData,
     genMergeDataMap,
     genMergeSimpleData,
-    parseEnv0,
     boundInt,
     isModKeyPress,
     incObs,

waveshell/pkg/packet/packet.go

@@ -927,14 +927,6 @@ func ParseJsonPacket(jsonBuf []byte) (PacketType, error) {
 	return pk, nil
 }
 
-func sanitizeBytes(buf []byte) {
-	for idx, b := range buf {
-		if b >= 127 || (b < 32 && b != 10 && b != 13) {
-			buf[idx] = '?'
-		}
-	}
-}
-
 type SendError struct {
 	IsWriteError   bool // fatal
 	IsMarshalError bool // not fatal
@@ -970,7 +962,6 @@ func MarshalPacket(packet PacketType) ([]byte, error) {
 	outBuf.Write(jsonBytes)
 	outBuf.WriteByte('\n')
 	outBytes := outBuf.Bytes()
-	sanitizeBytes(outBytes)
 	return outBytes, nil
 }
 

wavesrv/pkg/remote/remote.go

@@ -1126,6 +1126,9 @@ func addScVarsToState(state *packet.ShellState) *packet.ShellState {
 	envMap := shexec.DeclMapFromState(&rtn)
 	envMap["PROMPT"] = &shexec.DeclareDeclType{Name: "PROMPT", Value: "1", Args: "x"}
 	envMap["PROMPT_VERSION"] = &shexec.DeclareDeclType{Name: "PROMPT_VERSION", Value: scbase.WaveVersion, Args: "x"}
+	if _, exists := envMap["LANG"]; !exists {
+		envMap["LANG"] = &shexec.DeclareDeclType{Name: "LANG", Value: scbase.DetermineLang(), Args: "x"}
+	}
 	rtn.ShellVars = shexec.SerializeDeclMap(envMap)
 	return &rtn
 }

wavesrv/pkg/scbase/scbase.go

@@ -377,3 +377,30 @@ func MacOSRelease() string {
 	})
 	return osRelease
 }
+
+var osLangOnce = &sync.Once{}
+var osLang string
+
+func determineLang() string {
+	ctx, cancelFn := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancelFn()
+	if runtime.GOOS == "darwin" {
+		out, err := exec.CommandContext(ctx, "defaults", "read", "-g", "AppleLocale").CombinedOutput()
+		if err != nil {
+			log.Printf("error executing 'defaults read -g AppleLocale': %v\n", err)
+			return ""
+		}
+		return strings.TrimSpace(string(out)) + ".UTF-8"
+	} else {
+		// this is specifically to get the wavesrv LANG so waveshell
+		// on a remote uses the same LANG
+		return os.Getenv("LANG")
+	}
+}
+
+func DetermineLang() string {
+	osLangOnce.Do(func() {
+		osLang = determineLang()
+	})
+	return osLang
+}