Merge pull request #617 from sfall-team/feature/nested-array-expressions

Support for nested array expressions (#616)

artifacts/scripting/headers/lib.arrays.h

@@ -35,8 +35,11 @@ procedure array_keys(variable array);
 // list of array values (useful for maps)
 procedure array_values(variable array);
 
+// fix_array for multi-dimensional arrays
+procedure fix_array_deep(variable array, variable levels := 1);
+
 // makes given array permanent and returns it
-procedure array_fixed(variable array);
+procedure array_fixed(variable array, variable levels := 1);
 
 // returns temp array containing a subarray starting from $index with $count elements
 // negative $index means index from the end of array
@@ -246,13 +249,28 @@ end
 /**
  * Sets given array as permanent and returns it.
  * @arg {array} array
+ * @arg {int} levels - Number of depth levels for a multi-dimensional array
  * @ret {array}
  */
-procedure array_fixed(variable array) begin
-   fix_array(array);
+procedure array_fixed(variable array, variable levels) begin
+   call fix_array_deep(array, levels);
    return array;
 end
 
+/**
+ * Makes a multi-dimensional temp array permenant.
+ * @arg {array} array
+ * @arg {int} levels - Number of depth levels for a multi-dimensional array
+ */
+procedure fix_array_deep(variable array, variable levels) begin
+   fix_array(array);
+   if (levels > 1) then begin
+      foreach (variable subArray in array) begin
+         call fix_array_deep(subArray, levels - 1);
+      end
+   end
+end
+
 /**
  * Returns a slice of a given list array as a new temp array.
  * @arg {list} array
@@ -757,7 +775,7 @@ end
 procedure debug_array_str_deep(variable arr, variable levels, variable prefix := false) begin
 #define _newline if (levels > 1) then s += "\n";
 #define _indent ii := 0; while (ii < levels - 1) do begin s += "   "; ii++; end
-#define _value(v) (v if (levels <= 1 or not array_exists(v)) else debug_array_str_deep(v, levels - 1))
+#define _value(v) (v if (levels <= 1 or typeof(v) != VALTYPE_INT or not array_exists(v)) else debug_array_str_deep(v, levels - 1))
    variable i := 0, ii, k, v, s, len;
    len := len_array(arr);
    if (array_is_map(arr)) then begin  // print assoc array
@@ -776,7 +794,6 @@ procedure debug_array_str_deep(variable arr, variable levels, variable prefix :=
       s += "}";
    end else begin  // print list
       s := ("List("+len+"): [") if prefix else "[";
-      _newline
       while i < len do begin
          _newline
          v := get_array(arr, i);

artifacts/scripting/tests/gl_arrays_testcase.ssl

@@ -0,0 +1,211 @@
+#include "../headers/sfall.h"
+#include "../headers/lib.arrays.h"
+#include "../headers/lib.strings.h"
+
+variable test_suite_errors := 0;
+variable test_suite_verbose := true;
+
+#ifndef assertEquals
+procedure assertEquals(variable desc, variable a, variable b) begin
+   if (a != b or typeof(a) != typeof(b)) then begin
+      display_msg("Assertion failed \""+desc+"\": "+a+" != "+b);
+      test_suite_errors ++;
+   end else if (test_suite_verbose) then begin
+      display_msg("Assert \""+desc+"\" ok");
+   end
+end
+
+procedure assertNotEquals(variable desc, variable a, variable b) begin
+   if (a == b) then begin
+      display_msg("Assertion failed \""+desc+"\": "+a+" == "+b);
+      test_suite_errors ++;
+   end else if (test_suite_verbose) then begin
+      display_msg("Assert \""+desc+"\" ok");
+   end
+end
+#endif
+
+#define ARRAY_MAX_STRING	   (1024)
+#define ARRAY_MAX_SIZE		 (100000)
+
+procedure array_test_suite begin
+   variable arr, i, arr2, map, k, v, s;
+   test_suite_errors := 0;
+   
+   display_msg("Testing utility functions...");
+   call assertEquals("strlen", strlen("testing"), 7);
+   call assertEquals("substr 1", substr("testing", 4, 2), "in");
+   call assertEquals("substr 2", substr("testing", 1, -1), "estin");
+   call assertEquals("substr 3", substr("testing", -5, 5), "sting");
+   call assertEquals("typeof 1", typeof(1), VALTYPE_INT);
+   call assertEquals("typeof 2", typeof(1.0), VALTYPE_FLOAT);
+   call assertEquals("typeof 3", typeof("1.0"), VALTYPE_STR);
+      
+   // Basic array functionality
+   display_msg("Testing basic arrays functionality...");
+   arr := create_array(5,0);
+   call assertEquals("array size", len_array(arr), 5);
+   arr[0]:=100;
+   call assertEquals("get array", get_array(arr, 0), 100);
+   arr[1]:=5.555;
+   call assertEquals("get array float", get_array(arr, 1), 5.555);
+   arr[2]:="hello";
+   call assertEquals("get array str", get_array(arr, 2), "hello");
+   call assertEquals("get array invalid index", get_array(arr, -1), 0);
+   call assertEquals("get array invalid index", get_array(arr, 6), 0);
+   resize_array(arr, 77);
+   call assertEquals("list resize", len_array(arr), 77);
+   resize_array(arr, ARRAY_MAX_SIZE + 100);
+   call assertEquals("resize max", len_array(arr), ARRAY_MAX_SIZE);
+   free_array(arr);
+   call assertEquals("not exists check", len_array(arr), -1);
+   arr := create_array(ARRAY_MAX_SIZE + 100, 0);
+   call assertEquals("create max", len_array(arr), ARRAY_MAX_SIZE);
+      
+   free_array(arr);
+   arr := [6, 1, 10, "one", 10, "two", 5, "three"];
+   arr[0] := "wtf";
+   //display_msg(debug_array_str(arr));
+   call assertEquals("scan list", scan_array(arr, 10), 2);
+   call assertEquals("scan list 2", scan_array(arr, "two"), 5);
+   call assertEquals("array_key", array_key(arr, 5), 5);
+   call assertEquals("array_key out of range", array_key(arr, 9), 0);
+   call assertEquals("is list", array_key(arr, -1), 0);
+   
+   call assertEquals("get array as substr", get_array("NiCe", 1), "i");
+      
+   arr := [78, 12, 99, 1, -5];
+   sort_array(arr);
+   call assertEquals("sort ASC", arrays_equal(arr, [-5, 1, 12, 78, 99]), true);
+
+   arr := ["Albert", "John", "Mike", "David"];
+   sort_array_reverse(arr);
+   call assertEquals("sort DESC", arrays_equal(arr, ["Mike", "John", "David", "Albert"]), true);
+   reverse_array(arr);
+   call assertEquals("reverse list", arrays_equal(arr, ["Albert", "David", "John", "Mike"]), true);
+   shuffle_array(arr);
+   call assertEquals("shuffle list 1", arrays_equal(arr, ["Albert", "David", "John", "Mike"]), false);
+   call assertEquals("shuffle list 2", len_array(arr), 4);
+   if (test_suite_verbose) then display_array(arr);
+   
+   // some additional stuff
+   call assertEquals("string_split", get_array(string_split("this+is+good", "+"), 2), "good");
+   call assertEquals("string_split 2", len_array(string_split("advice", "")), 6);
+   s := "";
+   for (i := 0; i < ARRAY_MAX_STRING+30; i+=10) begin
+      s += "Verbosity.";
+   end
+   arr[0] := s;
+   call assertEquals("array max string", strlen(arr[0]), ARRAY_MAX_STRING-1);
+   
+   // ASSOC ARRAYS TEST
+   display_msg("Testing associative arrays...");
+   arr := create_array(-1,0);
+   call assertEquals("is map", array_key(arr, -1), 1);
+   call assertEquals("exists check", len_array(arr), 0);
+   arr["123"] := 100;
+   call assertEquals("set/get str=>int", arr["123"], 100);
+   arr["123"] := 50;
+   call assertEquals("overwrite 1", arr["123"], 50);
+   call assertEquals("overwrite 2", len_array(arr), 1);
+   arr[-1] := "wtf";
+   call assertEquals("set/get int=>str", arr[-1], "wtf");
+   arr[3.14] := 0.00001;
+   call assertEquals("set/get float=>float", arr[3.14], 0.00001);
+   arr["fourth"] := "elem";
+   call assertEquals("map size", len_array(arr), 4);
+   arr[-1] := 0;
+   call assertEquals("unset key: length", len_array(arr), 3);
+   call assertEquals("unset key: hashmap", arr[3.14], 0.00001);
+   call assertEquals("key not exist", arr[777], 0);
+   free_array(arr);
+   call assertEquals("not exists check", len_array(arr), -1);
+   arr := {6:5, 1.001:0.5, 10:"A", 5:0, "wtf":1.1, 10:0.0001, "some":"What"}; // 7 here
+   // 10:"A" will be overwritten by 10:0.0001 
+   call assertEquals("assoc array expr", len_array(arr), 6); // 6 actual
+   
+   call assertEquals("scan map 1", scan_array(arr, 1.1), "wtf");
+   call assertEquals("scan map 2", scan_array(arr, 5), 6);
+   call assertEquals("scan map 3", scan_array(arr, "What"), "some");
+   call assertEquals("array_key 1", array_key(arr, 0), 6);
+   call assertEquals("array_key 2", array_key(arr, 4), "wtf");
+   call assertEquals("array_key 3", array_key(arr, 1), 1.001);
+   
+   resize_array(arr, 2);
+   call assertEquals("map resize 1", len_array(arr), 2);
+   call assertEquals("map resize 2", arrays_equal(arr, {6:5, 1.001:0.5}), true);
+   resize_array(arr, 0);
+   call assertEquals("map clear", len_array(arr), 0);
+   
+   display_msg("Testing foreach...");
+   s := "ar";
+   arr := [6, 10, 0.5, "wtf"];
+   foreach v in arr begin
+      s += v;
+   end
+   call assertEquals("foreach 1", s, "ar6100.50000wtf");
+   s := "ar2=";
+   arr := {"name": "John", "hp": 25, 0: 5.5};
+   foreach k: v in arr begin
+      s += k+":"+v+";";
+   end
+   call assertEquals("foreach 2", s, "ar2=name:John;hp:25;0:5.50000;");
+   
+   
+   display_msg("Testing save/load...");
+   arr := [2,1];
+   arr2 := {1:2};
+   s := "wtf";
+   k := 1;
+   if (arr and arr2[k]) then s := "ok";
+   call assertEquals("bracket syntax", s, "ok");
+   save_array("myarray", arr);
+   call assertEquals("load 1", load_array("myarray"), arr);
+   save_array("myarray", arr2);
+   call assertEquals("load 2", load_array("myarray"), arr2);
+   free_array(arr2);
+   call assertEquals("not exists check", len_array(arr2), -1);
+   call assertEquals("load fail", load_array("myarray"), 0);
+   save_array(0.1, arr);
+   call assertEquals("save as float", load_array(0.1), arr);
+   arr2 := list_saved_arrays; // list of array names
+   //display_msg(debug_array_str(arr2));
+   call assertNotEquals("saved arrays 1", scan_array(arr2, 0.1), -1);
+   call assertEquals("saved arrays 2", scan_array(arr2, "myarray"), -1);
+   save_array(0, arr);
+   call assertEquals("unsave array 1", load_array(0.1), 0);
+   call assertEquals("unsave array 2", len_array(arr), 2);
+   call assertEquals("saved arrays 3", len_array(list_saved_arrays), len_array(arr2) - 1);
+
+   display_msg("Testing nested expressions...");
+   arr := [["one", "two"], {"three": "four"}];
+   call assertEquals("nested 1", arr[0][1], "two");
+   call assertEquals("nested 2", arr[1].three, "four");
+
+   display_msg("All tests finished with "+test_suite_errors+" errors.");
+end
+
+
+procedure arrays_lib_tests begin
+   variable arr, i, arr2, map, k, v, s;
+   test_suite_errors := 0;
+   
+   call assertEquals("array_equals 1", arrays_equal([9, 7, 2, 1], [9, 7, 2, 1]), true);
+   call assertEquals("array_equals 2", arrays_equal([9, 7, 0, 1], [9, 7, 2, 1]), false);
+   call assertEquals("array_equals 3", arrays_equal([1, 1, 1, 1], [1, 1, 1]), false);
+   call assertEquals("array_equals 4", arrays_equal([], []), true);
+   call assertEquals("array_equals 5", arrays_equal({}, []), false);
+   call assertEquals("array_equals 6", arrays_equal({1: 1.0, 2: 2.0}, {1: 1.000, 2: 2.000}), true);
+   call assertEquals("array_equals 7", arrays_equal({"name": "John", "dept": 15.20}, {"name": "John"}), false);
+
+   arr := array_transform(array_filter(string_split(",23,1,ghh 6 6,77.1 ", ","), @string_null_or_empty, true), @string_to_float);
+   call assertEquals("array_filter 1", arrays_equal(arr, [23.0,1.0,0.0,77.1]), true);
+   display_msg("All tests finished with "+test_suite_errors+" errors.");
+end
+
+procedure start begin
+   if not game_loaded then return;
+
+   call array_test_suite;
+   call arrays_lib_tests;
+end

sfall/Modules/Scripting/Arrays.cpp

@@ -45,7 +45,9 @@ ArrayKeysMap savedArrays;
 // auto-incremented ID
 DWORD nextArrayID = 1;
 // special array ID for array expressions, contains the ID number of the currently created array
-DWORD stackArrayId;
+DWORD expressionArrayId;
+// special stack for array expressions, contains ID numbers of the currently created arrays
+std::vector<DWORD> arrayExpressionStack;
 
 static char get_all_arrays_special_key[] = "...all_arrays...";
 
@@ -429,7 +431,19 @@ DWORD CreateArray(int len, DWORD flags) {
 	//	var.key = sArrayElement(nextArrayID, DataType::INT);
 	//	savedArrays[var.key] = nextArrayID;
 	//}
-	stackArrayId = nextArrayID;
+	if ((flags & ARRAYFLAG_EXPR_PUSH) != 0) {
+		// When creating array for sub-expression, make sure to add array for base expression to stack
+		// This is messy, but required to support older scripts:
+		// - We must always assign expressionArrayId for one-layer expressions from older scripts to work like they did before
+		// - We can't directly push first arrayID into stack b/c no way to distinguish between start of an expression and normal temp_array call
+		// - Compiler will only add this flag for temp_array call generated from a sub-expression
+		// - So only on this second call we know we are in expression and expressionArrayId definitely contains arrayId of the first layer
+		if (arrayExpressionStack.empty() && expressionArrayId != 0) {
+			arrayExpressionStack.push_back(expressionArrayId);
+		}
+		arrayExpressionStack.push_back(nextArrayID);
+	}
+	expressionArrayId = nextArrayID;
 	arrays[nextArrayID] = var;
 	return nextArrayID++;
 }
@@ -757,16 +771,31 @@ void SaveArray(const ScriptValue& key, DWORD id) {
 
 	Should always return 0!
 */
-long StackArray(const ScriptValue& key, const ScriptValue& val) {
-	if (stackArrayId == 0 || !ArrayExists(stackArrayId)) return 0;
+void SetArrayFromExpression(const ScriptValue& key, const ScriptValue& val) {
+	DWORD arrayId = !arrayExpressionStack.empty()
+		? arrayExpressionStack.back()
+		: expressionArrayId;
 
-	if (!arrays[stackArrayId].isAssoc()) { // automatically resize array to fit one new element
-		size_t size = arrays[stackArrayId].val.size();
-		if (size >= ARRAY_MAX_SIZE) return 0;
-		if (key.rawValue() >= size) arrays[stackArrayId].val.resize(size + 1);
+	if (arrayId == 0 || !ArrayExists(arrayId)) return;
+
+	if (!arrays[arrayId].isAssoc()) { // automatically resize array to fit one new element
+		size_t size = arrays[arrayId].val.size();
+		if (size >= ARRAY_MAX_SIZE) return;
+		if (key.rawValue() >= size) arrays[arrayId].val.resize(size + 1);
+	}
+	SetArray(arrayId, key, val, false);
+}
+
+void PopExpressionArray() {
+	if (arrayExpressionStack.empty()) return;
+
+	arrayExpressionStack.pop_back();
+
+	// Reversing the hack from CreateArray
+	if (arrayExpressionStack.size() == 1) {
+		expressionArrayId = arrayExpressionStack.back();
+		arrayExpressionStack.pop_back();
 	}
-	SetArray(stackArrayId, key, val, false);
-	return 0;
 }
 
 sArrayVar* GetRawArray(DWORD id) {

sfall/Modules/Scripting/Arrays.h

@@ -109,6 +109,8 @@ struct sArrayVarOld
 #define ARRAYFLAG_ASSOC		(1) // is map
 #define ARRAYFLAG_CONSTVAL	(2) // don't update value of key if the key exists in map
 #define ARRAYFLAG_RESERVED	(4)
+#define ARRAYFLAG_EXPR_PUSH (32) // is created as part of array sub-expression
+#define ARRAYFLAG_EXPR_POP  (64) // is used to indicate end of array sub-expression, not used in actual array
 
 typedef std::unordered_map<sArrayElement, DWORD, sArrayElement_HashFunc, sArrayElement_EqualFunc> ArrayKeysMap;
 
@@ -224,8 +226,11 @@ DWORD LoadArray(const ScriptValue& key);
 // make array saved into the savegame with associated key
 void SaveArray(const ScriptValue& key, DWORD id);
 
-// special function that powers array expressions
-long StackArray(const ScriptValue& key, const ScriptValue& val);
+// sets array element from array expression
+void SetArrayFromExpression(const ScriptValue& key, const ScriptValue& val);
+
+// used to indicate the end of array sub-expression
+void PopExpressionArray();
 
 sArrayVar* GetRawArray(DWORD id);
 

sfall/Modules/Scripting/Handlers/Arrays.cpp

@@ -90,7 +90,15 @@ void op_resize_array(OpcodeContext& ctx) {
 }
 
 void op_temp_array(OpcodeContext& ctx) {
-	auto arrayId = CreateTempArray(ctx.arg(0).rawValue(), ctx.arg(1).rawValue());
+	const auto& flags = ctx.arg(1);
+
+	// Special case for array sub-expressions.
+	if ((flags.rawValue() & ARRAYFLAG_EXPR_POP) != 0) {
+		PopExpressionArray();
+		ctx.setReturn(0);
+		return;
+	}
+	auto arrayId = CreateTempArray(ctx.arg(0).rawValue(), flags.rawValue());
 	ctx.setReturn(arrayId);
 }
 
@@ -120,10 +128,9 @@ void op_get_array_key(OpcodeContext& ctx) {
 	);
 }
 
-void op_stack_array(OpcodeContext& ctx) {
-	ctx.setReturn(
-		StackArray(ctx.arg(0), ctx.arg(1))
-	);
+void op_arrayexpr(OpcodeContext& ctx) {
+	SetArrayFromExpression(ctx.arg(0), ctx.arg(1));
+	ctx.setReturn(0);
 }
 
 // object LISTS

sfall/Modules/Scripting/Handlers/Arrays.h

@@ -51,7 +51,7 @@ void op_load_array(OpcodeContext&);
 
 void op_get_array_key(OpcodeContext&);
 
-void op_stack_array(OpcodeContext&);
+void op_arrayexpr(OpcodeContext&);
 
 void op_list_begin(OpcodeContext&);
 

sfall/Modules/Scripting/Opcodes.cpp

@@ -193,9 +193,9 @@ static SfallOpcodeInfo opcodeInfoArray[] = {
 	// 0x252 // RESERVED
 	{0x253, "typeof",                     op_typeof,                    1, true},
 	{0x254, "save_array",                 op_save_array,                2, false,  0, {ARG_ANY, ARG_OBJECT}},
-	{0x255, "load_array",                 op_load_array,                1, true,  -1, {ARG_INTSTR}},
+	{0x255, "load_array",                 op_load_array,                1, true,  -1, {ARG_ANY}},
 	{0x256, "array_key",                  op_get_array_key,             2, true,   0, {ARG_INT, ARG_INT}},
-	{0x257, "arrayexpr",                  op_stack_array,               2, true},
+	{0x257, "arrayexpr",                  op_arrayexpr,                 2, true},
 	// 0x258 // RESERVED for arrays
 	// 0x259 // RESERVED for arrays
 	{0x25a, "reg_anim_destroy",           op_reg_anim_destroy,          1, false,  0, {ARG_OBJECT}},