Implement symbols for XML elements

UIXC/Commands/DebugCommand.cs

@@ -1,4 +1,5 @@
 using Microsoft.Iris.Debug;
+using Microsoft.Iris.Debug.Symbols;
 using Microsoft.Iris.Debug.SystemNet;
 using Spectre.Console;
 using Spectre.Console.Cli;
@@ -24,7 +25,7 @@ public class DebugCommand : Command<DebugCommand.Settings>
         if (settings.SymbolDir is not null)
         {
             Directory.CreateDirectory(settings.SymbolDir);
-            symbolResolver = new(settings.SymbolDir);
+            symbolResolver = new(settings.SymbolDir, settings.SourceDir);
         }
 
         AnsiConsole.MarkupLineInterpolated($"Connecting to '{settings.ServerUri}'...");
@@ -86,11 +87,26 @@ public class DebugCommand : Command<DebugCommand.Settings>
 
                         var line = int.Parse(inputParts[2]);
                         var column = int.Parse(inputParts[3]);
-                        breakOffset = fsym!.OffsetByLineAndColumn(line, column);
+                        var position = new SourcePosition(line, column);
+
+                        var location = fsym!.SourceMap.GetLocationFromPosition(position);
+                        if (location is null)
+                        {
+                            AnsiConsole.MarkupLineInterpolated($"[red]Line {line}, column {column} has known offset in '{fileName}'.[/]");
+                            break;
+                        }
+
+                        breakOffset = location.Value.Offset;
+
+                        if (fsym.HasSourceCode())
+                        {
+                            var sourceCode = fsym.GetSourceSubstring(location.Value.Span);
+                            AnsiConsole.Write(new Panel(sourceCode));
+                        }
                     }
 
-                    client.UpdateBreakpoint(new(path, breakOffset));
                     AnsiConsole.MarkupLineInterpolated($"[green]Breakpoint set in '{fileName}' at offset 0x{breakOffset:X4}.[/]");
+                    client.UpdateBreakpoint(new(path, breakOffset));
 
                     break;
 
@@ -115,6 +131,10 @@ public class DebugCommand : Command<DebugCommand.Settings>
         [CommandOption("-s|--symbols <symbolDir>")]
         public string? SymbolDir { get; init; }
 
+        [Description("The directory containing source code.")]
+        [CommandOption("--sources <sourceDir>")]
+        public string? SourceDir { get; init; }
+
         [Description("Whether to decompile the current file when a breakpoint is hit. Generated symbols will be written to the symbol directory if specified.")]
         [CommandOption("-d|--decompile")]
         public bool Decompile { get; init; }

UIXC/DebugSymbolResolver.cs

@@ -2,15 +2,15 @@
 
 namespace UIXC;
 
-public class DebugSymbolResolver(string symbolDir)
+public class DebugSymbolResolver(string symbolDir, string? sourceDir)
 {
     private readonly DirectoryInfo _symbolDir = new(symbolDir);
+    private readonly DirectoryInfo? _sourceDir = sourceDir is not null ? new(sourceDir) : null;
 
     public ApplicationDebugSymbols? GetForApplication(string application)
     {
         var fileName = $"{application}.asym.json";
-        var asymFile = _symbolDir.EnumerateFiles()
-            .FirstOrDefault(f => f.Name == fileName);
+        var asymFile = FindFile(fileName, _symbolDir);
 
         if (asymFile is null)
             return null;
@@ -23,13 +23,23 @@ public class DebugSymbolResolver(string symbolDir)
     public FileDebugSymbols? GetForFile(string file, string? application = null)
     {
         var fileName = $"{file}.fsym.json";
-        var fsymFile = FindFile(fileName);
+        var fsymFile = FindFile(fileName, _symbolDir);
 
         if (fsymFile is not null)
         {
             using var stream = fsymFile.OpenRead();
             using var reader = new StreamReader(stream);
-            return DebugSymbolsJsonParser.ParseForFile(reader.ReadToEnd());
+            var fsym = DebugSymbolsJsonParser.ParseForFile(reader.ReadToEnd());
+
+            var sourceFile = FindFile(file, _sourceDir);
+            if (sourceFile is not null)
+            {
+                using var sourceStream = sourceFile.OpenRead();
+                using var sourceReader = new StreamReader(sourceStream);
+                fsym.SetSourceCode(sourceReader.ReadToEnd());
+            }
+
+            return fsym;
         }
         
         if (application is null)
@@ -42,9 +52,9 @@ public class DebugSymbolResolver(string symbolDir)
         return asym.Files.FirstOrDefault(f => f.CompiledFileName == file || f.SourceFileName == file);
     }
 
-    private FileInfo? FindFile(string fileName)
+    private static FileInfo? FindFile(string fileName, DirectoryInfo? dir)
     {
-        return _symbolDir
+        return dir?
             .EnumerateFiles()
             .FirstOrDefault(f => AreEquivalentFileNames(f.Name, fileName));
     }

UIXC/Properties/launchSettings.json

@@ -3,11 +3,11 @@
         "UIXC": {
             "commandName": "Project",
 
-            "commandLineArgs": "debug --symbols E:\\Repos\\ZuneDev\\ZuneUIXTools\\test\\syms",
+            //"commandLineArgs": "debug --symbols E:\\Repos\\ZuneDev\\ZuneUIXTools\\test\\syms --sources E:\\Repos\\ZuneDev\\ZuneUIXTools\\test",
+            "commandLineArgs": "decompile E:\\Documents\\REProj\\Zune\\Resources\\ZuneShellResources48\\NOWPLAYINGMUSICBACKGROUND.UIX -l xml -A ZuneShell.dll -A UIXControls.dll -A ZuneDBApi.dll -o E:\\Repos\\ZuneDev\\ZuneUIXTools\\test --symbols E:\\Repos\\ZuneDev\\ZuneUIXTools\\test\\syms",
 
             //"commandLineArgs": "decompile E:\\Documents\\REProj\\Zune\\Resources\\ZuneMarketplaceResources48\\MarketplaceData.schema.xml -l xml -A ZuneShell.dll -A UIXControls.dll -A ZuneDBApi.dll -o E:\\Repos\\ZuneDev\\ZuneUIXTools\\test",
             //"commandLineArgs": "decompile E:\\Documents\\REProj\\Zune\\Resources\\ZuneShellResources48\\DeviceLandElements.uix -l xml -A ZuneShell.dll -A UIXControls.dll -A ZuneDBApi.dll -o E:\\Repos\\ZuneDev\\ZuneUIXTools\\test",
-            //"commandLineArgs": "decompile E:\\Documents\\REProj\\Zune\\Resources\\ZuneShellResources48\\NOWPLAYINGMUSICBACKGROUND.UIX -l xml -A ZuneShell.dll -A UIXControls.dll -A ZuneDBApi.dll -o E:\\Repos\\ZuneDev\\ZuneUIXTools\\test --symbols E:\\Repos\\ZuneDev\\ZuneUIXTools\\test\\syms",
             //"commandLineArgs": "decompile E:\\Repos\\ZuneDev\\ZuneShell.dll\\libs\\MicrosoftIris\\UIXcontrols\\Resources\\RCDATA\\STYLES.UIX -l xml -A ZuneShell.dll -A UIXControls.dll -A ZuneDBApi.dll -o E:\\Repos\\ZuneDev\\ZuneShell.dll\\ZuneShell\\Resources\\RCDATA\\Dark\\Controls",
             //"commandLineArgs": "compile E:\\Repos\\ZuneDev\\ZuneUIXTools\\test\\ADDTOCOLLECTION.31_48.UIX -A ZuneShell.dll -A UIXControls.dll -A ZuneDBApi.dll -o E:\\Repos\\ZuneDev\\ZuneUIXTools\\test",
             //"commandLineArgs": "compile E:\\Documents\\REProj\\Zune\\Resources\\ZuneMarketplaceResources31\\MARKETPLACEDATA.SCHEMA.31_48.XML -A ZuneShell.dll -A UIXControls.dll -A ZuneDBApi.dll -o E:\\Repos\\ZuneDev\\ZuneUIXTools\\test",

libs/UIX.Asm/UIX.Asm.csproj

@@ -33,7 +33,7 @@
             <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
         </PackageReference>
 
-        <PackageReference Include="System.Memory" Version="4.6.0" />
+        <PackageReference Include="System.Memory" Version="4.6.3" />
     </ItemGroup>
 
 </Project>

libs/UIX.DecompXml/Decompiler.Script.cs

@@ -615,34 +615,17 @@ partial class Decompiler
         return SyntaxTree(root);
     }
 
-    public string FormatInlineExpression(ExpressionSyntax expr, CancellationToken token = default) => '{' + FormatSyntaxNode(expr, token) + '}';
+    public static string FormatInlineExpression(ExpressionSyntax expr, CancellationToken token = default) => '{' + FormatSyntaxNode(expr, token) + '}';
 
-    public string FormatScript(SyntaxTree tree, CancellationToken token = default) => FormatSyntaxNode(tree.GetRoot(token), token);
+    public static string FormatScript(SyntaxTree tree, CancellationToken token = default) => FormatSyntaxNode(tree.GetRoot(token), token);
 
-    public string FormatSyntaxNode(SyntaxNode root, CancellationToken cancellationToken = default)
+    public static string FormatSyntaxNode(SyntaxNode root, CancellationToken cancellationToken = default)
     {
-        uint? scriptOffset = null;
-
-        root = root.NormalizeWhitespace();
-
-        foreach (var node in root.GetAnnotatedNodes(UIBOffsetAnnotation.Kind))
-        {
-            var offsetAnnotation = node.GetAnnotations(UIBOffsetAnnotation.Kind).Single();
-            var offset = UIBOffsetAnnotation.GetOffset(offsetAnnotation);
-
-            var location = node.GetLocation();
-
-            //debugSymbols.SourceMap[offset] = new(location.SourceSpan.Start, location.SourceSpan.End);
-
-            if (scriptOffset is null && node is CompilationUnitSyntax)
-                scriptOffset = offset;
-        }
-
-        var sourceText = root
+        return root
+            .NormalizeWhitespace()
             .SyntaxTree
-            .GetText(cancellationToken);
-
-        return sourceText.ToString();
+            .GetText(cancellationToken)
+            .ToString();
     }
 
     private bool TryDecompileExpression(Instruction instruction, Stack<object> stack, ControlFlowAnalyzer cfa = null)

libs/UIX.DecompXml/Decompiler.cs

@@ -100,28 +100,28 @@ public partial class Decompiler
 
             foreach (var syntaxTree in _context.GetScriptContents(export))
             {
-                var scriptText = FormatScript(syntaxTree);
-                XElement xScript = new(_nsUix + "Script", new XCData(Environment.NewLine + scriptText + Environment.NewLine));
+                XElement xScript = new(_nsUix + "Script", new XIrisScriptContent(syntaxTree.GetRoot()));
                 xScripts.Add(xScript);
             }
 
             if (!xScripts.HasElements)
                 xScripts.Remove();
 
-            List<string> methodsContent = [];
+            List<MethodDeclarationSyntax> methodsContent = [];
 
             foreach (var method in export.Methods.OfType<MarkupMethodSchema>())
             {
                 var methodSyntax = DecompileMethodDeclaration(method, export);
-                methodsContent.Add(FormatSyntaxNode(methodSyntax));
+                methodsContent.Add(methodSyntax);
             }
 
             if (methodsContent.Count > 0)
             {
-                var methodsContentStr = Environment.NewLine
-                    + string.Join(Environment.NewLine + Environment.NewLine, methodsContent)
-                    + Environment.NewLine;
-                XElement xMethods = new(_nsUix + "Methods", new XCData(methodsContentStr));
+                var methodsSyntax = CodeAnalysis.CSharp.SyntaxFactory
+                    .CompilationUnit()
+                    .AddMembers([.. methodsContent]);
+
+                XElement xMethods = new(_nsUix + "Methods", new XIrisScriptContent(methodsSyntax));
                 xExport.Add(xMethods);
             }
 
@@ -147,25 +147,9 @@ public partial class Decompiler
     {
         var xmlDoc = Decompile();
 
-        XmlWriterSettings writerSettings = new()
-        {
-            Indent = true,
-            NamespaceHandling = NamespaceHandling.OmitDuplicates,
-            Encoding = Encoding.UTF8,
-        };
+        using IrisXmlWriter writer = new(xmlDoc, generateDebugSymbols ? DebugSymbols : null);
 
-        StringBuilder sb = new();
-        using (XmlWriter writer = XmlWriter.Create(sb, writerSettings))
-        {
-            xmlDoc.WriteTo(writer);
-        }
-
-        var xmlStr = sb.ToString();
-
-        if (generateDebugSymbols)
-            PopulateDebugSymbolsWithXml(xmlDoc, xmlStr);
-
-        return xmlStr;
+        return writer.WriteToString();
     }
 
     public void PopulateDebugSymbolsWithXml(XDocument xmlDoc, string xmlStr)

libs/UIX.DecompXml/IrisTextWriter.cs

@@ -0,0 +1,76 @@
+using Microsoft.Iris.Debug.Symbols;
+using System;
+using System.IO;
+
+namespace Microsoft.Iris.DecompXml;
+
+internal class IrisTextWriter : StringWriter
+{
+    private int _currentLineIndex, _currentColumnIndex;
+
+    public SourcePosition Position => new(_currentLineIndex + 1, _currentColumnIndex + 1);
+
+    public override void Write(char[] buffer, int index, int count)
+    {
+        UpdatePosition(buffer.AsSpan(index, count));
+        base.Write(buffer, index, count);
+    }
+
+    public override void Write(string value)
+    {
+        if (value is not null)
+            UpdatePosition(value.AsSpan());
+
+        base.Write(value);
+    }
+
+#if NETCOREAPP2_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER
+    public override void Write(ReadOnlySpan<char> buffer)
+    {
+        UpdatePosition(buffer);
+        base.Write(buffer);
+    }
+#endif
+
+    private void UpdatePosition(ReadOnlySpan<char> buffer)
+    {
+        int lastNewLineIndex = 0;
+        for (int c = 0; c < buffer.Length;)
+        {
+            if (!IsAtNewLine(buffer, c))
+            {
+                ++c;
+                continue;
+            }
+
+            c += NewLine.Length;
+            IncrementLine();
+            lastNewLineIndex = c;
+        }
+
+        _currentColumnIndex += buffer.Length - lastNewLineIndex;
+    }
+
+    private bool IsAtNewLine(ReadOnlySpan<char> buffer, int index)
+    {
+        for (int n = 0; n < NewLine.Length; ++n)
+        {
+            var b = n + index;
+            if (b >= buffer.Length)
+                return false;
+
+            var nch = NewLine[n];
+            var bch = buffer[b];
+            if (nch != bch)
+                return false;
+        }
+
+        return true;
+    }
+
+    private void IncrementLine()
+    {
+        ++_currentLineIndex;
+        _currentColumnIndex = 0;
+    }
+}

libs/UIX.DecompXml/IrisXmlWriter.cs

@@ -0,0 +1,100 @@
+using Microsoft.Iris.Debug.Symbols;
+using System;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Xml;
+using System.Xml.Linq;
+
+namespace Microsoft.Iris.DecompXml;
+
+internal class IrisXmlWriter : IDisposable
+{
+    private readonly IrisTextWriter _textWriter;
+    private readonly XmlWriter _xmlWriter;
+    private readonly XDocument _xDoc;
+    private readonly FileDebugSymbols _symbols;
+
+    public IrisXmlWriter(XDocument xDoc, FileDebugSymbols symbols)
+    {
+        _xDoc = xDoc;
+        _symbols = symbols;
+
+        _textWriter = new();
+        _xmlWriter = XmlWriter.Create(_textWriter, new XmlWriterSettings
+        {
+            Indent = true,
+            NamespaceHandling = NamespaceHandling.OmitDuplicates,
+            Encoding = Encoding.UTF8,
+        });
+    }
+
+    public void Dispose() => _xmlWriter.Dispose();
+
+    public string WriteToString()
+    {
+        Write();
+        return _textWriter.ToString();
+    }
+
+    private void Write()
+    {
+        if (_symbols is null)
+        {
+            // If we're not generating symbols, there's no reason to
+            // use our custom implementation.
+            _xDoc.WriteTo(_xmlWriter);
+            _xmlWriter.Flush();
+            return;
+        }
+
+        _xmlWriter.WriteStartDocument();
+
+        foreach (var node in _xDoc.Nodes())
+            WriteXObject(node);
+
+        _xmlWriter.WriteEndDocument();
+        _xmlWriter.Flush();
+    }
+
+    private void WriteXObject(XObject obj)
+    {
+        _xmlWriter.Flush();
+        var startPos = _textWriter.Position;
+
+        if (obj is XElement element)
+        {
+            _xmlWriter.WriteStartElement(element.Name.LocalName, element.Name.NamespaceName);
+
+            foreach (var attribute in element.Attributes())
+                WriteXObject(attribute);
+
+            foreach (var childNode in element.Nodes())
+                WriteXObject(childNode);
+
+            _xmlWriter.WriteEndElement();
+        }
+        else if (obj is XIrisScriptContent script)
+        {
+            _xmlWriter.WriteCData(script.Value);
+        }
+        else if (obj is XCData cdata)
+        {
+            _xmlWriter.WriteCData(cdata.Value);
+        }
+        else if (obj is XAttribute attribute)
+        {
+            _xmlWriter.WriteAttributeString(attribute.Name.LocalName, attribute.Name.NamespaceName, attribute.Value);
+        }
+
+
+        var offset = obj.GetOffset();
+        if (offset is not null)
+        {
+            _xmlWriter.Flush();
+            var endPos = _textWriter.Position;
+
+            var span = new SourceSpan(startPos, endPos);
+            _symbols.SourceMap.Xml[offset.Value] = span;
+        }
+    }
+}

libs/UIX.DecompXml/XIrisScriptContent.cs

@@ -0,0 +1,16 @@
+using Microsoft.CodeAnalysis;
+using System;
+using System.Xml.Linq;
+
+namespace Microsoft.Iris.DecompXml;
+
+internal class XIrisScriptContent : XCData
+{
+    public SyntaxNode Syntax { get; }
+
+    public XIrisScriptContent(SyntaxNode syntax) : base("")
+    {
+        Syntax = syntax.NormalizeWhitespace();
+        Value = $"{Environment.NewLine}{Syntax.GetText()}{Environment.NewLine}";
+    }
+}