izzy/UnrealEngineUWP
0
0
Fork 0
mirror of https://github.com/izzy2lost/UnrealEngineUWP.git synced 2026-03-26 18:15:20 -07:00
Code Issues Packages Projects Releases Wiki Activity
Files
140ec947f35cd6569874fe2ecd182f64e4fddc36
UnrealEngineUWP/Engine/Source/Programs/IncludeTool/IncludeTool/Preprocessor.cs
Ben Marsh d528bb8440 IncludeTool: Detect invalid newlines in string literals, and add support for parsing raw string literals. 
#preflight 63693446d0174259ccdcf94a

[CL 23012594 by Ben Marsh in ue5-main branch]
2022-11-07 12:05:31 -05:00

2270 lines
85 KiB
C#
Raw Blame History

// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using IncludeTool;
using IncludeTool.Support;
using EpicGames.Core;
using System.Diagnostics;
using System.IO;
using System.Collections;
using System.Collections.Concurrent;
namespace IncludeTool
{
/// <summary>
/// Exception class for the preprocessor, which contains the file and position of the code causing an error
/// </summary>
class PreprocessorException : Exception
{
/// <summary>
/// Constructor
/// </summary>
/// <param name="Format">Format string, to be passed to String.Format</param>
/// <param name="Args">Optional argument list for the format string</param>
public PreprocessorException(string Format, params object[] Args)
: base(String.Format(Format, Args))
{
}
}
/// <summary>
/// Stores information about a defined preprocessor macro
/// </summary>
class PreprocessorMacro
{
/// <summary>
/// Name of the macro
/// </summary>
public string Name;
/// <summary>
/// Parameter names for the macro. The '...' placeholder is represented by the VariadicParameter string.
/// </summary>
public List<string> Parameters;
/// <summary>
/// Raw list of tokens for this macro
/// </summary>
public List<Token> Tokens;
/// <summary>
/// The parameter name used for the variadic parameter. This appears in place of '...' in the parameters list.
/// </summary>
public const string VariadicParameter = "__VA_ARGS__";
/// <summary>
/// Construct a preprocessor macro
/// </summary>
/// <param name="InName">Name of the macro</param>
/// <param name="InParameters">Parameter list for the macro. Should be null for object macros. Ownership of this list is transferred.</param>
/// <param name="InTokens">Tokens for the macro. Ownership of this list is transferred.</param>
public PreprocessorMacro(string InName, List<string> InParameters, List<Token> InTokens)
{
Name = InName;
Parameters = InParameters;
Tokens = InTokens;
}
/// <summary>
/// Finds the index of a parameter in the parameter list
/// </summary>
/// <param name="Parameter">Parameter name to look for</param>
/// <returns>Index of the parameter, or -1 if it's not found.</returns>
public int FindParameterIndex(string Parameter)
{
for (int Idx = 0; Idx < Parameters.Count; Idx++)
{
if (Parameters[Idx] == Parameter)
{
return Idx;
}
}
return -1;
}
/// <summary>
/// True if the macro is an object macro
/// </summary>
public bool IsObjectMacro
{
get { return Parameters == null; }
}
/// <summary>
/// True if the macro is a function macro
/// </summary>
public bool IsFunctionMacro
{
get { return Parameters != null; }
}
/// <summary>
/// The number of required parameters. For variadic macros, the last parameter is optional.
/// </summary>
public int MinRequiredParameters
{
get { return HasVariableArgumentList? (Parameters.Count - 1) : Parameters.Count; }
}
/// <summary>
/// True if the macro has a variable argument list
/// </summary>
public bool HasVariableArgumentList
{
get { return Parameters.Count > 0 && Parameters[Parameters.Count - 1] == VariadicParameter; }
}
/// <summary>
/// Converts this macro to a string for debugging
/// </summary>
/// <returns>The tokens in this macro</returns>
public override string ToString()
{
StringBuilder Result = new StringBuilder();
Result.AppendFormat("{0}=", Name);
if (Tokens.Count > 0)
{
Result.Append(Tokens[0].Text);
for (int Idx = 1; Idx < Tokens.Count; Idx++)
{
if(Tokens[Idx].HasLeadingSpace)
{
Result.Append(" ");
}
Result.Append(Tokens[Idx].Text);
}
}
return Result.ToString();
}
}
/// <summary>
/// Record for a file context within the preprocessor stack
/// </summary>
class PreprocessorFile
{
/// <summary>
/// Filename as specified when included
/// </summary>
public string FileName;
/// <summary>
/// Reference to the current file
/// </summary>
public FileReference Location;
/// <summary>
/// The workspace file
/// </summary>
public WorkspaceFile WorkspaceFile;
/// <summary>
/// Construct a new preprocessor file from the given name
/// </summary>
/// <param name="FileName">The filename of the new file</param>
public PreprocessorFile(string FileName, WorkspaceFile WorkspaceFile)
{
this.FileName = FileName;
this.Location = WorkspaceFile.Location;
this.WorkspaceFile = WorkspaceFile;
}
/// <summary>
/// Returns the name of this file for visualization in the debugger
/// </summary>
/// <returns>Name of this file</returns>
public override string ToString()
{
return FileName;
}
}
/// <summary>
/// Implementation of a C++ preprocessor. Should be complete and accurate.
/// </summary>
class Preprocessor
{
/// <summary>
/// Flags indicating the state of each nested conditional block in a stack.
/// </summary>
[Flags]
enum BranchState
{
/// <summary>
/// The current conditional branch is active.
/// </summary>
Active = 0x01,
/// <summary>
/// A branch within the current conditional block has been taken. Any #else/#elif blocks will not be taken.
/// </summary>
Taken = 0x02,
/// <summary>
/// The #else directive for this conditional block has been encountered. An #endif directive is expected next.
/// </summary>
Complete = 0x04,
/// <summary>
/// This branch is within an active parent branch
/// </summary>
ParentIsActive = 0x08,
/// <summary>
/// The first condition in this branch was an #if directive (as opposed to an #ifdef or #ifndef directive)
/// </summary>
HasIfDirective = 0x10,
/// <summary>
/// The first condition in this branch was an #if directive (as opposed to an #ifdef or #ifndef directive)
/// </summary>
HasIfdefDirective = 0x20,
/// <summary>
/// The first condition in this branch was an #ifndef directive (as opposed to an #ifdef or #if directive)
/// </summary>
HasIfndefDirective = 0x40,
/// <summary>
/// The branch has an #elif directive
/// </summary>
HasElifDirective = 0x80,
}
/// <summary>
/// A directory to search for include paths
/// </summary>
[DebuggerDisplay("{Location}")]
class IncludeDirectory
{
/// <summary>
/// The directory, as specified as a parameter. We keep the original spelling for formatting the preprocessor output to match MSVC.
/// </summary>
public DirectoryReference Location;
/// <summary>
/// The corresponding workspace directory
/// </summary>
public WorkspaceDirectory WorkspaceDirectory;
}
/// <summary>
/// Stack of conditional branch states
/// </summary>
Stack<BranchState> Branches = new Stack<BranchState>();
/// <summary>
/// Mapping of name to macro definition
/// </summary>
Dictionary<string, PreprocessorMacro> NameToMacro = new Dictionary<string, PreprocessorMacro>();
/// <summary>
/// The stack of included source files
/// </summary>
List<PreprocessorFile> Files = new List<PreprocessorFile>();
/// <summary>
/// Include paths to look in
/// </summary>
List<IncludeDirectory> IncludeDirectories = new List<IncludeDirectory>();
/// <summary>
/// Set of all included files with the #pragma once directive
/// </summary>
HashSet<FileReference> PragmaOnceFiles = new HashSet<FileReference>();
/// <summary>
/// Value of the __COUNTER__ variable
/// </summary>
int Counter;
/// <summary>
/// Default constructor
/// </summary>
/// <param name="Definitions">Definitions passed in on the command-line</param>
/// <param name="IncludePaths">Paths to search when resolving include directives</param>
public Preprocessor(FileReference PreludeFile, IEnumerable<string> Definitions, IEnumerable<DirectoryReference> IncludePaths)
{
DateTime Now = DateTime.Now;
AddSingleTokenMacro("__DATE__", TokenType.StringLiteral, String.Format("\"{0} {1,2} {2}\"", Now.ToString("MMM"), Now.Day, Now.Year));
AddSingleTokenMacro("__TIME__", TokenType.StringLiteral, "\"" + Now.ToString("HH:mm:ss") + "\"");
AddSingleTokenMacro("__FILE__", TokenType.StringLiteral, "\"<unknown>\"");
AddSingleTokenMacro("__LINE__", TokenType.NumericLiteral, "-1");
AddSingleTokenMacro("__COUNTER__", TokenType.NumericLiteral, "-1");
AddSingleTokenMacro("CHAR_BIT", TokenType.NumericLiteral, "8"); // Workaround for #include_next not being supported on Linux for limit.h
AddMacro("__has_cpp_attribute", new List<string> { "_" }, new List<Token> { new Token("1", TokenType.NumericLiteral, TokenFlags.None) }); // Workaround for `__has_cpp_attribute` (a core C++ feature) being defined by a 3rd-party library
if(PreludeFile != null)
{
TextBuffer PreludeText = TextBuffer.FromFile(PreludeFile.FullName);
PreprocessorMarkup[] PreludeMarkup = PreprocessorMarkup.ParseArray("prelude", PreludeText);
foreach(PreprocessorMarkup Markup in PreludeMarkup)
{
ParseMarkup(Markup.Type, Markup.Tokens, Markup.Location.LineIdx);
}
}
foreach(string Definition in Definitions)
{
// Create a token reader for this definition
TokenReader Reader = new TokenReader($"(predefined value for {Definition})", TextBuffer.FromString(Definition), TextLocation.Origin);
// Read it into a buffer
List<Token> Tokens = new List<Token>();
while (Reader.MoveNext() && Reader.Current.Text != "\n")
{
Tokens.Add(Reader.Current);
}
// Create the macro for it
if(Tokens.Count == 0 || Tokens[0].Type != TokenType.Identifier)
{
throw new PreprocessorException("Invalid token for define");
}
else if(Tokens.Count == 1)
{
NameToMacro[Tokens[0].Text] = new PreprocessorMacro(Tokens[0].Text, null, new List<Token> { new Token("1", TokenType.NumericLiteral, TokenFlags.None) });
}
else if(Tokens[1].Text == "=")
{
NameToMacro[Tokens[0].Text] = new PreprocessorMacro(Tokens[0].Text, null, Tokens.Skip(2).ToList());
}
else if(Tokens[1].Text != "(")
{
throw new PreprocessorException("Expected '=' token for definition on command-line");
}
}
foreach(DirectoryReference IncludePath in IncludePaths)
{
if(Directory.Exists(IncludePath.FullName))
{
IncludeDirectories.Add(new IncludeDirectory() { Location = IncludePath, WorkspaceDirectory = Workspace.GetDirectory(IncludePath) });
}
}
}
/// <summary>
/// Returns the path to the current file
/// </summary>
public PreprocessorFile CurrentFile
{
get { return Files[Files.Count - 1]; }
}
/// <summary>
/// Returns an array consisting of the current include stack
/// </summary>
/// <returns>Array of the current include stack</returns>
public FileReference[] CaptureIncludeStack()
{
return Files.Select(x => x.Location).ToArray();
}
/// <summary>
/// Determines if the branch that the preprocessor is in is active
/// </summary>
/// <returns>True if the branch is active, false otherwise</returns>
public bool IsBranchActive()
{
return Branches.Count == 0 || Branches.Peek().HasFlag(BranchState.Active);
}
/// <summary>
/// Push a file onto the preprocessor stack
/// </summary>
/// <param name="File">The file to add</param>
public bool PushFile(PreprocessorFile File)
{
if(!PragmaOnceFiles.Contains(File.Location))
{
Files.Add(File);
if (File.WorkspaceFile.ReadSourceFile().HasHeaderGuard)
{
PragmaOnceFiles.Add(File.Location);
}
return true;
}
return false;
}
/// <summary>
/// Remove the topmost file from the preprocessor stack
/// </summary>
public void PopFile()
{
Files.RemoveAt(Files.Count - 1);
}
/// <summary>
/// Format a #line directive at the given line
/// </summary>
/// <param name="CurrentLine">The current line being parsed</param>
/// <returns></returns>
string FormatLineDirective(int CurrentLine)
{
return String.Format("#line {0} \"{1}\"", CurrentLine, CurrentFile.FileName.Replace("\\", "\\\\"));
}
/// <summary>
/// Validate and add a macro using the given parameter and token list
/// </summary>
/// <param name="Name">Name of the macro</param>
/// <param name="Parameters">Parameter list for the macro</param>
/// <param name="Tokens">List of tokens</param>
void AddMacro(string Name, List<string> Parameters, List<Token> Tokens)
{
if(Tokens.Count == 0)
{
Tokens.Add(new Token("", TokenType.Placemarker, TokenFlags.None));
}
else
{
if(Tokens[0].HasLeadingSpace)
{
Tokens[0] = new Token(Tokens[0].Text, Tokens[0].Type, Tokens[0].Flags & ~TokenFlags.HasLeadingSpace);
}
if (Tokens[0].Text == "##" || Tokens[Tokens.Count - 1].Text == "##")
{
throw new PreprocessorException("Invalid use of concatenation at start or end of token sequence");
}
if (Parameters == null || Parameters.Count == 0 || Parameters[Parameters.Count - 1] != PreprocessorMacro.VariadicParameter)
{
if(Tokens.Any(x => x.Text == PreprocessorMacro.VariadicParameter))
{
throw new PreprocessorException("Invalid reference to {0}", PreprocessorMacro.VariadicParameter);
}
}
}
NameToMacro[Name] = new PreprocessorMacro(Name, Parameters, Tokens);
}
/// <summary>
/// Set a predefined macro to a given value
/// </summary>
/// <param name="Name">Name of the macro</param>
/// <param name="Type">Type of the macro token</param>
/// <param name="Value">Value of the macro</param>
/// <returns>The created macro</returns>
void AddSingleTokenMacro(string Name, TokenType Type, string Value)
{
Token Token = new Token(Value, Type, TokenFlags.None);
PreprocessorMacro Macro = new PreprocessorMacro(Name, null, new List<Token> { Token });
NameToMacro[Name] = Macro;
}
/// <summary>
/// Tries to resolve an include
/// </summary>
/// <param name="Tokens">List of include tokens</param>
/// <returns>The resolved include file</returns>
public PreprocessorFile ResolveInclude(List<Token> Tokens, int CurrentLine)
{
// Expand macros in the given tokens
List<Token> ExpandedTokens = new List<Token>();
ExpandMacros(Tokens, ExpandedTokens, false, CurrentLine);
// Expand any macros in them and resolve it
string IncludeToken = Token.Format(ExpandedTokens).Replace("//", "/");
if (IncludeToken.Length >= 2 && IncludeToken.StartsWith("\"") && IncludeToken.EndsWith("\""))
{
return ResolveQuotedInclude(IncludeToken.Substring(1, IncludeToken.Length - 2));
}
else if(IncludeToken.StartsWith("<") && IncludeToken.EndsWith(">"))
{
return ResolveSystemInclude(IncludeToken.Substring(1, IncludeToken.Length - 2));
}
else
{
throw new PreprocessorException("Couldn't resolve include '{0}'", IncludeToken);
}
}
/// <summary>
/// Try to resolve an quoted include against the list of include directories. Uses search order described by https://msdn.microsoft.com/en-us/library/36k2cdd4.aspx.
/// </summary>
/// <param name="IncludeText">The path appearing in quotes in an #include directive</param>
/// <returns>The resolved file</returns>
public PreprocessorFile ResolveQuotedInclude(string IncludeText)
{
// If it's an absolute path, return it immediately
if(Path.IsPathRooted(IncludeText))
{
WorkspaceFile WorkspaceFile = Workspace.GetFile(new FileReference(IncludeText));
return new PreprocessorFile(IncludeText, WorkspaceFile);
}
// Otherwise search through the open file directories
string[] RelativePathFragments = IncludeText.Split('/', '\\');
for(int Idx = Files.Count - 1; Idx >= 0; Idx--)
{
WorkspaceFile WorkspaceFile;
if(TryResolveRelativePath(Files[Idx].WorkspaceFile.Directory, RelativePathFragments, out WorkspaceFile))
{
string FileName = Path.Combine(Utility.RemoveRelativePaths(Path.GetDirectoryName(Files[Idx].FileName)).ToLowerInvariant(), IncludeText);
return new PreprocessorFile(FileName, WorkspaceFile);
}
}
// Otherwise fall back to the system search
return ResolveSystemInclude(IncludeText, RelativePathFragments);
}
/// <summary>
/// Try to resolve a system include against the list of include directories
/// </summary>
/// <param name="IncludeText">The path appearing in quotes in an #include directive</param>
/// <returns>The resolved file</returns>
public PreprocessorFile ResolveSystemInclude(string IncludeText)
{
// If it's an absolute path, return it immediately
if(Path.IsPathRooted(IncludeText))
{
WorkspaceFile WorkspaceFile = Workspace.GetFile(new FileReference(IncludeText));
return new PreprocessorFile(IncludeText, WorkspaceFile);
}
// Otherwise try to resolve a relative path
return ResolveSystemInclude(IncludeText, IncludeText.Split('/', '\\'));
}
/// <summary>
/// Try to resolve a system include against the list of include directories
/// </summary>
/// <param name="IncludeText">The path appearing in quotes in an #include directive</param>
/// <param name="RelativePathFragments">The components of a relative path, without directory separators</param>
/// <returns>The resolved file</returns>
public PreprocessorFile ResolveSystemInclude(string IncludeText, string[] RelativePathFragments)
{
foreach(IncludeDirectory IncludeDirectory in IncludeDirectories)
{
WorkspaceFile WorkspaceFile;
if(TryResolveRelativePath(IncludeDirectory.WorkspaceDirectory, RelativePathFragments, out WorkspaceFile))
{
string FileName = Path.Combine(IncludeDirectory.Location.FullName, IncludeText);
return new PreprocessorFile(FileName, WorkspaceFile);
}
}
throw new PreprocessorException("Couldn't resolve include {0}", IncludeText);
}
/// <summary>
/// Tries to get a file at the relative path from a base directory
/// </summary>
/// <param name="BaseDirectory">The base directory to search from</param>
/// <param name="RelativePathFragments">Fragments of the relative path to follow</param>
/// <param name="Result">The file that was found, if successful</param>
/// <returns>True if the file was located</returns>
static bool TryResolveRelativePath(WorkspaceDirectory BaseDirectory, string[] RelativePathFragments, out WorkspaceFile Result)
{
WorkspaceDirectory NextDirectory = BaseDirectory;
for(int Idx = 0; Idx < RelativePathFragments.Length - 1; Idx++)
{
if(!NextDirectory.TryGetDirectory(RelativePathFragments[Idx], out NextDirectory))
{
Result = null;
return false;
}
}
return NextDirectory.TryGetFile(RelativePathFragments[RelativePathFragments.Length - 1], out Result);
}
/// <summary>
/// Parse a marked up directive from a file
/// </summary>
/// <param name="Type">The markup type</param>
/// <param name="Tokens">Tokens for the directive</param>
/// <param name="CurrentLine">The line number being processed</param>
public void ParseMarkup(PreprocessorMarkupType Type, List<Token> Tokens, int CurrentLine)
{
switch(Type)
{
case PreprocessorMarkupType.Include:
throw new PreprocessorException("Include directives should be handled by the caller.");
case PreprocessorMarkupType.Define:
ParseDefineDirective(Tokens);
break;
case PreprocessorMarkupType.Undef:
ParseUndefDirective(Tokens);
break;
case PreprocessorMarkupType.If:
ParseIfDirective(Tokens, CurrentLine);
break;
case PreprocessorMarkupType.Ifdef:
ParseIfdefDirective(Tokens);
break;
case PreprocessorMarkupType.Ifndef:
ParseIfndefDirective(Tokens);
break;
case PreprocessorMarkupType.Elif:
ParseElifDirective(Tokens, CurrentLine);
break;
case PreprocessorMarkupType.Else:
ParseElseDirective(Tokens);
break;
case PreprocessorMarkupType.Endif:
ParseEndifDirective(Tokens);
break;
case PreprocessorMarkupType.Pragma:
ParsePragmaDirective(Tokens);
break;
}
}
/// <summary>
/// Read a macro definition
/// </summary>
/// <param name="Tokens">List of tokens in the directive</param>
public void ParseDefineDirective(List<Token> Tokens)
{
if(IsBranchActive())
{
// Check there's a name token
if(Tokens.Count < 1 || Tokens[0].Type != TokenType.Identifier || Tokens[0].Text == "defined")
{
throw new PreprocessorException("Invalid macro name");
}
// Read the macro name
string Name = Tokens[0].Text;
int TokenIdx = 1;
// Read the macro parameter list, if there is one
List<string> Parameters = null;
if (TokenIdx < Tokens.Count && !Tokens[TokenIdx].HasLeadingSpace && Tokens[TokenIdx].Type == TokenType.LeftParen)
{
Parameters = new List<string>();
if(++TokenIdx == Tokens.Count)
{
throw new PreprocessorException("Unexpected end of macro parameter list");
}
if(Tokens[TokenIdx].Type != TokenType.RightParen)
{
for(;;TokenIdx++)
{
// Check there's enough tokens left for a parameter name, plus ',' or ')'
if(TokenIdx + 2 > Tokens.Count)
{
throw new PreprocessorException("Unexpected end of macro parameter list");
}
// Check it's a valid name, and add it to the list
Token NameToken = Tokens[TokenIdx++];
if(NameToken.Text == "...")
{
if(Tokens[TokenIdx].Type != TokenType.RightParen)
{
throw new PreprocessorException("Variadic macro arguments must be last in list");
}
else
{
NameToken = new Token(PreprocessorMacro.VariadicParameter, TokenType.Identifier, NameToken.Flags & TokenFlags.HasLeadingSpace);
}
}
else
{
if (NameToken.Type != TokenType.Identifier || NameToken.Text == PreprocessorMacro.VariadicParameter)
{
throw new PreprocessorException("Invalid preprocessor token: {0}", NameToken);
}
if (Parameters.Contains(NameToken.Text))
{
throw new PreprocessorException("'{0}' has already been used as an argument name", NameToken);
}
}
Parameters.Add(NameToken.Text);
// Read the separator
Token SeparatorToken = Tokens[TokenIdx];
if (SeparatorToken.Type == TokenType.RightParen)
{
break;
}
if (SeparatorToken.Type != TokenType.Comma)
{
throw new PreprocessorException("Expected ',' or ')'");
}
}
}
TokenIdx++;
}
// Read the macro tokens
AddMacro(Name, Parameters, Tokens.Skip(TokenIdx).ToList());
}
}
/// <summary>
/// Parse an #undef directive
/// </summary>
/// <param name="Tokens">List of tokens in the directive</param>
/// <param name="CurrentLine">The line number being processed</param>
public void ParseUndefDirective(List<Token> Tokens)
{
if(IsBranchActive())
{
// Check there's a name token
if (Tokens.Count != 1)
{
throw new PreprocessorException("Expected a single token after #undef");
}
// Remove the macro from the list of definitions
NameToMacro.Remove(Tokens[0].Text);
}
}
/// <summary>
/// Parse an #if directive
/// </summary>
/// <param name="Tokens">List of tokens in the directive</param>
public void ParseIfDirective(List<Token> Tokens, int CurrentLine)
{
BranchState State = BranchState.HasIfDirective;
if (IsBranchActive())
{
// The new branch is within an active branch
State |= BranchState.ParentIsActive;
// Read a line into the buffer and expand the macros in it
List<Token> ExpandedTokens = new List<Token>();
ExpandMacros(Tokens, ExpandedTokens, true, CurrentLine);
// Evaluate the condition
long Result = PreprocessorExpression.Evaluate(ExpandedTokens);
if (Result != 0)
{
State |= BranchState.Active | BranchState.Taken;
}
}
Branches.Push(State);
}
/// <summary>
/// Parse an #ifdef directive
/// </summary>
/// <param name="Tokens">List of tokens in the directive</param>
public void ParseIfdefDirective(List<Token> Tokens)
{
BranchState State = BranchState.HasIfdefDirective;
if (IsBranchActive())
{
// The new branch is within an active branch
State |= BranchState.ParentIsActive;
// Make sure there's only one token
if(Tokens.Count != 1 || Tokens[0].Type != TokenType.Identifier)
{
throw new PreprocessorException("Missing or invalid macro name for #ifdef directive");
}
// Check if the macro is defined
if(NameToMacro.ContainsKey(Tokens[0].Text))
{
State |= BranchState.Active | BranchState.Taken;
}
}
Branches.Push(State);
}
/// <summary>
/// Parse an #ifndef directive
/// </summary>
/// <param name="Tokens">List of tokens for this directive</param>
public void ParseIfndefDirective(List<Token> Tokens)
{
BranchState State = BranchState.HasIfndefDirective;
if (IsBranchActive())
{
// The new branch is within an active branch
State |= BranchState.ParentIsActive;
// Make sure there's only one token
if (Tokens.Count != 1 || Tokens[0].Type != TokenType.Identifier)
{
throw new PreprocessorException("Missing or invalid macro name for #ifndef directive");
}
// Check if the macro is defined
if(!NameToMacro.ContainsKey(Tokens[0].Text))
{
State |= BranchState.Active | BranchState.Taken;
}
}
Branches.Push(State);
}
/// <summary>
/// Parse an #elif directive
/// </summary>
/// <param name="Tokens">List of tokens for this directive</param>
/// <param name="CurrentLine">The line number being processed</param>
public void ParseElifDirective(List<Token> Tokens, int CurrentLine)
{
// Check we're in a branch, and haven't already read an #else directive
if (Branches.Count == 0)
{
throw new PreprocessorException("#elif directive outside conditional block");
}
if (Branches.Peek().HasFlag(BranchState.Complete))
{
throw new PreprocessorException("#elif directive cannot appear after #else");
}
// Pop the current branch state at this depth, so we can test against whether the parent state is enabled
BranchState State = (Branches.Pop() | BranchState.HasElifDirective) & ~BranchState.Active;
if (IsBranchActive())
{
// Read a line into the buffer and expand the macros in it
List<Token> ExpandedTokens = new List<Token>();
ExpandMacros(Tokens, ExpandedTokens, true, CurrentLine);
// Check we're at the end of a conditional block
if (!State.HasFlag(BranchState.Taken))
{
long Result = PreprocessorExpression.Evaluate(ExpandedTokens);
if (Result != 0)
{
State |= BranchState.Active | BranchState.Taken;
}
}
}
Branches.Push(State);
}
/// <summary>
/// Parse an #else directive
/// </summary>
/// <param name="Tokens">List of tokens in the directive</param>
public void ParseElseDirective(List<Token> Tokens)
{
// Make sure there's nothing else on the line
if (Tokens.Count > 0)
{
throw new PreprocessorException("Garbage after #else directive");
}
// Check we're in a branch, and haven't already read an #else directive
if (Branches.Count == 0)
{
throw new PreprocessorException("#else directive without matching #if directive");
}
if (Branches.Peek().HasFlag(BranchState.Complete))
{
throw new PreprocessorException("Only one #else directive can appear in a conditional block");
}
// Check whether to take this branch, but only allow activating if the parent state is active.
BranchState State = Branches.Pop() & ~BranchState.Active;
if(IsBranchActive() && !State.HasFlag(BranchState.Taken))
{
State |= BranchState.Active | BranchState.Taken;
}
Branches.Push(State | BranchState.Complete);
}
/// <summary>
/// Parse an #endif directive
/// </summary>
/// <param name="Tokens">List of tokens in the directive</param>
public void ParseEndifDirective(List<Token> Tokens)
{
// Check we're in a branch
if (Branches.Count == 0)
{
throw new PreprocessorException("#endif directive without matching #if/#ifdef/#ifndef directive");
}
// Pop the branch off the stack
Branches.Pop();
}
/// <summary>
/// Parse a #pragma directive
/// </summary>
/// <param name="Tokens">List of tokens in the directive</param>
public void ParsePragmaDirective(List<Token> Tokens)
{
if(IsBranchActive())
{
if(Tokens.Count == 1 && Tokens[0].Text == "once")
{
PragmaOnceFiles.Add(CurrentFile.Location);
}
}
}
/// <summary>
/// Expand macros in the given sequence.
/// </summary>
/// <param name="InputTokens">Sequence of input tokens</param>
/// <param name="OutputTokens">List to receive the expanded tokens</param>
/// <param name="bIsConditional">Whether a conditional expression is being evaluated (and 'defined' expressions are valid)</param>
public void ExpandMacros(IEnumerable<Token> InputTokens, List<Token> OutputTokens, bool bIsConditional, int CurrentLine)
{
List<PreprocessorMacro> IgnoreMacros = new List<PreprocessorMacro>();
ExpandMacrosRecursively(InputTokens, OutputTokens, bIsConditional, CurrentLine, IgnoreMacros);
}
/// <summary>
/// Expand macros in the given sequence, ignoring previously expanded macro names from a list.
/// </summary>
/// <param name="InputTokens">Sequence of input tokens</param>
/// <param name="OutputTokens">List to receive the expanded tokens</param>
/// <param name="bIsConditional">Whether a conditional expression is being evaluated (and 'defined' expressions are valid)</param>
/// <param name="CurrentLine">The current line being expanded which can return the current line at any point</param>
/// <param name="IgnoreMacros">List of macros to ignore</param>
void ExpandMacrosRecursively(IEnumerable<Token> InputTokens, List<Token> OutputTokens, bool bIsConditional, int CurrentLine, List<PreprocessorMacro> IgnoreMacros)
{
IEnumerator<Token> InputEnumerator = InputTokens.GetEnumerator();
if(InputEnumerator.MoveNext())
{
for(;;)
{
if(!ReadExpandedToken(InputEnumerator, OutputTokens, bIsConditional, () => CurrentLine, IgnoreMacros))
{
break;
}
}
}
}
/// <summary>
/// Merges an optional leading space flag into the given token (recycling the original token if possible).
/// </summary>
/// <param name="Token">The token to merge a leading space into</param>
/// <param name="bHasLeadingSpace">The leading space flag</param>
/// <returns>New token with the leading space flag set, or the existing token</returns>
Token MergeLeadingSpace(Token Token, bool bHasLeadingSpace)
{
Token Result = Token;
if(bHasLeadingSpace && !Result.HasLeadingSpace)
{
Result = new Token(Result.Text, Result.Type, Result.Flags | TokenFlags.HasLeadingSpace);
}
return Result;
}
/// <summary>
/// Read a token from an enumerator and substitute it if it's a macro or 'defined' expression (reading more tokens as necessary to complete the expression).
/// </summary>
/// <param name="InputEnumerator">The enumerator of input tokens</param>
/// <param name="OutputTokens">List to receive the expanded tokens</param>
/// <param name="bIsConditional">Whether a conditional expression is being evaluated (and 'defined' expressions are valid)</param>
/// <param name="CurrentLine">Callback which can return the current line at any point</param>
/// <param name="IgnoreMacros">List of macros to ignore</param>
/// <returns>Result from calling the enumerator's MoveNext() method</returns>
bool ReadExpandedToken(IEnumerator<Token> InputEnumerator, List<Token> OutputTokens, bool bIsConditional, Func<int> CurrentLine, List<PreprocessorMacro> IgnoreMacros)
{
// Capture the first token, then move to the next
OutputTokens.Add(InputEnumerator.Current);
bool bMoveNext = InputEnumerator.MoveNext();
// If it's an identifier, try to resolve it as a macro
if (OutputTokens[OutputTokens.Count - 1].Text == "defined" && bIsConditional)
{
// Remove the 'defined' keyword
OutputTokens.RemoveAt(OutputTokens.Count - 1);
// Make sure there's another token
if (!bMoveNext)
{
throw new PreprocessorException("Invalid syntax for 'defined' expression");
}
// Check for the form 'defined identifier'
Token NameToken;
if(InputEnumerator.Current.Type == TokenType.Identifier)
{
NameToken = InputEnumerator.Current;
}
else
{
// Otherwise assume the form 'defined ( identifier )'
if(InputEnumerator.Current.Type != TokenType.LeftParen || !InputEnumerator.MoveNext() || InputEnumerator.Current.Type != TokenType.Identifier)
{
throw new PreprocessorException("Invalid syntax for 'defined' expression");
}
NameToken = InputEnumerator.Current;
if(!InputEnumerator.MoveNext() || InputEnumerator.Current.Type != TokenType.RightParen)
{
throw new PreprocessorException("Invalid syntax for 'defined' expression");
}
}
// Insert a token for whether it's defined or not
OutputTokens.Add(new Token(NameToMacro.ContainsKey(NameToken.Text)? "1" : "0", TokenType.NumericLiteral, TokenFlags.None));
bMoveNext = InputEnumerator.MoveNext();
}
else
{
// Repeatedly try to expand the last token into the list
while(OutputTokens[OutputTokens.Count - 1].Type == TokenType.Identifier && !OutputTokens[OutputTokens.Count - 1].Flags.HasFlag(TokenFlags.DisableExpansion))
{
// Try to get a macro for the current token
PreprocessorMacro Macro;
if (!NameToMacro.TryGetValue(OutputTokens[OutputTokens.Count - 1].Text, out Macro) || IgnoreMacros.Contains(Macro))
{
break;
}
if(Macro.IsFunctionMacro && (!bMoveNext || InputEnumerator.Current.Type != TokenType.LeftParen))
{
break;
}
// Remove the macro name from the output list
bool bHasLeadingSpace = OutputTokens[OutputTokens.Count - 1].HasLeadingSpace;
OutputTokens.RemoveAt(OutputTokens.Count - 1);
// Save the initial number of tokens in the list, so we can tell if it expanded
int NumTokens = OutputTokens.Count;
// If it's an object macro, expand it immediately into the output buffer
if(Macro.IsObjectMacro)
{
// Expand the macro tokens into the output buffer
ExpandObjectMacro(Macro, OutputTokens, bIsConditional, CurrentLine(), IgnoreMacros);
}
else
{
// Read balanced token for argument list
List<Token> ArgumentTokens = new List<Token>();
bMoveNext = ReadBalancedToken(InputEnumerator, ArgumentTokens);
// Expand the macro tokens into the output buffer
ExpandFunctionMacro(Macro, ArgumentTokens, OutputTokens, bIsConditional, CurrentLine(), IgnoreMacros);
}
// If the macro expanded to nothing, quit
if(OutputTokens.Count <= NumTokens)
{
break;
}
// Make sure the space is propagated to the expanded macro
OutputTokens[NumTokens] = MergeLeadingSpace(OutputTokens[NumTokens], bHasLeadingSpace);
// Mark any tokens matching the macro name as not to be expanded again. This can happen with recursive object macros, eg. #define DWORD ::DWORD
for(int Idx = NumTokens; Idx < OutputTokens.Count; Idx++)
{
if(OutputTokens[Idx].Type == TokenType.Identifier && OutputTokens[Idx].Text == Macro.Name)
{
OutputTokens[Idx] = new Token(OutputTokens[Idx].Text, TokenType.Identifier, OutputTokens[Idx].Flags | TokenFlags.DisableExpansion);
}
}
}
}
return bMoveNext;
}
/// <summary>
/// Expand an object macro
/// </summary>
/// <param name="Macro">The functional macro</param>
/// <param name="OutputTokens">The list to receive the output tokens</param>
/// <param name="bIsConditional">Whether the macro is being expanded in a conditional context, allowing use of the 'defined' keyword</param>
/// <param name="CurrentLine">The line number being processed</param>
/// <param name="IgnoreMacros">List of macros currently being expanded, which should be ignored for recursion</param>
void ExpandObjectMacro(PreprocessorMacro Macro, List<Token> OutputTokens, bool bIsConditional, int CurrentLine, List<PreprocessorMacro> IgnoreMacros)
{
// Special handling for the __LINE__ directive, since we need an updated line number for the current token
if(Macro.Name == "__FILE__")
{
Token Token = new Token(String.Format("\"{0}\"", CurrentFile.FileName.Replace("\\", "\\\\")), TokenType.StringLiteral, TokenFlags.None);
OutputTokens.Add(Token);
}
else if(Macro.Name == "__LINE__")
{
Token Token = new Token((CurrentLine + 1).ToString(), TokenType.NumericLiteral, TokenFlags.None);
OutputTokens.Add(Token);
}
else if(Macro.Name == "__COUNTER__")
{
Token Token = new Token((Counter++).ToString(), TokenType.NumericLiteral, TokenFlags.None);
OutputTokens.Add(Token);
}
else
{
int OutputTokenCount = OutputTokens.Count;
// Expand all the macros
IgnoreMacros.Add(Macro);
ExpandMacrosRecursively(Macro.Tokens, OutputTokens, bIsConditional, CurrentLine, IgnoreMacros);
IgnoreMacros.RemoveAt(IgnoreMacros.Count - 1);
// Concatenate any adjacent tokens
for(int Idx = OutputTokenCount + 1; Idx < OutputTokens.Count - 1; Idx++)
{
if(OutputTokens[Idx].Text == "##")
{
OutputTokens[Idx - 1] = Token.Concatenate(OutputTokens[Idx - 1], OutputTokens[Idx + 1]);
OutputTokens.RemoveRange(Idx, 2);
Idx--;
}
}
}
}
/// <summary>
/// Expand a function macro
/// </summary>
/// <param name="Macro">The functional macro</param>
/// <param name="OutputTokens">The list to receive the output tokens</param>
/// <param name="bIsConditional">Whether the macro is being expanded in a conditional context, allowing use of the 'defined' keyword</param>
/// <param name="CurrentLine">The line number being processed</param>
/// <param name="IgnoreMacros">List of macros currently being expanded, which should be ignored for recursion</param>
void ExpandFunctionMacro(PreprocessorMacro Macro, List<Token> ArgumentListTokens, List<Token> OutputTokens, bool bIsConditional, int CurrentLine, List<PreprocessorMacro> IgnoreMacros)
{
// Replace any newlines with spaces, and merge them with the following token
for(int Idx = 0; Idx < ArgumentListTokens.Count; Idx++)
{
if(ArgumentListTokens[Idx].Text == "\n")
{
if(Idx + 1 < ArgumentListTokens.Count)
{
ArgumentListTokens[Idx + 1] = MergeLeadingSpace(ArgumentListTokens[Idx + 1], true);
}
ArgumentListTokens.RemoveAt(Idx--);
}
}
// Split the arguments out into separate lists
List<List<Token>> Arguments = new List<List<Token>>();
if(ArgumentListTokens.Count > 2)
{
for(int Idx = 1;;Idx++)
{
if (!Macro.HasVariableArgumentList || Arguments.Count < Macro.Parameters.Count)
{
Arguments.Add(new List<Token>());
}
List<Token> Argument = Arguments[Arguments.Count - 1];
int InitialIdx = Idx;
while (Idx < ArgumentListTokens.Count - 1 && ArgumentListTokens[Idx].Text != ",")
{
if (!ReadBalancedToken(ArgumentListTokens, ref Idx, Argument))
{
throw new PreprocessorException("Invalid argument");
}
}
if(Argument.Count > 0 && Arguments[Arguments.Count - 1][0].HasLeadingSpace)
{
Argument[0] = new Token(Argument[0].Text, Argument[0].Type, Argument[0].Flags & ~TokenFlags.HasLeadingSpace);
}
bool bHasLeadingSpace = false;
for(int TokenIdx = 0; TokenIdx < Argument.Count; TokenIdx++)
{
if(Argument[TokenIdx].Text.Length == 0)
{
bHasLeadingSpace |= Argument[TokenIdx].HasLeadingSpace;
Argument.RemoveAt(TokenIdx--);
}
else
{
Argument[TokenIdx] = MergeLeadingSpace(Argument[TokenIdx], bHasLeadingSpace);
bHasLeadingSpace = false;
}
}
if (Argument.Count == 0)
{
Argument.Add(new Token("", TokenType.Placemarker, TokenFlags.None));
Argument.Add(new Token("", TokenType.Placemarker, bHasLeadingSpace? TokenFlags.HasLeadingSpace : TokenFlags.None));
}
if(Idx == ArgumentListTokens.Count - 1)
{
break;
}
if (ArgumentListTokens[Idx].Text != ",")
{
throw new PreprocessorException("Expected ',' between arguments");
}
if (Macro.HasVariableArgumentList && Arguments.Count == Macro.Parameters.Count && Idx < ArgumentListTokens.Count - 1)
{
Arguments[Arguments.Count - 1].Add(ArgumentListTokens[Idx]);
}
}
}
// Add an empty variable argument if one was not specified
if (Macro.HasVariableArgumentList && Arguments.Count == Macro.Parameters.Count - 1)
{
Arguments.Add(new List<Token> { new Token("", TokenType.Placemarker, TokenFlags.None) });
}
// Validate the argument list
if (Arguments.Count != Macro.Parameters.Count)
{
throw new PreprocessorException("Incorrect number of arguments to macro");
}
// Expand each one of the arguments
List<List<Token>> ExpandedArguments = new List<List<Token>>();
for (int Idx = 0; Idx < Arguments.Count; Idx++)
{
List<Token> NewArguments = new List<Token>();
ExpandMacrosRecursively(Arguments[Idx], NewArguments, bIsConditional, CurrentLine, IgnoreMacros);
ExpandedArguments.Add(NewArguments);
}
// Substitute all the argument tokens
List<Token> ExpandedTokens = new List<Token>();
for(int Idx = 0; Idx < Macro.Tokens.Count; Idx++)
{
Token Token = Macro.Tokens[Idx];
if(Token.Text == "#" && Idx + 1 < Macro.Tokens.Count)
{
// Stringizing operator
int ParamIdx = Macro.FindParameterIndex(Macro.Tokens[++Idx].Text);
if (ParamIdx == -1)
{
throw new PreprocessorException("{0} is not an argument name", Macro.Tokens[Idx].Text);
}
ExpandedTokens.Add(new Token(String.Format("\"{0}\"", Token.Format(Arguments[ParamIdx]).Replace("\\", "\\\\").Replace("\"", "\\\"")), TokenType.StringLiteral, Token.Flags & TokenFlags.HasLeadingSpace));
}
else if(Macro.HasVariableArgumentList && Idx + 2 < Macro.Tokens.Count && Token.Text == "," && Macro.Tokens[Idx + 1].Text == "##" && Macro.Tokens[Idx + 2].Text == PreprocessorMacro.VariadicParameter)
{
// Special MSVC/GCC extension: ', ## __VA_ARGS__' removes the comma if __VA_ARGS__ is empty. MSVC seems to format the result with a forced space.
List<Token> ExpandedArgument = ExpandedArguments[ExpandedArguments.Count - 1];
if(ExpandedArgument.Any(x => x.Text.Length > 0))
{
ExpandedTokens.Add(Token);
AppendTokensWithWhitespace(ExpandedTokens, ExpandedArgument, false);
Idx += 2;
}
else
{
ExpandedTokens.Add(new Token("", TokenType.Placemarker, Token.Flags & TokenFlags.HasLeadingSpace));
ExpandedTokens.Add(new Token("", TokenType.Placemarker, TokenFlags.HasLeadingSpace));
Idx += 2;
}
}
else if (Token.Type == TokenType.Identifier)
{
// Expand a parameter
int ParamIdx = Macro.FindParameterIndex(Token.Text);
if(ParamIdx == -1)
{
ExpandedTokens.Add(Token);
}
else if(Idx > 0 && Macro.Tokens[Idx - 1].Text == "##")
{
AppendTokensWithWhitespace(ExpandedTokens, Arguments[ParamIdx], Token.HasLeadingSpace);
}
else
{
AppendTokensWithWhitespace(ExpandedTokens, ExpandedArguments[ParamIdx], Token.HasLeadingSpace);
}
}
else
{
ExpandedTokens.Add(Token);
}
}
// Concatenate adjacent tokens
for (int Idx = 1; Idx < ExpandedTokens.Count - 1; Idx++)
{
if(ExpandedTokens[Idx].Text == "##")
{
Token ConcatenatedToken = Token.Concatenate(ExpandedTokens[Idx - 1], ExpandedTokens[Idx + 1]);
ExpandedTokens.RemoveRange(Idx, 2);
ExpandedTokens[--Idx] = ConcatenatedToken;
}
}
// Finally, return the expansion of this
IgnoreMacros.Add(Macro);
ExpandMacrosRecursively(ExpandedTokens, OutputTokens, bIsConditional, CurrentLine, IgnoreMacros);
IgnoreMacros.RemoveAt(IgnoreMacros.Count - 1);
}
/// <summary>
/// Appends a list of tokens to another list, setting the leading whitespace flag to the given value
/// </summary>
/// <param name="OutputTokens">List to receive the appended tokens</param>
/// <param name="InputTokens">List of tokens to append</param>
/// <param name="bHasLeadingSpace">Whether there is space before the first token</param>
void AppendTokensWithWhitespace(List<Token> OutputTokens, List<Token> InputTokens, bool bHasLeadingSpace)
{
if(InputTokens.Count > 0)
{
OutputTokens.Add(MergeLeadingSpace(InputTokens[0], bHasLeadingSpace));
OutputTokens.AddRange(InputTokens.Skip(1));
}
}
/// <summary>
/// Copies a single token from one list of tokens to another, or if it's an opening parenthesis, the entire subexpression until the closing parenthesis.
/// </summary>
/// <param name="InputTokens">The input token list</param>
/// <param name="InputIdx">First token index in the input token list. Set to the last uncopied token index on return.</param>
/// <param name="OutputTokens">List to recieve the output tokens</param>
/// <returns>True if a balanced expression was read, or false if the end of the list was encountered before finding a matching token</returns>
bool ReadBalancedToken(List<Token> InputTokens, ref int InputIdx, List<Token> OutputTokens)
{
// Copy a single token to the output list
Token Token = InputTokens[InputIdx++];
OutputTokens.Add(Token);
// If it was the start of a subexpression, copy until the closing parenthesis
if (Token.Type == TokenType.LeftParen)
{
// Copy the contents of the subexpression
for (;;)
{
if (InputIdx == InputTokens.Count)
{
return false;
}
if (InputTokens[InputIdx].Type == TokenType.RightParen)
{
break;
}
if (!ReadBalancedToken(InputTokens, ref InputIdx, OutputTokens))
{
return false;
}
}
// Copy the closing parenthesis
Token = InputTokens[InputIdx++];
OutputTokens.Add(Token);
}
return true;
}
/// <summary>
/// Copies a single token from one list of tokens to another, or if it's an opening parenthesis, the entire subexpression until the closing parenthesis.
/// </summary>
/// <param name="InputTokens">The input token list</param>
/// <param name="InputIdx">First token index in the input token list. Set to the last uncopied token index on return.</param>
/// <param name="OutputTokens">List to recieve the output tokens</param>
/// <returns>True if a balanced expression was read, or false if the end of the list was encountered before finding a matching token</returns>
bool ReadBalancedToken(IEnumerator<Token> InputEnumerator, List<Token> OutputTokens)
{
// Copy a single token to the output list
Token Token = InputEnumerator.Current;
bool bMoveNext = InputEnumerator.MoveNext();
OutputTokens.Add(Token);
// If it was the start of a subexpression, copy until the closing parenthesis
if (Token.Type == TokenType.LeftParen)
{
// Copy the contents of the subexpression
for (;;)
{
if (!bMoveNext)
{
throw new PreprocessorException("Unbalanced token sequence");
}
if (InputEnumerator.Current.Type == TokenType.RightParen)
{
OutputTokens.Add(InputEnumerator.Current);
bMoveNext = InputEnumerator.MoveNext();
break;
}
bMoveNext = ReadBalancedToken(InputEnumerator, OutputTokens);
}
}
return bMoveNext;
}
/// <summary>
/// Create a file which represents the compiler definitions
/// </summary>
/// <param name="PreludeFile"></param>
/// <param name="CompileEnvironment"></param>
/// <param name="WorkingDir"></param>
/// <param name="Definitions"></param>
public static void CreatePreludeFile(FileReference PreludeFile, CompileEnvironment CompileEnvironment, DirectoryReference WorkingDir)
{
// Spawn the compiler and write the prelude file
string CompilerName = CompileEnvironment.Compiler.GetFileName().ToLowerInvariant();
if(CompileEnvironment.CompilerType == CompilerType.Clang)
{
// Create a dummy input file
FileReference InputFile = PreludeFile.ChangeExtension(".in.cpp");
File.WriteAllText(InputFile.FullName, "");
// Spawn clang and parse the output
using (StreamWriter Writer = new StreamWriter(PreludeFile.FullName))
{
Utility.Run(CompileEnvironment.Compiler, String.Join(" ", CompileEnvironment.Options.Where(x => x.Name == "-target").Select(x => x.ToString())) + " -dM -E -x c++ " + InputFile.FullName, WorkingDir, new LineBasedTextWriterWrapper(Writer));
}
}
else if(CompileEnvironment.CompilerType == CompilerType.VisualC)
{
// Create a query file
FileReference InputFile = PreludeFile.ChangeExtension(".in");
using (StreamWriter InputWriter = new StreamWriter(InputFile.FullName))
{
InputWriter.WriteLine("#define STRINGIZE_2(x) #x");
InputWriter.WriteLine("#define STRINGIZE(x) STRINGIZE_2(x)");
string[] MacroNames = Properties.Resources.Prelude.Split('\n').Select(x => x.Trim()).Where(x => x.Length > 0 && !x.StartsWith(";")).ToArray();
foreach (string MacroName in MacroNames)
{
InputWriter.WriteLine("");
InputWriter.WriteLine("#ifdef {0}", MacroName);
InputWriter.WriteLine("#pragma message(\"#define {0} \" STRINGIZE({0}))", MacroName);
InputWriter.WriteLine("#endif");
}
}
// Invoke the compiler and capture the output into a string
StringWriter OutputWriter = new StringWriter();
string FullCommandLine = String.Format("{0} /Zs /TP {1}", CompileEnvironment.GetCommandLine(), InputFile.FullName);
Utility.Run(CompileEnvironment.Compiler, FullCommandLine, WorkingDir, new LineBasedTextWriterWrapper(OutputWriter));
// Filter the output
using (StreamWriter Writer = new StreamWriter(PreludeFile.FullName))
{
foreach(string Line in OutputWriter.ToString().Split('\n').Select(x => x.Trim()).Where(x => x.Length > 0))
{
if(Line.StartsWith("#define"))
{
Writer.WriteLine(Line);
}
else if(Line.Trim() != InputFile.GetFileName() && !Line.StartsWith("time("))
{
throw new Exception(String.Format("Unexpected output from compiler: {0}", OutputWriter.ToString()));
}
}
}
}
else
{
throw new NotImplementedException("Unknown compiler: " + CompilerName);
}
}
/// <summary>
/// Preprocess the current file and write it to the given output
/// </summary>
/// <param name="InputFile">The file to read from</param>
/// <param name="OutputFile">File to output to</param>
public void PreprocessFile(string InputFileName, FileReference OutputFile, IEnumerable<FileReference> ForceIncludeFiles)
{
using (StreamWriter Writer = new StreamWriter(OutputFile.FullName))
{
List<FileReference> InputFiles = new List<FileReference>();
InputFiles.AddRange(ForceIncludeFiles);
InputFiles.Add(new FileReference(InputFileName));
foreach(FileReference InputFile in InputFiles)
{
WorkspaceFile InputWorkspaceFile = Workspace.GetFile(InputFile);
PreprocessFile(new PreprocessorFile(InputFile.FullName, InputWorkspaceFile), Writer);
}
}
}
/// <summary>
/// Preprocess the current file to the given output stream
/// </summary>
/// <param name="InputFile">The file to read from</param>
/// <param name="Writer">Stream to output to</param>
bool PreprocessFile(PreprocessorFile InputFile, TextWriter Writer)
{
bool bResult = false;
if(PushFile(InputFile))
{
FileReference InputFileLocation = InputFile.Location;
SourceFile SourceFile = InputFile.WorkspaceFile.ReadSourceFile();
// Always write a #line directive at the start of the file
Writer.WriteLine(FormatLineDirective(1));
// Write the rest of the file
TextLocation Location = TextLocation.Origin;
foreach (PreprocessorMarkup Markup in SourceFile.Markup)
{
// Write everything since the last item
if (Markup.Location > Location)
{
if (IsBranchActive())
{
WriteText(SourceFile.Text, Location, Markup.Location, Writer);
}
else
{
WriteBlankedText(SourceFile.Text, Location, Markup.Location, Writer);
}
Location = Markup.Location;
}
// Check the type of this markup
if (Markup.Type == PreprocessorMarkupType.Text)
{
// Write the expanded macro tokens if the current block is active, or blank lines if not
if (IsBranchActive())
{
WriteExpandedText(SourceFile.Location.FullName, SourceFile.Text, Location, Markup.EndLocation, Writer);
}
else
{
WriteBlankedText(SourceFile.Text, Location, Markup.EndLocation, Writer);
}
Location = Markup.EndLocation;
}
else
{
// Capture initial state that we need later on
bool bActive = IsBranchActive();
// Check whether we want a #line directive after this point
bool bOutputLineDirective = false;
if(Markup.Type == PreprocessorMarkupType.Endif)
{
if(Branches.Peek().HasFlag(BranchState.HasIfDirective) && Branches.Peek().HasFlag(BranchState.ParentIsActive))
{
bOutputLineDirective = true;
}
if(Branches.Peek().HasFlag(BranchState.HasIfdefDirective) && Branches.Peek().HasFlag(BranchState.ParentIsActive) && Branches.Peek().HasFlag(BranchState.Taken))
{
bOutputLineDirective = true;
}
if(Branches.Peek().HasFlag(BranchState.HasIfndefDirective) && Branches.Peek().HasFlag(BranchState.ParentIsActive) && Branches.Peek().HasFlag(BranchState.Taken))
{
bOutputLineDirective = true;
}
}
// Parse the markup
if (Markup.Type == PreprocessorMarkupType.Include)
{
bOutputLineDirective |= WriteInclude(Markup.Tokens, Markup.Location.LineIdx, Writer);
}
else
{
ParseMarkup(Markup.Type, Markup.Tokens, Markup.Location.LineIdx);
}
// Find the newline token (note - may be skipping over escaped newlines here)
Location = Markup.EndLocation;
// Write the contents of this markup, if necessary
if (IsBranchActive())
{
if(Markup.Type == PreprocessorMarkupType.Else)
{
if(Branches.Peek().HasFlag(BranchState.HasIfDirective) || Branches.Peek().HasFlag(BranchState.HasElifDirective))
{
bOutputLineDirective = true;
}
}
else if (Markup.Type == PreprocessorMarkupType.Elif)
{
if(!Branches.Peek().HasFlag(BranchState.HasIfdefDirective))
{
bOutputLineDirective = true;
}
}
}
else
{
if(Markup.Type == PreprocessorMarkupType.Elif)
{
if(!Branches.Peek().HasFlag(BranchState.Taken) && Branches.Peek().HasFlag(BranchState.ParentIsActive) && !Branches.Peek().HasFlag(BranchState.HasIfdefDirective))
{
bOutputLineDirective = true;
}
}
if(Markup.Type == PreprocessorMarkupType.Else)
{
if(!Branches.Peek().HasFlag(BranchState.Taken) && Branches.Peek().HasFlag(BranchState.ParentIsActive))// && !Branches.Peek().HasFlag(BranchState.InitialIfCondition))
{
bOutputLineDirective = true;
}
}
}
// Write any text or escaped newlines for this markup
if (bActive && Markup.Type == PreprocessorMarkupType.Pragma && (Markup.Tokens[0].Text != "pop_macro" && Markup.Tokens[0].Text != "push_macro"))
{
List<Token> ExpandedTokens = new List<Token>();
ExpandMacros(Markup.Tokens, ExpandedTokens, false, Markup.Location.LineIdx);
Writer.Write("#pragma {0}", Token.Format(ExpandedTokens));
}
else if(!bActive || Markup.Type == PreprocessorMarkupType.Define)
{
for (int EscapedLineIdx = Markup.Location.LineIdx; EscapedLineIdx < Markup.EndLocation.LineIdx - 1; EscapedLineIdx++)
{
Writer.WriteLine();
}
}
// Output the line directive
if(bOutputLineDirective)
{
Writer.WriteLine(FormatLineDirective(Markup.EndLocation.LineIdx + 1));
}
else
{
Writer.WriteLine();
}
}
}
WriteText(SourceFile.Text, Location, SourceFile.Text.End, Writer);
// Automatically treat header guards as #pragma once style directives
if (SourceFile.HasHeaderGuard)
{
PragmaOnceFiles.Add(SourceFile.Location);
}
// Pop the file from the include stack
PopFile();
bResult = true;
}
return bResult;
}
/// <summary>
/// Resolve an included file and write it to an output stream
/// </summary>
/// <param name="Tokens">Tokens for the include directive</param>
/// <param name="CurrentLine">The current line</param>
/// <param name="Writer">Writer for the output text</param>
/// <returns>True if an include was written, false if the current branch is not active, or the file has already been included</returns>
bool WriteInclude(List<Token> Tokens, int CurrentLine, TextWriter Writer)
{
bool bResult = false;
if (IsBranchActive())
{
PreprocessorFile IncludeFile = ResolveInclude(Tokens, CurrentLine);
bResult = PreprocessFile(IncludeFile, Writer);
}
return bResult;
}
/// <summary>
/// Preprocess the current file to the given output stream
/// </summary>
/// <param name="Text">The text buffer for the current file</param>
/// <param name="Location">Start of the region to copy to the output stream</param>
/// <param name="EndLocation">End of the region to copy to the output stream</param>
/// <param name="Writer">Writer for the output text</param>
void WriteText(TextBuffer Text, TextLocation Location, TextLocation EndLocation, TextWriter Writer)
{
// Write the whitespace
while(EndLocation.LineIdx > Location.LineIdx)
{
if(Text.Lines[Location.LineIdx].EndsWith("*\\"))
{
// Hacky fix to make preprocessed output the same as MSVC. When writing a multiline comment ending with *\ (as appears in copyright boilerplate for
// headers like winver.h), both characters are removed and the * is pushed into the start of the next line.
Writer.WriteLine(Text.Lines[Location.LineIdx].Substring(Location.ColumnIdx, Text.Lines[Location.LineIdx].Length - 2));
Writer.Write("*");
}
else
{
Writer.WriteLine(Text.Lines[Location.LineIdx].Substring(Location.ColumnIdx));
}
Location.ColumnIdx = 0;
Location.LineIdx++;
}
// Write the last line
if(Location.ColumnIdx < EndLocation.ColumnIdx)
{
Writer.Write(Text.Lines[Location.LineIdx].Substring(Location.ColumnIdx, EndLocation.ColumnIdx - Location.ColumnIdx));
}
}
/// <summary>
/// Preprocess the current file to the given output stream
/// </summary>
/// <param name="Text">The text buffer for the current file</param>
/// <param name="Location">Start of the region to copy to the output stream</param>
/// <param name="EndLocation">End of the region to copy to the output stream</param>
/// <param name="Writer">Writer for the output text</param>
void WriteBlankedText(TextBuffer Text, TextLocation Location, TextLocation EndLocation, TextWriter Writer)
{
// Write the whitespace
while(EndLocation.LineIdx > Location.LineIdx)
{
Writer.WriteLine();
Location.ColumnIdx = 0;
Location.LineIdx++;
}
// Write the last line
if(Location.ColumnIdx < EndLocation.ColumnIdx)
{
//Writer.Write(Text.Lines[Location.LineIdx].Substring(Location.ColumnIdx, EndLocation.ColumnIdx - Location.ColumnIdx));
}
}
/// <summary>
/// Write a portion of the given file to the output stream, expanding macros in it
/// </summary>
/// <param name="FileName">The file being parsed</param>
/// <param name="Text">The text buffer for the current file</param>
/// <param name="Location">Start of the region to copy to the output stream</param>
/// <param name="EndLocation">End of the region to copy to the output stream</param>
/// <param name="Writer">Writer for the output text</param>
void WriteExpandedText(string FileName, TextBuffer Text, TextLocation Location, TextLocation EndLocation, TextWriter Writer)
{
int ExpectedLineIdx = Location.LineIdx;
// Expand any macros in this block of tokens
TokenReader Reader = new TokenReader(FileName, Text, Location);
for(bool bMoveNext = Reader.MoveNext(); bMoveNext; )
{
// Write the whitespace to the output stream
WriteText(Text, Reader.TokenWhitespaceLocation, Reader.TokenLocation, Writer);
ExpectedLineIdx += Reader.TokenLocation.LineIdx - Reader.TokenWhitespaceLocation.LineIdx;
// Check if we've reached the end
if(Reader.TokenLocation >= EndLocation)
{
break;
}
// Write out the current token
if(Reader.Current.Text == "\n")
{
Writer.WriteLine();
if(++ExpectedLineIdx != Reader.TokenEndLocation.LineIdx)
{
ExpectedLineIdx = Reader.TokenEndLocation.LineIdx;
Writer.WriteLine(FormatLineDirective(ExpectedLineIdx + 1));
}
bMoveNext = Reader.MoveNext();
}
else
{
// MSVC includes comments in any macro arguments before any expanded tokens. Save the potential first argument location in case this happens.
TextLocation FirstArgumentLocation = Reader.TokenEndLocation;
// Expand all the tokens
List<Token> ExpandedTokens = new List<Token>();
bMoveNext = ReadExpandedToken(Reader, ExpandedTokens, false, () => Reader.TokenLocation.LineIdx, new List<PreprocessorMacro>());
// If we expanded a macro, insert all the comment tokens
if(Reader.TokenWhitespaceLocation > FirstArgumentLocation)
{
TokenReader CommentReader = new TokenReader(FileName, Text, FirstArgumentLocation);
while(CommentReader.MoveNext() && CommentReader.TokenLocation < Reader.TokenWhitespaceLocation)
{
string[] Comments = Text.Extract(CommentReader.TokenWhitespaceLocation, CommentReader.TokenLocation).Select(x => x.EndsWith("\\")? x.Substring(0, x.Length - 1) : x).ToArray();
Comments[0] = Comments[0].TrimStart();
for(int Idx = 0; Idx < Comments.Length - 1; Idx++)
{
Writer.WriteLine(Comments[Idx]);
}
Writer.Write(Comments[Comments.Length - 1].TrimEnd());
}
}
// Now write the expanded tokens
Writer.Write(Token.Format(ExpandedTokens));
}
}
}
}
/// <summary>
/// Tests for the preprocessor
/// </summary>
static class PreprocessorTests
{
public static void Run(bool bTestFailureCases)
{
RunTest("token", "token\n");
RunTest("#", "");
RunTest("#define X token\nX", "token\n");
RunTest("#define X(x) token x other\nX(and one).", "token and one other.\n");
RunTest("#define X(x,y) token x and y\nX(1, 2).", "token 1 and 2.\n");
RunTest("#define INC(x) (x + 2)\nINC", "INC\n");
RunTest("#define TEST(x) x\\\n?\nTEST(A)", "A?\n");
//RunTest("%:define X token\nX", "<token>", Preprocess);
RunTest("#define A B C D\nA", "B C D\n");
RunTest("#define A B ## D\nA", "BD\n");
RunTest("#define F(A) <A>\nF(x)", "<x>\n");
RunTest("#define F(A,B) <A,B>\nF(x,y) + 1", "<x,y> + 1\n");
RunTest("#define F(A,B,C) <A,B,C>\nF(x,y,z)", "<x,y,z>\n");
RunTest("#define F(...) <__VA_ARGS__>\nF(x)", "<x>\n");
RunTest("#define F(...) <__VA_ARGS__>\nF(x,y)", "<x,y>\n");
RunTest("#define F(A,...) <A,__VA_ARGS__>\nF(x,y)", "<x,y>\n");
RunTest("#define F(A,...) <A, __VA_ARGS__>\nF(x,y)", "<x, y>\n");
RunTest("#define F(A,...) <A, __VA_ARGS__>\nF(x,y, z)", "<x, y, z>\n");
RunTest("#define X list of tokens\nX", "list of tokens\n");
RunTest("#define LST list\n#define TOKS tokens\n#define LOTS LST of TOKS\nLOTS LOTS", "list of tokens list of tokens\n");
RunTest("#define LOTS LST of TOKS\n#define LST list\n#define TOKS tokens\nLOTS LOTS", "list of tokens list of tokens\n");
RunTest("#define FUNC(x) arg=x.\nFUNC(var) FUNC(2)", "arg=var. arg=2.\n");
RunTest("#define FUNC(x,y,z) int n = z+y*x;\nFUNC(1,2,3)", "int n = 3+2*1;\n");
RunTest("#define X 20\n#define FUNC(x,y) x+y\nx=FUNC(X,Y);", "x=20+Y;\n");
RunTest("#define FA(x,y) FB(x,y)\n#define FB(x,y) x + y\nFB(1,2);", "1 + 2;\n");
RunTest("#define PRINTF(...) printf(__VA_ARGS__)\nPRINTF()", "printf()\n");
RunTest("#define PRINTF(...) printf(__VA_ARGS__)\nPRINTF(\"hello\")", "printf(\"hello\")\n");
RunTest("#define PRINTF(...) printf(__VA_ARGS__)\nPRINTF(\"%d\", 1)", "printf(\"%d\", 1)\n");
RunTest("#define PRINTF(FORMAT, ...) printf(FORMAT, __VA_ARGS__)\nPRINTF(\"test\")", "printf(\"test\", )\n");
RunTest("#define PRINTF(FORMAT, ...) printf(FORMAT, __VA_ARGS__)\nPRINTF(\"test %s\", \"hello\")", "printf(\"test %s\", \"hello\")\n");
RunTest("#define PRINTF(FORMAT, ...) printf(FORMAT, __VA_ARGS__)\nPRINTF(\"test %s %d\", \"hello\", 1)", "printf(\"test %s %d\", \"hello\", 1)\n");
RunTest("#define PRINTF(FORMAT, ...) printf(FORMAT, ## __VA_ARGS__)\nPRINTF(\"test\")", "printf(\"test\" )\n");
RunTest("#define INC(x) (x + 2)\nINC(\n1\n)", "(1 + 2)\n");
RunTest("#define STRINGIZE(ARG) #ARG\nSTRINGIZE(+=)", "\"+=\"\n");
RunTest("#define STRINGIZE(ARG) #ARG\nSTRINGIZE(:>)", "\":>\"\n");
RunTest("#define STRINGIZE(ARG) #ARG\nSTRINGIZE(3.1415)", "\"3.1415\"\n");
RunTest("#define CONCAT(X, Y) X ## Y\nCONCAT(+, =)", "+=\n");
RunTest("#define CONCAT(X, Y) X ## Y\nCONCAT(3, .14159)", "3.14159\n");
RunTest("#define CONCAT(X, Y) X ## Y\nCONCAT(3, hello)", "3hello\n");
RunTest("#define CONCAT(X, Y) X ## #Y\nCONCAT(u, hello)", "u\"hello\"\n");
RunTest("#define CONCAT(X, ...) X ## __VA_ARGS__\nCONCAT(hello) there", "hello there\n");
RunTest("#define CONCAT(X, ...) X ## __VA_ARGS__\nCONCAT(hello, there)", "hellothere\n");
RunTest("#if 1\nfirst_branch\n#endif\nend", "first_branch\nend\n");
RunTest("#if 0\nfirst_branch\n#endif\nend", "end\n");
RunTest("#if 1\nbranch_1\n#else\nbranch_2\n#endif", "branch_1\n");
RunTest("#if 0\nbranch_1\n#else\nbranch_2\n#endif", "branch_2\n");
RunTest("#define A\n#ifdef A\nYes\n#endif", "Yes\n");
RunTest("#define B\n#ifdef A\nYes\n#endif\nNo", "No\n");
RunTest("#define A\n#ifndef A\nYes\n#endif\nNo", "No\n");
RunTest("#define B\n#ifndef A\nYes\n#endif", "Yes\n");
RunTest("#define A\n#undef A\n#ifdef A\nYes\n#endif\nNo", "No\n");
RunTest("#define A\n#undef A\n#ifndef A\nYes\n#endif", "Yes\n");
RunTest("#define A\n#ifdef A\nYes\n#else\nNo\n#endif", "Yes\n");
RunTest("#define B\n#ifdef A\nYes\n#else\nNo\n#endif", "No\n");
RunTest("#define A MACRO\nA", "MACRO\n");
RunTest("#define A MACRO\n#undef A\nA", "A\n");
RunTest("#define A(X) MACRO X\nA(x)", "MACRO x\n");
RunTest("#define A(X) MACRO X\n#undef A\nA(x)", "A(x)\n");
RunTest("#if 1 + 2 > 3\nYes\n#endif\nNo", "No\n");
RunTest("#if 1 + 2 >= 3\nYes\n#endif", "Yes\n");
RunTest("#define ONE 1\n#define TWO 2\n#define PLUS(x, y) x + y\n#if PLUS(ONE, TWO) > 3\nYes\n#endif\nNo", "No\n");
RunTest("#define ONE 1\n#define TWO 2\n#define PLUS(x, y) x + y\n#if PLUS(ONE, TWO) >= 3\nYes\n#endif", "Yes\n");
RunTest("#define ONE 1\n#if defined ONE\nOne\n#elif defined TWO\nTwo\n#else\nThree\n#endif", "One\n");
RunTest("#define TWO 1\n#if defined ONE\nOne\n#elif defined TWO\nTwo\n#else\nThree\n#endif", "Two\n");
RunTest("#define ONE 0\n#if defined(ONE) + defined(TWO) >= 1\nYes\n#else\nNo\n#endif", "Yes\n");
RunTest("#define ONE 0\n#if defined(ONE) + defined(TWO) >= 2\nYes\n#else\nNo\n#endif", "No\n");
RunTest("#define ONE 0\n#define TWO\n#if defined(ONE) + defined(TWO) >= 2\nYes\n#else\nNo\n#endif", "Yes\n");
RunTest("#define REGISTER_NAME(num,name) NAME_##name = num,\nREGISTER_NAME(201, TRUE)", "NAME_TRUE = 201,\n");
RunTest("#define FUNC_2(X) VALUE=X\n#define FUNC_N(X) FUNC_##X\n#define OBJECT FUNC_N(2)\nOBJECT(1234)", "VALUE=1234\n");
RunTest("#define GCC_EXTENSION(...) 123, ## __VA_ARGS__\nGCC_EXTENSION(456)", "123,456\n");
RunTest("#define FUNC(fmt,...) (fmt, ## __VA_ARGS__)\nFUNC(a)", "(a )\n");
RunTest("#define FUNC(fmt,...) (fmt, ## __VA_ARGS__)\nFUNC(a,b)", "(a,b)\n");
RunTest("#define FUNC(fmt,...) (fmt, ## __VA_ARGS__)\nFUNC(a,b )", "(a,b)\n");
RunTest("#define FUNC(fmt, ...) (fmt, ## __VA_ARGS__)\nFUNC(a)", "(a )\n");
RunTest("#define FUNC(fmt, ...) (fmt, ## __VA_ARGS__)\nFUNC(a,b)", "(a,b)\n");
RunTest("#define FUNC(fmt, ...) (fmt, ## __VA_ARGS__)\nFUNC(a,b )", "(a,b)\n");
RunTest("#define EMPTY_TOKEN\n#define FUNC(_FuncName, _HType1, _HArg1) _FuncName(_HType1 _HArg1);\nFUNC(hello, EMPTY_TOKEN, int)", "hello( int);\n");
RunTest("#define EMPTY_TOKEN\n#define GCC_EXTENSION(...) 123 , ## __VA_ARGS__\nGCC_EXTENSION(EMPTY_TOKEN)", "123 \n");
RunTest("#define FUNC(x) Value = x\n#define PP_JOIN(x, y) x ## y\n#define RESULT(x) PP_JOIN(FU, NC)(x)\nRESULT(1234)", "Value = 1234\n");
RunTest("#define VALUE 1234\n#define PP_JOIN(x, y) x ## y\n#define RESULT PP_JOIN(V, ALUE)\nRESULT", "1234\n");
RunTest("#define EMPTY_TOKEN\n#define FUNC(x) (x)\nFUNC(EMPTY_TOKEN A)", "( A)\n");
RunTest("#define EMPTY_TOKEN\n#define FUNC(x,y) (x y)\nFUNC(EMPTY_TOKEN,A)", "( A)\n");
// RunTest("#define EMPTY_TOKEN\n#define FUNC(x) (x)\n#define FUNC_2(x,y) FUNC(x y)\nFUNC_2(EMPTY_TOKEN,A)", "( A)\n");
// RunTest("#define EMPTY_TOKEN\n#define FUNC(x) (x EMPTY_TOKEN)\nFUNC(A)", "(A )\n");
// RunTest("#define EMPTY_TOKEN\n#define FUNC(x,y) (x y)\nFUNC(A,)", "(A)\n");
RunTest("#define EMPTY\n#define FUNC(x) (x)\nFUNC(x y)", "(x y)\n");
RunTest("#define EMPTY\n#define FUNC(x) (x)\nFUNC(x EMPTY y)", "(x y)\n");
RunTest("#define EMPTY\n#define FUNC(x) (x)\nFUNC(x EMPTY y EMPTY)", "(x y )\n");
RunTest("#define EMPTY\n#define FUNC(x) (x)\nFUNC(EMPTY x EMPTY y)", "( x y)\n");
RunTest("#define EMPTY\n#define FUNC(x) (x)\nFUNC( EMPTY x EMPTY y)", "( x y)\n");
RunTest("#define EMPTY\n#define FUNC(x) (x)\nFUNC(EMPTY x EMPTY y )", "( x y)\n");
RunTest("#define EMPTY\n#define FUNC(x) (x)\n#define FUNC_2(x) FUNC(x)\nFUNC_2(x y)", "(x y)\n");
RunTest("#define EMPTY\n#define FUNC(x) (x)\n#define FUNC_2(x) FUNC(x)\nFUNC_2(x EMPTY y)", "(x y)\n");
RunTest("#define EMPTY\n#define FUNC(x) (x)\n#define FUNC_2(x) FUNC(x)\nFUNC_2(x EMPTY y EMPTY)", "(x y)\n");
RunTest("#define EMPTY\n#define FUNC(x) (x)\n#define FUNC_2(x) FUNC(x)\nFUNC_2(EMPTY x EMPTY y)", "( x y)\n");
RunTest("#define EMPTY\n#define FUNC(x) (x)\n#define FUNC_2(x) FUNC(x)\nFUNC_2( EMPTY x EMPTY y)", "( x y)\n");
RunTest("#define EMPTY\n#define FUNC(x) (x)\n#define FUNC_2(x) FUNC(x)\nFUNC_2(EMPTY x EMPTY y )", "( x y)\n");
RunTest("#define EMPTY\n#define FUNC(x) ( x )\nFUNC(EMPTY EMPTY)", "( )\n");
RunTest("#define EMPTY\n#define FUNC(x) ( x)\nFUNC(EMPTY EMPTY x)", "( x)\n");
RunTest("#define EMPTY\n#define FUNC(x,y) ( x y)\n#define FUNC2(x,y) FUNC(x,y)\nFUNC2(EMPTY EMPTY EMPTY, x)", "( x)\n");
RunTest("#define EMPTY\n#define DOUBLE_EMPTY EMPTY EMPTY\n#define FUNC(x) ( x)\nFUNC(DOUBLE_EMPTY x)", "( x)\n");
RunTest("#define EMPTY\n#define DOUBLE_EMPTY EMPTY EMPTY\n#define FUNC(x,y) ( x y )\n#define FUNC2(x,y) FUNC(x,y)\nFUNC2(DOUBLE_EMPTY EMPTY, x)", "( x )\n");
RunTest("#define EMPTY\n#define FUNC(x,y) ( x y )\nFUNC(EMPTY EMPTY EMPTY EMPTY EMPTY EMPTY, x)", "( x )\n");
RunTest("#define _NON_MEMBER_CALL(FUNC, CV_REF_OPT) FUNC(X,CV_REF_OPT)\n#define _NON_MEMBER_CALL_CV(FUNC, REF_OPT) _NON_MEMBER_CALL(FUNC, REF_OPT) _NON_MEMBER_CALL(FUNC, const REF_OPT)\n#define _NON_MEMBER_CALL_CV_REF(FUNC) _NON_MEMBER_CALL_CV(FUNC, ) _NON_MEMBER_CALL_CV(FUNC, &)\n#define _IS_FUNCTION(X,CV_REF_OPT) (CV_REF_OPT)\n_NON_MEMBER_CALL_CV_REF(_IS_FUNCTION)", "() (const) (&) (const &)\n");
RunTest("#define TEXT(x) L ## x\n#define checkf(expr, format, ...) FDebug::AssertFailed(#expr, format, ##__VA_ARGS__)\ncheckf( true, TEXT( \"hello world\" ) );", "FDebug::AssertFailed(\"true\", L\"hello world\" );\n");
if (bTestFailureCases)
{
RunTest("#define", null);
RunTest("#define INC(x) (x + 2)\nINC(", null);
RunTest("#define +", null);
RunTest("#define A B __VA_ARGS__ D\nA", null);
RunTest("#define A ## B\nA", null);
RunTest("#define A B ##\nA", null);
RunTest("#define defined not_defined", null);
RunTest("#define F(A) <A>\nF(x,y) + 1", null);
RunTest("#define F(A,) <A>\nF(x)", null);
RunTest("#define F(A+) <A>\nF(x)", null);
RunTest("#define F(A,A) <A>\nF(x,y)", null);
RunTest("#define F(A,B) <A,B>\nF(x) + 1", null);
RunTest("#define F(A,B,) <A,B>\nF(x,y) + 1", null);
RunTest("#define F(...) <__VA_ARGS__>\nF(x,)", null);
RunTest("#define F(A...) <A, __VA_ARGS__>\nF(x)", null);
RunTest("#define F(A...) <A, __VA_ARGS__>\nF(x,y)", null);
RunTest("#define F(A,__VA_ARGS__) <A, __VA_ARGS__>\nF(x,y)", null);
RunTest("#define F(A) #+\nF(x)", null);
RunTest("#define F(A) #B\nF(x)", null);
RunTest("#define F(A) ## A\nF(x)", null);
RunTest("#define F(A) A ##\nF(x)", null);
RunTest("#define F(A) <__VA_ARGS__>\nF(x)", null);
RunTest("#define INC(x\n)", null);
RunTest("#if 1\nbranch_1\n#else garbage\nbranch_2\n#endif\nend", null);
RunTest("#if 1\nbranch_1\n#else\nbranch_2\n#endif garbage\nend", null);
RunTest("#if 1\nbranch_1\n#else\nbranch_2\n#elif 1\nbranch_3\n#endif\nend", null);
RunTest("#if 1\nbranch_1\n#else\nbranch_2\n#else\nbranch_3\n#endif", null);
RunTest("#if 0\nbranch_1\n#else\nbranch_2\n#else\nbranch_3\n#endif", null);
RunTest("#elif\nbranch_1\n#else\nbranch_2\n#endif\nend", null);
RunTest("#ifdef +\nbranch_1\n#else\nbranch_2\n#endif", null);
RunTest("#ifdef A 1\nbranch_1\n#else\nbranch_2\n#endif", null);
RunTest("#define A()\n#if A(\n)\nOK\n#endif", null);
RunTest("#define A MACRO\n#undef A B\nA", null);
}
}
/// <summary>
/// Preprocess a fragment of code, and check it results in an expected sequence of tokens
/// </summary>
/// <param name="Fragment">The code fragment to preprocess</param>
/// <param name="ExpectedResult">The expected sequence of tokens, as a string. Null to indicate that the input is invalid, and an exception is expected.</param>
static void RunTest(string Fragment, string ExpectedResult)
{
SourceFile File = new SourceFile(null, TextBuffer.FromString(Fragment), SourceFileFlags.Inline);
Preprocessor Instance = new Preprocessor(null, new string[] { }, new DirectoryReference[] { });
string Result;
try
{
List<Token> OutputTokens = new List<Token>();
foreach(PreprocessorMarkup Markup in File.Markup)
{
if(Markup.Type == PreprocessorMarkupType.Text)
{
if(Instance.IsBranchActive())
{
List<Token> Tokens = new List<Token>();
TokenReader Reader = new TokenReader("", File.Text, Markup.Location);
while(Reader.MoveNext() && Reader.TokenLocation < Markup.EndLocation)
{
Tokens.Add(Reader.Current);
}
Instance.ExpandMacros(Tokens, OutputTokens, false, 0);
}
}
else
{
Instance.ParseMarkup(Markup.Type, Markup.Tokens, 0);
}
}
Result = Token.Format(OutputTokens);
}
catch (PreprocessorException)
{
Result = null;
}
if((ExpectedResult == null && Result != null) || (ExpectedResult != null && Result == null) || (ExpectedResult != null && Result != null && ExpectedResult != Result))
{
string Prefix = "Failed test for: ";
foreach(string Line in Fragment.Split('\n'))
{
Console.WriteLine("{0}{1}", Prefix, Line);
Prefix = new string(' ', Prefix.Length);
}
Console.WriteLine("Expected: {0}", (ExpectedResult != null)? ExpectedResult.Replace("\n", "\\n") : "Exception");
Console.WriteLine("Received: {0}", (Result != null)? Result.Replace("\n", "\\n") : "Exception");
Console.WriteLine();
}
}
/// <summary>
/// Preprocess a set of source files
/// </summary>
/// <param name="WorkingDir">The working directory to invoke the compiler from</param>
/// <param name="PreprocessedDir">The output directory for the preprocessed files</param>
/// <param name="FilePairs">List of files and their compile environment</param>
/// <param name="Log">Writer for log output</param>
public static void WritePreprocessedFiles(DirectoryReference WorkingDir, DirectoryReference PreprocessedDir, IEnumerable<KeyValuePair<FileReference, CompileEnvironment>> FilePairs, LineBasedTextWriter Log)
{
DirectoryReference.CreateDirectory(PreprocessedDir);
// Create separate directories for the compiler and include tool
DirectoryReference CompilerDir = DirectoryReference.Combine(PreprocessedDir, "Compiler");
DirectoryReference.CreateDirectory(CompilerDir);
DirectoryReference CompilerNormalizedDir = DirectoryReference.Combine(PreprocessedDir, "CompilerNormalized");
DirectoryReference.CreateDirectory(CompilerNormalizedDir);
DirectoryReference ToolDir = DirectoryReference.Combine(PreprocessedDir, "Tool");
DirectoryReference.CreateDirectory(ToolDir);
// Write out the prelude file, which we can use to scrape the compiler settings
FileReference PreludeFile = FileReference.Combine(PreprocessedDir, "Prelude.cpp");
Preprocessor.CreatePreludeFile(PreludeFile, FilePairs.First().Value, WorkingDir);
// Process all the files
int MaxDegreeOfParallelism = -1;
if(MaxDegreeOfParallelism == 1)
{
foreach (KeyValuePair<FileReference, CompileEnvironment> Pair in FilePairs)
{
WritePreprocessedFile(Pair.Key, Pair.Value, PreludeFile, WorkingDir, CompilerDir, CompilerNormalizedDir, ToolDir, Log);
}
}
else
{
ParallelOptions Options = new ParallelOptions() { MaxDegreeOfParallelism = MaxDegreeOfParallelism };
Parallel.ForEach(FilePairs, Options, Pair => WritePreprocessedFile(Pair.Key, Pair.Value, PreludeFile, WorkingDir, CompilerDir, CompilerNormalizedDir, ToolDir, Log));
}
}
/// <summary>
/// Preprocess a single file, with this tool and the default compiler
/// </summary>
/// <param name="InputFile">The source file to preprocess</param>
/// <param name="Environment">The environment to compile with</param>
/// <param name="PreludeFile">The prelude source file to use to scrape the compiler's default definitions</param>
/// <param name="WorkingDir">Working directory for the compiler</param>
/// <param name="CompilerDir">Output directory for files preprocessed by the compiler</param>
/// <param name="CompilerNormalizedDir">Output directory for normalized versions of files preprocessed by the compiler</param>
/// <param name="ToolDir">Output directory for files preprocessed by this tool</param>
/// <param name="FileCache">Cache of file locations to their source file</param>
/// <param name="Log">Output log writer</param>
static void WritePreprocessedFile(FileReference InputFile, CompileEnvironment Environment, FileReference PreludeFile, DirectoryReference WorkingDir, DirectoryReference CompilerDir, DirectoryReference CompilerNormalizedDir, DirectoryReference ToolDir, LineBasedTextWriter Log)
{
BufferedTextWriter BufferedLog = new BufferedTextWriter();
// Run the preprocessor on it
Preprocessor ToolPreprocessor = new Preprocessor(PreludeFile, Environment.Definitions, Environment.IncludePaths);
FileReference ToolOutputFile = FileReference.Combine(ToolDir, InputFile.GetFileNameWithoutExtension() + ".i");
ToolPreprocessor.PreprocessFile(InputFile.FullName, ToolOutputFile, Environment.ForceIncludeFiles);
// Generate the compiler output
FileReference CompilerOutputFile = FileReference.Combine(CompilerDir, InputFile.GetFileNameWithoutExtension() + ".i");
string CommandLine = String.Format("{0} {1} /P /C /utf-8 /Fi{2}", Environment.GetCommandLine(), Utility.QuoteArgument(InputFile.FullName), Utility.QuoteArgument(CompilerOutputFile.FullName));
Utility.Run(Environment.Compiler, CommandLine, WorkingDir, BufferedLog);
// Normalize the compiler output
FileReference NormalizedCompilerOutputFile = FileReference.Combine(CompilerNormalizedDir, InputFile.GetFileNameWithoutExtension() + ".i");
NormalizePreprocessedFile(ToolOutputFile, CompilerOutputFile, NormalizedCompilerOutputFile, BufferedLog);
// Copy everything to the output log
lock (Log)
{
BufferedLog.CopyTo(Log, " ");
}
}
/// <summary>
/// Takes an output file generated by this tool, and removes any formatting differences from a compiler-generated preprocessed file.
/// </summary>
/// <param name="ToolFile">File preprocessed by this tool</param>
/// <param name="CompilerFile">Path to the preprocessed file generated by the compiler</param>
/// <param name="NormalizedCompilerFile">Output file for the normalized compiler file</param>
/// <param name="Log"></param>
static int NormalizePreprocessedFile(FileReference ToolFile, FileReference CompilerFile, FileReference NormalizedCompilerFile, LineBasedTextWriter Log)
{
// Read the tool file
string[] ToolLines = File.ReadAllLines(ToolFile.FullName);
// Read the compiler file
string[] CompilerLines = File.ReadAllLines(CompilerFile.FullName);
// Write the output file
int NumDifferences = 0;
using (StreamWriter Writer = new StreamWriter(NormalizedCompilerFile.FullName))
{
int ToolIdx = 0;
int CompilerIdx = 0;
while (CompilerIdx < CompilerLines.Length && ToolIdx < ToolLines.Length)
{
string ToolLine = ToolLines[ToolIdx];
string CompilerLine = CompilerLines[CompilerIdx];
// Weird inconsistency with MSVC output where it sometimes adds an extra newline at the end of the file, even if one already exists. Skip over these lines in the compiler output.
if(CompilerLine.Length == 0 && ToolLine.Length > 0 && (CompilerIdx + 1 == CompilerLines.Length || ComparePreprocessedLines(CompilerLines[CompilerIdx + 1], ToolLine)))
{
CompilerIdx++;
continue;
}
// If the lines are semantically equivalent, just ignore the whitespace differences and take the tool line
if (ComparePreprocessedLines(CompilerLine, ToolLine))
{
Writer.WriteLine(ToolLine);
ToolIdx++;
CompilerIdx++;
continue;
}
// Output the difference
if(NumDifferences < 5)
{
Log.WriteLine(" Compiler ({0}): {1}", CompilerIdx, CompilerLine);
Log.WriteLine(" Tool ({0}): {1}", ToolIdx, ToolLine);
Log.WriteLine();
NumDifferences++;
}
// Otherwise try to find a matching line
int PrevCompilerIdx = CompilerIdx;
if(!FindMatchingLines(ToolLines, ref ToolIdx, CompilerLines, ref CompilerIdx))
{
ToolIdx++;
CompilerIdx++;
}
while(PrevCompilerIdx < CompilerIdx)
{
Writer.WriteLine(CompilerLines[PrevCompilerIdx]);
PrevCompilerIdx++;
}
}
while(CompilerIdx < CompilerLines.Length && (CompilerIdx < CompilerLines.Length - 1 || CompilerLines[CompilerIdx].Length > 0))
{
Writer.WriteLine(CompilerLines[CompilerIdx]);
CompilerIdx++;
}
}
return NumDifferences;
}
/// <summary>
/// Update indices into two line lists until we find two matching lines
/// </summary>
/// <param name="ToolLines">Array of lines from tool output</param>
/// <param name="ToolIdx">Index into tool lines</param>
/// <param name="CompilerLines">Array of lines from compiler output</param>
/// <param name="CompilerIdx">Index into compiler lines</param>
/// <returns>True if a matching line was found</returns>
static bool FindMatchingLines(string[] ToolLines, ref int ToolIdx, string[] CompilerLines, ref int CompilerIdx)
{
for(int Offset = 1; ToolIdx + Offset < ToolLines.Length || CompilerIdx + Offset < CompilerLines.Length; Offset++)
{
// Try to match the tool line at this offset with a compiler line at any equal or smaller offset
if(ToolIdx + Offset < ToolLines.Length)
{
string ToolLine = ToolLines[ToolIdx + Offset];
for(int OtherCompilerIdx = CompilerIdx; OtherCompilerIdx <= CompilerIdx + Offset && OtherCompilerIdx < CompilerLines.Length; OtherCompilerIdx++)
{
if(ComparePreprocessedLines(ToolLine, CompilerLines[OtherCompilerIdx]))
{
ToolIdx = ToolIdx + Offset;
CompilerIdx = OtherCompilerIdx;
return true;
}
}
}
// Try to match the compiler line at this offset with a tool line at any equal or smaller offset
if (CompilerIdx + Offset < CompilerLines.Length)
{
string CompilerLine = CompilerLines[CompilerIdx + Offset];
for(int OtherToolIdx = ToolIdx; OtherToolIdx <= ToolIdx + Offset && OtherToolIdx < ToolLines.Length; OtherToolIdx++)
{
if(ComparePreprocessedLines(ToolLines[OtherToolIdx], CompilerLine))
{
ToolIdx = OtherToolIdx;
CompilerIdx = CompilerIdx + Offset;
return true;
}
}
}
}
return false;
}
/// <summary>
/// Compares two preprocessed source file lines, and returns true if they are the same. Ignores leading, trailing, and consecutive whitespace.
/// </summary>
/// <param name="First">First string to compare</param>
/// <param name="Second">Second string to compare</param>
/// <returns>True if the two lines are equivalent</returns>
static bool ComparePreprocessedLines(string First, string Second)
{
// Skip any leading whitespace
int FirstIdx = SkipWhitespace(First, 0);
int SecondIdx = SkipWhitespace(Second, 0);
// Scan to the end of the line
while(FirstIdx < First.Length && SecondIdx < Second.Length)
{
if (IsWhitespace(First[FirstIdx]) && IsWhitespace(Second[SecondIdx]))
{
FirstIdx = SkipWhitespace(First, FirstIdx);
SecondIdx = SkipWhitespace(Second, SecondIdx);
}
else if (First[FirstIdx] == Second[SecondIdx])
{
FirstIdx++;
SecondIdx++;
}
else
{
return false;
}
}
// Skip any trailing whitespace
FirstIdx = SkipWhitespace(First, FirstIdx);
SecondIdx = SkipWhitespace(Second, SecondIdx);
return FirstIdx == First.Length && SecondIdx == Second.Length;
}
/// <summary>
/// Checks whether the given character is whitespace
/// </summary>
/// <param name="Character">The character to check</param>
/// <returns>True if the character is whitespace</returns>
static bool IsWhitespace(char Character)
{
return Character == ' ' || Character == '\t' || Character == '\f';
}
/// <summary>
/// Skips past a sequence of whitespace characters
/// </summary>
/// <param name="Line">The line to check</param>
/// <param name="Idx">Current character index within the line</param>
/// <returns>Index after the run of whitespace characters</returns>
static int SkipWhitespace(string Line, int Idx)
{
while(Idx < Line.Length && IsWhitespace(Line[Idx]))
{
Idx++;
}
return Idx;
}
}
}
Reference in New Issue View Git Blame Copy Permalink