Merge from //UE4/Main@4203495

#rb no.one

[CL 4208755 by Mike Beach in Dev-VR branch]
This commit is contained in:
Mike Beach
2018-07-12 19:05:28 -04:00
parent 3721f28384
commit eacc4047d8
395 changed files with 8201 additions and 3988 deletions

View File

@@ -1,10 +1,11 @@
// Copyright 1998-2018 Epic Games, Inc. All Rights Reserved.
#include "PyOnlineDocsWriter.h"
#include "PyUtil.h"
#include "PythonScriptPlugin.h"
#include "PyGenUtil.h"
#include "HAL/FileManager.h"
#include "Logging/MessageLog.h"
#include "Misc/CommandLine.h"
#include "Misc/FileHelper.h"
#include "Misc/Paths.h"
#include "Runtime/Launch/Resources/Version.h"
@@ -72,6 +73,11 @@ FString FPyOnlineDocsWriter::GetSourcePath() const
return GetSphinxDocsPath() / TEXT("source");
}
FString FPyOnlineDocsWriter::GetBuildPath() const
{
return GetSphinxDocsPath() / TEXT("build");
}
FString FPyOnlineDocsWriter::GetTemplatePath() const
{
return GetSourcePath() / TEXT("_templates");
@@ -123,13 +129,16 @@ void FPyOnlineDocsWriter::GenerateIndexFile()
Section->TypeNames.StableSort();
}
FString SectionList;
FString TableOfContents;
// Accumulate all the modules into the table of contents
if (Modules.Num() > 0)
{
SectionList += TEXT("* :ref:`Modules`") LINE_TERMINATOR;
TableOfContents +=
LINE_TERMINATOR
TEXT(".. _Modules:") LINE_TERMINATOR
TEXT(".. toctree::") LINE_TERMINATOR
TEXT(" :maxdepth: 1") LINE_TERMINATOR
TEXT(" :caption: Modules") LINE_TERMINATOR LINE_TERMINATOR;
@@ -143,8 +152,19 @@ void FPyOnlineDocsWriter::GenerateIndexFile()
// Accumulate all the classes for each section into the table of contents
for (const TSharedRef<FPyOnlineDocsSection>& Section : Sections)
{
FString SectionRef(Section->Name);
SectionRef.ReplaceInline(TEXT(" "), TEXT("-"));
SectionList += TEXT("* :ref:`");
SectionList += SectionRef;
SectionList += TEXT("`") LINE_TERMINATOR;
TableOfContents +=
LINE_TERMINATOR
TEXT(".. _");
TableOfContents += SectionRef;
TableOfContents +=
TEXT(":") LINE_TERMINATOR
TEXT(".. toctree::") LINE_TERMINATOR
TEXT(" :maxdepth: 1") LINE_TERMINATOR
TEXT(" :caption: ");
@@ -157,8 +177,11 @@ void FPyOnlineDocsWriter::GenerateIndexFile()
}
}
// Replace {{TableOfContents}} with actual list
// Replace {{SectionList}} with actual list
FString IndexText = IndexTemplate;
IndexText.ReplaceInline(TEXT("{{SectionList}}"), *SectionList, ESearchCase::CaseSensitive);
// Replace {{TableOfContents}} with actual list
IndexText.ReplaceInline(TEXT("{{TableOfContents}}"), *TableOfContents, ESearchCase::CaseSensitive);
// Save out index file
@@ -258,7 +281,7 @@ void FPyOnlineDocsWriter::GenerateClassFiles()
void FPyOnlineDocsWriter::GenerateFiles(const FString& InPythonStubPath)
{
UE_LOG(LogPython, Log, TEXT("Generating Python API online docs used by Sphinx to generate static HTML..."));
UE_LOG(LogPython, Display, TEXT("Generating Python API online docs used by Sphinx to generate static HTML..."));
// Copy generated unreal module stub file to PythonScriptPlugin/SphinxDocs/modules
{
@@ -283,10 +306,112 @@ void FPyOnlineDocsWriter::GenerateFiles(const FString& InPythonStubPath)
GenerateModuleFiles();
GenerateClassFiles();
UE_LOG(LogPython, Log, TEXT(
"... Finished generating Python API online docs.\n"
"In the OS command prompt in PythonPlugin/SphinxDocs call `sphinx-build -b html source/ build/` to generate the HTML."
));
UE_LOG(LogPython, Display, TEXT(
" ... finished generating Sphinx files."));
FString Commandline = FCommandLine::Get();
bool bUseSphinx = Commandline.Contains(TEXT("-NoHTML")) == false;
if (bUseSphinx)
{
// Call Sphinx to generate online Python API docs. (Default)
// Running as internal Python calls on the version embedded in UE4 rather than as an
// executed external process since other installs, paths and environment variables may act
// in unexpected ways. Could potentially use Python C++ API calls rather than Python scripts,
// though this keeps it clear and if the calls evolve over time the vast number of examples
// online are in Python rather than C++.
// Update pip and then install Sphinx if needed. If Sphinx and its dependencies are already
// installed then it will determine that quickly and move on to using it.
// More info on using pip within Python here:
// https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
FString PyCommandStr;
FString PythonPath = FPaths::ConvertRelativePathToFull(FPaths::EngineSourceDir()) / TEXT("ThirdParty/Python/Win64/python.exe");
PyCommandStr += TEXT(
"import sys\n"
"import subprocess\n"
"subprocess.check_call(['");
PyCommandStr += PythonPath;
PyCommandStr += TEXT(
"', '-m', 'pip', 'install', '-q', '-U', 'pip'])\n"
"subprocess.check_call(['");
PyCommandStr += PythonPath;
PyCommandStr += TEXT(
"', '-m', 'pip', 'install', '-q', '--no-warn-script-location', 'sphinx'])\n"
"import sphinx\n");
// Alternate technique calling pip as a Python command though above is recommended by pip.
//
//FString PyCommandStr = TEXT(
// "import pip\n"
// "pip.main(['install', 'sphinx'])\n"
// "import sphinx\n");
// Un-import full unreal module so Sphinx will use generated stub version of unreal module
PyCommandStr += TEXT(
"del unreal\n"
"del sys.modules['unreal']\n");
// Add on Sphinx build command
PyCommandStr += TEXT("sphinx.build_main(['sphinx-build', '-b', 'html', '");
PyCommandStr += GetSourcePath();
PyCommandStr += TEXT("', '");
PyCommandStr += GetBuildPath();
PyCommandStr += TEXT("'])");
UE_LOG(LogPython, Display, TEXT(
"Calling Sphinx in PythonPlugin/SphinxDocs to generate the HTML...\n\n"
"%s\n\n"
"This can take a long time - 16+ minutes for full build on test system...\n"),
*PyCommandStr);
bool bLogSphinx = Commandline.Contains(TEXT("-HTMLLog"));
ELogVerbosity::Type OldVerbosity = LogPython.GetVerbosity();
if (!bLogSphinx)
{
// Disable Python logging (default)
LogPython.SetVerbosity(ELogVerbosity::NoLogging);
}
// Run the Python commands
bool PyRunSuccess = FPythonScriptPlugin::Get()->RunString(*PyCommandStr);
if (!bLogSphinx)
{
// Re-enable Python logging
LogPython.SetVerbosity(OldVerbosity);
}
// The running of the Python commands seem to think there are errors no matter what so not much use to make this notification.
//if (PyRunSuccess)
//{
// UE_LOG(LogPython, Display, TEXT(
// "\n"
// " There was an internal Python error while generating the docs, though it may not be an issue.\n\n"
// " You could call the commandlet again with the '-HTMLLog' option to see more information.\n"));
//}
UE_LOG(LogPython, Display, TEXT(
" ... finished generating Python API online docs!\n\n"
"Find them in the following directory:\n"
" %s\n\n"
"See additional instructions and information in:\n"
" PythonScriptPlugin/SphinxDocs/PythonAPI_docs_readme.txt\n"),
*GetBuildPath());
}
else
{
// Prompt to manually call Sphinx to generate online Python API docs.
UE_LOG(LogPython, Display, TEXT(
"To build the Python API online docs manually follow the instructions in:\n"
" PythonScriptPlugin/SphinxDocs/PythonAPI_docs_readme.txt\n\n"
"And then call:"
" PythonScriptPlugin/SphinxDocs/sphinx-build -b html source/ build/"));
}
}
#endif // WITH_PYTHON

View File

@@ -3,9 +3,24 @@
#pragma once
#include "CoreMinimal.h"
#include "Misc/EnumClassFlags.h"
#if WITH_PYTHON
/**
* Flags controlling which data is included in the Python API online docs.
*/
enum class EPyOnlineDocsFilterFlags : uint8
{
IncludeNone = 0,
IncludeEngine = 1<<0,
IncludeEnterprise = 1<<1,
IncludeInternal = 1<<2,
IncludeProject = 1<<3,
IncludeAll = IncludeEngine | IncludeEnterprise | IncludeInternal | IncludeProject,
};
ENUM_CLASS_FLAGS(EPyOnlineDocsFilterFlags);
/**
* A single module in the Python API online docs.
* Hosts a series of function names that belong to this module and will be used for indexing purposes later.
@@ -48,6 +63,9 @@ public:
/** Store class name in this section to generate files later. */
void AccumulateClass(const TCHAR* InTypeName);
//** Get name of section. */
const FString& GetName() const { return Name; }
private:
/** Section name. */
FString Name;
@@ -76,6 +94,9 @@ public:
/** Get the directory for the Sphinx source files. */
FString GetSourcePath() const;
/** Get the directory for the Sphinx build files. */
FString GetBuildPath() const;
/** Get the directory for the Sphinx template files. */
FString GetTemplatePath() const;

View File

@@ -70,6 +70,12 @@ struct FPyWrapperBaseMetaData
/** Get the ID associated with this meta-data type */
virtual FGuid GetTypeId() const = 0;
/** Get the reflection meta data type object associated with this wrapper type if there is one or nullptr if not. */
virtual const UField* GetMetaType() const
{
return nullptr;
}
/** Add object references from the given Python object to the given collector */
virtual void AddReferencedObjects(FPyWrapperBase* Instance, FReferenceCollector& Collector)
{

View File

@@ -111,6 +111,12 @@ struct TPyWrapperDelegateMetaData : public FPyWrapperBaseMetaData
Collector.AddReferencedObject(PythonCallableForDelegateClass);
}
/** Get the reflection meta data type object associated with this wrapper type if there is one or nullptr if not. */
virtual const UField* GetMetaType() const override
{
return DelegateSignature.Func;
}
/** Unreal function representing the signature for the delegate */
PyGenUtil::FGeneratedWrappedFunction DelegateSignature;

View File

@@ -85,6 +85,12 @@ struct FPyWrapperEnumMetaData : public FPyWrapperBaseMetaData
/** Check to see if the enum is finalized */
static bool IsEnumFinalized(FPyWrapperEnum* Instance);
/** Get the reflection meta data type object associated with this wrapper type if there is one or nullptr if not. */
virtual const UField* GetMetaType() const override
{
return Enum;
}
/** Unreal enum */
UEnum* Enum;

View File

@@ -149,6 +149,12 @@ struct FPyWrapperObjectMetaData : public FPyWrapperBaseMetaData
/** Add object references from the given Python object to the given collector */
virtual void AddReferencedObjects(FPyWrapperBase* Instance, FReferenceCollector& Collector) override;
/** Get the reflection meta data type object associated with this wrapper type if there is one or nullptr if not. */
virtual const UField* GetMetaType() const override
{
return Class;
}
/** Unreal class */
UClass* Class;

View File

@@ -331,6 +331,12 @@ struct FPyWrapperStructMetaData : public FPyWrapperBaseMetaData
/** Add object references from the given Python object to the given collector */
virtual void AddReferencedObjects(FPyWrapperBase* Instance, FReferenceCollector& Collector) override;
/** Get the reflection meta data type object associated with this wrapper type if there is one or nullptr if not. */
virtual const UField* GetMetaType() const override
{
return Struct;
}
/** Unreal struct */
UScriptStruct* Struct;

View File

@@ -16,11 +16,11 @@
#include "PyGIL.h"
#include "PyCore.h"
#include "PyFileWriter.h"
#include "PyOnlineDocsWriter.h"
#include "PythonScriptPluginSettings.h"
#include "ProfilingDebugging/ScopedTimers.h"
#include "Misc/Paths.h"
#include "Misc/FileHelper.h"
#include "SourceCodeNavigation.h"
#include "UObject/UnrealType.h"
#include "UObject/UObjectHash.h"
#include "UObject/Class.h"
@@ -1934,8 +1934,10 @@ void FPyWrapperTypeRegistry::GatherWrappedTypesForPropertyReferences(const UProp
}
}
void FPyWrapperTypeRegistry::GenerateStubCodeForWrappedTypes() const
void FPyWrapperTypeRegistry::GenerateStubCodeForWrappedTypes(const EPyOnlineDocsFilterFlags InDocGenFlags) const
{
UE_LOG(LogPython, Display, TEXT("Generating Python API stub file..."));
FPyFileWriter PythonScript;
TUniquePtr<FPyOnlineDocsWriter> OnlineDocsWriter;
@@ -1946,15 +1948,15 @@ void FPyWrapperTypeRegistry::GenerateStubCodeForWrappedTypes() const
TSharedPtr<FPyOnlineDocsSection> OnlineDocsStructTypesSection;
TSharedPtr<FPyOnlineDocsSection> OnlineDocsClassTypesSection;
if (GetDefault<UPythonScriptPluginSettings>()->bGenerateOnlineDocs)
if (EnumHasAnyFlags(InDocGenFlags, EPyOnlineDocsFilterFlags::IncludeAll))
{
OnlineDocsWriter = MakeUnique<FPyOnlineDocsWriter>();
OnlineDocsUnrealModule = OnlineDocsWriter->CreateModule(TEXT("unreal"));
OnlineDocsNativeTypesSection = OnlineDocsWriter->CreateSection(TEXT("Native Types"));
OnlineDocsEnumTypesSection = OnlineDocsWriter->CreateSection(TEXT("Enum Types"));
OnlineDocsDelegateTypesSection = OnlineDocsWriter->CreateSection(TEXT("Delegate Types"));
OnlineDocsStructTypesSection = OnlineDocsWriter->CreateSection(TEXT("Struct Types"));
OnlineDocsClassTypesSection = OnlineDocsWriter->CreateSection(TEXT("Class Types"));
OnlineDocsEnumTypesSection = OnlineDocsWriter->CreateSection(TEXT("Enum Types"));
OnlineDocsDelegateTypesSection = OnlineDocsWriter->CreateSection(TEXT("Delegate Types"));
}
// Process additional Python files
@@ -2047,6 +2049,10 @@ void FPyWrapperTypeRegistry::GenerateStubCodeForWrappedTypes() const
}
// Process native glue code
UE_LOG(LogPython, Display, TEXT(" ...generating Python API: glue code"));
PythonScript.WriteLine(TEXT("##### Glue Code #####"));
PythonScript.WriteNewLine();
for (const PyGenUtil::FNativePythonModule& NativePythonModule : NativePythonModules)
{
for (PyMethodDef* MethodDef = NativePythonModule.PyModuleMethods; MethodDef && MethodDef->ml_name; ++MethodDef)
@@ -2072,11 +2078,83 @@ void FPyWrapperTypeRegistry::GenerateStubCodeForWrappedTypes() const
}
// Process generated glue code
auto ProcessWrappedDataArray = [this, &PythonScript](const TMap<FName, PyTypeObject*>& WrappedData, const TSharedPtr<FPyOnlineDocsSection>& OnlineDocsSection)
// Also excludes types that don't pass the filters specified in InDocGenFlags using the information about
// which module it came from and where that module exists on disk.
auto ProcessWrappedDataArray = [this, &PythonScript, InDocGenFlags](const TMap<FName, PyTypeObject*>& WrappedData, const TSharedPtr<FPyOnlineDocsSection>& OnlineDocsSection)
{
if (InDocGenFlags == EPyOnlineDocsFilterFlags::IncludeNone)
{
return;
}
UE_LOG(LogPython, Display, TEXT(" ...generating Python API: %s"), *OnlineDocsSection->GetName());
PythonScript.WriteLine(FString::Printf(TEXT("##### %s #####"), *OnlineDocsSection->GetName()));
PythonScript.WriteNewLine();
FString ProjectTopDir;
if (FPaths::IsProjectFilePathSet())
{
ProjectTopDir / FPaths::GetCleanFilename(FPaths::ProjectDir());
}
for (const auto& WrappedDataPair : WrappedData)
{
TSharedPtr<PyGenUtil::FGeneratedWrappedType> GeneratedWrappedType = GeneratedWrappedTypes.FindRef(WrappedDataPair.Key);
if ((InDocGenFlags != EPyOnlineDocsFilterFlags::IncludeAll) && GeneratedWrappedType.IsValid())
{
const UField* MetaType = GeneratedWrappedType->MetaData->GetMetaType();
FString ModulePath;
if (MetaType)
{
FSourceCodeNavigation::FindModulePath(MetaType->GetTypedOuter<UPackage>(), ModulePath);
}
if (!ModulePath.IsEmpty())
{
// Is Project class?
if (!ProjectTopDir.IsEmpty()
&& (ModulePath.Contains(ProjectTopDir)))
{
// Optionally exclude Project classes
if (!EnumHasAnyFlags(InDocGenFlags, EPyOnlineDocsFilterFlags::IncludeProject))
{
continue;
}
}
// Is Enterprise class
else if (ModulePath.Contains(TEXT("/Enterprise/")))
{
// Optionally exclude Enterprise classes
if (!EnumHasAnyFlags(InDocGenFlags, EPyOnlineDocsFilterFlags::IncludeEnterprise))
{
continue;
}
}
// is internal class
else if (FPaths::IsRestrictedPath(ModulePath))
{
// Optionally exclude internal classes
if (!EnumHasAnyFlags(InDocGenFlags, EPyOnlineDocsFilterFlags::IncludeInternal))
{
continue;
}
}
// Everything else is considered an "Engine" class
else
{
// Optionally exclude engine classes
if (!EnumHasAnyFlags(InDocGenFlags, EPyOnlineDocsFilterFlags::IncludeEngine))
{
continue;
}
}
}
// else if cannot determine origin then include
}
GenerateStubCodeForWrappedType(WrappedDataPair.Value, GeneratedWrappedType.Get(), PythonScript, OnlineDocsSection.Get());
}
};
@@ -2087,13 +2165,18 @@ void FPyWrapperTypeRegistry::GenerateStubCodeForWrappedTypes() const
ProcessWrappedDataArray(PythonWrappedClasses, OnlineDocsClassTypesSection);
// Append any additional Python code now that all the reflected API has been exported
UE_LOG(LogPython, Display, TEXT(" ...generating Python API: additional code"));
PythonScript.WriteLine(TEXT("##### Additional Code #####"));
PythonScript.WriteNewLine();
for (const FString& AdditionalPythonLine : AdditionalPythonCode)
{
PythonScript.WriteLine(AdditionalPythonLine);
}
const FString PythonSourceFilename = FPaths::ProjectIntermediateDir() / TEXT("PythonStub") / TEXT("unreal.py");
const FString PythonSourceFilename = FPaths::ConvertRelativePathToFull(FPaths::ProjectIntermediateDir()) / TEXT("PythonStub") / TEXT("unreal.py");
PythonScript.SaveFile(*PythonSourceFilename);
UE_LOG(LogPython, Display, TEXT(" ...generated: %s"), *PythonSourceFilename);
if (OnlineDocsWriter.IsValid())
{

View File

@@ -8,6 +8,7 @@
#include "PyUtil.h"
#include "PyGenUtil.h"
#include "PyConversionMethod.h"
#include "PyOnlineDocsWriter.h"
#include "UObject/WeakObjectPtr.h"
#include "Templates/Function.h"
@@ -30,7 +31,6 @@ class UPythonGeneratedStruct;
class IPyWrapperInlineStructFactory;
class FPyFileWriter;
class FPyOnlineDocsSection;
/** Type conversion for TPyWrapperTypeFactory */
template <typename UnrealType, typename KeyType>
@@ -428,7 +428,7 @@ public:
PyTypeObject* GetWrappedDelegateType(const UFunction* InDelegateSignature) const;
/** Generate stub Python code for our wrapped types */
void GenerateStubCodeForWrappedTypes() const;
void GenerateStubCodeForWrappedTypes(const EPyOnlineDocsFilterFlags InDocGenFlags = EPyOnlineDocsFilterFlags::IncludeNone) const;
private:
/** Gather any types referenced by the given property are still need to be wrapped for use in Python */

View File

@@ -0,0 +1,56 @@
// Copyright 1998-2018 Epic Games, Inc. All Rights Reserved.
#include "PythonOnlineDocsCommandlet.h"
#include "PyWrapperTypeRegistry.h"
DEFINE_LOG_CATEGORY_STATIC(LogPythonOnlineDocsCommandlet, Log, All);
UPythonOnlineDocsCommandlet::UPythonOnlineDocsCommandlet()
{
IsServer = true;
IsClient = true;
IsEditor = true;
LogToConsole = false;
ShowErrorCount = false;
}
int32 UPythonOnlineDocsCommandlet::Main(const FString& Params)
{
TArray<FString> Tokens;
TArray<FString> Switches;
TMap<FString, FString> ParamVals;
UCommandlet::ParseCommandLine(*Params, Tokens, Switches, ParamVals);
#if WITH_PYTHON
EPyOnlineDocsFilterFlags DocGenFlags = EPyOnlineDocsFilterFlags::IncludeNone;
// Apply the filter flags from the command line
#define APPLY_FILTER_FLAG(NAME) \
if (Switches.Contains(TEXT(#NAME))) \
{ \
DocGenFlags |= EPyOnlineDocsFilterFlags::NAME; \
}
APPLY_FILTER_FLAG(IncludeEngine)
APPLY_FILTER_FLAG(IncludeEnterprise)
APPLY_FILTER_FLAG(IncludeInternal)
APPLY_FILTER_FLAG(IncludeProject)
#undef APPLY_FILTER_FLAG
// If we weren't given any filter flags, include everything
if (DocGenFlags == EPyOnlineDocsFilterFlags::IncludeNone)
{
DocGenFlags = EPyOnlineDocsFilterFlags::IncludeAll;
}
UE_LOG(LogPythonOnlineDocsCommandlet, Display, TEXT("\n\nGenerating Python documentation..."));
FPyWrapperTypeRegistry::Get().GenerateStubCodeForWrappedTypes(DocGenFlags);
#else // WITH_PYTHON
UE_LOG(LogPythonOnlineDocsCommandlet, Error, TEXT("Python docs cannot be generated as the plugin was built as a stub!"));
return -1;
#endif // WITH_PYTHON
return 0;
}

View File

@@ -0,0 +1,28 @@
// Copyright 1998-2018 Epic Games, Inc. All Rights Reserved.
#pragma once
#include "Commandlets/Commandlet.h"
#include "PythonOnlineDocsCommandlet.generated.h"
/**
* Minimal commandlet to format and write Python API online docs.
* Can be passed the following flags to filter which types are included in the docs:
* -IncludeEngine
* -IncludeEnterprise
* -IncludeInternal
* -IncludeProject
*/
UCLASS()
class UPythonOnlineDocsCommandlet : public UCommandlet
{
GENERATED_BODY()
public:
/** Default constructor. */
UPythonOnlineDocsCommandlet();
//~ Begin UCommandlet Interface
virtual int32 Main(const FString& Params) override;
//~ End UCommandlet Interface
};

View File

@@ -34,11 +34,4 @@ public:
/** Should Developer Mode be enabled on the Python interpreter (will enable extra warnings (eg, for deprecated code), and enable stub code generation for use with external IDEs). */
UPROPERTY(config, EditAnywhere, Category=Python, meta=(ConfigRestartRequired=true))
bool bDeveloperMode;
/**
* Should files used for Unreal Python API doc generation be created?
* [Also needs Developer Mode above enabled.]
*/
UPROPERTY(config, EditAnywhere, Category = Python, meta = (ConfigRestartRequired = true))
bool bGenerateOnlineDocs;
};

View File

@@ -22,6 +22,7 @@ namespace UnrealBuildTool.Rules
"Python",
"Slate",
"SlateCore",
"InputCore",
}
);