Files
acceptance-tests
data
debian
docs
external
Newtonsoft.Json
api-doc-tools
api-snapshot
aspnetwebstack
bdwgc
binary-reference-assemblies
bockbuild
boringssl
cecil
cecil-legacy
corefx
corert
helix-binaries
ikdasm
ikvm
illinker-test-assets
linker
llvm-project
clang
clang-tools-extra
compiler-rt
libcxx
libcxxabi
libunwind
lld
lldb
llvm
bindings
cmake
docs
examples
include
lib
Analysis
AsmParser
BinaryFormat
Bitcode
CodeGen
DebugInfo
Demangle
ExecutionEngine
FuzzMutate
Fuzzer
IR
IRReader
LTO
LineEditor
Linker
MC
Object
ObjectYAML
Option
Passes
ProfileData
Support
Unix
Windows
AMDGPUMetadata.cpp
APFloat.cpp.REMOVED.git-id
APInt.cpp
APSInt.cpp
ARMAttributeParser.cpp
ARMBuildAttrs.cpp
ARMWinEH.cpp
Allocator.cpp
Atomic.cpp
BinaryStreamError.cpp
BinaryStreamReader.cpp
BinaryStreamRef.cpp
BinaryStreamWriter.cpp
BlockFrequency.cpp
BranchProbability.cpp
CMakeLists.txt
COM.cpp
COPYRIGHT.regex
CachePruning.cpp
Chrono.cpp
CodeGenCoverage.cpp
CommandLine.cpp
Compression.cpp
ConvertUTF.cpp
ConvertUTFWrapper.cpp
CrashRecoveryContext.cpp
DAGDeltaAlgorithm.cpp
DataExtractor.cpp
Debug.cpp
DebugCounter.cpp
DeltaAlgorithm.cpp
DynamicLibrary.cpp
Errno.cpp
Error.cpp
ErrorHandling.cpp
FileOutputBuffer.cpp
FileUtilities.cpp
FoldingSet.cpp
FormatVariadic.cpp
FormattedStream.cpp
GlobPattern.cpp
GraphWriter.cpp
Hashing.cpp
Host.cpp
IntEqClasses.cpp
IntervalMap.cpp
JamCRC.cpp
KnownBits.cpp
LEB128.cpp
LLVMBuild.txt
LineIterator.cpp
Locale.cpp
LockFileManager.cpp
LowLevelType.cpp
MD5.cpp
ManagedStatic.cpp
MathExtras.cpp
Memory.cpp
MemoryBuffer.cpp
Mutex.cpp
NativeFormatting.cpp
Options.cpp
Parallel.cpp
Path.cpp
PluginLoader.cpp
PrettyStackTrace.cpp
Process.cpp
Program.cpp
README.txt.system
RWMutex.cpp
RandomNumberGenerator.cpp
Regex.cpp
SHA1.cpp
ScaledNumber.cpp
ScopedPrinter.cpp
Signals.cpp
SmallPtrSet.cpp
SmallVector.cpp
SourceMgr.cpp
SpecialCaseList.cpp
Statistic.cpp
StringExtras.cpp
StringMap.cpp
StringPool.cpp
StringRef.cpp
StringSaver.cpp
SystemUtils.cpp
TarWriter.cpp
TargetParser.cpp
TargetRegistry.cpp
ThreadLocal.cpp
ThreadPool.cpp
Threading.cpp
Timer.cpp
ToolOutputFile.cpp
TrigramIndex.cpp
Triple.cpp
Twine.cpp
Unicode.cpp
Valgrind.cpp
Watchdog.cpp
YAMLParser.cpp
YAMLTraits.cpp
circular_raw_ostream.cpp
raw_os_ostream.cpp
raw_ostream.cpp
regcomp.c
regengine.inc
regerror.c
regex2.h
regex_impl.h
regexec.c
regfree.c
regstrlcpy.c
regutils.h
xxhash.cpp
TableGen
Target
Testing
ToolDrivers
Transforms
WindowsManifest
XRay
CMakeLists.txt
LLVMBuild.txt
projects
resources
runtimes
scripts
test
tools
unittests
utils
.arcconfig
.clang-format
.clang-tidy
.gitattributes
.gitignore
CMakeLists.txt
CODE_OWNERS.TXT
CREDITS.TXT
LICENSE.TXT
LLVMBuild.txt
README.txt
RELEASE_TESTERS.TXT
configure
llvm.spec.in
openmp
polly
nuget-buildtasks
nunit-lite
roslyn-binaries
rx
xunit-binaries
how-to-bump-roslyn-binaries.md
ikvm-native
llvm
m4
man
mcs
mono
msvc
netcore
po
runtime
samples
scripts
support
tools
COPYING.LIB
LICENSE
Makefile.am
Makefile.in
NEWS
README.md
acinclude.m4
aclocal.m4
autogen.sh
code_of_conduct.md
compile
config.guess
config.h.in
config.rpath
config.sub
configure.REMOVED.git-id
configure.ac.REMOVED.git-id
depcomp
install-sh
ltmain.sh.REMOVED.git-id
missing
mkinstalldirs
mono-uninstalled.pc.in
test-driver
winconfig.h
linux-packaging-mono/external/llvm-project/llvm/lib/Support/CachePruning.cpp
Xamarin Public Jenkins (auto-signing) 468663ddbb Imported Upstream version 6.10.0.49
Former-commit-id: 1d6753294b2993e1fbf92de9366bb9544db4189b
2020-01-16 16:38:04 +00:00

276 lines
9.8 KiB
C++

//===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements the pruning of a directory based on least recently used.
//
//===----------------------------------------------------------------------===//
#include "llvm/Support/CachePruning.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
#define DEBUG_TYPE "cache-pruning"
#include <set>
#include <system_error>
using namespace llvm;
/// Write a new timestamp file with the given path. This is used for the pruning
/// interval option.
static void writeTimestampFile(StringRef TimestampFile) {
std::error_code EC;
raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
}
static Expected<std::chrono::seconds> parseDuration(StringRef Duration) {
if (Duration.empty())
return make_error<StringError>("Duration must not be empty",
inconvertibleErrorCode());
StringRef NumStr = Duration.slice(0, Duration.size()-1);
uint64_t Num;
if (NumStr.getAsInteger(0, Num))
return make_error<StringError>("'" + NumStr + "' not an integer",
inconvertibleErrorCode());
switch (Duration.back()) {
case 's':
return std::chrono::seconds(Num);
case 'm':
return std::chrono::minutes(Num);
case 'h':
return std::chrono::hours(Num);
default:
return make_error<StringError>("'" + Duration +
"' must end with one of 's', 'm' or 'h'",
inconvertibleErrorCode());
}
}
Expected<CachePruningPolicy>
llvm::parseCachePruningPolicy(StringRef PolicyStr) {
CachePruningPolicy Policy;
std::pair<StringRef, StringRef> P = {"", PolicyStr};
while (!P.second.empty()) {
P = P.second.split(':');
StringRef Key, Value;
std::tie(Key, Value) = P.first.split('=');
if (Key == "prune_interval") {
auto DurationOrErr = parseDuration(Value);
if (!DurationOrErr)
return DurationOrErr.takeError();
Policy.Interval = *DurationOrErr;
} else if (Key == "prune_after") {
auto DurationOrErr = parseDuration(Value);
if (!DurationOrErr)
return DurationOrErr.takeError();
Policy.Expiration = *DurationOrErr;
} else if (Key == "cache_size") {
if (Value.back() != '%')
return make_error<StringError>("'" + Value + "' must be a percentage",
inconvertibleErrorCode());
StringRef SizeStr = Value.drop_back();
uint64_t Size;
if (SizeStr.getAsInteger(0, Size))
return make_error<StringError>("'" + SizeStr + "' not an integer",
inconvertibleErrorCode());
if (Size > 100)
return make_error<StringError>("'" + SizeStr +
"' must be between 0 and 100",
inconvertibleErrorCode());
Policy.MaxSizePercentageOfAvailableSpace = Size;
} else if (Key == "cache_size_bytes") {
uint64_t Mult = 1;
switch (tolower(Value.back())) {
case 'k':
Mult = 1024;
Value = Value.drop_back();
break;
case 'm':
Mult = 1024 * 1024;
Value = Value.drop_back();
break;
case 'g':
Mult = 1024 * 1024 * 1024;
Value = Value.drop_back();
break;
}
uint64_t Size;
if (Value.getAsInteger(0, Size))
return make_error<StringError>("'" + Value + "' not an integer",
inconvertibleErrorCode());
Policy.MaxSizeBytes = Size * Mult;
} else if (Key == "cache_size_files") {
if (Value.getAsInteger(0, Policy.MaxSizeFiles))
return make_error<StringError>("'" + Value + "' not an integer",
inconvertibleErrorCode());
} else {
return make_error<StringError>("Unknown key: '" + Key + "'",
inconvertibleErrorCode());
}
}
return Policy;
}
/// Prune the cache of files that haven't been accessed in a long time.
bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
using namespace std::chrono;
if (Path.empty())
return false;
bool isPathDir;
if (sys::fs::is_directory(Path, isPathDir))
return false;
if (!isPathDir)
return false;
Policy.MaxSizePercentageOfAvailableSpace =
std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
if (Policy.Expiration == seconds(0) &&
Policy.MaxSizePercentageOfAvailableSpace == 0 &&
Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
DEBUG(dbgs() << "No pruning settings set, exit early\n");
// Nothing will be pruned, early exit
return false;
}
// Try to stat() the timestamp file.
SmallString<128> TimestampFile(Path);
sys::path::append(TimestampFile, "llvmcache.timestamp");
sys::fs::file_status FileStatus;
const auto CurrentTime = system_clock::now();
if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
if (EC == errc::no_such_file_or_directory) {
// If the timestamp file wasn't there, create one now.
writeTimestampFile(TimestampFile);
} else {
// Unknown error?
return false;
}
} else {
if (!Policy.Interval)
return false;
if (Policy.Interval != seconds(0)) {
// Check whether the time stamp is older than our pruning interval.
// If not, do nothing.
const auto TimeStampModTime = FileStatus.getLastModificationTime();
auto TimeStampAge = CurrentTime - TimeStampModTime;
if (TimeStampAge <= *Policy.Interval) {
DEBUG(dbgs() << "Timestamp file too recent ("
<< duration_cast<seconds>(TimeStampAge).count()
<< "s old), do not prune.\n");
return false;
}
}
// Write a new timestamp file so that nobody else attempts to prune.
// There is a benign race condition here, if two processes happen to
// notice at the same time that the timestamp is out-of-date.
writeTimestampFile(TimestampFile);
}
// Keep track of space. Needs to be kept ordered by size for determinism.
std::set<std::pair<uint64_t, std::string>> FileSizes;
uint64_t TotalSize = 0;
// Walk the entire directory cache, looking for unused files.
std::error_code EC;
SmallString<128> CachePathNative;
sys::path::native(Path, CachePathNative);
// Walk all of the files within this directory.
for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
File != FileEnd && !EC; File.increment(EC)) {
// Ignore any files not beginning with the string "llvmcache-". This
// includes the timestamp file as well as any files created by the user.
// This acts as a safeguard against data loss if the user specifies the
// wrong directory as their cache directory.
if (!sys::path::filename(File->path()).startswith("llvmcache-"))
continue;
// Look at this file. If we can't stat it, there's nothing interesting
// there.
ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
if (!StatusOrErr) {
DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
continue;
}
// If the file hasn't been used recently enough, delete it
const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
auto FileAge = CurrentTime - FileAccessTime;
if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
DEBUG(dbgs() << "Remove " << File->path() << " ("
<< duration_cast<seconds>(FileAge).count() << "s old)\n");
sys::fs::remove(File->path());
continue;
}
// Leave it here for now, but add it to the list of size-based pruning.
TotalSize += StatusOrErr->getSize();
FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())});
}
auto FileAndSize = FileSizes.rbegin();
size_t NumFiles = FileSizes.size();
auto RemoveCacheFile = [&]() {
// Remove the file.
sys::fs::remove(FileAndSize->second);
// Update size
TotalSize -= FileAndSize->first;
NumFiles--;
DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
<< FileAndSize->first << "), new occupancy is " << TotalSize
<< "%\n");
++FileAndSize;
};
// Prune for number of files.
if (Policy.MaxSizeFiles)
while (NumFiles > Policy.MaxSizeFiles)
RemoveCacheFile();
// Prune for size now if needed
if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
if (!ErrOrSpaceInfo) {
report_fatal_error("Can't get available size");
}
sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
auto AvailableSpace = TotalSize + SpaceInfo.free;
if (Policy.MaxSizePercentageOfAvailableSpace == 0)
Policy.MaxSizePercentageOfAvailableSpace = 100;
if (Policy.MaxSizeBytes == 0)
Policy.MaxSizeBytes = AvailableSpace;
auto TotalSizeTarget = std::min<uint64_t>(
AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
Policy.MaxSizeBytes);
DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
<< "% target is: " << Policy.MaxSizePercentageOfAvailableSpace
<< "%, " << Policy.MaxSizeBytes << " bytes\n");
// Remove the oldest accessed files first, till we get below the threshold.
while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend())
RemoveCacheFile();
}
return true;
}