2022-02-15 04:30:27 -05:00
// Copyright Epic Games, Inc. All Rights Reserved.
2023-08-29 19:18:18 -04:00
# include "UnsyncAuth.h"
2022-02-15 04:30:27 -05:00
# include "UnsyncCmdDiff.h"
# include "UnsyncCmdHash.h"
2023-08-29 19:18:18 -04:00
# include "UnsyncCmdLogin.h"
2023-06-21 15:01:35 -04:00
# include "UnsyncCmdMount.h"
2023-10-29 11:26:35 -04:00
# include "UnsyncCmdPack.h"
2022-02-15 04:30:27 -05:00
# include "UnsyncCmdPatch.h"
# include "UnsyncCmdPush.h"
2022-03-01 05:53:29 -05:00
# include "UnsyncCmdQuery.h"
2023-06-21 15:01:35 -04:00
# include "UnsyncCmdSync.h"
2024-06-03 18:26:41 -04:00
# include "UnsyncCmdInfo.h"
2022-02-15 04:30:27 -05:00
# include "UnsyncCore.h"
# include "UnsyncFile.h"
# include "UnsyncMemory.h"
# include "UnsyncProxy.h"
# include "UnsyncTest.h"
# include "UnsyncThread.h"
# include "UnsyncUtil.h"
2024-05-09 15:37:00 -04:00
# include "UnsyncScheduler.h"
2024-05-26 02:04:36 -04:00
# include "UnsyncVersion.h"
2024-05-26 13:37:37 -04:00
# include "UnsyncSource.h"
# include "UnsyncFilter.h"
2024-06-20 18:37:09 -04:00
# include "UnsyncHorde.h"
2022-02-15 04:30:27 -05:00
2022-02-24 06:20:58 -05:00
UNSYNC_THIRD_PARTY_INCLUDES_START
2022-02-15 04:30:27 -05:00
# if UNSYNC_PLATFORM_WINDOWS
# include <io.h>
2022-02-24 06:20:58 -05:00
# include <shellapi.h>
2022-02-15 04:30:27 -05:00
# endif // UNSYNC_PLATFORM_WINDOWS
# include <fcntl.h>
2023-06-21 15:01:35 -04:00
# include <CLI/CLI.hpp>
2022-02-15 04:30:27 -05:00
# include <filesystem>
# include <fstream>
# include <iostream>
# include <system_error>
2022-02-24 06:20:58 -05:00
UNSYNC_THIRD_PARTY_INCLUDES_END
2022-02-15 04:30:27 -05:00
namespace unsync {
2022-08-05 18:51:23 -04:00
static FPath GExePath ;
2022-02-15 04:30:27 -05:00
int
InnerMain ( int Argc , char * * Argv )
{
2023-10-31 16:34:04 -04:00
LogSaveCommandLineUtf8 ( Argc , Argv ) ;
2022-02-15 04:30:27 -05:00
2023-09-11 22:49:17 -04:00
std : : string AppDescription = " UNSYNC v " ;
2022-02-15 04:30:27 -05:00
AppDescription + = GetVersionString ( ) ;
AppDescription + =
" -- Differential binary synchronization tool. \n "
" Copyright Epic Games, Inc. All Rights Reserved. \n " ;
CLI : : App Cli ( AppDescription , " unsync " ) ;
2023-10-24 23:10:40 -04:00
Cli . allow_windows_style_options ( false ) ; // never allow /flag syntax, only --flag
2023-09-11 22:49:17 -04:00
Cli . set_version_flag ( " --version " , GetVersionString ( ) ) ;
2022-02-15 04:30:27 -05:00
2022-03-01 05:53:29 -05:00
std : : vector < CLI : : App * > SubCommands ;
2022-02-15 04:30:27 -05:00
std : : string InputFilenameUtf8 ;
std : : string OutputFilenameUtf8 ;
std : : string BaseFilenameUtf8 ;
std : : string SourceFilenameUtf8 ;
std : : string TargetFilenameUtf8 ;
std : : string PatchFilenameUtf8 ;
std : : string InputFilename2Utf8 ;
std : : string SourceManifestFilenameUtf8 ;
2022-08-22 14:00:37 -04:00
std : : vector < std : : string > IncludeFilterArrayUtf8 ;
2022-02-15 04:30:27 -05:00
std : : vector < std : : string > ExcludeFilterArrayUtf8 ;
2022-07-22 14:45:34 -04:00
std : : vector < std : : string > CleanupExcludeFilterArrayUtf8 ;
2022-07-07 22:20:00 -04:00
std : : vector < std : : string > OverlayArrayUtf8 ;
2022-02-15 04:30:27 -05:00
std : : string RemoteAddressUtf8 ;
std : : string PreferredDfsUtf8 ;
std : : string WeakHashUtf8 = " buzhash " ;
std : : string StrongHashUtf8 = " blake3.128 " ;
std : : string PresetUtf8 = " all " ;
std : : string ChunkModeUtf8 ;
std : : string CacertFilenameUtf8 ;
2024-06-07 13:00:34 -04:00
std : : string ProtocolName ;
2022-02-15 04:30:27 -05:00
std : : string HttpHeaderFilenameUtf8 ;
2022-03-01 05:53:29 -05:00
std : : string QueryStringUtf8 ;
2023-09-22 20:13:26 -04:00
std : : vector < std : : string > QueryArgsUtf8 ;
2023-04-25 17:41:49 -04:00
std : : string ScavengeRootUtf8 ;
2023-10-29 11:26:35 -04:00
std : : string P4HavePathUtf8 ;
2023-11-02 01:12:47 -04:00
std : : string StorePathUtf8 ;
2023-11-04 01:04:21 -04:00
std : : string SnapshotNameUtf8 ;
2023-11-29 16:19:56 -05:00
std : : string AuthTokenPathUtf8 ;
2023-11-02 23:41:06 -04:00
bool bRunP4Have = false ;
2022-07-07 22:20:00 -04:00
bool bForceOperation = false ;
bool bAllowInsecureTls = false ;
2024-04-19 22:35:37 -04:00
bool bRequireTls = false ;
2022-07-07 22:20:00 -04:00
bool bUseDebugMode = false ;
bool bIncrementalMode = false ;
bool bNoOutputValidation = false ;
bool bNoCleanupAfterSync = false ;
2022-10-12 20:45:24 -04:00
bool bNoSpaceValidation = false ;
2022-07-07 22:20:00 -04:00
bool bFullSourceScan = false ;
bool bFullDifference = false ;
2022-08-22 14:00:37 -04:00
bool bInfoFiles = false ;
2023-06-21 15:01:35 -04:00
bool bNoProxySelect = false ;
2023-08-29 19:18:18 -04:00
bool bInteractive = false ;
bool bDecode = false ;
bool bPrint = false ;
2023-09-07 23:20:15 -04:00
bool bPrintHttpHeader = false ;
2023-08-31 16:20:09 -04:00
bool bShouldLogin = false ;
2023-08-31 17:15:46 -04:00
bool bQuickLogin = false ;
bool bForceRefreshAuth = false ;
2023-09-15 22:24:38 -04:00
bool bNoSocketTimeout = false ;
2023-11-16 16:15:34 -05:00
bool bNoOutputFiles = false ;
bool bNoOutputRevisions = false ;
2024-05-24 20:21:04 -04:00
bool bPackOnlySmallFiles = false ;
2024-06-03 18:26:41 -04:00
bool bPackFiles = false ;
2024-06-18 15:36:15 -04:00
bool bNoCompression = false ;
2022-07-07 22:20:00 -04:00
int32 CompressionLevel = 3 ;
uint32 DiffBlockSize = uint32 ( 4 _KB ) ;
uint32 HashOrSyncBlockSize = uint32 ( 64 _KB ) ;
2023-11-13 20:57:57 -05:00
uint32 BackgroundTaskMemoryBudgetGB = 2 ;
2022-02-15 04:30:27 -05:00
2024-06-04 17:15:24 -04:00
const std : : string HiddenGroupId ; // CLI11 uses an empty string group name to mark arguments that should be hidden
const std : : string ExperimentalGroupId = " Experimental " ;
const std : : string DangerousGroupId = " Dangerous " ;
2022-05-24 13:55:09 -04:00
struct FDeprecatedOptions
{
2023-06-21 15:01:35 -04:00
bool bQuickSyncMode = false ;
bool bQuickDifference = false ;
2022-05-24 13:55:09 -04:00
bool bQuickSourceValidation = false ;
} DeprecatedOptions ;
2024-06-04 17:15:24 -04:00
auto AddTlsOptions = [ & CacertFilenameUtf8 , & bRequireTls , & bAllowInsecureTls , & DangerousGroupId ] ( CLI : : App * App )
{
2022-03-01 05:53:29 -05:00
App - > add_option ( " --cacert " , CacertFilenameUtf8 , " Certificate authority file to use for TLS validation (.pem) " ) ;
2024-04-19 22:35:37 -04:00
App - > add_flag ( " --tls " , bRequireTls , " Force TLS when connecting to remote server " ) ;
2024-06-04 17:15:24 -04:00
App - > add_flag ( " --insecure " , bAllowInsecureTls , " Skip remote server TLS certificate validation " ) - > group ( DangerousGroupId ) ;
2022-03-01 05:53:29 -05:00
} ;
2024-06-18 15:36:15 -04:00
auto AddProxyOptions = [ & RemoteAddressUtf8 , & ProtocolName , & bNoProxySelect , & bNoCompression ] ( CLI : : App * App )
2024-06-07 13:00:34 -04:00
{
2023-08-18 21:41:29 -04:00
App - > add_option ( " --proxy, --remote, --server " ,
2023-06-21 15:01:35 -04:00
RemoteAddressUtf8 ,
2024-06-07 13:00:34 -04:00
" Download server address ([protocol+][transport://]address[:port][/request][#namespace]) " ) ;
2023-08-16 16:34:36 -04:00
App - > add_flag ( " --no-proxy-select " ,
bNoProxySelect ,
" Skip automatic server selection and use the exact one specified by command line or environment variable " ) ;
2024-06-18 15:36:15 -04:00
App - > add_flag ( " --no-compression " ,
bNoCompression ,
" Disable compression when downloading blocks from the server, if possible (intended for debugging) " ) ;
2024-06-07 13:00:34 -04:00
App - > add_option ( " --protocol " , ProtocolName , " Explicitly specify server protocol instead of inferring it from URL " )
- > required ( false )
- > check ( CLI : : IsMember ( { " unsync " , " jupiter " , " horde " } ) ) ;
2023-06-21 13:12:26 -04:00
} ;
2023-08-29 19:18:18 -04:00
// Configure hash
2022-02-15 04:30:27 -05:00
CLI : : App * SubHash = Cli . add_subcommand ( " hash " , " Generate hash manifest for a file or directory " ) ;
SubHash - > add_option ( " Input " , InputFilenameUtf8 , " Input file or directory path " ) - > required ( ) ;
SubHash - > add_flag ( " -f, --force " , bForceOperation , " Force the operation even if hash is already computed for the input " ) ;
SubHash
- > add_option ( " --mode " ,
ChunkModeUtf8 ,
" Specify chunking mode. Fixed chunking is faster and may produce smaller patches. Variable chunking allows block "
" reuse between files. " )
- > check ( CLI : : IsMember ( { " fixed " , " variable " } ) )
- > default_str ( " variable " ) ;
SubHash - > add_option ( " -o " , OutputFilenameUtf8 , " Output file name " ) ;
SubHash - > add_option ( " --strong " , StrongHashUtf8 , " Specify strong hash algorithm instead) " )
- > check ( CLI : : IsMember ( { " blake3.128 " , " blake3.160 " , " iohash " , " blake3.256 " , " md5 " } ) )
- > default_str ( StrongHashUtf8 ) ;
SubHash - > add_option ( " --weak " , WeakHashUtf8 , " Specify weak hash algorithm instead) " )
- > check ( CLI : : IsMember ( { " naive " , " buzhash " } ) )
- > default_str ( WeakHashUtf8 ) ;
SubHash - > add_option ( " -b, --block " , HashOrSyncBlockSize , " Block size in bytes (default=64KB) " ) ;
SubHash - > add_flag (
" --update " ,
bIncrementalMode ,
" Create a directory manifest incrementally, by updating an existing manifest if one exists (only process changed files) " ) ;
2024-05-24 20:21:04 -04:00
SubHash - > add_flag (
" --pack-small-files " ,
bPackOnlySmallFiles ,
" Small files will be copied to compressed pack files during manifest generation and stored next to the manifest. "
" This can make syncing more efficient by reducing the number of remote file handles that must be opened. Implies --pack. "
" Files less than 4MB in size are considered small. " ) ;
SubHash - > add_flag (
" --pack " ,
bPackFiles ,
" Input files will be copied to compressed pack files during manifest generation and stored next to the manifest file. " ) ;
2022-03-01 05:53:29 -05:00
SubCommands . push_back ( SubHash ) ;
2022-02-15 04:30:27 -05:00
2023-10-29 11:26:35 -04:00
// Configure pack
2023-11-02 23:41:06 -04:00
CLI : : App * SubPack = nullptr ;
{
SubPack =
Cli . add_subcommand ( " pack " ,
2024-06-04 17:15:24 -04:00
" EXPERIMENTAL: Generate manifest for a directory and store all referenced data in a compressed pack file " ) - > group ( HiddenGroupId ) ;
2023-11-02 23:41:06 -04:00
SubPack - > add_option ( " Input " , InputFilenameUtf8 , " Input directory path " ) - > required ( ) ;
auto P4HaveFileOpt =
SubPack - > add_option ( " --p4havefile " ,
P4HavePathUtf8 ,
" Use `p4 have` output from a given file to explicitly specify files included in the manifest " ) ;
auto RunP4HaveOpt = SubPack - > add_flag ( " --p4have " , bRunP4Have , " Run `p4 have` when generating the dirctory pack " ) ;
SubPack - > add_option ( " --store " , StorePathUtf8 , " Use this location to store pack data (default: <Input>/.unsync/pack) " ) ;
2023-11-16 21:05:20 -05:00
SubPack - > add_option ( " --snapshot " , SnapshotNameUtf8 , " Custom name for the snapshot (will overwrite an existing tag) " ) ;
2023-11-02 23:41:06 -04:00
RunP4HaveOpt - > excludes ( P4HaveFileOpt ) ;
SubCommands . push_back ( SubPack ) ;
}
2023-10-29 11:26:35 -04:00
2023-11-04 01:04:21 -04:00
CLI : : App * SubUnpack = nullptr ;
{
2024-06-04 17:15:24 -04:00
SubUnpack = Cli . add_subcommand ( " unpack " , " EXPERIMENTAL: Sync directory based on package snapshot " ) - > group ( HiddenGroupId ) ;
2023-11-04 01:04:21 -04:00
SubUnpack - > add_option ( " Output " , OutputFilenameUtf8 , " Output directory path " ) - > required ( ) ;
SubUnpack - > add_option ( " --store " , StorePathUtf8 , " Pack storage path " ) - > required ( ) ;
SubUnpack - > add_option ( " --snapshot " , SnapshotNameUtf8 , " Directory snapshot ID " ) - > required ( ) ;
2023-11-16 16:15:34 -05:00
SubUnpack - > add_option ( " --p4havefile " , P4HavePathUtf8 , " Write revision control data in `p4 have` format into this file " ) ;
SubUnpack - > add_flag ( " --no-revisions " , bNoOutputRevisions , " Skip writing revision control data to <output>/.unsync/revisions.txt " ) ;
SubUnpack - > add_flag ( " --no-files " ,
bNoOutputFiles ,
" Skip actually unpacking the snapshot files, but attempt to reconstruct and verify the manifest. "
" Can be used in combination with --p4havefile option to only extract the `p4 have` list. " ) ;
2023-11-04 01:04:21 -04:00
SubCommands . push_back ( SubUnpack ) ;
}
2023-08-29 19:18:18 -04:00
// Configure push
2022-02-15 04:30:27 -05:00
CLI : : App * SubPush = Cli . add_subcommand ( " push " , " Loads a manifest from a directory and uploads referenced blocks to the remote server " ) ;
SubPush - > add_option ( " Input " , InputFilenameUtf8 , " Input file or directory path " ) - > required ( ) ;
SubPush
- > add_option ( " Remote " ,
RemoteAddressUtf8 ,
" Remote storage that will receive blocks ([transport://]address[:port][/request][#namespace]) " )
- > required ( ) ;
SubPush - > add_option ( " --http-header-file " ,
HttpHeaderFilenameUtf8 ,
" Text file that contains any extra HTTP headers to pass to the remote server (auth tokens, etc.) " ) ;
SubPush - > add_flag ( " --insecure " , bAllowInsecureTls , " Skip remote server TLS certificate validation " ) ;
2022-03-01 05:53:29 -05:00
SubCommands . push_back ( SubPush ) ;
2022-02-15 04:30:27 -05:00
CLI : : App * SubInfo = Cli . add_subcommand ( " info " , " Display information about a manifest file or diff two manifests " ) ;
SubInfo - > add_option ( " Input 1 " , InputFilenameUtf8 , " Input manifest file or root directory " ) - > required ( ) ;
SubInfo - > add_option ( " Input 2 " , InputFilename2Utf8 , " Optional input manifest file or root directory " ) ;
2022-08-22 14:00:37 -04:00
SubInfo - > add_flag ( " --files " , bInfoFiles , " List all files in the manifest " ) ;
2023-05-11 18:34:56 -04:00
SubInfo - > add_option (
" --include " ,
IncludeFilterArrayUtf8 ,
" Include filenames that contain specified words (comma separated). If this is not present, all files will be included. " ) ;
SubInfo - > add_option ( " --exclude " ,
ExcludeFilterArrayUtf8 ,
" Exclude filenames that contain specified words (comma separated). Filter is run after --include. " ) ;
2024-06-03 18:26:41 -04:00
SubInfo - > add_flag ( " --decode " , bDecode , " Decode binary manifest into json " ) ;
2022-03-01 05:53:29 -05:00
SubCommands . push_back ( SubInfo ) ;
2022-02-15 04:30:27 -05:00
2023-08-29 19:18:18 -04:00
// Configure diff
2022-02-15 04:30:27 -05:00
CLI : : App * SubDiff = Cli . add_subcommand ( " diff " , " Compute difference required to transform BaseFile into SourceFile " ) ;
SubDiff - > add_option ( " Base " , BaseFilenameUtf8 , " Base file name (local data) " ) - > required ( ) ;
SubDiff - > add_option ( " Source " , SourceFilenameUtf8 , " Source file name (remote data) " ) - > required ( ) ;
SubDiff - > add_option ( " -o " , OutputFilenameUtf8 , " Output patch file name (data required to transform base into source) " ) ;
SubDiff - > add_option ( " --level " , CompressionLevel , " ZSTD compression level (default=3) " ) ;
SubDiff - > add_option ( " -b, --block " , DiffBlockSize , " Block size in bytes (default=4KB) " ) ;
2022-03-01 05:53:29 -05:00
SubCommands . push_back ( SubDiff ) ;
2022-02-15 04:30:27 -05:00
2023-08-29 19:18:18 -04:00
// Configure sync
2022-02-15 04:30:27 -05:00
CLI : : App * SubSync = Cli . add_subcommand ( " sync " , " Synchronize files, transforming target file/directory into source " ) ;
SubSync
- > add_option ( " Source " ,
SourceFilenameUtf8 ,
" Source path, object name, hash or full URL ([transport://]address[:port]#namespace/object) " )
- > required ( ) ;
SubSync - > add_option ( " Target " , TargetFilenameUtf8 , " Target path " ) - > required ( ) ;
SubSync - > add_option ( " -m, --manifest " , SourceManifestFilenameUtf8 , " Override manifest path for Source " ) ;
2023-06-21 13:12:26 -04:00
AddProxyOptions ( SubSync ) ;
2024-06-04 17:15:24 -04:00
SubSync - > add_option ( " --dfs " , PreferredDfsUtf8 , " DEPRECATED: Preferred DFS mirror (matched by sub-string) " ) - > group ( HiddenGroupId ) ;
2023-06-21 15:01:35 -04:00
SubSync - > add_option (
" --overlay " ,
OverlayArrayUtf8 ,
" Additional source directory to sync (keep unique files from all sources, overwrite conflicting files with overlay source) " ) ;
SubSync - > add_option (
" --include " ,
IncludeFilterArrayUtf8 ,
" Include filenames that contain specified words (comma separated). If this is not present, all files will be included. " ) ;
SubSync - > add_option ( " --exclude " ,
ExcludeFilterArrayUtf8 ,
" Exclude filenames that contain specified words (comma separated). Filter is run after --include. " ) ;
2022-03-01 05:53:29 -05:00
AddTlsOptions ( SubSync ) ;
2023-08-29 19:18:18 -04:00
2022-02-15 04:30:27 -05:00
SubSync - > add_option ( " --http-header-file " ,
HttpHeaderFilenameUtf8 ,
" Text file that contains any extra HTTP headers to pass to the remote server (auth tokens, etc.) " ) ;
SubSync - > add_flag ( " --no-cleanup " , bNoCleanupAfterSync , " Do not delete local files that aren't in the manifest after a successful sync " ) ;
2023-06-21 15:01:35 -04:00
SubSync - > add_option ( " --cleanup-exclude " ,
CleanupExcludeFilterArrayUtf8 ,
" Exclude filenames that contain specified words from cleanup process (comma separated) " ) ;
2022-05-24 13:55:09 -04:00
// Deprecated --quick flag
SubSync
- > add_flag ( " --quick " ,
DeprecatedOptions . bQuickSyncMode ,
" Quick sync mode that skips some of the validation steps (enables all '--quick-*' options) " )
- > group ( HiddenGroupId ) ;
// Deprecated --quick-source-validation flag
SubSync
- > add_flag ( " --quick-source-validation " ,
DeprecatedOptions . bQuickSourceValidation ,
" Skip checking if all source files are present before starting a sync " )
- > group ( HiddenGroupId ) ;
// Deprecatred --quick-difference flag
SubSync
- > add_flag ( " --quick-difference " ,
DeprecatedOptions . bQuickDifference ,
" Allow computing file difference based on previous sync manifest and file timestamps " )
- > group ( HiddenGroupId ) ;
SubSync - > add_flag ( " --full-diff " ,
bFullDifference ,
2022-05-28 10:41:29 -04:00
" Run the full binary differencing algorithm on local files, even if there is a compatible local directory "
2022-05-24 13:55:09 -04:00
" manifest and file timestamps/sizes match. This is an extra precaution that will handle any unexpected local file "
" modifications, but it should not be needed in a common case. " ) ;
SubSync - > add_flag ( " --full-source-scan " ,
bFullSourceScan ,
" Perform a full scan of the source directory to check that all files are present and their timestamps/sizes match "
" the manifest. This is an extra precaution that will detect any missing or invalid remote files before running the "
" sync process, however this can be very slow when dealing with large numbers of files and directories. " ) ;
2024-06-04 17:15:24 -04:00
SubSync - > add_flag ( " --no-output-validation " , bNoOutputValidation , " Skip final patched file block hash validation (DANGEROUS) " )
- > group ( DangerousGroupId ) ;
SubSync - > add_flag ( " --no-space-validation " , bNoSpaceValidation , " Skip checking available disk space before sync (DANGEROUS) " )
- > group ( DangerousGroupId ) ;
SubSync - > add_option ( " --scavenge " , ScavengeRootUtf8 , " Search for unsync manifests and reusable blocks in this directory (EXPERIMENTAL) " )
- > group ( ExperimentalGroupId ) ;
2023-08-31 16:20:09 -04:00
SubSync - > add_flag ( " --login " , bShouldLogin , " Use user authentication when accessing unsync server " ) ;
2023-11-29 16:19:56 -05:00
SubSync - > add_option ( " --token " , AuthTokenPathUtf8 , " Explicit path to the authentication token file to use " ) ;
2023-09-15 22:24:38 -04:00
SubSync - > add_flag ( " --no-timeout " , bNoSocketTimeout , " Disable the default 60 second timeout on network socket operations " ) ;
2023-04-25 17:41:49 -04:00
2023-11-13 20:57:57 -05:00
CLI : : Option * BackgroundMemoryBudgetOption = SubSync - > add_option ( " --background-task-memory " ,
BackgroundTaskMemoryBudgetGB ,
" Set memory budget that background tasks in gigabytes (default: 2 GB) " ) ;
2022-03-01 05:53:29 -05:00
SubCommands . push_back ( SubSync ) ;
2022-02-15 04:30:27 -05:00
CLI : : App * SubPatch = Cli . add_subcommand ( " patch " , " Applies a patch generated with 'diff' on top of base file " ) ;
SubPatch - > add_option ( " Base " , BaseFilenameUtf8 , " Base file name " ) - > required ( ) ;
SubPatch - > add_option ( " Patch " , PatchFilenameUtf8 , " Patch file name " ) - > required ( ) ;
SubPatch - > add_option ( " -o " , OutputFilenameUtf8 , " Output file name " ) - > required ( ) ;
2022-03-01 05:53:29 -05:00
SubCommands . push_back ( SubPatch ) ;
2022-02-15 04:30:27 -05:00
CLI : : App * SubTest = Cli . add_subcommand ( " test " , " Run internal tests " ) ;
SubTest - > add_option ( " --preset " , PresetUtf8 , " Test preset " ) - > default_str ( PresetUtf8 ) ;
2022-03-01 05:53:29 -05:00
SubCommands . push_back ( SubTest ) ;
2022-02-15 04:30:27 -05:00
2023-08-29 19:18:18 -04:00
// Configure query
2022-03-01 05:53:29 -05:00
CLI : : App * SubQuery = Cli . add_subcommand ( " query " , " Run a query command on the remote server " ) ;
2023-08-29 19:18:18 -04:00
SubQuery - > add_option ( " QueryString " , QueryStringUtf8 , " Query to run: mirrors, login, list " ) - > required ( ) ;
2023-08-24 10:44:55 -04:00
SubQuery - > add_option ( " QueryArgs " , QueryArgsUtf8 , " Query arguments " ) ;
2023-08-31 16:20:09 -04:00
SubQuery - > add_option ( " -o " , OutputFilenameUtf8 , " Output file name " ) ;
2023-06-21 13:12:26 -04:00
AddProxyOptions ( SubQuery ) ;
2022-03-01 05:53:29 -05:00
AddTlsOptions ( SubQuery ) ;
SubCommands . push_back ( SubQuery ) ;
2023-08-29 19:18:18 -04:00
// Configure login
CLI : : App * SubLogin = Cli . add_subcommand ( " login " , " Authenticate with the remote server (acquire access and refresh tokens) " ) ;
SubLogin - > add_flag ( " --interactive " , bInteractive , " Allow user interaction through modal dialogs " ) ;
SubLogin - > add_flag ( " --decode " , bDecode , " Decode authentication token (implies --print) " ) ;
SubLogin - > add_flag ( " --print " , bPrint , " Print authentication token to standard output " ) ;
2023-09-07 23:20:15 -04:00
SubLogin - > add_flag ( " --print-http-header " , bPrintHttpHeader , " Print authentication token to standard output as HTTP Authorization header that could be used with curl, etc. " ) ;
2023-08-31 17:15:46 -04:00
SubLogin - > add_flag ( " --refresh " , bForceRefreshAuth , " Force authentication refresh even if access token has not yet expired " ) ;
SubLogin - > add_flag ( " --quick " , bQuickLogin , " Skip token validation using remote server (fast path when cached acess token is expected to be valid) " ) ;
2023-08-29 19:18:18 -04:00
AddTlsOptions ( SubLogin ) ;
AddProxyOptions ( SubLogin ) ;
SubCommands . push_back ( SubLogin ) ;
// Configure mount
2024-06-04 17:15:24 -04:00
CLI : : App * SubMount = Cli . add_subcommand ( " mount " , " Mount directory manifest as a virtual file system (EXPERIMENTAL) " ) - > group ( HiddenGroupId ) ;
2023-06-21 13:12:26 -04:00
SubMount
- > add_option ( " Source " ,
SourceFilenameUtf8 ,
" Source path, object name, hash or full URL ([transport://]address[:port]#namespace/object) " )
- > required ( ) ;
AddProxyOptions ( SubMount ) ;
SubCommands . push_back ( SubMount ) ;
2022-03-01 05:53:29 -05:00
for ( CLI : : App * Subcommand : SubCommands )
2022-02-15 04:30:27 -05:00
{
Subcommand - > add_flag ( " -d, --dry, --dry-run " , GDryRun , " Don't write any outputs to disk " ) ;
2023-10-31 16:34:04 -04:00
auto VerboseFlag = Subcommand - > add_flag ( " -v, --verbose " , GLogVerbose , " Verbose logging " ) ;
auto VeryVerboseFlag = Subcommand - > add_flag ( " --very-verbose " , GLogVeryVerbose , " Very verbose logging " ) ;
auto SilentFlag = Subcommand - > add_flag ( " --silent " , GLogSilent , " Skip all console logging except errors and warnings " ) ;
2022-02-15 04:30:27 -05:00
Subcommand - > add_flag ( " --progress " , GLogProgress , " Output @progress and @status markers " ) ;
Subcommand - > add_option ( " --threads " , GMaxThreads , " Limit worker threads to specified number " ) ;
Subcommand - > add_flag ( " --buffered-files " , GForceBufferedFiles , " Always use buffered file IO " ) ;
Subcommand - > add_flag ( " --debug " , bUseDebugMode , " Enable extra debugging features, such as extra memory safety validation " ) ;
2024-06-04 17:15:24 -04:00
Subcommand - > add_flag ( " --experimental " , GExperimental , " Enable experimental code paths " ) - > group ( HiddenGroupId ) ;
2023-10-31 16:34:04 -04:00
SilentFlag - > excludes ( VerboseFlag ) ;
SilentFlag - > excludes ( VeryVerboseFlag ) ;
VeryVerboseFlag - > excludes ( VerboseFlag ) ;
2022-02-15 04:30:27 -05:00
}
2023-08-29 19:18:18 -04:00
// Run the command
2023-09-11 22:49:17 -04:00
try
{
Cli . parse ( Argc , Argv ) ;
}
catch ( CLI : : Error & E )
{
std : : stringstream OutputStream ;
const int32 ReturnCode = Cli . exit ( E , OutputStream , OutputStream ) ;
std : : wstring Output = ConvertUtf8ToWide ( OutputStream . str ( ) ) ;
2023-09-13 18:33:59 -04:00
wprintf ( L " %ls " , Output . c_str ( ) ) ;
2023-09-11 22:49:17 -04:00
return ReturnCode ;
}
2022-02-15 04:30:27 -05:00
2024-05-24 20:21:04 -04:00
if ( bPackOnlySmallFiles )
{
bPackFiles = true ;
}
2022-02-15 04:30:27 -05:00
if ( Cli . get_subcommands ( ) . size ( ) = = 0 )
{
2023-09-11 22:49:17 -04:00
wprintf ( L " %hs " , Cli . help ( ) . c_str ( ) ) ;
2022-02-15 04:30:27 -05:00
}
2023-10-31 16:34:04 -04:00
FTimingLogger TimingLogger ( " Total time " , ELogLevel : : Info ) ;
2023-08-31 17:15:46 -04:00
// Configure default output mehtod based on subcommand.
// In machine-readable mode, all verbose logging is directed to stderr.
2023-08-29 19:18:18 -04:00
if ( Cli . got_subcommand ( SubQuery ) | | Cli . got_subcommand ( SubLogin ) )
{
GLogMachineReadable = true ;
}
2024-06-03 18:26:41 -04:00
else if ( Cli . got_subcommand ( SubInfo ) )
{
GLogMachineReadable = bDecode ;
}
2023-08-29 19:18:18 -04:00
2023-09-11 22:49:17 -04:00
UNSYNC_VERBOSE ( L " UNSYNC v%hs " , GetVersionString ( ) . c_str ( ) ) ;
2023-08-31 18:07:30 -04:00
UnsyncMallocInit ( bUseDebugMode ? EMallocType : : Debug : EMallocType : : Default ) ;
if ( bUseDebugMode )
{
UNSYNC_LOG ( L " *** Debug mode enabled *** " ) ;
}
2023-08-31 17:15:46 -04:00
// Augment configuration based on environment variables if corresponding command line arguments are missing.
if ( const char * EnvCleanupExclude = getenv ( " UNSYNC_CLEANUP_EXCLUDE " ) )
{
2023-10-31 16:34:04 -04:00
UNSYNC_LOG ( L " Using UNSYNC_CLEANUP_EXCLUDE environment: '%hs' " , EnvCleanupExclude ) ;
2023-08-31 17:15:46 -04:00
CleanupExcludeFilterArrayUtf8 . push_back ( EnvCleanupExclude ) ;
}
if ( PreferredDfsUtf8 . empty ( ) )
{
const char * EnvDfs = getenv ( " UNSYNC_DFS " ) ;
if ( EnvDfs )
{
2023-10-31 16:34:04 -04:00
UNSYNC_LOG ( L " Using UNSYNC_DFS environment: '%hs' " , EnvDfs ) ;
2023-08-31 17:15:46 -04:00
PreferredDfsUtf8 = std : : string ( EnvDfs ) ;
}
}
if ( RemoteAddressUtf8 . empty ( ) )
{
const char * EnvProxy = getenv ( " UNSYNC_PROXY " ) ;
if ( EnvProxy )
{
2023-10-31 16:34:04 -04:00
UNSYNC_LOG ( L " Using UNSYNC_PROXY environment: '%hs' " , EnvProxy ) ;
2023-08-31 17:15:46 -04:00
RemoteAddressUtf8 = std : : string ( EnvProxy ) ;
}
}
if ( CacertFilenameUtf8 . empty ( ) )
{
const char * EnvCacert = getenv ( " UNSYNC_CACERT " ) ;
if ( EnvCacert )
{
2023-10-31 16:34:04 -04:00
UNSYNC_LOG ( L " Using UNSYNC_CACERT environment: '%hs' " , EnvCacert ) ;
2023-08-31 17:15:46 -04:00
CacertFilenameUtf8 = std : : string ( EnvCacert ) ;
}
}
if ( HttpHeaderFilenameUtf8 . empty ( ) )
{
const char * EnvHttpHeaderFile = getenv ( " UNSYNC_HTTP_HEADER_FILE " ) ;
if ( EnvHttpHeaderFile )
{
2023-10-31 16:34:04 -04:00
UNSYNC_LOG ( L " Using UNSYNC_HTTP_HEADER_FILE environment: '%hs' " , EnvHttpHeaderFile ) ;
2023-08-31 17:15:46 -04:00
HttpHeaderFilenameUtf8 = std : : string ( EnvHttpHeaderFile ) ;
}
}
2022-05-24 13:55:09 -04:00
if ( GLogVeryVerbose )
2022-02-15 04:30:27 -05:00
{
2022-05-24 13:55:09 -04:00
// Force verbose mode if very-verbose flag is present
GLogVerbose = true ;
}
if ( DeprecatedOptions . bQuickSyncMode )
{
UNSYNC_WARNING (
L " Quick mode is now the default and --quick flag is deprecated. Use --full-source-scan and --full-diff options to enable legacy "
L " default behavior. " ) ;
}
if ( DeprecatedOptions . bQuickSourceValidation )
{
UNSYNC_WARNING (
L " Quick mode is now the default and --quick-source-validation flag is deprecated. Use --full-source-scan to enable legacy behavior "
L " that scans source directory. " ) ;
}
if ( DeprecatedOptions . bQuickDifference )
{
UNSYNC_WARNING (
L " Quick mode is now the default and --quick-difference flag is deprecated. Use --full-diff to enable legacy behavior performs "
L " full binary difference of local files even if timestamps and sizes match. " ) ;
2022-02-15 04:30:27 -05:00
}
EWeakHashAlgorithmID DefaultWeakHasher = EWeakHashAlgorithmID : : BuzHash ;
if ( WeakHashUtf8 = = " naive " )
{
DefaultWeakHasher = EWeakHashAlgorithmID : : Naive ;
}
else if ( WeakHashUtf8 = = " buzhash " )
{
DefaultWeakHasher = EWeakHashAlgorithmID : : BuzHash ;
}
EStrongHashAlgorithmID DefaultStrongHasher = EStrongHashAlgorithmID : : Blake3_128 ;
if ( StrongHashUtf8 = = " md5 " )
{
DefaultStrongHasher = EStrongHashAlgorithmID : : MD5 ;
}
else if ( StrongHashUtf8 = = " blake3.128 " )
{
DefaultStrongHasher = EStrongHashAlgorithmID : : Blake3_128 ;
}
else if ( StrongHashUtf8 = = " blake3.160 " | | StrongHashUtf8 = = " iohash " )
{
DefaultStrongHasher = EStrongHashAlgorithmID : : Blake3_160 ;
}
else if ( StrongHashUtf8 = = " blake3.256 " )
{
DefaultStrongHasher = EStrongHashAlgorithmID : : Blake3_256 ;
}
EChunkingAlgorithmID DefaultChunkingAlgorithm = EChunkingAlgorithmID : : VariableBlocks ;
if ( ChunkModeUtf8 = = " fixed " )
{
DefaultChunkingAlgorithm = EChunkingAlgorithmID : : FixedBlocks ;
}
else if ( ChunkModeUtf8 = = " variable " )
{
DefaultChunkingAlgorithm = EChunkingAlgorithmID : : VariableBlocks ;
}
FRemoteDesc RemoteDesc ;
2023-11-21 18:25:48 -05:00
FAuthDesc AuthDesc ;
2022-02-15 04:30:27 -05:00
2024-05-10 00:54:55 -04:00
bool bFilesystemSource = true ;
std : : string_view PossibleUrl ;
if ( Cli . got_subcommand ( SubSync ) )
2022-02-15 04:30:27 -05:00
{
2024-05-10 00:54:55 -04:00
PossibleUrl = SourceFilenameUtf8 ;
}
else if ( Cli . got_subcommand ( SubQuery ) & & ! QueryArgsUtf8 . empty ( ) )
{
PossibleUrl = QueryArgsUtf8 [ 0 ] ;
}
2022-02-15 04:30:27 -05:00
2024-06-07 13:00:34 -04:00
EProtocolFlavor ProtocolFlavorHint = EProtocolFlavor : : Unknown ;
if ( ! ProtocolName . empty ( ) )
{
ProtocolFlavorHint = ProtocolFlavorFromString ( ProtocolName ) ;
}
2024-05-10 00:54:55 -04:00
if ( RemoteAddressUtf8 . empty ( ) & & LooksLikeUrl ( PossibleUrl ) & & ( Cli . got_subcommand ( SubSync ) | | Cli . got_subcommand ( SubQuery ) ) )
{
// Derive remote server address from source name if explicit --proxy or --remote option is not provided for sync or query
2024-06-07 13:00:34 -04:00
TResult < FRemoteDesc > ParsedRemoteDesc = FRemoteDesc : : FromUrl ( PossibleUrl , ProtocolFlavorHint ) ;
2024-05-10 00:54:55 -04:00
if ( ParsedRemoteDesc . IsOk ( ) )
2022-02-15 04:30:27 -05:00
{
2024-05-10 00:54:55 -04:00
RemoteDesc = * ParsedRemoteDesc ;
bFilesystemSource = false ;
2022-02-15 04:30:27 -05:00
2024-05-10 00:54:55 -04:00
if ( RemoteDesc . Protocol = = EProtocolFlavor : : Jupiter )
{
2022-02-15 04:30:27 -05:00
size_t SlashPos = RemoteDesc . StorageNamespace . find_first_of ( ' / ' ) ;
if ( SlashPos = = std : : string : : npos )
{
2024-05-10 00:54:55 -04:00
UNSYNC_ERROR ( L " Jupiter URL source is expected to follow [transport://]address[:port]#namespace/object format " ) ;
2022-02-15 04:30:27 -05:00
return 1 ;
}
else
{
SourceFilenameUtf8 = RemoteDesc . StorageNamespace . substr ( SlashPos + 1 ) ;
RemoteDesc . StorageNamespace = RemoteDesc . StorageNamespace . substr ( 0 , SlashPos ) ;
}
}
2024-06-07 00:03:58 -04:00
else if ( RemoteDesc . Protocol = = EProtocolFlavor : : Unsync | | RemoteDesc . Protocol = = EProtocolFlavor : : Horde )
2022-02-15 04:30:27 -05:00
{
2024-05-10 00:54:55 -04:00
bShouldLogin = true ; // Try to authenticate by default when source is a valid URL
if ( Cli . got_subcommand ( SubQuery ) )
{
QueryArgsUtf8 [ 0 ] = RemoteDesc . RequestPath ;
}
else if ( Cli . got_subcommand ( SubSync ) )
{
SourceFilenameUtf8 = RemoteDesc . RequestPath ;
}
2022-02-15 04:30:27 -05:00
}
}
2024-05-10 00:54:55 -04:00
else
{
UNSYNC_ERROR ( L " Failed to parse remote address '%hs': %ls " ,
RemoteAddressUtf8 . c_str ( ) ,
ParsedRemoteDesc . TryError ( ) - > Context . c_str ( ) ) ;
return 1 ;
}
2022-02-15 04:30:27 -05:00
}
else
{
2024-06-07 13:00:34 -04:00
TResult < FRemoteDesc > ParsedRemoteDesc = FRemoteDesc : : FromUrl ( RemoteAddressUtf8 , ProtocolFlavorHint ) ;
2022-02-15 04:30:27 -05:00
if ( ParsedRemoteDesc . IsOk ( ) )
{
RemoteDesc = * ParsedRemoteDesc ;
}
else
{
UNSYNC_ERROR ( L " Failed to parse remote address '%hs': %ls " ,
RemoteAddressUtf8 . c_str ( ) ,
ParsedRemoteDesc . TryError ( ) - > Context . c_str ( ) ) ;
return 1 ;
}
}
2024-06-20 18:37:09 -04:00
// Derive artifact request path when syncing from Horde, if it wasn't specified via URL source syntax
if ( RemoteDesc . Protocol = = EProtocolFlavor : : Horde & & Cli . got_subcommand ( SubSync ) )
{
bFilesystemSource = false ;
if ( RemoteDesc . RequestPath . empty ( ) )
{
2024-06-25 18:31:35 -04:00
TResult < FHordeArtifactQuery > Query = FHordeArtifactQuery : : FromString ( SourceFilenameUtf8 ) ;
if ( Query . IsError ( ) )
2024-06-20 18:37:09 -04:00
{
2024-06-27 00:49:36 -04:00
LogError ( Query . GetError ( ) , L " Could not parse sync source path " ) ;
2024-06-20 18:37:09 -04:00
return 1 ;
}
2024-06-25 18:31:35 -04:00
if ( Query - > Id . empty ( ) )
2024-06-20 18:37:09 -04:00
{
UNSYNC_ERROR ( L " Could not parse sync source path. Artifact ID is expected, i.e. '#123456abcdef'. " ) ;
return 1 ;
}
2024-06-25 18:31:35 -04:00
SourceFilenameUtf8 = " api/v2/artifacts/ " + Query - > Id ;
2024-06-20 18:37:09 -04:00
RemoteDesc . RequestPath = SourceFilenameUtf8 ;
}
}
2024-06-18 15:36:15 -04:00
if ( bNoCompression )
{
UNSYNC_VERBOSE ( L " Uncompressed data transfer is preferred " ) ;
RemoteDesc . bPreferCompression = false ;
}
2022-03-08 08:17:08 -05:00
FPath InputFilename = NormalizeFilenameUtf8 ( InputFilenameUtf8 ) ;
FPath InputFilename2 = NormalizeFilenameUtf8 ( InputFilename2Utf8 ) ;
FPath OutputFilename = NormalizeFilenameUtf8 ( OutputFilenameUtf8 ) ;
FPath BaseFilename = NormalizeFilenameUtf8 ( BaseFilenameUtf8 ) ;
FPath TargetFilename = NormalizeFilenameUtf8 ( TargetFilenameUtf8 ) ;
FPath PatchFilename = NormalizeFilenameUtf8 ( PatchFilenameUtf8 ) ;
2023-04-25 17:41:49 -04:00
FPath ScavengeRoot = NormalizeFilenameUtf8 ( ScavengeRootUtf8 ) ;
2022-03-08 08:17:08 -05:00
FPath SourceManifestFilename = NormalizeFilenameUtf8 ( SourceManifestFilenameUtf8 ) ;
2022-02-15 04:30:27 -05:00
2024-05-10 00:54:55 -04:00
FPath SourceFilename = bFilesystemSource ? NormalizeFilenameUtf8 ( SourceFilenameUtf8 ) : FPath ( SourceFilenameUtf8 ) ;
2024-06-04 17:15:24 -04:00
if ( GDryRun )
{
UNSYNC_LOG ( L " >>> DRY RUN <<< " ) ;
}
if ( GExperimental )
{
UNSYNC_LOG ( L " >>> EXPERIMENTAL MODE <<< " ) ;
}
2022-05-24 13:55:09 -04:00
if ( GLogVeryVerbose )
{
2023-10-31 16:34:04 -04:00
UNSYNC_LOG ( L " Very verbose logging is enabled " ) ;
2022-05-24 13:55:09 -04:00
}
else if ( GLogVerbose )
{
2023-10-31 16:34:04 -04:00
UNSYNC_LOG ( L " Verbose logging is enabled " ) ;
2022-05-24 13:55:09 -04:00
}
2022-02-15 04:30:27 -05:00
if ( GForceBufferedFiles )
{
UNSYNC_VERBOSE ( L " Using buffered file IO " ) ;
}
GMaxThreads = std : : max ( 1u , GMaxThreads ) ;
2024-05-09 15:37:00 -04:00
UNSYNC_VERBOSE ( L " Using threads: %d " , GMaxThreads ) ;
// Don't count the main thread when starting the thread pool
const uint32 NumWorkerThreads = GMaxThreads - 1 ;
static FScheduler MainScheduler ( NumWorkerThreads ) ;
UNSYNC_ASSERT ( GScheduler = = nullptr ) ;
GScheduler = & MainScheduler ;
2023-10-31 16:34:04 -04:00
if ( Cli . got_subcommand ( SubHash ) | | Cli . got_subcommand ( SubPack ) )
2022-02-15 04:30:27 -05:00
{
2023-11-02 23:41:06 -04:00
UNSYNC_VERBOSE ( L " Using block size: %d KB " , HashOrSyncBlockSize / 1024 ) ;
2022-02-15 04:30:27 -05:00
}
if ( Cli . got_subcommand ( SubDiff ) )
{
2023-11-02 23:41:06 -04:00
UNSYNC_VERBOSE ( L " Using block size: %d KB " , DiffBlockSize / 1024 ) ;
2022-02-15 04:30:27 -05:00
}
if ( Cli . got_subcommand ( SubDiff ) )
{
DefaultWeakHasher = EWeakHashAlgorithmID : : Naive ;
DefaultChunkingAlgorithm = EChunkingAlgorithmID : : FixedBlocks ;
}
2023-11-04 01:04:21 -04:00
if ( Cli . got_subcommand ( SubHash ) | | Cli . got_subcommand ( SubDiff ) )
2022-02-15 04:30:27 -05:00
{
if ( ! bIncrementalMode )
{
UNSYNC_VERBOSE ( L " Using weak hash: %hs " , ToString ( DefaultWeakHasher ) ) ;
UNSYNC_VERBOSE ( L " Using strong hash: %hs " , ToString ( DefaultStrongHasher ) ) ;
UNSYNC_VERBOSE ( L " Using chunking mode: %hs " , ToString ( DefaultChunkingAlgorithm ) ) ;
}
}
FAlgorithmOptions Algorithm ;
Algorithm . ChunkingAlgorithmId = DefaultChunkingAlgorithm ;
Algorithm . StrongHashAlgorithmId = DefaultStrongHasher ;
Algorithm . WeakHashAlgorithmId = DefaultWeakHasher ;
2022-07-22 14:45:34 -04:00
FSyncFilter SyncFilter ;
2022-02-15 04:30:27 -05:00
for ( const std : : string & Str : ExcludeFilterArrayUtf8 )
{
2022-07-22 14:45:34 -04:00
SyncFilter . ExcludeFromSync ( ConvertUtf8ToWide ( Str ) ) ;
2022-02-15 04:30:27 -05:00
}
2022-08-22 14:00:37 -04:00
for ( const std : : string & Str : IncludeFilterArrayUtf8 )
{
SyncFilter . IncludeInSync ( ConvertUtf8ToWide ( Str ) ) ;
}
2022-02-15 04:30:27 -05:00
2022-07-22 14:45:34 -04:00
for ( const std : : string & Str : CleanupExcludeFilterArrayUtf8 )
{
SyncFilter . ExcludeFromCleanup ( ConvertUtf8ToWide ( Str ) ) ;
}
2022-02-15 04:30:27 -05:00
if ( ! PreferredDfsUtf8 . empty ( ) & & ! SourceFilenameUtf8 . empty ( ) )
{
LogGlobalStatus ( L " Enumerating DFS " ) ;
2023-10-31 16:34:04 -04:00
UNSYNC_LOG ( L " Enumerating DFS " ) ;
2022-02-15 04:30:27 -05:00
std : : wstring PreferredDfs = ConvertUtf8ToWide ( PreferredDfsUtf8 ) ;
auto DfsEntries = DfsEnumerate ( SourceFilename ) ;
const FDfsStorageInfo * FoundDfsStorage = nullptr ;
size_t FoundDfsSubstringPos = std : : numeric_limits < size_t > : : max ( ) ;
for ( const FDfsStorageInfo & DfsStorage : DfsEntries . Storages )
{
size_t Pos = DfsStorage . Server . find ( PreferredDfs ) ;
if ( Pos < FoundDfsSubstringPos )
{
FoundDfsSubstringPos = Pos ;
FoundDfsStorage = & DfsStorage ;
}
}
if ( FoundDfsStorage )
{
2023-10-31 16:34:04 -04:00
UNSYNC_LOG ( L " Found preferred DFS storage server '%ls' with share '%ls' " ,
2022-02-15 04:30:27 -05:00
FoundDfsStorage - > Server . c_str ( ) ,
FoundDfsStorage - > Share . c_str ( ) ) ;
FDfsAlias DfsAlias ;
DfsAlias . Source = DfsEntries . Root ;
2022-03-08 08:17:08 -05:00
DfsAlias . Target = FPath ( L " \\ \\ " ) / FoundDfsStorage - > Server / FoundDfsStorage - > Share ;
2022-02-15 04:30:27 -05:00
2023-10-31 16:34:04 -04:00
UNSYNC_LOG ( L " Using DFS alias '%ls' -> '%ls' " , DfsAlias . Source . wstring ( ) . c_str ( ) , DfsAlias . Target . wstring ( ) . c_str ( ) ) ;
2022-02-15 04:30:27 -05:00
if ( ! DfsAlias . Source . empty ( ) )
{
SyncFilter . DfsAliases . push_back ( std : : move ( DfsAlias ) ) ;
}
}
}
if ( ! HttpHeaderFilenameUtf8 . empty ( ) )
{
2022-03-08 08:17:08 -05:00
FPath Filename = NormalizeFilenameUtf8 ( HttpHeaderFilenameUtf8 ) ;
FBuffer HttpHeadersBuffer = ReadFileToBuffer ( Filename ) ;
2022-02-15 04:30:27 -05:00
const uint8 Bom [ 2 ] = { 0xFF , 0xFE } ;
if ( HttpHeadersBuffer . Size ( ) > 2 & & ! memcmp ( HttpHeadersBuffer . Data ( ) , Bom , 2 ) )
{
std : : wstring_view View ( ( const wchar_t * ) ( HttpHeadersBuffer . Data ( ) + 2 ) , ( HttpHeadersBuffer . Size ( ) - 2 ) / 2 ) ;
RemoteDesc . HttpHeaders = ConvertWideToUtf8 ( View ) ;
}
else
{
RemoteDesc . HttpHeaders = std : : string ( ( const char * ) HttpHeadersBuffer . Data ( ) , HttpHeadersBuffer . Size ( ) ) ;
}
}
2024-04-19 22:35:37 -04:00
if ( bRequireTls )
2022-03-01 05:53:29 -05:00
{
2024-04-19 22:35:37 -04:00
RemoteDesc . TlsRequirement = ETlsRequirement : : Required ;
2022-03-01 05:53:29 -05:00
}
2022-02-15 04:30:27 -05:00
if ( bAllowInsecureTls )
{
RemoteDesc . bTlsVerifyCertificate = false ;
2023-10-24 15:07:43 -04:00
RemoteDesc . bTlsVerifySubject = false ;
2022-02-15 04:30:27 -05:00
UNSYNC_WARNING ( L " Remote server certificate verification is disabled. " ) ;
}
2023-09-08 12:19:57 -04:00
else
{
RemoteDesc . bTlsVerifyCertificate = true ;
2023-10-24 15:07:43 -04:00
RemoteDesc . bTlsVerifySubject = true ;
2023-09-08 12:19:57 -04:00
}
2022-02-15 04:30:27 -05:00
if ( bNoOutputValidation )
{
UNSYNC_WARNING ( L " Final file validation is disabled. Data corruptions will not be detected or reported! " ) ;
}
if ( ! CacertFilenameUtf8 . empty ( ) )
{
2022-03-08 08:17:08 -05:00
FPath CacertPath = NormalizeFilenameUtf8 ( CacertFilenameUtf8 ) ;
FBuffer CacertBuffer = ReadFileToBuffer ( CacertPath ) ;
2022-02-15 04:30:27 -05:00
RemoteDesc . TlsCacert = std : : make_shared < FBuffer > ( std : : move ( CacertBuffer ) ) ;
2022-08-05 18:51:23 -04:00
RemoteDesc . TlsCacert - > PushBack ( ' \n ' ) ;
}
{
2023-06-21 15:01:35 -04:00
FPath ExtraCertPath = GExePath . parent_path ( ) / " unsync.cer " ;
FBuffer CertBuffer = ReadFileToBuffer ( ExtraCertPath ) ;
2022-08-05 18:51:23 -04:00
if ( ! CertBuffer . Empty ( ) )
{
UNSYNC_LOG ( L " Using trusted certificates from '%ls' " , ExtraCertPath . wstring ( ) . c_str ( ) ) ;
if ( ! RemoteDesc . TlsCacert )
{
RemoteDesc . TlsCacert = std : : make_shared < FBuffer > ( std : : move ( CertBuffer ) ) ;
}
else
{
RemoteDesc . TlsCacert - > Append ( CertBuffer ) ;
}
RemoteDesc . TlsCacert - > PushBack ( ' \n ' ) ;
}
2022-02-15 04:30:27 -05:00
}
2023-10-24 15:07:43 -04:00
if ( bShouldLogin )
{
2024-06-27 00:49:36 -04:00
RemoteDesc . PrimaryHost = RemoteDesc . Host ;
RemoteDesc . bAuthenticationRequired = true ;
2023-10-24 15:07:43 -04:00
}
2023-08-31 16:20:09 -04:00
FRemoteDesc RootRemoteDesc = RemoteDesc ;
2023-08-29 19:18:18 -04:00
if ( ! bNoProxySelect
& & Cli . got_subcommand ( SubSync )
2023-08-16 16:34:36 -04:00
& & RemoteDesc . IsValid ( ) & & RemoteDesc . Protocol = = EProtocolFlavor : : Unsync )
2023-06-21 15:01:35 -04:00
{
2023-11-21 18:25:48 -05:00
UNSYNC_LOG ( L " Selecting server using root '%hs' " , RemoteDesc . Host . Address . c_str ( ) ) ;
2023-06-21 15:01:35 -04:00
TResult < FMirrorInfo > MirrorResult = FindClosestMirror ( RemoteDesc ) ;
if ( const FMirrorInfo * Mirror = MirrorResult . TryData ( ) )
{
UNSYNC_LOG ( L " Closest server: '%hs', ping: %.2f ms " , Mirror - > Address . c_str ( ) , Mirror - > Ping * 1000.0 ) ;
2023-11-21 18:25:48 -05:00
RemoteDesc . Host . Address = Mirror - > Address ;
RemoteDesc . Host . Port = Mirror - > Port ;
2024-04-19 22:35:37 -04:00
if ( Mirror - > Port = = 443 )
{
RemoteDesc . TlsRequirement = ETlsRequirement : : Required ;
}
else if ( RemoteDesc . TlsRequirement < ETlsRequirement : : Required )
{
RemoteDesc . TlsRequirement = ETlsRequirement : : Preferred ;
}
2023-06-21 15:01:35 -04:00
}
else
{
UNSYNC_WARNING ( L " Failed to find closest proxy using root server '%hs': %ls " ,
RemoteAddressUtf8 . c_str ( ) ,
MirrorResult . TryError ( ) - > Context . c_str ( ) ) ;
}
}
2022-02-15 04:30:27 -05:00
if ( Cli . got_subcommand ( SubHash ) )
{
FCmdHashOptions HashOptions ;
2024-05-24 20:21:04 -04:00
HashOptions . Input = InputFilename ;
HashOptions . Output = OutputFilename ;
HashOptions . BlockSize = HashOrSyncBlockSize ;
HashOptions . Algorithm = Algorithm ;
HashOptions . bForce = bForceOperation ;
HashOptions . bIncremental = bIncrementalMode ;
HashOptions . bPackFiles = bPackFiles ;
if ( bPackOnlySmallFiles )
{
HashOptions . MaxFileSizeToPack = 4 _MB ;
}
2022-02-15 04:30:27 -05:00
return CmdHash ( HashOptions ) ;
}
2023-11-04 01:04:21 -04:00
else if ( Cli . got_subcommand ( SubPack ) )
2023-10-29 11:26:35 -04:00
{
FCmdPackOptions PackOptions ;
PackOptions . RootPath = InputFilename ;
PackOptions . P4HavePath = NormalizeFilenameUtf8 ( P4HavePathUtf8 ) ;
2023-11-02 01:12:47 -04:00
PackOptions . StorePath = NormalizeFilenameUtf8 ( StorePathUtf8 ) ;
2023-11-02 23:41:06 -04:00
PackOptions . bRunP4Have = bRunP4Have ;
2023-10-29 11:26:35 -04:00
PackOptions . BlockSize = HashOrSyncBlockSize ;
PackOptions . Algorithm = Algorithm ;
2023-11-16 21:05:20 -05:00
PackOptions . SnapshotName = SnapshotNameUtf8 ;
2023-10-29 11:26:35 -04:00
return CmdPack ( PackOptions ) ;
}
2023-11-04 01:04:21 -04:00
else if ( Cli . got_subcommand ( SubUnpack ) )
{
FCmdUnpackOptions UnpackOptions ;
2023-11-16 16:15:34 -05:00
UnpackOptions . OutputPath = OutputFilename ;
UnpackOptions . SnapshotName = SnapshotNameUtf8 ;
UnpackOptions . P4HaveOutputPath = NormalizeFilenameUtf8 ( P4HavePathUtf8 ) ;
UnpackOptions . StorePath = NormalizeFilenameUtf8 ( StorePathUtf8 ) ;
UnpackOptions . bOutputFiles = ! bNoOutputFiles ;
UnpackOptions . bOutputRevisions = ! bNoOutputRevisions ;
2023-11-04 01:04:21 -04:00
return CmdUnpack ( UnpackOptions ) ;
}
2022-02-15 04:30:27 -05:00
else if ( Cli . got_subcommand ( SubDiff ) )
{
FCmdDiffOptions DiffOptions ;
DiffOptions . Source = SourceFilename ;
DiffOptions . Base = BaseFilename ;
DiffOptions . Output = OutputFilename ;
DiffOptions . BlockSize = DiffBlockSize ;
DiffOptions . WeakHasher = DefaultWeakHasher ;
DiffOptions . StrongHasher = DefaultStrongHasher ;
DiffOptions . CompressionLevel = CompressionLevel ;
return CmdDiff ( DiffOptions ) ;
}
else if ( Cli . got_subcommand ( SubSync ) )
{
2023-05-03 17:28:38 -04:00
if ( ! ScavengeRoot . empty ( ) & & ! IsDirectory ( ScavengeRoot ) )
{
UNSYNC_WARNING ( L " Scavenge directory '%ls' does not exist " , ScavengeRoot . wstring ( ) . c_str ( ) ) ;
ScavengeRoot = FPath { } ;
}
2023-08-31 16:20:09 -04:00
if ( bShouldLogin )
2023-08-18 21:41:29 -04:00
{
2023-10-31 16:34:04 -04:00
UNSYNC_LOG ( L " Attempting to authenticate " ) ;
2023-08-18 21:41:29 -04:00
UNSYNC_LOG_INDENT ;
2023-11-21 18:25:48 -05:00
TResult < FAuthDesc > AuthDescResult = GetRemoteAuthDesc ( RemoteDesc ) ;
if ( AuthDescResult . IsOk ( ) )
2023-08-18 21:41:29 -04:00
{
2023-11-21 18:25:48 -05:00
AuthDesc = AuthDescResult . GetData ( ) ;
2023-11-29 16:19:56 -05:00
AuthDesc . TokenPath = NormalizeFilenameUtf8 ( AuthTokenPathUtf8 ) ;
2023-11-21 18:25:48 -05:00
// Note: since tokens can expire during a long operation,
// we can only save the auth descriptor and re-authenticate later if necessary
2024-06-20 18:37:09 -04:00
TResult < FAuthToken > AuthTokenResult = Authenticate ( AuthDesc ) ;
2023-11-21 18:25:48 -05:00
if ( AuthTokenResult . IsError ( ) )
{
UNSYNC_ERROR ( " Failed to authenticate with server '%hs' " , RemoteDesc . Host . Address . c_str ( ) ) ;
LogError ( AuthTokenResult . GetError ( ) ) ;
return - 1 ;
}
// Authentication requires encrypted connection
2024-04-19 22:35:37 -04:00
RemoteDesc . TlsRequirement = ETlsRequirement : : Required ;
2023-11-21 18:25:48 -05:00
RemoteDesc . bAuthenticationRequired = true ;
UNSYNC_LOG ( L " Authentication enabled " )
2023-08-18 21:41:29 -04:00
}
}
2023-09-15 22:24:38 -04:00
if ( bNoSocketTimeout )
{
RemoteDesc . RecvTimeoutSeconds = 0 ;
}
else
{
RemoteDesc . RecvTimeoutSeconds = 60 ;
}
2023-11-13 20:57:57 -05:00
// Try to derive default memory budget
if ( BackgroundMemoryBudgetOption - > empty ( ) )
{
FSystemMemoryInfo MemoryInfo ;
if ( QueryMemoryInfo ( MemoryInfo ) )
{
uint32 InstalledMemoryGB = CheckedNarrow ( MemoryInfo . InstalledPhysicalMemory > > 30 ) ;
UNSYNC_VERBOSE2 ( L " Detected memory: %llu GB " , InstalledMemoryGB ) ;
BackgroundTaskMemoryBudgetGB = std : : max < uint32 > ( 2 , InstalledMemoryGB / 4 ) ;
}
2023-11-29 16:19:56 -05:00
else
{
UNSYNC_VERBOSE2 ( L " Could not detect system memory size " ) ;
}
2023-11-13 20:57:57 -05:00
UNSYNC_VERBOSE2 ( L " Using automatic background task memory budget: %llu GB " , BackgroundTaskMemoryBudgetGB ) ;
}
else
{
UNSYNC_VERBOSE2 ( L " Using explicit background task memory budget: %llu GB " , BackgroundTaskMemoryBudgetGB ) ;
}
//
2022-02-15 04:30:27 -05:00
FCmdSyncOptions SyncOptions ;
2023-11-13 20:57:57 -05:00
SyncOptions . Algorithm = Algorithm ;
SyncOptions . Source = SourceFilename ;
SyncOptions . Target = TargetFilename ;
SyncOptions . SourceManifestOverride = SourceManifestFilename ;
SyncOptions . Remote = RemoteDesc ;
2023-11-21 18:25:48 -05:00
SyncOptions . AuthDesc = AuthDesc . IsValid ( ) ? & AuthDesc : nullptr ;
2023-11-13 20:57:57 -05:00
SyncOptions . bFullDifference = bFullDifference ;
SyncOptions . bFullSourceScan = bFullSourceScan ;
SyncOptions . bCleanup = ! bNoCleanupAfterSync ;
SyncOptions . Filter = & SyncFilter ;
SyncOptions . bValidateTargetFiles = ! bNoOutputValidation ;
SyncOptions . bCheckAvailableSpace = ! bNoSpaceValidation ;
SyncOptions . ScavengeRoot = ScavengeRoot ;
SyncOptions . BackgroundTaskMemoryBudget = uint64 ( BackgroundTaskMemoryBudgetGB ) < < 30ull ;
2022-02-15 04:30:27 -05:00
2022-07-07 22:20:00 -04:00
for ( const std : : string & Entry : OverlayArrayUtf8 )
{
SyncOptions . Overlays . push_back ( NormalizeFilenameUtf8 ( Entry ) ) ;
}
2022-02-15 04:30:27 -05:00
return CmdSync ( SyncOptions ) ;
}
else if ( Cli . got_subcommand ( SubPatch ) )
{
FCmdPatchOptions PatchOptions ;
PatchOptions . Base = BaseFilename ;
PatchOptions . Output = OutputFilename ;
PatchOptions . Patch = PatchFilename ;
return CmdPatch ( PatchOptions ) ;
}
else if ( Cli . got_subcommand ( SubPush ) )
{
FCmdPushOptions PushOptions ;
PushOptions . Input = InputFilename ;
PushOptions . Remote = RemoteDesc ;
return CmdPush ( PushOptions ) ;
}
else if ( Cli . got_subcommand ( SubTest ) )
{
UNSYNC_LOG ( L " Running internal tests ... " ) ;
RunTests ( PresetUtf8 ) ;
}
else if ( Cli . got_subcommand ( SubInfo ) )
{
2023-05-11 18:34:56 -04:00
FCmdInfoOptions Options ;
2023-06-21 15:01:35 -04:00
Options . InputA = InputFilename ;
Options . InputB = InputFilename2 ;
2023-05-11 18:34:56 -04:00
Options . bListFiles = bInfoFiles ;
Options . SyncFilter = & SyncFilter ;
2024-06-03 18:26:41 -04:00
Options . bDecode = bDecode ;
2023-05-11 18:34:56 -04:00
return CmdInfo ( Options ) ;
2022-02-15 04:30:27 -05:00
}
2022-03-01 05:53:29 -05:00
else if ( Cli . got_subcommand ( SubQuery ) )
{
FCmdQueryOptions QueryOptions ;
2023-08-31 16:20:09 -04:00
QueryOptions . Query = QueryStringUtf8 ;
QueryOptions . Args = QueryArgsUtf8 ;
QueryOptions . Remote = RemoteDesc ;
QueryOptions . OutputPath = OutputFilename ;
2022-03-01 05:53:29 -05:00
return CmdQuery ( QueryOptions ) ;
}
2023-08-29 19:18:18 -04:00
else if ( Cli . got_subcommand ( SubLogin ) )
{
2023-09-07 23:20:15 -04:00
if ( bDecode | | bPrintHttpHeader )
2023-08-29 19:18:18 -04:00
{
bPrint = true ;
}
FCmdLoginOptions LoginOptions ;
2023-10-24 23:10:40 -04:00
LoginOptions . Remote = RemoteDesc ;
LoginOptions . bInteractive = bInteractive ;
LoginOptions . bDecode = bDecode ;
LoginOptions . bPrint = bPrint ;
2023-09-07 23:20:15 -04:00
LoginOptions . bPrintHttpHeader = bPrintHttpHeader ;
2023-10-24 23:10:40 -04:00
LoginOptions . bForceRefresh = bForceRefreshAuth ;
LoginOptions . bQuick = bQuickLogin ;
2023-08-29 19:18:18 -04:00
return CmdLogin ( LoginOptions ) ;
}
2023-06-21 13:12:26 -04:00
else if ( Cli . got_subcommand ( SubMount ) )
{
FCmdMountOptions MountOptions ;
MountOptions . Path = SourceFilename ;
return CmdMount ( MountOptions ) ;
}
2022-02-15 04:30:27 -05:00
return 0 ;
}
# if UNSYNC_PLATFORM_WINDOWS // TODO: Ctrl-C signal handler for Linux
static BOOL WINAPI
ConsoleCtrlHandler ( int Signal )
{
FLogFlushScope FlushScope ;
const char * TerminateReason = nullptr ;
switch ( Signal )
{
case CTRL_C_EVENT :
TerminateReason = " Ctrl-C " ;
break ;
case CTRL_BREAK_EVENT :
TerminateReason = " Ctrl-Break " ;
break ;
case CTRL_CLOSE_EVENT :
TerminateReason = " console closed " ;
break ;
default :
TerminateReason = nullptr ;
}
if ( TerminateReason )
{
UNSYNC_LOG ( L " \n Terminating process on request: %hs \n " , TerminateReason ) ;
TerminateProcess ( GetCurrentProcess ( ) , 1 ) ;
}
return true ;
}
static LONG
ExceptionFilter ( _EXCEPTION_POINTERS * ExceptionPointers )
{
FLogFlushScope FlushScope ;
PEXCEPTION_RECORD Record = ExceptionPointers - > ExceptionRecord ;
2023-11-16 21:05:20 -05:00
if ( Record - > ExceptionCode = = EXCEPTION_BREAKPOINT )
{
LogPrintf ( ELogLevel : : Error , L " Break point at address 0x%016X \n " , Record - > ExceptionAddress ) ;
}
else
{
LogPrintf ( ELogLevel : : Error , L " Unhandled exception 0x%08X at address 0x%016X \n " , Record - > ExceptionCode , Record - > ExceptionAddress ) ;
}
2022-02-15 04:30:27 -05:00
LogWriteCrashDump ( ExceptionPointers ) ;
return EXCEPTION_EXECUTE_HANDLER ;
}
# endif // UNSYNC_PLATFORM_WINDOWS
} // namespace unsync
int
main ( int argc , char * * argv )
{
using namespace unsync ;
FLogFlushScope FlushScope ;
# if UNSYNC_PLATFORM_WINDOWS
SetConsoleCtrlHandler ( ( PHANDLER_ROUTINE ) ConsoleCtrlHandler , TRUE ) ;
SetUnhandledExceptionFilter ( ExceptionFilter ) ;
_setmode ( _fileno ( stdout ) , _O_U8TEXT ) ;
LPCWSTR WideCmdLine = GetCommandLineW ( ) ;
int NumWideArgs = 0 ;
LPWSTR * ArgvWide = CommandLineToArgvW ( WideCmdLine , & NumWideArgs ) ;
UNSYNC_ASSERT ( argc = = NumWideArgs ) ;
std : : vector < std : : string > ArgvStringsUtf8 ;
ArgvStringsUtf8 . reserve ( NumWideArgs ) ;
for ( int32 I = 0 ; I < NumWideArgs ; + + I )
{
ArgvStringsUtf8 . push_back ( ConvertWideToUtf8 ( ArgvWide [ I ] ) ) ;
}
2022-08-05 18:51:23 -04:00
GExePath = FPath ( ArgvWide [ 0 ] ) ;
2022-02-15 04:30:27 -05:00
LocalFree ( ArgvWide ) ;
std : : vector < char * > ArgvUtf8 ;
ArgvUtf8 . reserve ( NumWideArgs ) ;
for ( int32 I = 0 ; I < argc ; + + I )
{
ArgvUtf8 . push_back ( ArgvStringsUtf8 [ I ] . data ( ) ) ;
}
2023-06-21 15:01:35 -04:00
# else // UNSYNC_PLATFORM_WINDOWS
2022-08-05 18:51:23 -04:00
GExePath = FPath ( argv [ 0 ] ) ;
2022-02-15 04:30:27 -05:00
# endif // UNSYNC_PLATFORM_WINDOWS
2022-08-05 18:51:23 -04:00
GExePath = std : : filesystem : : weakly_canonical ( GExePath ) ;
2022-09-13 14:13:45 -04:00
GExePath = GetAbsoluteNormalPath ( GExePath ) ;
2022-08-05 18:51:23 -04:00
2022-02-15 04:30:27 -05:00
# if UNSYNC_PLATFORM_UNIX
2022-02-22 02:09:06 -05:00
std : : vector < char * > ArgvUtf8 ;
ArgvUtf8 . reserve ( argc ) ;
2022-02-15 04:30:27 -05:00
for ( int32 i = 0 ; i < argc ; + + i )
{
2022-02-22 02:09:06 -05:00
ArgvUtf8 . push_back ( argv [ i ] ) ;
2022-02-15 04:30:27 -05:00
}
# endif // UNSYNC_PLATFORM_UNIX
if ( GBreakOnError )
{
return InnerMain ( ( int ) ArgvUtf8 . size ( ) , ArgvUtf8 . data ( ) ) ;
}
else
{
try
{
return InnerMain ( ( int ) ArgvUtf8 . size ( ) , ArgvUtf8 . data ( ) ) ;
}
catch ( const std : : system_error & E )
{
UNSYNC_ERROR ( L " System error %d: %hs " , E . code ( ) . value ( ) , E . what ( ) )
}
}
return 1 ;
2024-05-09 15:37:00 -04:00
}