macports/macports-base
0
0
Fork 0
mirror of https://github.com/macports/macports-base.git synced 2026-07-12 18:18:43 -07:00
Code Issues Packages Projects Releases Wiki Activity

Don't rely on env vars for curl in worker threads

Browse Source
This commit is contained in:
Joshua Root
2026-05-04 13:08:50 +10:00
parent 6d4444ab20
commit 6d9edb07a8
3 changed files with 151 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/macports1.0/macports.tcl
+22
View File
@@ -101,6 +101,10 @@ namespace eval macports {
variable portinterp_deferred_options [list developer_dir xcodeversion xcodebuildcmd \
xcodecltversion xcode_license_unaccepted]
# Network proxies to use
variable proxies [dict create]
variable no_proxy {}
# maps porturls to the list of open mports with each url
variable open_mports [dict create]
@@ -1914,6 +1918,9 @@ match macports.conf.default."
set env(http_proxy) $sysConfProxies(proxy_http)
}
}
if {[info exists env(http_proxy)]} {
dict set ::macports::proxies http $env(http_proxy)
}
if {(![info exists env(https_proxy)] && ![info exists env(HTTPS_PROXY)]) || $proxy_override_env} {
unset -nocomplain env(https_proxy)
if {[info exists proxy_https]} {
@@ -1930,6 +1937,16 @@ match macports.conf.default."
set env(FTP_PROXY) $sysConfProxies(proxy_ftp)
}
}
foreach proxytype {https ftp all} {
if {[info exists env(${proxytype}_proxy)]} {
dict set ::macports::proxies $proxytype $env(${proxytype}_proxy)
continue
}
set proxytype_upper [string toupper $proxytype]
if {[info exists env(${proxytype_upper}_PROXY)]} {
dict set ::macports::proxies $proxytype $env(${proxytype_upper}_PROXY)
}
}
if {![info exists env(RSYNC_PROXY)] || $proxy_override_env} {
if {[info exists proxy_rsync]} {
set env(RSYNC_PROXY) $proxy_rsync
@@ -1943,6 +1960,11 @@ match macports.conf.default."
set env(NO_PROXY) $sysConfProxies(proxy_skip)
}
}
if {[info exists env(no_proxy)]} {
set ::macports::no_proxy $env(no_proxy)
} elseif {[info exists env(NO_PROXY)]} {
set ::macports::no_proxy $env(NO_PROXY)
}
# add ccache to environment
set env(CCACHE_DIR) $ccache_dir
src/macports1.0/mport_fetch_thread.tcl
+40 -10
View File
@@ -76,6 +76,26 @@ namespace eval mport_fetch_thread {
}
}
proc get_proxy_args {url} {
global proxies no_proxy
set ret [list]
if {[dict size $proxies] > 0} {
set colon [string first : $url]
if {$colon > 0} {
set scheme [string tolower [string range $url 0 ${colon}-1]]
if {[dict exists $proxies $scheme]} {
lappend ret --proxy [dict get $proxies $scheme]
} elseif {[dict exists $proxies all]} {
lappend ret --proxy [dict get $proxies all]
}
}
}
if {[info exists no_proxy]} {
lappend ret --no-proxy $no_proxy
}
return $ret
}
# Perform a curl-based operation and return the result.
# Result format: 2 element list: status, body
# status = 0: success, body is the actual result
@@ -96,10 +116,11 @@ namespace eval mport_fetch_thread {
set result [list 0 0]
lassign $opargs fixed_args credential_args urls
foreach {url sigurl} $urls {creds sigcreds} $credential_args {
set proxy_args [get_proxy_args $url]
# curl getsize can return -1 instead of throwing an error for
# nonexistent files on FTP sites.
if {![catch {curl getsize {*}$creds {*}$fixed_args $url} size] && $size > 0
&& ![catch {curl getsize {*}$sigcreds {*}$fixed_args $sigurl} sigsize] && $sigsize > 0} {
if {![catch {curl getsize {*}$proxy_args {*}$creds {*}$fixed_args $url} size] && $size > 0
&& ![catch {curl getsize {*}$proxy_args {*}$sigcreds {*}$fixed_args $sigurl} sigsize] && $sigsize > 0} {
set result [list 0 1]
break
}
@@ -122,6 +143,7 @@ namespace eval mport_fetch_thread {
set failed_sites 0
set logphase archivefetch
foreach url $urls creds $credential_args {
set proxy_args [get_proxy_args $url]
if {!$archive_fetched} {
try {
if {$show_progress} {
@@ -131,7 +153,7 @@ namespace eval mport_fetch_thread {
set current_url $url
}
progress_handler $msg_priority $logphase "Attempting to fetch $url"
curl fetch --progress progress_handler {*}$creds {*}$fixed_args $url ${outpath}.TMP
curl fetch --progress progress_handler {*}$proxy_args {*}$creds {*}$fixed_args $url ${outpath}.TMP
set archive_fetched 1
} on error {eMessage} {
progress_handler debug $logphase "Fetching $url failed: $eMessage"
@@ -155,7 +177,7 @@ namespace eval mport_fetch_thread {
set current_url $sigurl
}
progress_handler $msg_priority $logphase "Attempting to fetch $sigurl"
curl fetch --progress progress_handler {*}$creds {*}$fixed_args $sigurl $signature
curl fetch --progress progress_handler {*}$proxy_args {*}$creds {*}$fixed_args $sigurl $signature
set sig_fetched 1
set fetched_sigtype $sigtype
set result [list 0 1]
@@ -197,7 +219,7 @@ namespace eval mport_fetch_thread {
set current_url $url
}
progress_handler $msg_priority $logphase "Attempting to fetch $url"
curl fetch --progress progress_handler {*}$creds {*}$fixed_args $url ${outpath}.TMP
curl fetch --progress progress_handler {*}[get_proxy_args $url] {*}$creds {*}$fixed_args $url ${outpath}.TMP
set fetched 1
set result [list 0 1]
break
@@ -243,8 +265,8 @@ namespace eval mport_fetch_thread {
}
# End worker_init_script
proc init_max_threads {fetch_threads} {
global max_threads max_fetches
proc init_globals {fetch_threads proxies_in no_proxy_in} {
global max_threads max_fetches proxies no_proxy
if {![catch {sysctl hw.activecpu} ncpus] && $ncpus > $fetch_threads} {
set max_threads [expr {$ncpus * 2}]
} else {
@@ -254,6 +276,10 @@ namespace eval mport_fetch_thread {
set max_threads 8
}
set max_fetches $fetch_threads
set proxies $proxies_in
if {$no_proxy_in ne {}} {
set no_proxy $no_proxy_in
}
}
set available_threads [list]
set thread_count 0
@@ -281,9 +307,13 @@ namespace eval mport_fetch_thread {
# Create a new thread if possible.
global max_threads thread_count
if {$thread_count < $max_threads} {
global worker_init_script
global worker_init_script proxies no_proxy
set free_tid [thread::create -preserved $worker_init_script]
thread::send -async $free_tid [list set management_tid [thread::id]]
thread::send -async $free_tid [list set proxies $proxies]
if {[info exists no_proxy]} {
thread::send -async $free_tid [list set no_proxy $no_proxy]
}
incr thread_count
return $free_tid
}
@@ -384,9 +414,9 @@ proc mport_fetch_thread::init_management_thread {args} {
trace remove variable management_thread read mport_fetch_thread::init_management_thread
variable init_script
set management_thread [thread::create -preserved $init_script]
global macports::fetch_threads
global macports::fetch_threads macports::proxies macports::no_proxy
set max_fetches [expr {[info exists fetch_threads] && $fetch_threads > 1 ? $fetch_threads : 1}]
thread::send -async $management_thread [list init_max_threads $max_fetches]
thread::send -async $management_thread [list init_globals $max_fetches $proxies $no_proxy]
}
proc mport_fetch_thread::record_request {op opargs id} {
src/pextlib1.0/curl.c
+89 -9
View File
@@ -410,6 +410,8 @@ CurlFetchCmd(Tcl_Interp* interp, int objc, Tcl_Obj* const objv[])
const int MAXHTTPHEADERS = 100;
int numHTTPHeaders = 0;
const char* httpHeaders[MAXHTTPHEADERS];
char *proxy = NULL;
char *no_proxy = NULL;
int optioncrsr;
int lastoption;
const char* theURL;
@@ -519,6 +521,30 @@ CurlFetchCmd(Tcl_Interp* interp, int objc, Tcl_Obj* const objv[])
}
} else if (strcmp(theOption, "--enable-compression") == 0) {
acceptEncoding = "";
} else if (strcmp(theOption, "--proxy") == 0) {
/* check we also have the parameter */
if (optioncrsr < lastoption) {
optioncrsr++;
proxy = Tcl_GetString(objv[optioncrsr]);
} else {
Tcl_SetResult(interp,
"curl fetch: --proxy option requires a parameter",
TCL_STATIC);
theResult = TCL_ERROR;
break;
}
} else if (strcmp(theOption, "--no-proxy") == 0) {
/* check we also have the parameter */
if (optioncrsr < lastoption) {
optioncrsr++;
no_proxy = Tcl_GetString(objv[optioncrsr]);
} else {
Tcl_SetResult(interp,
"curl fetch: --no-proxy option requires a parameter",
TCL_STATIC);
theResult = TCL_ERROR;
break;
}
} else {
Tcl_ResetResult(interp);
Tcl_AppendResult(interp, "curl fetch: unknown option ", theOption, NULL);
@@ -604,17 +630,24 @@ CurlFetchCmd(Tcl_Interp* interp, int objc, Tcl_Obj* const objv[])
#if LIBCURL_VERSION_NUM >= 0x071304 && LIBCURL_VERSION_NUM <= 0x071307
/* FTP_PROXY workaround for Snow Leopard */
if (strncmp(theURL, "ftp:", 4) == 0) {
char *ftp_proxy = getenv("FTP_PROXY");
if (ftp_proxy) {
theCurlCode = curl_easy_setopt(theHandle, CURLOPT_PROXY, ftp_proxy);
if (theCurlCode != CURLE_OK) {
theResult = SetResultFromCurlErrorCode(interp, theCurlCode);
break;
}
}
if (proxy == NULL && strncmp(theURL, "ftp:", 4) == 0) {
proxy = getenv("FTP_PROXY");
}
#endif
if (proxy) {
theCurlCode = curl_easy_setopt(theHandle, CURLOPT_PROXY, proxy);
if (theCurlCode != CURLE_OK) {
theResult = SetResultFromCurlErrorCode(interp, theCurlCode);
break;
}
}
if (no_proxy) {
theCurlCode = curl_easy_setopt(theHandle, CURLOPT_NOPROXY, no_proxy);
if (theCurlCode != CURLE_OK) {
theResult = SetResultFromCurlErrorCode(interp, theCurlCode);
break;
}
}
/* -L option */
theCurlCode = curl_easy_setopt(theHandle, CURLOPT_FOLLOWLOCATION, 1);
@@ -1247,6 +1280,8 @@ CurlGetSizeCmd(Tcl_Interp* interp, int objc, Tcl_Obj* const objv[])
do {
int ignoresslcert = 0;
const char* theUserPassString = NULL;
char *proxy = NULL;
char *no_proxy = NULL;
int optioncrsr;
int lastoption;
const char* theURL;
@@ -1288,6 +1323,30 @@ CurlGetSizeCmd(Tcl_Interp* interp, int objc, Tcl_Obj* const objv[])
theResult = TCL_ERROR;
break;
}
} else if (strcmp(theOption, "--proxy") == 0) {
/* check we also have the parameter */
if (optioncrsr < lastoption) {
optioncrsr++;
proxy = Tcl_GetString(objv[optioncrsr]);
} else {
Tcl_SetResult(interp,
"curl getsize: --proxy option requires a parameter",
TCL_STATIC);
theResult = TCL_ERROR;
break;
}
} else if (strcmp(theOption, "--no-proxy") == 0) {
/* check we also have the parameter */
if (optioncrsr < lastoption) {
optioncrsr++;
no_proxy = Tcl_GetString(objv[optioncrsr]);
} else {
Tcl_SetResult(interp,
"curl getsize: --no-proxy option requires a parameter",
TCL_STATIC);
theResult = TCL_ERROR;
break;
}
} else {
Tcl_ResetResult(interp);
Tcl_AppendResult(interp, "curl getsize: unknown option ", theOption, NULL);
@@ -1364,6 +1423,27 @@ CurlGetSizeCmd(Tcl_Interp* interp, int objc, Tcl_Obj* const objv[])
break;
}
#if LIBCURL_VERSION_NUM >= 0x071304 && LIBCURL_VERSION_NUM <= 0x071307
/* FTP_PROXY workaround for Snow Leopard */
if (proxy == NULL && strncmp(theURL, "ftp:", 4) == 0) {
proxy = getenv("FTP_PROXY");
}
#endif
if (proxy) {
theCurlCode = curl_easy_setopt(theHandle, CURLOPT_PROXY, proxy);
if (theCurlCode != CURLE_OK) {
theResult = SetResultFromCurlErrorCode(interp, theCurlCode);
break;
}
}
if (no_proxy) {
theCurlCode = curl_easy_setopt(theHandle, CURLOPT_NOPROXY, no_proxy);
if (theCurlCode != CURLE_OK) {
theResult = SetResultFromCurlErrorCode(interp, theCurlCode);
break;
}
}
/* -L option */
theCurlCode = curl_easy_setopt(theHandle, CURLOPT_FOLLOWLOCATION, 1);
if (theCurlCode != CURLE_OK) {