From 6d9edb07a82817f7bb47e0ab7620bf0a8e8fc204 Mon Sep 17 00:00:00 2001 From: Joshua Root Date: Mon, 4 May 2026 13:08:50 +1000 Subject: [PATCH] Don't rely on env vars for curl in worker threads --- src/macports1.0/macports.tcl | 22 ++++++ src/macports1.0/mport_fetch_thread.tcl | 50 ++++++++++--- src/pextlib1.0/curl.c | 98 +++++++++++++++++++++++--- 3 files changed, 151 insertions(+), 19 deletions(-) diff --git a/src/macports1.0/macports.tcl b/src/macports1.0/macports.tcl index 7bde58dcc..c69f5da33 100644 --- a/src/macports1.0/macports.tcl +++ b/src/macports1.0/macports.tcl @@ -101,6 +101,10 @@ namespace eval macports { variable portinterp_deferred_options [list developer_dir xcodeversion xcodebuildcmd \ xcodecltversion xcode_license_unaccepted] + # Network proxies to use + variable proxies [dict create] + variable no_proxy {} + # maps porturls to the list of open mports with each url variable open_mports [dict create] @@ -1914,6 +1918,9 @@ match macports.conf.default." set env(http_proxy) $sysConfProxies(proxy_http) } } + if {[info exists env(http_proxy)]} { + dict set ::macports::proxies http $env(http_proxy) + } if {(![info exists env(https_proxy)] && ![info exists env(HTTPS_PROXY)]) || $proxy_override_env} { unset -nocomplain env(https_proxy) if {[info exists proxy_https]} { @@ -1930,6 +1937,16 @@ match macports.conf.default." set env(FTP_PROXY) $sysConfProxies(proxy_ftp) } } + foreach proxytype {https ftp all} { + if {[info exists env(${proxytype}_proxy)]} { + dict set ::macports::proxies $proxytype $env(${proxytype}_proxy) + continue + } + set proxytype_upper [string toupper $proxytype] + if {[info exists env(${proxytype_upper}_PROXY)]} { + dict set ::macports::proxies $proxytype $env(${proxytype_upper}_PROXY) + } + } if {![info exists env(RSYNC_PROXY)] || $proxy_override_env} { if {[info exists proxy_rsync]} { set env(RSYNC_PROXY) $proxy_rsync @@ -1943,6 +1960,11 @@ match macports.conf.default." set env(NO_PROXY) $sysConfProxies(proxy_skip) } } + if {[info exists env(no_proxy)]} { + set ::macports::no_proxy $env(no_proxy) + } elseif {[info exists env(NO_PROXY)]} { + set ::macports::no_proxy $env(NO_PROXY) + } # add ccache to environment set env(CCACHE_DIR) $ccache_dir diff --git a/src/macports1.0/mport_fetch_thread.tcl b/src/macports1.0/mport_fetch_thread.tcl index 7983ce2be..50ab5e549 100644 --- a/src/macports1.0/mport_fetch_thread.tcl +++ b/src/macports1.0/mport_fetch_thread.tcl @@ -76,6 +76,26 @@ namespace eval mport_fetch_thread { } } + proc get_proxy_args {url} { + global proxies no_proxy + set ret [list] + if {[dict size $proxies] > 0} { + set colon [string first : $url] + if {$colon > 0} { + set scheme [string tolower [string range $url 0 ${colon}-1]] + if {[dict exists $proxies $scheme]} { + lappend ret --proxy [dict get $proxies $scheme] + } elseif {[dict exists $proxies all]} { + lappend ret --proxy [dict get $proxies all] + } + } + } + if {[info exists no_proxy]} { + lappend ret --no-proxy $no_proxy + } + return $ret + } + # Perform a curl-based operation and return the result. # Result format: 2 element list: status, body # status = 0: success, body is the actual result @@ -96,10 +116,11 @@ namespace eval mport_fetch_thread { set result [list 0 0] lassign $opargs fixed_args credential_args urls foreach {url sigurl} $urls {creds sigcreds} $credential_args { + set proxy_args [get_proxy_args $url] # curl getsize can return -1 instead of throwing an error for # nonexistent files on FTP sites. - if {![catch {curl getsize {*}$creds {*}$fixed_args $url} size] && $size > 0 - && ![catch {curl getsize {*}$sigcreds {*}$fixed_args $sigurl} sigsize] && $sigsize > 0} { + if {![catch {curl getsize {*}$proxy_args {*}$creds {*}$fixed_args $url} size] && $size > 0 + && ![catch {curl getsize {*}$proxy_args {*}$sigcreds {*}$fixed_args $sigurl} sigsize] && $sigsize > 0} { set result [list 0 1] break } @@ -122,6 +143,7 @@ namespace eval mport_fetch_thread { set failed_sites 0 set logphase archivefetch foreach url $urls creds $credential_args { + set proxy_args [get_proxy_args $url] if {!$archive_fetched} { try { if {$show_progress} { @@ -131,7 +153,7 @@ namespace eval mport_fetch_thread { set current_url $url } progress_handler $msg_priority $logphase "Attempting to fetch $url" - curl fetch --progress progress_handler {*}$creds {*}$fixed_args $url ${outpath}.TMP + curl fetch --progress progress_handler {*}$proxy_args {*}$creds {*}$fixed_args $url ${outpath}.TMP set archive_fetched 1 } on error {eMessage} { progress_handler debug $logphase "Fetching $url failed: $eMessage" @@ -155,7 +177,7 @@ namespace eval mport_fetch_thread { set current_url $sigurl } progress_handler $msg_priority $logphase "Attempting to fetch $sigurl" - curl fetch --progress progress_handler {*}$creds {*}$fixed_args $sigurl $signature + curl fetch --progress progress_handler {*}$proxy_args {*}$creds {*}$fixed_args $sigurl $signature set sig_fetched 1 set fetched_sigtype $sigtype set result [list 0 1] @@ -197,7 +219,7 @@ namespace eval mport_fetch_thread { set current_url $url } progress_handler $msg_priority $logphase "Attempting to fetch $url" - curl fetch --progress progress_handler {*}$creds {*}$fixed_args $url ${outpath}.TMP + curl fetch --progress progress_handler {*}[get_proxy_args $url] {*}$creds {*}$fixed_args $url ${outpath}.TMP set fetched 1 set result [list 0 1] break @@ -243,8 +265,8 @@ namespace eval mport_fetch_thread { } # End worker_init_script - proc init_max_threads {fetch_threads} { - global max_threads max_fetches + proc init_globals {fetch_threads proxies_in no_proxy_in} { + global max_threads max_fetches proxies no_proxy if {![catch {sysctl hw.activecpu} ncpus] && $ncpus > $fetch_threads} { set max_threads [expr {$ncpus * 2}] } else { @@ -254,6 +276,10 @@ namespace eval mport_fetch_thread { set max_threads 8 } set max_fetches $fetch_threads + set proxies $proxies_in + if {$no_proxy_in ne {}} { + set no_proxy $no_proxy_in + } } set available_threads [list] set thread_count 0 @@ -281,9 +307,13 @@ namespace eval mport_fetch_thread { # Create a new thread if possible. global max_threads thread_count if {$thread_count < $max_threads} { - global worker_init_script + global worker_init_script proxies no_proxy set free_tid [thread::create -preserved $worker_init_script] thread::send -async $free_tid [list set management_tid [thread::id]] + thread::send -async $free_tid [list set proxies $proxies] + if {[info exists no_proxy]} { + thread::send -async $free_tid [list set no_proxy $no_proxy] + } incr thread_count return $free_tid } @@ -384,9 +414,9 @@ proc mport_fetch_thread::init_management_thread {args} { trace remove variable management_thread read mport_fetch_thread::init_management_thread variable init_script set management_thread [thread::create -preserved $init_script] - global macports::fetch_threads + global macports::fetch_threads macports::proxies macports::no_proxy set max_fetches [expr {[info exists fetch_threads] && $fetch_threads > 1 ? $fetch_threads : 1}] - thread::send -async $management_thread [list init_max_threads $max_fetches] + thread::send -async $management_thread [list init_globals $max_fetches $proxies $no_proxy] } proc mport_fetch_thread::record_request {op opargs id} { diff --git a/src/pextlib1.0/curl.c b/src/pextlib1.0/curl.c index d986e9bc5..0c20827d7 100644 --- a/src/pextlib1.0/curl.c +++ b/src/pextlib1.0/curl.c @@ -410,6 +410,8 @@ CurlFetchCmd(Tcl_Interp* interp, int objc, Tcl_Obj* const objv[]) const int MAXHTTPHEADERS = 100; int numHTTPHeaders = 0; const char* httpHeaders[MAXHTTPHEADERS]; + char *proxy = NULL; + char *no_proxy = NULL; int optioncrsr; int lastoption; const char* theURL; @@ -519,6 +521,30 @@ CurlFetchCmd(Tcl_Interp* interp, int objc, Tcl_Obj* const objv[]) } } else if (strcmp(theOption, "--enable-compression") == 0) { acceptEncoding = ""; + } else if (strcmp(theOption, "--proxy") == 0) { + /* check we also have the parameter */ + if (optioncrsr < lastoption) { + optioncrsr++; + proxy = Tcl_GetString(objv[optioncrsr]); + } else { + Tcl_SetResult(interp, + "curl fetch: --proxy option requires a parameter", + TCL_STATIC); + theResult = TCL_ERROR; + break; + } + } else if (strcmp(theOption, "--no-proxy") == 0) { + /* check we also have the parameter */ + if (optioncrsr < lastoption) { + optioncrsr++; + no_proxy = Tcl_GetString(objv[optioncrsr]); + } else { + Tcl_SetResult(interp, + "curl fetch: --no-proxy option requires a parameter", + TCL_STATIC); + theResult = TCL_ERROR; + break; + } } else { Tcl_ResetResult(interp); Tcl_AppendResult(interp, "curl fetch: unknown option ", theOption, NULL); @@ -604,17 +630,24 @@ CurlFetchCmd(Tcl_Interp* interp, int objc, Tcl_Obj* const objv[]) #if LIBCURL_VERSION_NUM >= 0x071304 && LIBCURL_VERSION_NUM <= 0x071307 /* FTP_PROXY workaround for Snow Leopard */ - if (strncmp(theURL, "ftp:", 4) == 0) { - char *ftp_proxy = getenv("FTP_PROXY"); - if (ftp_proxy) { - theCurlCode = curl_easy_setopt(theHandle, CURLOPT_PROXY, ftp_proxy); - if (theCurlCode != CURLE_OK) { - theResult = SetResultFromCurlErrorCode(interp, theCurlCode); - break; - } - } + if (proxy == NULL && strncmp(theURL, "ftp:", 4) == 0) { + proxy = getenv("FTP_PROXY"); } #endif + if (proxy) { + theCurlCode = curl_easy_setopt(theHandle, CURLOPT_PROXY, proxy); + if (theCurlCode != CURLE_OK) { + theResult = SetResultFromCurlErrorCode(interp, theCurlCode); + break; + } + } + if (no_proxy) { + theCurlCode = curl_easy_setopt(theHandle, CURLOPT_NOPROXY, no_proxy); + if (theCurlCode != CURLE_OK) { + theResult = SetResultFromCurlErrorCode(interp, theCurlCode); + break; + } + } /* -L option */ theCurlCode = curl_easy_setopt(theHandle, CURLOPT_FOLLOWLOCATION, 1); @@ -1247,6 +1280,8 @@ CurlGetSizeCmd(Tcl_Interp* interp, int objc, Tcl_Obj* const objv[]) do { int ignoresslcert = 0; const char* theUserPassString = NULL; + char *proxy = NULL; + char *no_proxy = NULL; int optioncrsr; int lastoption; const char* theURL; @@ -1288,6 +1323,30 @@ CurlGetSizeCmd(Tcl_Interp* interp, int objc, Tcl_Obj* const objv[]) theResult = TCL_ERROR; break; } + } else if (strcmp(theOption, "--proxy") == 0) { + /* check we also have the parameter */ + if (optioncrsr < lastoption) { + optioncrsr++; + proxy = Tcl_GetString(objv[optioncrsr]); + } else { + Tcl_SetResult(interp, + "curl getsize: --proxy option requires a parameter", + TCL_STATIC); + theResult = TCL_ERROR; + break; + } + } else if (strcmp(theOption, "--no-proxy") == 0) { + /* check we also have the parameter */ + if (optioncrsr < lastoption) { + optioncrsr++; + no_proxy = Tcl_GetString(objv[optioncrsr]); + } else { + Tcl_SetResult(interp, + "curl getsize: --no-proxy option requires a parameter", + TCL_STATIC); + theResult = TCL_ERROR; + break; + } } else { Tcl_ResetResult(interp); Tcl_AppendResult(interp, "curl getsize: unknown option ", theOption, NULL); @@ -1364,6 +1423,27 @@ CurlGetSizeCmd(Tcl_Interp* interp, int objc, Tcl_Obj* const objv[]) break; } +#if LIBCURL_VERSION_NUM >= 0x071304 && LIBCURL_VERSION_NUM <= 0x071307 + /* FTP_PROXY workaround for Snow Leopard */ + if (proxy == NULL && strncmp(theURL, "ftp:", 4) == 0) { + proxy = getenv("FTP_PROXY"); + } +#endif + if (proxy) { + theCurlCode = curl_easy_setopt(theHandle, CURLOPT_PROXY, proxy); + if (theCurlCode != CURLE_OK) { + theResult = SetResultFromCurlErrorCode(interp, theCurlCode); + break; + } + } + if (no_proxy) { + theCurlCode = curl_easy_setopt(theHandle, CURLOPT_NOPROXY, no_proxy); + if (theCurlCode != CURLE_OK) { + theResult = SetResultFromCurlErrorCode(interp, theCurlCode); + break; + } + } + /* -L option */ theCurlCode = curl_easy_setopt(theHandle, CURLOPT_FOLLOWLOCATION, 1); if (theCurlCode != CURLE_OK) {