libzypp: zypp/media/MediaCurl.cc Source File

libzypp 17.38.11

MediaCurl.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
12
13#include <iostream>
14#include <chrono>
15#include <list>
16
17#include <zypp-core/base/Logger.h>
18#include <zypp-core/ExternalProgram.h>
19#include <zypp-core/base/String.h>
20#include <zypp-core/base/Gettext.h>
21#include <utility>
22#include <zypp-core/parser/Sysconfig>
23#include <zypp-core/base/Gettext.h>
24
25#include <zypp/media/MediaCurl.h>
26#include <zypp-core/ng/base/private/linuxhelpers_p.h>
27#include <zypp-core/base/userrequestexception.h>
28#include <zypp-curl/ProxyInfo>
29#include <zypp-curl/auth/CurlAuthData>
30#include <zypp-media/auth/CredentialManager>
31#include <zypp-curl/CurlConfig>
32#include <zypp/Target.h>
33#include <zypp/ZYppFactory.h>
34#include <zypp/ZConfig.h>
35#include <zypp/zypp_detail/ZYppImpl.h> // for zypp_poll
36
37#include <cstdlib>
38#include <sys/types.h>
39#include <sys/stat.h>
40#include <sys/mount.h>
41#include <dirent.h>
42#include <unistd.h>
43#include <glib.h>
44
45#include "detail/OptionalDownloadProgressReport.h"
46
47using std::endl;
48
49namespace internal {
50 using namespace zypp;
51 struct ProgressData
52 {
53 ProgressData( AutoFILE file, CURL *curl, time_t timeout = 0, zypp::Url url = zypp::Url(),
54 zypp::ByteCount expectedFileSize_r = 0,
55 zypp::callback::SendReport<zypp::media::DownloadProgressReport> *_report = nullptr );
56
57 void updateStats( curl_off_t dltotal = 0.0, curl_off_t dlnow = 0.0 );
58
59 int reportProgress() const;
60
61 CURL * curl()
62 { return _curl; }
63
64 bool timeoutReached() const
65 { return _timeoutReached; }
66
67 bool fileSizeExceeded() const
68 { return _fileSizeExceeded; }
69
70 ByteCount expectedFileSize() const
71 { return _expectedFileSize; }
72
73 void expectedFileSize( ByteCount newval_r )
74 { _expectedFileSize = newval_r; }
75
76 zypp::Url url() const
77 { return _url; }
78
79 FILE* file()
80 { return _file.value(); }
81
82 size_t writeBytes( char *ptr, ByteCount bytes );
83
84 ByteCount bytesWritten() const {
85 return _bytesWritten;
86 }
87
88 private:
89 CURL * _curl;
90 AutoFILE _file;
91 zypp::Url _url;
92 time_t _timeout;
93 bool _timeoutReached;
94 bool _fileSizeExceeded;
95 ByteCount _expectedFileSize;
96 zypp::callback::SendReport<zypp::media::DownloadProgressReport> *report;
97
98 time_t _timeStart = 0;
99 time_t _timeLast = 0;
100 time_t _timeRcv = 0;
101 time_t _timeNow = 0;
102
103 curl_off_t _dnlTotal = 0.0;
104 curl_off_t _dnlLast = 0.0;
105 curl_off_t _dnlNow = 0.0;
106
107 ByteCount _bytesWritten = 0;
108
109 int _dnlPercent= 0;
110
111 double _drateTotal= 0.0;
112 double _drateLast = 0.0;
113 };
114
115
116
117 ProgressData::ProgressData(AutoFILE file, CURL *curl, time_t timeout, Url url, ByteCount expectedFileSize_r, zypp::callback::SendReport< zypp::media::DownloadProgressReport> *_report)
118 : _curl( curl )
119 , _file( std::move(file) )
120 , _url(std::move( url ))
121 , _timeout( timeout )
122 , _timeoutReached( false )
123 , _fileSizeExceeded ( false )
124 , _expectedFileSize( expectedFileSize_r )
125 , report( _report )
126 {}
127
128 void ProgressData::updateStats( curl_off_t dltotal, curl_off_t dlnow )
129 {
130 time_t now = _timeNow = time(0);
131
132 // If called without args (0.0), recompute based on the last values seen
133 if ( dltotal && dltotal != _dnlTotal )
134 _dnlTotal = dltotal;
135
136 if ( dlnow && dlnow != _dnlNow )
137 {
138 _timeRcv = now;
139 _dnlNow = dlnow;
140 }
141
142 // init or reset if time jumps back
143 if ( !_timeStart || _timeStart > now )
144 _timeStart = _timeLast = _timeRcv = now;
145
146 // timeout condition
147 if ( _timeout )
148 _timeoutReached = ( (now - _timeRcv) > _timeout );
149
150 // percentage:
151 if ( _dnlTotal )
152 _dnlPercent = int( _dnlNow * 100 / _dnlTotal );
153
154 // download rates:
155 _drateTotal = double(_dnlNow) / std::max( int(now - _timeStart), 1 );
156
157 if ( _timeLast < now )
158 {
159 _drateLast = double(_dnlNow - _dnlLast) / int(now - _timeLast);
160 // start new period
161 _timeLast = now;
162 _dnlLast = _dnlNow;
163 }
164 else if ( _timeStart == _timeLast )
165 _drateLast = _drateTotal;
166 }
167
168 int ProgressData::reportProgress() const
169 {
170 if ( _fileSizeExceeded )
171 return 1;
172 if ( _timeoutReached )
173 return 1; // no-data timeout
174 if ( report && !(*report)->progress( _dnlPercent, _url, _drateTotal, _drateLast ) )
175 return 1; // user requested abort
176 return 0;
177 }
178
179 size_t ProgressData::writeBytes(char *ptr, ByteCount bytes)
180 {
181 // check if the downloaded data is already bigger than what we expected
182 _fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < ( _bytesWritten + bytes );
183 if ( _fileSizeExceeded )
184 return 0;
185
186 auto written = fwrite( ptr, 1, bytes, _file );
187 _bytesWritten += written;
188 return written;
189 }
190
194 class MediaCurlExceptionMayRetryInternaly : public media::MediaCurlException
195 {
196 public:
197 MediaCurlExceptionMayRetryInternaly( const Url & url_r,
198 const std::string & err_r,
199 const std::string & msg_r )
200 : media::MediaCurlException( url_r, err_r, msg_r )
201 {}
202 //~MediaCurlExceptionMayRetryInternaly() noexcept {}
203 };
204
205}
206
207
208using namespace internal;
209using namespace zypp::base;
210
211namespace zypp {
212
213 namespace media {
214
215Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
216
217// we use this define to unbloat code as this C setting option
218// and catching exception is done frequently.
220#define SET_OPTION(opt,val) do { \
221 ret = curl_easy_setopt ( curl, opt, val ); \
222 if ( ret != 0) { \
223 ZYPP_THROW(MediaCurlSetOptException(_origin.at(rData.mirror).url(), _curlError)); \
224 } \
225 } while ( false )
226
227#define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
228#define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
229#define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
230
231MediaCurl::MediaCurl(const MirroredOrigin &origin_r,
232 const Pathname & attach_point_hint_r )
233 : MediaNetworkCommonHandler( origin_r, attach_point_hint_r,
234 "/", // urlpath at attachpoint
235 true ), // does_download
236 _customHeaders(0L)
237{
238 _multi = curl_multi_init();
239
240 _curlError[0] = '\0';
241
242 MIL << "MediaCurl::MediaCurl(" << origin_r.authority().url() << ", " << attach_point_hint_r << ")" << endl;
243
244 globalInitCurlOnce();
245
246 if( !attachPoint().empty())
247 {
248 PathInfo ainfo(attachPoint());
249 Pathname apath(attachPoint() + "XXXXXX");
250 char *atemp = ::strdup( apath.asString().c_str());
251 char *atest = NULL;
252 if( !ainfo.isDir() || !ainfo.userMayRWX() ||
253 atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
254 {
255 WAR << "attach point " << ainfo.path()
256 << " is not useable for " << origin_r.authority().url().getScheme() << endl;
257 setAttachPoint("", true);
258 }
259 else if( atest != NULL)
260 ::rmdir(atest);
261
262 if( atemp != NULL)
263 ::free(atemp);
264 }
265}
266
267MediaCurl::~MediaCurl()
268{
269 try { release(); } catch(...) {}
270 if (_multi)
271 curl_multi_cleanup(_multi);
272}
273
274 void MediaCurl::setCookieFile( const Pathname &fileName )
275{
276 _cookieFile = fileName;
277}
278
279 void MediaCurl::setCurlError(const char* error)
280{
281 // FIXME(dmllr): Use strlcpy if available for better performance
282 strncpy(_curlError, error, sizeof(_curlError)-1);
283 _curlError[sizeof(_curlError)-1] = '\0';
284}
285
287
288 void MediaCurl::checkProtocol(const Url &url) const
289{
290 curl_version_info_data *curl_info = NULL;
291 curl_info = curl_version_info(CURLVERSION_NOW);
292 // curl_info does not need any free (is static)
293 if (curl_info->protocols)
294 {
295 const char * const *proto = nullptr;
296 std::string scheme( url.getScheme());
297 bool found = false;
298 for(proto=curl_info->protocols; !found && *proto; ++proto)
299 {
300 if( scheme == std::string((const char *)*proto))
301 found = true;
302 }
303 if( !found)
304 {
305 std::string msg("Unsupported protocol '");
306 msg += scheme;
307 msg += "'";
308 ZYPP_THROW(MediaBadUrlException( url, msg));
309 }
310 }
311}
312
313 void MediaCurl::setupEasy(RequestData &rData, TransferSettings &settings )
314{
315 CURL *curl = rData.curl;
316
317 // kill old settings
318 curl_easy_reset ( curl );
319
320 ::internal::setupZYPP_MEDIA_CURL_DEBUG( curl );
321 curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
322 curl_easy_setopt(curl, CURLOPT_HEADERDATA, &_lastRedirect);
323 CURLcode ret = curl_easy_setopt( curl, CURLOPT_ERRORBUFFER, _curlError );
324 if ( ret != 0 ) {
325 ZYPP_THROW(MediaCurlSetOptException( _origin.at(rData.mirror).url(), "Error setting error buffer"));
326 }
327
328 SET_OPTION(CURLOPT_FAILONERROR, 1L);
329 SET_OPTION(CURLOPT_NOSIGNAL, 1L);
330
332 switch ( env::ZYPP_MEDIA_CURL_IPRESOLVE() )
333 {
334 case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
335 case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
336 }
337
341 SET_OPTION(CURLOPT_CONNECTTIMEOUT, settings.connectTimeout());
342 // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
343 // just in case curl does not trigger its progress callback frequently
344 // enough.
345 if ( settings.timeout() )
346 {
347 SET_OPTION(CURLOPT_TIMEOUT, 3600L);
348 }
349
350 // follow any Location: header that the server sends as part of
351 // an HTTP header (#113275)
352 SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
353 // 3 redirects seem to be too few in some cases (bnc #465532)
354 SET_OPTION(CURLOPT_MAXREDIRS, 6L);
355
356 if ( _origin.at(rData.mirror).url().getScheme() == "https" )
357 {
358 if ( :: internal::setCurlRedirProtocols ( curl ) != CURLE_OK ) {
359 ZYPP_THROW(MediaCurlSetOptException(_origin.at(rData.mirror).url(), _curlError));
360 }
361
362 if( settings.verifyPeerEnabled() ||
363 settings.verifyHostEnabled() )
364 {
365 SET_OPTION(CURLOPT_CAPATH, settings.certificateAuthoritiesPath().c_str());
366 }
367
368 if( ! settings.clientCertificatePath().empty() )
369 {
370 SET_OPTION(CURLOPT_SSLCERT, settings.clientCertificatePath().c_str());
371 }
372 if( ! settings.clientKeyPath().empty() )
373 {
374 SET_OPTION(CURLOPT_SSLKEY, settings.clientKeyPath().c_str());
375 }
376
377#ifdef CURLSSLOPT_ALLOW_BEAST
378 // see bnc#779177
379 ret = curl_easy_setopt( curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
380 if ( ret != 0 ) {
381 disconnectFrom();
382 ZYPP_THROW(MediaCurlSetOptException(_origin.at(rData.mirror).url(), _curlError));
383 }
384#endif
385 SET_OPTION(CURLOPT_SSL_VERIFYPEER, settings.verifyPeerEnabled() ? 1L : 0L);
386 SET_OPTION(CURLOPT_SSL_VERIFYHOST, settings.verifyHostEnabled() ? 2L : 0L);
387 // bnc#903405 - POODLE: libzypp should only talk TLS
388 SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
389 }
390
391 SET_OPTION(CURLOPT_USERAGENT, settings.userAgentString().c_str() );
392
393 /* Fixes bsc#1174011 "auth=basic ignored in some cases"
394 * We should proactively add the password to the request if basic auth is configured
395 * and a password is available in the credentials but not in the URL.
396 *
397 * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
398 * and ask the server first about the auth method
399 */
400 if ( settings.authType() == "basic" && not settings.username().empty() && settings.password().empty() ) {
401 CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
402 const auto cred = cm.getCred( _origin.at(rData.mirror).url() );
403 if ( cred && cred->valid() ) {
404 settings.setPassword(cred->password());
405 }
406 }
407
408 /*---------------------------------------------------------------*
409 CURLOPT_USERPWD: [user name]:[password]
410
411 Url::username/password -> CURLOPT_USERPWD
412 If not provided, anonymous FTP identification
413 *---------------------------------------------------------------*/
414
415 if ( settings.hasCredentials() )
416 {
417 settings.logCredentials( DBG << "Credentials: " ) << endl;
418 SET_OPTION(CURLOPT_USERNAME, settings.username().c_str());
419 SET_OPTION(CURLOPT_PASSWORD, settings.password().c_str());
420 std::string use_auth = settings.authType();
421 if (use_auth.empty())
422 use_auth = "digest,basic"; // our default
423 long auth = CurlAuthData::auth_type_str2long(use_auth);
424 if( auth != CURLAUTH_NONE)
425 {
426 DBG << "Enabling HTTP authentication methods: " << use_auth
427 << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
428 SET_OPTION(CURLOPT_HTTPAUTH, auth);
429 }
430 }
431
432 if ( settings.proxyEnabled() && ! settings.proxy().empty() )
433 {
434 DBG << "Proxy: '" << settings.proxy() << "'" << endl;
435 SET_OPTION(CURLOPT_PROXY, settings.proxy().c_str());
436 SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
437 /*---------------------------------------------------------------*
438 * CURLOPT_PROXYUSERPWD: [user name]:[password]
439 *
440 * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
441 * If not provided, $HOME/.curlrc is evaluated
442 *---------------------------------------------------------------*/
443
444 std::string proxyuserpwd = settings.proxyUserPassword();
445
446 if ( proxyuserpwd.empty() )
447 {
448 CurlConfig curlconf;
449 CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
450 if ( curlconf.proxyuserpwd.empty() )
451 DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
452 else
453 {
454 proxyuserpwd = curlconf.proxyuserpwd;
455 DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
456 }
457 }
458 else
459 {
460 DBG << "Proxy: using provided proxy-user '" << settings.proxyUsername() << "'" << endl;
461 }
462
463 if ( ! proxyuserpwd.empty() )
464 {
465 SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
466 }
467 }
468#if CURLVERSION_AT_LEAST(7,19,4)
469 else if ( settings.proxy() == EXPLICITLY_NO_PROXY )
470 {
471 // Explicitly disabled in URL (see fillSettingsFromUrl()).
472 // This should also prevent libcurl from looking into the environment.
473 DBG << "Proxy: explicitly NOPROXY" << endl;
474 SET_OPTION(CURLOPT_NOPROXY, "*");
475 }
476#endif
477 else
478 {
479 DBG << "Proxy: not explicitly set, libcurl may look into the environment" << endl;
480 }
481
483 if ( settings.minDownloadSpeed() != 0 )
484 {
485 SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, settings.minDownloadSpeed());
486 // default to 10 seconds at low speed
487 SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
488 }
489
490#if CURLVERSION_AT_LEAST(7,15,5)
491 if ( settings.maxDownloadSpeed() != 0 )
492 SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, settings.maxDownloadSpeed());
493#endif
494
495 /*---------------------------------------------------------------*
496 *---------------------------------------------------------------*/
497
498 _currentCookieFile = _cookieFile.asString();
499 if ( ::geteuid() == 0 || PathInfo(_currentCookieFile).owner() == ::geteuid() )
500 filesystem::assert_file_mode( _currentCookieFile, 0600 );
501
502 const auto &cookieFileParam = _origin.at(rData.mirror).url().getQueryParam( "cookies" );
503 if ( !cookieFileParam.empty() && str::strToBool( cookieFileParam, true ) )
504 SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
505 else
506 MIL << "No cookies requested" << endl;
507 SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
508 SET_OPTION(CURLOPT_XFERINFOFUNCTION, &progressCallback );
509 SET_OPTION(CURLOPT_NOPROGRESS, 0L);
510
511#if CURLVERSION_AT_LEAST(7,18,0)
512 // bnc #306272
513 SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
514#endif
515 // Append settings custom headers to curl.
516 // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
517 if ( _customHeaders ) {
518 curl_slist_free_all(_customHeaders);
519 _customHeaders = 0L;
520 }
521 for ( const auto &header : settings.headers() ) {
522 _customHeaders = curl_slist_append(_customHeaders, header.c_str());
523 if ( !_customHeaders )
524 ZYPP_THROW(MediaCurlInitException(_origin.at(rData.mirror).url()));
525 }
526 SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
527}
528
530
531 void MediaCurl::disconnectFrom()
532{
533 if ( _customHeaders ) {
534 curl_slist_free_all(_customHeaders);
535 _customHeaders = 0L;
536 }
537
538 // clear effective settings
539 clearTransferSettings();
540}
541
543
544 void MediaCurl::releaseFrom( const std::string & ejectDev )
545{
546 disconnect();
547}
548
550
551 void MediaCurl::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
552{
553 // we need a non const pointer to work around the current API
554 auto that = const_cast<MediaCurl *>(this);
555 std::exception_ptr lastErr;
556 const auto &mirrOrder = mirrorOrder (srcFile);
557 for ( unsigned mirr : mirrOrder ) {
558 try {
559 return that->getFileCopyFromMirror ( mirr, srcFile, target );
560
561 } catch (MediaException & excpt_r) {
562 if ( !canTryNextMirror ( excpt_r ) )
563 ZYPP_RETHROW(excpt_r);
564
565 that->deprioritizeMirror( mirr );
566
567 lastErr = ZYPP_FWD_CURRENT_EXCPT();
568 }
569 }
570 if ( lastErr ) {
571 ZYPP_RETHROW( lastErr );
572 }
573
574 // should not happen
575 ZYPP_THROW( MediaException("No usable mirror available.") );
576
577}
578
579 void MediaCurl::getFileCopyFromMirror(const int mirror, const OnMediaLocation &srcFile, const Pathname &target)
580{
581 const auto &filename = srcFile.filename();
582
583 // Optional files will send no report until data are actually received (we know it exists).
584 OptionalDownloadProgressReport reportfilter( srcFile.optional() );
585 callback::SendReport<DownloadProgressReport> report;
586
587 auto &endpoint = _origin[mirror];
588 auto &settings = endpoint.getConfig<TransferSettings>(MIRR_SETTINGS_KEY.data());
589
590 AutoDispose<CURL*> curl( curl_easy_init(), []( CURL *hdl ) { if ( hdl ) { curl_easy_cleanup(hdl); } } );
591
592 RequestData rData;
593 rData.mirror = mirror;
594 rData.curl = curl.value ();
595
596 if( !endpoint.url().isValid() )
597 ZYPP_THROW(MediaBadUrlException(endpoint.url()));
598
599 if( endpoint.url().getHost().empty() )
600 ZYPP_THROW(MediaBadUrlEmptyHostException(endpoint.url()));
601
602 Url fileurl( getFileUrl(mirror, filename) );
603
604 bool firstAuth = true; // bsc#1210870: authenticate must not return stored credentials more than once.
605 unsigned internalTry = 0;
606 static constexpr unsigned maxInternalTry = 3;
607
608 do
609 {
610 try
611 {
612 Pathname dest = target.absolutename();
613 if( assert_dir( dest.dirname() ) )
614 {
615 DBG << "assert_dir " << dest.dirname() << " failed" << endl;
616 ZYPP_THROW( MediaSystemException(fileurl, "System error on " + dest.dirname().asString()) );
617 }
618
619 ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
620 AutoFILE file;
621 {
622 AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
623 if( ! buf )
624 {
625 ERR << "out of memory for temp file name" << endl;
626 ZYPP_THROW(MediaSystemException(fileurl, "out of memory for temp file name"));
627 }
628
629 AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
630 if( tmp_fd == -1 )
631 {
632 ERR << "mkstemp failed for file '" << destNew << "'" << endl;
633 ZYPP_THROW(MediaWriteException(destNew));
634 }
635 destNew = ManagedFile( (*buf), filesystem::unlink );
636
637 file = ::fdopen( tmp_fd, "we" );
638 if ( ! file )
639 {
640 ERR << "fopen failed for file '" << destNew << "'" << endl;
641 ZYPP_THROW(MediaWriteException(destNew));
642 }
643 tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
644 }
645
646 DBG << "dest: " << dest << endl;
647 DBG << "temp: " << destNew << endl;
648
649 setupEasy( rData, settings );
650
651 // set IFMODSINCE time condition (no download if not modified)
652 if( PathInfo(target).isExist() )
653 {
654 curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
655 curl_easy_setopt(curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
656 }
657 else
658 {
659 curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
660 curl_easy_setopt(curl, CURLOPT_TIMEVALUE, 0L);
661 }
662
663 zypp_defer{
664 curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
665 curl_easy_setopt(curl, CURLOPT_TIMEVALUE, 0L);
666 };
667
668 DBG << srcFile.filename().asString() << endl;
669
670 DBG << "URL: " << fileurl.asString() << endl;
671 // Use URL without options and without username and passwd
672 // (some proxies dislike them in the URL).
673 // Curl seems to need the just scheme, hostname and a path;
674 // the rest was already passed as curl options (in attachTo).
675 Url curlUrl( clearQueryString(fileurl) );
676
677 //
678 // See also Bug #154197 and ftp url definition in RFC 1738:
679 // The url "ftp://user@host/foo/bar/file" contains a path,
680 // that is relative to the user's home.
681 // The url "ftp://user@host//foo/bar/file" (or also with
682 // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
683 // contains an absolute path.
684 //
685 _lastRedirect.clear();
686 std::string urlBuffer( curlUrl.asString());
687 CURLcode ret = curl_easy_setopt( curl, CURLOPT_URL,
688 urlBuffer.c_str() );
689 if ( ret != 0 ) {
690 ZYPP_THROW(MediaCurlSetOptException(fileurl, _curlError));
691 }
692
693 // Set callback and perform.
694 internal::ProgressData progressData( file, curl, settings.timeout(), fileurl, srcFile.downloadSize(), &report );
695
696 ret = curl_easy_setopt( curl, CURLOPT_WRITEDATA, &progressData );
697 if ( ret != 0 ) {
698 ZYPP_THROW(MediaCurlSetOptException(fileurl, _curlError));
699 }
700
701 ret = curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, &MediaCurl::writeCallback );
702 if ( ret != 0 ) {
703 ZYPP_THROW(MediaCurlSetOptException(fileurl, _curlError));
704 }
705
706 report->start(fileurl, dest);
707
708 if ( curl_easy_setopt( curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
709 WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
710 }
711
712 ret = executeCurl( rData );
713
714 // flush buffers
715 fflush ( file );
716
717 #if CURLVERSION_AT_LEAST(7,19,4)
718 // bnc#692260: If the client sends a request with an If-Modified-Since header
719 // with a future date for the server, the server may respond 200 sending a
720 // zero size file.
721 // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
722 if ( ftell(file) == 0 && ret == 0 )
723 {
724 long httpReturnCode = 33;
725 if ( curl_easy_getinfo( curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
726 {
727 long conditionUnmet = 33;
728 if ( curl_easy_getinfo( curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
729 {
730 WAR << "TIMECONDITION unmet - retry without." << endl;
731 curl_easy_setopt( curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
732 curl_easy_setopt( curl, CURLOPT_TIMEVALUE, 0L);
733 ret = executeCurl( rData );
734 }
735 }
736 }
737 #endif
738
739 if ( curl_easy_setopt( curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
740 WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
741 }
742
743 if ( ret != 0 ) {
744 ERR << "curl error: " << ret << ": " << _curlError
745 << ", temp file size " << ftell(file)
746 << " bytes." << endl;
747
748 // the timeout is determined by the progress data object
749 // which holds whether the timeout was reached or not,
750 // otherwise it would be a user cancel
751
752 if ( progressData.fileSizeExceeded() )
753 ZYPP_THROW(MediaFileSizeExceededException(fileurl, progressData.expectedFileSize()));
754
755 evaluateCurlCode( rData, srcFile.filename(), ret, progressData.timeoutReached() );
756 }
757
758 long httpReturnCode = 0;
759 CURLcode infoRet = curl_easy_getinfo(curl,
760 CURLINFO_RESPONSE_CODE,
761 &httpReturnCode);
762 bool modified = true;
763 if (infoRet == CURLE_OK)
764 {
765 DBG << "HTTP response: " + str::numstring(httpReturnCode);
766 if ( httpReturnCode == 304
767 || ( httpReturnCode == 213 && (endpoint.url().getScheme() == "ftp" || endpoint.url().getScheme() == "tftp") ) ) // not modified
768 {
769 DBG << " Not modified.";
770 modified = false;
771 }
772 DBG << endl;
773 }
774 else
775 {
776 WAR << "Could not get the response code." << endl;
777 }
778
779 if (modified || infoRet != CURLE_OK)
780 {
781 // apply umask
782 if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
783 {
784 ERR << "Failed to chmod file " << destNew << endl;
785 }
786
787 file.resetDispose(); // we're going to close it manually here
788 if ( ::fclose( file ) )
789 {
790 ERR << "Fclose failed for file '" << destNew << "'" << endl;
791 ZYPP_THROW(MediaWriteException(destNew));
792 }
793
794 // move the temp file into dest
795 if ( rename( destNew, dest ) != 0 ) {
796 ERR << "Rename failed" << endl;
797 ZYPP_THROW(MediaWriteException(dest));
798 }
799 destNew.resetDispose(); // no more need to unlink it
800 }
801
802 DBG << "done: " << PathInfo(dest) << endl;
803 break; // success!
804 }
805 catch (MediaUnauthorizedException & ex_r)
806 {
807 if ( authenticate( endpoint.url(), settings, ex_r.hint(), firstAuth) ) {
808 firstAuth = false; // must not return stored credentials again
809 continue; // retry
810 }
811
812 report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
813 ZYPP_RETHROW(ex_r);
814 }
815 // unexpected exception
816 catch (MediaException & excpt_r)
817 {
818 if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
819 ++internalTry;
820 if ( internalTry < maxInternalTry ) {
821 // just report (NO_ERROR); no interactive request to the user
822 report->problem(fileurl, media::DownloadProgressReport::NO_ERROR, excpt_r.asUserHistory()+_("Will try again..."));
823 continue; // retry
824 }
825 excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
826 }
827
828 media::DownloadProgressReport::Error reason = media::DownloadProgressReport::ERROR;
829 if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
830 typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
831 {
832 reason = media::DownloadProgressReport::NOT_FOUND;
833 }
834 report->finish(fileurl, reason, excpt_r.asUserHistory());
835 ZYPP_RETHROW(excpt_r);
836 }
837 } while ( true );
838
839 report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
840}
841
843
844 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
845{
846 // we need a non const pointer to work around the current API
847 auto that = const_cast<MediaCurl *>(this);
848
849 std::exception_ptr lastErr;
850 for ( int i : mirrorOrder( OnMediaLocation(filename).setMirrorsAllowed(false) )) {
851 try {
852 return that->doGetDoesFileExist( i, filename );
853
854 } catch (MediaException & excpt_r) {
855 if ( !canTryNextMirror ( excpt_r ) )
856 ZYPP_RETHROW(excpt_r);
857
858 that->deprioritizeMirror( i );
859
860 lastErr = ZYPP_FWD_CURRENT_EXCPT();
861 }
862 }
863 if ( lastErr ) {
864 try {
865 ZYPP_RETHROW( lastErr );
866 } catch ( const MediaFileNotFoundException &e ) {
867 // on file not found we return false
868 ZYPP_CAUGHT(e);
869 return false;
870 }
871 }
872 return false;
873}
874
876
877 void MediaCurl::evaluateCurlCode(RequestData &rData,
878 const Pathname &filename,
879 CURLcode code,
880 bool timeout_reached) const
881{
882 if ( code != 0 )
883 {
884 const auto &baseMirr = _origin[rData.mirror];
885 Url url;
886 if (filename.empty())
887 url = baseMirr.url();
888 else
889 url = getFileUrl(rData.mirror, filename);
890
891 std::string err;
892 {
893 switch ( code )
894 {
895 case CURLE_UNSUPPORTED_PROTOCOL:
896 err = " Unsupported protocol";
897 if ( !_lastRedirect.empty() )
898 {
899 err += " or redirect (";
900 err += _lastRedirect;
901 err += ")";
902 }
903 break;
904 case CURLE_URL_MALFORMAT:
905 case CURLE_URL_MALFORMAT_USER:
906 err = " Bad URL";
907 break;
908 case CURLE_LOGIN_DENIED:
909 ZYPP_THROW(
910 MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
911 break;
912 case CURLE_HTTP_RETURNED_ERROR:
913 {
914 long httpReturnCode = 0;
915 CURLcode infoRet = curl_easy_getinfo( rData.curl,
916 CURLINFO_RESPONSE_CODE,
917 &httpReturnCode );
918 if ( infoRet == CURLE_OK )
919 {
920 std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
921 switch ( httpReturnCode )
922 {
923 case 401:
924 {
925 std::string auth_hint = getAuthHint( rData.curl );
926
927 DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
928 DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
929
930 ZYPP_THROW(MediaUnauthorizedException(
931 url, "Login failed.", _curlError, auth_hint
932 ));
933 }
934
935 case 502: // bad gateway (bnc #1070851)
936 case 503: // service temporarily unavailable (bnc #462545)
937 ZYPP_THROW(MediaTemporaryProblemException(url));
938 case 504: // gateway timeout
939 ZYPP_THROW(MediaTimeoutException(url));
940 case 403:
941 {
942 std::string msg403;
943 if ( url.getHost().find(".suse.com") != std::string::npos )
944 msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
945 else if (url.asString().find("novell.com") != std::string::npos)
946 msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
947 ZYPP_THROW(MediaForbiddenException(url, msg403));
948 }
949 case 404:
950 case 410:
951 ZYPP_THROW(MediaFileNotFoundException(baseMirr.url(), filename));
952 }
953
954 DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
955 ZYPP_THROW(MediaCurlException(url, msg, _curlError));
956 }
957 else
958 {
959 std::string msg = "Unable to retrieve HTTP response:";
960 DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
961 ZYPP_THROW(MediaCurlException(url, msg, _curlError));
962 }
963 }
964 break;
965 case CURLE_FTP_COULDNT_RETR_FILE:
966#if CURLVERSION_AT_LEAST(7,16,0)
967 case CURLE_REMOTE_FILE_NOT_FOUND:
968#endif
969 case CURLE_FTP_ACCESS_DENIED:
970 case CURLE_TFTP_NOTFOUND:
971 err = "File not found";
972 ZYPP_THROW(MediaFileNotFoundException(baseMirr.url(), filename));
973 break;
974 case CURLE_BAD_PASSWORD_ENTERED:
975 case CURLE_FTP_USER_PASSWORD_INCORRECT:
976 err = "Login failed";
977 break;
978 case CURLE_COULDNT_RESOLVE_PROXY:
979 case CURLE_COULDNT_RESOLVE_HOST:
980 case CURLE_COULDNT_CONNECT:
981 case CURLE_FTP_CANT_GET_HOST:
982 err = "Connection failed";
983 break;
984 case CURLE_WRITE_ERROR:
985 err = "Write error";
986 break;
987 case CURLE_PARTIAL_FILE:
988 case CURLE_OPERATION_TIMEDOUT:
989 timeout_reached = true; // fall though to TimeoutException
990 // fall though...
991 case CURLE_ABORTED_BY_CALLBACK:
992 if( timeout_reached )
993 {
994 err = "Timeout reached";
995 ZYPP_THROW(MediaTimeoutException(url));
996 }
997 else
998 {
999 err = "User abort";
1000 }
1001 break;
1002
1003 default:
1004 err = "Curl error " + str::numstring( code );
1005 break;
1006 }
1007
1008 // uhm, no 0 code but unknown curl exception
1009 ZYPP_THROW(MediaCurlException(url, err, _curlError));
1010 }
1011 }
1012 else
1013 {
1014 // actually the code is 0, nothing happened
1015 }
1016}
1017
1019
1020 bool MediaCurl::doGetDoesFileExist( const int mirror, const Pathname & filename )
1021{
1022 DBG << filename.asString() << endl;
1023
1024 AutoDispose<CURL*> curl( curl_easy_init(), []( CURL *hdl ) { if ( hdl ) { curl_easy_cleanup(hdl); } } );
1025 RequestData rData;
1026 rData.mirror = mirror;
1027 rData.curl = curl.value ();
1028
1029 auto &endpoint = _origin[mirror];
1030
1031 if( !endpoint.url().isValid() )
1032 ZYPP_THROW(MediaBadUrlException(endpoint.url()));
1033
1034 if( endpoint.url().getHost().empty() )
1035 ZYPP_THROW(MediaBadUrlEmptyHostException(endpoint.url()));
1036
1037 Url url(getFileUrl(mirror, filename));
1038
1039 DBG << "URL: " << url.asString() << endl;
1040 // Use URL without options and without username and passwd
1041 // (some proxies dislike them in the URL).
1042 // Curl seems to need the just scheme, hostname and a path;
1043 // the rest was already passed as curl options (in attachTo).
1044 Url curlUrl( clearQueryString(url) );
1045
1046 // See also Bug #154197 and ftp url definition in RFC 1738:
1047 // The url "ftp://user@host/foo/bar/file" contains a path,
1048 // that is relative to the user's home.
1049 // The url "ftp://user@host//foo/bar/file" (or also with
1050 // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1051 // contains an absolute path.
1052 //
1053 _lastRedirect.clear();
1054 std::string urlBuffer( curlUrl.asString());
1055
1056 CURLcode ok;
1057 bool canRetry = true;
1058 bool firstAuth = true;
1059 auto &settings = endpoint.getConfig<TransferSettings>( MIRR_SETTINGS_KEY.data() );
1060
1061 while ( canRetry ) {
1062 canRetry = false;
1063 setupEasy( rData, settings );
1064
1065 CURLcode ret = curl_easy_setopt( curl, CURLOPT_URL,
1066 urlBuffer.c_str() );
1067 if ( ret != 0 ) {
1068 ZYPP_THROW(MediaCurlSetOptException(url, _curlError));
1069 }
1070
1071 AutoFILE file { ::fopen( "/dev/null", "w" ) };
1072 if ( !file ) {
1073 ERR << "fopen failed for /dev/null" << endl;
1074 ZYPP_THROW(MediaWriteException("/dev/null"));
1075 }
1076
1077 ret = curl_easy_setopt( curl, CURLOPT_WRITEDATA, (*file) );
1078 if ( ret != 0 ) {
1079 ZYPP_THROW(MediaCurlSetOptException(url, _curlError));
1080 }
1081
1082 // If no head requests allowed (?head_requests=no):
1083 // Instead of returning no data with NOBODY, we return
1084 // little data, that works with broken servers, and
1085 // works for ftp as well, because retrieving only headers
1086 // ftp will return always OK code ?
1087 // See http://curl.haxx.se/docs/knownbugs.html #58
1088 const bool doHeadRequest = (endpoint.url().getScheme() == "http" || endpoint.url().getScheme() == "https") && settings.headRequestsAllowed();
1089 if ( doHeadRequest ) {
1090 curl_easy_setopt( curl, CURLOPT_NOBODY, 1L );
1091 } else {
1092 curl_easy_setopt( curl, CURLOPT_RANGE, "0-1" );
1093 }
1094
1095 try {
1096 ok = const_cast<MediaCurl *>(this)->executeCurl( rData );
1097 MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
1098
1099 // as we are not having user interaction, the user can't cancel
1100 // the file existence checking, a callback or timeout return code
1101 // will be always a timeout.
1102 evaluateCurlCode( rData, filename, ok, true /* timeout */);
1103 }
1104 catch ( const MediaFileNotFoundException &e ) {
1105 // if the file did not exist then we can return false
1106 return false;
1107 }
1108 catch ( const MediaUnauthorizedException &e ) {
1109 if ( authenticate( endpoint.url(), settings, e.hint(), firstAuth ) ) {
1110 firstAuth = false;
1111 canRetry = true;
1112 continue;
1113 }
1114 }
1115
1116 // exists
1117 return ( ok == CURLE_OK );
1118 }
1119
1120 return false;
1121}
1122
1124//
1125 int MediaCurl::aliveCallback( void *clientp, curl_off_t /*dltotal*/, curl_off_t dlnow, curl_off_t /*ultotal*/, curl_off_t /*ulnow*/ )
1126{
1127 internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1128 if( pdata )
1129 {
1130 // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1131 // prevent a percentage raise while downloading a metalink file. Download
1132 // activity however is indicated by propagating the download rate (via dlnow).
1133 pdata->updateStats( 0.0, dlnow );
1134 return pdata->reportProgress();
1135 }
1136 return 0;
1137}
1138
1139 int MediaCurl::progressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
1140{
1141 internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1142 if( pdata )
1143 {
1144 // work around curl bug that gives us old data
1145 long httpReturnCode = 0;
1146 if ( curl_easy_getinfo( pdata->curl(), CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 ) {
1147 return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1148 }
1149 pdata->updateStats( dltotal, dlnow );
1150 return pdata->reportProgress();
1151 }
1152 return 0;
1153}
1154
1155 size_t MediaCurl::writeCallback( char *ptr, size_t size, size_t nmemb, void *userdata )
1156{
1157 internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( userdata );
1158 if( pdata ) {
1159 return pdata->writeBytes ( ptr, size * nmemb );
1160 }
1161 return 0;
1162}
1163
1165
1166std::string MediaCurl::getAuthHint( CURL *curl ) const
1167{
1168 long auth_info = CURLAUTH_NONE;
1169
1170 CURLcode infoRet =
1171 curl_easy_getinfo(curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1172
1173 if(infoRet == CURLE_OK)
1174 {
1175 return CurlAuthData::auth_type_long2str(auth_info);
1176 }
1177
1178 return "";
1179}
1180
1185 void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1186{
1187 internal::ProgressData *data = reinterpret_cast<internal::ProgressData *>(clientp);
1188 if ( data ) {
1189 data->expectedFileSize( expectedFileSize );
1190 }
1191}
1192
1198CURLcode MediaCurl::executeCurl( RequestData &rData )
1199{
1200 CURL *curl = rData.curl;
1201 const auto &baseUrl = _origin.at(rData.mirror);
1202
1203 if (!_multi)
1204 ZYPP_THROW(MediaCurlInitException(baseUrl.url()));
1205
1206 internal::CurlPollHelper _curlHelper(*this);
1207
1208 // add the easy handle to the multi instance
1209 if ( curl_multi_add_handle( _multi, curl ) != CURLM_OK )
1210 ZYPP_THROW(MediaCurlException( baseUrl.url(), "curl_multi_add_handle", "unknown error"));
1211
1212 // make sure the handle is cleanly removed from the multi handle
1213 OnScopeExit autoRemove([&](){ curl_multi_remove_handle( _multi, curl ); });
1214
1215 // kickstart curl, this will cause libcurl to go over the added handles and register sockets and timeouts
1216 CURLMcode mcode = _curlHelper.handleTimout();
1217 if (mcode != CURLM_OK)
1218 ZYPP_THROW(MediaCurlException( baseUrl.url(), "curl_multi_socket_action", "unknown error"));
1219
1220 bool canContinue = true;
1221 while ( canContinue ) {
1222
1223 CURLMsg *msg = nullptr;
1224 int nqueue = 0;
1225 while ((msg = curl_multi_info_read( _multi, &nqueue)) != 0) {
1226 if ( msg->msg != CURLMSG_DONE ) continue;
1227 if ( msg->easy_handle != curl ) continue;
1228
1229 return msg->data.result;
1230 }
1231
1232 // copy watched sockets in case curl changes the vector as we go over the events later
1233 std::vector<GPollFD> requestedFds = _curlHelper.socks;
1234
1235 int r = zypp_detail::zypp_poll( requestedFds, _curlHelper.timeout_ms.value_or( -1 ) );
1236 if ( r == -1 )
1237 ZYPP_THROW( MediaCurlException(baseUrl.url(), "zypp_poll() failed", "unknown error") );
1238
1239 // run curl
1240 if ( r == 0 ) {
1241 CURLMcode mcode = _curlHelper.handleTimout();
1242 if (mcode != CURLM_OK)
1243 ZYPP_THROW(MediaCurlException(baseUrl.url(), "curl_multi_socket_action", "unknown error"));
1244 } else {
1245 CURLMcode mcode = _curlHelper.handleSocketActions( requestedFds );
1246 if (mcode != CURLM_OK)
1247 ZYPP_THROW(MediaCurlException(baseUrl.url(), "curl_multi_socket_action", "unknown error"));
1248 }
1249 }
1250 return CURLE_OK;
1251}
1252
1253
1254 } // namespace media
1255} // namespace zypp
1256//
zypp_defer
#define zypp_defer
Definition AutoDispose.h:293
ZYPP_RETHROW
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition Exception.h:479
ZYPP_CAUGHT
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition Exception.h:475
ZYPP_FWD_CURRENT_EXCPT
#define ZYPP_FWD_CURRENT_EXCPT()
Drops a logline and returns the current Exception as a std::exception_ptr.
Definition Exception.h:471
ZYPP_THROW
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition Exception.h:459
ExternalProgram.h
Gettext.h
_
#define _(MSG)
Definition Gettext.h:39
Logger.h
DBG
#define DBG
Definition Logger.h:102
MIL
#define MIL
Definition Logger.h:103
ERR
#define ERR
Definition Logger.h:105
WAR
#define WAR
Definition Logger.h:104
SET_OPTION_OFFT
#define SET_OPTION_OFFT(opt, val)
Definition MediaCurl.cc:227
SET_OPTION
#define SET_OPTION(opt, val)
Definition MediaCurl.cc:220
MediaCurl.h
OptionalDownloadProgressReport.h
String.h
Target.h
ZConfig.h
ZYppFactory.h
ZYppImpl.h
internal::MediaCurlExceptionMayRetryInternaly
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g.
Definition MediaCurl.cc:195
internal::MediaCurlExceptionMayRetryInternaly::MediaCurlExceptionMayRetryInternaly
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition MediaCurl.cc:197
zypp::AutoDispose
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
Definition AutoDispose.h:95
zypp::AutoDispose::value
reference value() const
Reference to the Tp object.
Definition AutoDispose.h:138
zypp::AutoDispose::resetDispose
void resetDispose()
Set no dispose function.
Definition AutoDispose.h:171
zypp::ByteCount
Store and operate with byte count.
Definition ByteCount.h:32
zypp::Exception::asUserHistory
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition Exception.cc:140
zypp::Exception::addHistory
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition Exception.cc:189
zypp::MirroredOrigin
Manages a data source characterized by an authoritative URL and a list of mirror URLs.
Definition MirroredOrigin.h:170
zypp::MirroredOrigin::authority
const OriginEndpoint & authority() const
Definition MirroredOrigin.cc:205
zypp::OnMediaLocation
Describes a resource file located on a medium.
Definition onmedialocation.h:37
zypp::OnMediaLocation::optional
bool optional() const
Whether this is an optional resource.
Definition onmedialocation.cc:111
zypp::OnMediaLocation::downloadSize
const ByteCount & downloadSize() const
The size of the resource on the server.
Definition onmedialocation.cc:117
zypp::OnMediaLocation::filename
const Pathname & filename() const
The path to the resource on the medium.
Definition onmedialocation.cc:90
zypp::OriginEndpoint::url
const zypp::Url & url() const
Definition MirroredOrigin.cc:53
zypp::ProgressData::ProgressData
ProgressData()
Ctor no range [0,0](0).
Definition progressdata.h:174
zypp::Url
Url manipulation class.
Definition Url.h:93
zypp::Url::getScheme
std::string getScheme() const
Returns the scheme name of the URL.
Definition Url.cc:560
zypp::Url::asString
std::string asString() const
Returns a default string representation of the Url object.
Definition Url.cc:524
zypp::ZConfig::instance
static ZConfig & instance()
Singleton ctor.
Definition ZConfig.cc:788
zypp::filesystem::PathInfo
Wrapper class for stat/lstat.
Definition PathInfo.h:226
zypp::filesystem::PathInfo::userMayRWX
bool userMayRWX() const
Definition PathInfo.h:361
zypp::filesystem::PathInfo::path
const Pathname & path() const
Return current Pathname.
Definition PathInfo.h:251
zypp::filesystem::PathInfo::isDir
bool isDir() const
Definition PathInfo.h:296
zypp::filesystem::Pathname
Pathname.
Definition Pathname.h:48
zypp::filesystem::Pathname::dirname
Pathname dirname() const
Return all but the last component od this path.
Definition Pathname.h:133
zypp::filesystem::Pathname::c_str
const char * c_str() const
String representation.
Definition Pathname.h:113
zypp::filesystem::Pathname::asString
const std::string & asString() const
String representation.
Definition Pathname.h:94
zypp::filesystem::Pathname::empty
bool empty() const
Test for an empty path.
Definition Pathname.h:117
zypp::media::CredentialManager
Definition credentialmanager.h:64
zypp::media::CredentialManager::getCred
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
Definition credentialmanager.cc:335
zypp::media::CurlAuthData::auth_type_long2str
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
Definition curlauthdata.cc:104
zypp::media::CurlAuthData::auth_type_str2long
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition curlauthdata.cc:50
zypp::media::MediaBadUrlEmptyHostException
Definition mediaexception.h:308
zypp::media::MediaBadUrlException
Definition mediaexception.h:292
zypp::media::MediaCurlException
Definition mediaexception.h:365
zypp::media::MediaCurlException::MediaCurlException
MediaCurlException(const Url &url_r, std::string err_r, std::string msg_r)
Definition mediaexception.h:367
zypp::media::MediaCurlInitException
Definition mediaexception.h:227
zypp::media::MediaCurlSetOptException
Definition mediaexception.h:385
zypp::media::MediaCurl::doGetDoesFileExist
bool doGetDoesFileExist(const int mirror, const Pathname &filename)
Definition MediaCurl.cc:1020
zypp::media::MediaCurl::writeCallback
static size_t writeCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
Callback writing the data into our file.
Definition MediaCurl.cc:1155
zypp::media::MediaCurl::checkProtocol
void checkProtocol(const Url &url) const override
check the url is supported by the curl library
Definition MediaCurl.cc:288
zypp::media::MediaCurl::setCookieFile
static void setCookieFile(const Pathname &)
Definition MediaCurl.cc:274
zypp::media::MediaCurl::getDoesFileExist
bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly,...
Definition MediaCurl.cc:844
zypp::media::MediaCurl::resetExpectedFileSize
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition MediaCurl.cc:1185
zypp::media::MediaCurl::_currentCookieFile
std::string _currentCookieFile
Definition MediaCurl.h:126
zypp::media::MediaCurl::_cookieFile
static Pathname _cookieFile
Definition MediaCurl.h:127
zypp::media::MediaCurl::getAuthHint
std::string getAuthHint(CURL *curl) const
Return a comma separated list of available authentication methods supported by server.
Definition MediaCurl.cc:1166
zypp::media::MediaCurl::getFileCopyFromMirror
void getFileCopyFromMirror(const int mirror, const OnMediaLocation &srcFile, const Pathname &target)
Definition MediaCurl.cc:579
zypp::media::MediaCurl::progressCallback
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback reporting download progress.
Definition MediaCurl.cc:1139
zypp::media::MediaCurl::_lastRedirect
std::string _lastRedirect
to log/report redirections
Definition MediaCurl.h:130
zypp::media::MediaCurl::_curlError
char _curlError[CURL_ERROR_SIZE]
Definition MediaCurl.h:128
zypp::media::MediaCurl::~MediaCurl
~MediaCurl() override
Definition MediaCurl.cc:267
zypp::media::MediaCurl::evaluateCurlCode
void evaluateCurlCode(RequestData &rData, const zypp::Pathname &fileName, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition MediaCurl.cc:877
zypp::media::MediaCurl::executeCurl
CURLcode executeCurl(RequestData &rData)
Definition MediaCurl.cc:1198
zypp::media::MediaCurl::MediaCurl
MediaCurl(const MirroredOrigin &origin_r, const Pathname &attach_point_hint_r)
Definition MediaCurl.cc:231
zypp::media::MediaCurl::setCurlError
void setCurlError(const char *error)
Definition MediaCurl.cc:279
zypp::media::MediaCurl::releaseFrom
void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition MediaCurl.cc:544
zypp::media::MediaCurl::aliveCallback
static int aliveCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition MediaCurl.cc:1125
zypp::media::MediaCurl::disconnectFrom
void disconnectFrom() override
Definition MediaCurl.cc:531
zypp::media::MediaCurl::setupEasy
void setupEasy(RequestData &rData, TransferSettings &settings)
initializes the curl easy handle with the data from the url
Definition MediaCurl.cc:313
zypp::media::MediaCurl::_customHeaders
curl_slist * _customHeaders
Definition MediaCurl.h:131
zypp::media::MediaCurl::getFileCopy
void getFileCopy(const OnMediaLocation &srcFile, const Pathname &target) const override
Definition MediaCurl.cc:551
zypp::media::MediaException
Just inherits Exception to separate media exceptions.
Definition mediaexception.h:37
zypp::media::MediaFileNotFoundException
Definition mediaexception.h:164
zypp::media::MediaFileSizeExceededException
Definition mediaexception.h:513
zypp::media::MediaForbiddenException
Definition mediaexception.h:485
zypp::media::MediaHandler::_origin
MirroredOrigin _origin
Contains the authority URL and mirrors.
Definition MediaHandler.h:112
zypp::media::MediaHandler::url
Url url() const
Primary Url used.
Definition MediaHandler.h:507
zypp::media::MediaHandler::disconnect
void disconnect()
Use concrete handler to isconnect media.
Definition MediaHandler.cc:687
zypp::media::MediaHandler::release
void release(const std::string &ejectDev="")
Use concrete handler to release the media.
Definition MediaHandler.cc:704
zypp::media::MediaHandler::setAttachPoint
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
Definition MediaHandler.cc:200
zypp::media::MediaHandler::attachPoint
Pathname attachPoint() const
Return the currently used attach point.
Definition MediaHandler.cc:185
zypp::media::MediaNetworkCommonHandler::clearTransferSettings
void clearTransferSettings()
Definition MediaNetworkCommonHandler.cc:314
zypp::media::MediaNetworkCommonHandler::authenticate
bool authenticate(const Url &url, TransferSettings &settings, const std::string &availAuthTypes, bool firstTry)
Definition MediaNetworkCommonHandler.cc:322
zypp::media::MediaNetworkCommonHandler::canTryNextMirror
static bool canTryNextMirror(const Excpt &excpt_r)
Definition MediaNetworkCommonHandler.h:108
zypp::media::MediaNetworkCommonHandler::mirrorOrder
std::vector< unsigned > mirrorOrder(const OnMediaLocation &loc) const
Definition MediaNetworkCommonHandler.cc:329
zypp::media::MediaNetworkCommonHandler::clearQueryString
Url clearQueryString(const Url &url) const
Definition MediaNetworkCommonHandler.cc:73
zypp::media::MediaNetworkCommonHandler::getFileUrl
Url getFileUrl(int mirrorIdx, const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
Definition MediaNetworkCommonHandler.cc:209
zypp::media::MediaNetworkCommonHandler::MediaNetworkCommonHandler
MediaNetworkCommonHandler(const MirroredOrigin &origin_r, const Pathname &attach_point_r, const Pathname &urlpath_below_attachpoint_r, const bool does_download_r)
Definition MediaNetworkCommonHandler.cc:36
zypp::media::MediaNetworkCommonHandler::MIRR_SETTINGS_KEY
static constexpr std::string_view MIRR_SETTINGS_KEY
Definition MediaNetworkCommonHandler.h:37
zypp::media::MediaNotAFileException
Definition mediaexception.h:258
zypp::media::MediaSystemException
Definition mediaexception.h:241
zypp::media::MediaTemporaryProblemException
For HTTP 503 and similar.
Definition mediaexception.h:531
zypp::media::MediaTimeoutException
Definition mediaexception.h:499
zypp::media::MediaUnauthorizedException
Definition mediaexception.h:450
zypp::media::MediaUnauthorizedException::hint
const std::string & hint() const
comma separated list of available authentication types
Definition mediaexception.h:474
zypp::media::MediaWriteException
Definition mediaexception.h:185
zypp::media::TransferSettings
Holds transfer setting.
Definition transfersettings.h:34
zypp::media::TransferSettings::password
const std::string & password() const
auth password
Definition transfersettings.cc:175
zypp::media::TransferSettings::maxDownloadSpeed
long maxDownloadSpeed() const
Maximum download speed (bytes per second).
Definition transfersettings.cc:261
zypp::media::TransferSettings::connectTimeout
long connectTimeout() const
connection timeout
Definition transfersettings.cc:240
zypp::media::TransferSettings::authType
const std::string & authType() const
get the allowed authentication types
Definition transfersettings.cc:326
zypp::media::TransferSettings::timeout
long timeout() const
transfer timeout
Definition transfersettings.cc:233
zypp::media::TransferSettings::clientCertificatePath
const Pathname & clientCertificatePath() const
SSL client certificate file.
Definition transfersettings.cc:291
zypp::media::TransferSettings::minDownloadSpeed
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
Definition transfersettings.cc:254
zypp::media::TransferSettings::headers
const Headers & headers() const
returns a list of all added headers (trimmed)
Definition transfersettings.cc:141
zypp::media::TransferSettings::proxy
const std::string & proxy() const
proxy host
Definition transfersettings.cc:198
zypp::media::TransferSettings::clientKeyPath
const Pathname & clientKeyPath() const
SSL client key file.
Definition transfersettings.cc:301
zypp::media::TransferSettings::proxyUserPassword
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
Definition transfersettings.cc:220
zypp::media::TransferSettings::verifyHostEnabled
bool verifyHostEnabled() const
Whether to verify host for ssl.
Definition transfersettings.cc:275
zypp::media::TransferSettings::userAgentString
const std::string & userAgentString() const
user agent string (trimmed)
Definition transfersettings.cc:153
zypp::media::TransferSettings::hasCredentials
bool hasCredentials() const
has a username, maybe even the password
Definition transfersettings.cc:157
zypp::media::TransferSettings::setPassword
void setPassword(const std::string &val_r)
sets the auth password
Definition transfersettings.cc:169
zypp::media::TransferSettings::proxyEnabled
bool proxyEnabled() const
proxy is enabled
Definition transfersettings.cc:188
zypp::media::TransferSettings::username
const std::string & username() const
auth username
Definition transfersettings.cc:166
zypp::media::TransferSettings::proxyUsername
const std::string & proxyUsername() const
proxy auth username
Definition transfersettings.cc:208
zypp::media::TransferSettings::certificateAuthoritiesPath
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs ).
Definition transfersettings.cc:316
zypp::media::TransferSettings::logCredentials
std::ostream & logCredentials(std::ostream &str) const
log credentials to stream hiding the password.
Definition transfersettings.h:67
zypp::media::TransferSettings::verifyPeerEnabled
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
Definition transfersettings.cc:282
EXPLICITLY_NO_PROXY
#define EXPLICITLY_NO_PROXY
Definition curlhelper_p.h:23
linuxhelpers_p.h
internal
Definition OptionalDownloadProgressReport.cc:12
internal::log_redirects_curl
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition curlhelper.cc:143
internal::globalInitCurlOnce
void globalInitCurlOnce()
Definition curlhelper.cc:64
internal::curlUnEscape
std::string curlUnEscape(const std::string &text_r)
Definition curlhelper.cc:394
internal::setupZYPP_MEDIA_CURL_DEBUG
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition curlhelper.cc:130
internal::setCurlRedirProtocols
CURLcode setCurlRedirProtocols(CURL *curl)
Definition curlhelper.cc:544
std
Definition ansi.h:855
zypp::base
Definition Random.cc:7
zypp::env::ZYPP_MEDIA_CURL_IPRESOLVE
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition curlhelper.cc:45
zypp::filesystem::applyUmaskTo
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition PathInfo.h:813
zypp::filesystem::assert_file_mode
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition PathInfo.cc:1224
zypp::filesystem::unlink
int unlink(const Pathname &path)
Like 'unlink'.
Definition PathInfo.cc:719
zypp::media
Definition MediaNetworkRequestExecutor.h:22
zypp::str::numstring
std::string numstring(char n, int w=0)
Definition String.h:290
zypp::str::strToBool
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition String.h:500
zypp::target
Definition KeyRing.h:325
zypp::zypp_detail::zypp_poll
int zypp_poll(std::vector< GPollFD > &fds, int timeout)
Small wrapper around g_poll that additionally listens to the shutdown FD returned by ZYpp::shutdownSi...
Definition ZYppImpl.cc:323
zypp
Easy-to use interface to the ZYPP dependency resolver.
Definition CodePitfalls.doc:2
zypp::ManagedFile
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition ManagedFile.h:27
zypp::OnScopeExit
AutoDispose< void > OnScopeExit
Definition AutoDispose.h:276
internal::CurlPollHelper::CurlPoll::_multi
CURLM * _multi
Definition curlhelper_p.h:81
internal::CurlPollHelper
Definition curlhelper_p.h:79
internal::CurlPollHelper::handleSocketActions
CURLMcode handleSocketActions(const std::vector< GPollFD > &actionsFds, int first=0)
Definition curlhelper.cc:501
internal::CurlPollHelper::handleTimout
CURLMcode handleTimout()
Definition curlhelper.cc:528
internal::CurlPollHelper::socks
std::vector< GPollFD > socks
Definition curlhelper_p.h:109
internal::CurlPollHelper::timeout_ms
std::optional< long > timeout_ms
Definition curlhelper_p.h:110
internal::OptionalDownloadProgressReport
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
Definition OptionalDownloadProgressReport.h:22
internal::ProgressData
Definition MediaCurl.cc:52
internal::ProgressData::bytesWritten
ByteCount bytesWritten() const
Definition MediaCurl.cc:84
internal::ProgressData::_expectedFileSize
ByteCount _expectedFileSize
Definition MediaCurl.cc:95
internal::ProgressData::_fileSizeExceeded
bool _fileSizeExceeded
Definition MediaCurl.cc:94
internal::ProgressData::file
FILE * file()
Definition MediaCurl.cc:79
internal::ProgressData::_timeoutReached
bool _timeoutReached
Definition MediaCurl.cc:93
internal::ProgressData::_url
zypp::Url _url
Definition MediaCurl.cc:91
internal::ProgressData::_dnlNow
curl_off_t _dnlNow
Bytes downloaded now.
Definition MediaCurl.cc:105
internal::ProgressData::curl
CURL * curl()
Definition MediaCurl.cc:61
internal::ProgressData::_dnlPercent
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition MediaCurl.cc:109
internal::ProgressData::_timeRcv
time_t _timeRcv
Start of no-data timeout.
Definition MediaCurl.cc:100
internal::ProgressData::expectedFileSize
ByteCount expectedFileSize() const
Definition MediaCurl.cc:70
internal::ProgressData::_timeLast
time_t _timeLast
Start last period(~1sec).
Definition MediaCurl.cc:99
internal::ProgressData::reportProgress
int reportProgress() const
Definition MediaCurl.cc:168
internal::ProgressData::_drateLast
double _drateLast
Download rate in last period.
Definition MediaCurl.cc:112
internal::ProgressData::timeoutReached
bool timeoutReached() const
Definition MediaCurl.cc:64
internal::ProgressData::expectedFileSize
void expectedFileSize(ByteCount newval_r)
Definition MediaCurl.cc:73
internal::ProgressData::_timeNow
time_t _timeNow
Now.
Definition MediaCurl.cc:101
internal::ProgressData::_bytesWritten
ByteCount _bytesWritten
Bytes actually written into the file.
Definition MediaCurl.cc:107
internal::ProgressData::url
zypp::Url url() const
Definition MediaCurl.cc:76
internal::ProgressData::_timeout
time_t _timeout
Definition MediaCurl.cc:92
internal::ProgressData::_dnlLast
curl_off_t _dnlLast
Bytes downloaded at period start.
Definition MediaCurl.cc:104
internal::ProgressData::fileSizeExceeded
bool fileSizeExceeded() const
Definition MediaCurl.cc:67
internal::ProgressData::updateStats
void updateStats(curl_off_t dltotal=0.0, curl_off_t dlnow=0.0)
Definition MediaCurl.cc:128
internal::ProgressData::_drateTotal
double _drateTotal
Download rate so far.
Definition MediaCurl.cc:111
internal::ProgressData::_file
AutoFILE _file
Definition MediaCurl.cc:90
internal::ProgressData::report
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
Definition MediaCurl.cc:96
internal::ProgressData::writeBytes
size_t writeBytes(char *ptr, ByteCount bytes)
Definition MediaCurl.cc:179
internal::ProgressData::_dnlTotal
curl_off_t _dnlTotal
Bytes to download or 0 if unknown.
Definition MediaCurl.cc:103
internal::ProgressData::_curl
CURL * _curl
Definition MediaCurl.cc:89
internal::ProgressData::ProgressData
ProgressData(AutoFILE file, CURL *curl, time_t timeout=0, zypp::Url url=zypp::Url(), zypp::ByteCount expectedFileSize_r=0, zypp::callback::SendReport< zypp::media::DownloadProgressReport > *_report=nullptr)
Definition MediaCurl.cc:117
internal::ProgressData::_timeStart
time_t _timeStart
Start total stats.
Definition MediaCurl.cc:98
zypp::AutoFD
AutoDispose<int> calling close
Definition AutoDispose.h:310
zypp::AutoFILE
AutoDispose<FILE*> calling fclose
Definition AutoDispose.h:321
zypp::AutoFREE
Definition AutoDispose.h:332
zypp::callback::SendReport
Definition Callback.h:237
zypp::media::CredManagerOptions
Definition credentialmanager.h:40
zypp::media::CurlConfig
Structure holding values of curlrc options.
Definition curlconfig.h:27
zypp::media::CurlConfig::proxyuserpwd
std::string proxyuserpwd
Definition curlconfig.h:49
zypp::media::CurlConfig::parseConfig
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition curlconfig.cc:24
zypp::media::DownloadProgressReport::Error
Error
Definition ZYppCallbacks.h:370
zypp::media::DownloadProgressReport::NO_ERROR
@ NO_ERROR
Definition ZYppCallbacks.h:371
zypp::media::DownloadProgressReport::ACCESS_DENIED
@ ACCESS_DENIED
Definition ZYppCallbacks.h:374
zypp::media::DownloadProgressReport::ERROR
@ ERROR
Definition ZYppCallbacks.h:375
zypp::media::DownloadProgressReport::NOT_FOUND
@ NOT_FOUND
Definition ZYppCallbacks.h:372
zypp::media::MediaCurl::RequestData
Definition MediaCurl.h:34
zypp::media::MediaCurl::RequestData::mirror
int mirror
Definition MediaCurl.h:35
zypp::media::MediaCurl::RequestData::curl
CURL * curl
Definition MediaCurl.h:36
zypp::str::Format
Convenient building of std::string with boost::format.
Definition String.h:254
userrequestexception.h