libzypp 17.25.7
MediaCurl.cc
Go to the documentation of this file.
1/*---------------------------------------------------------------------\
2| ____ _ __ __ ___ |
3| |__ / \ / / . \ . \ |
4| / / \ V /| _/ _/ |
5| / /__ | | | | | | |
6| /_____||_| |_| |_| |
7| |
8\---------------------------------------------------------------------*/
13#include <iostream>
14#include <list>
15
16#include <zypp/base/Logger.h>
18#include <zypp/base/String.h>
19#include <zypp/base/Gettext.h>
20#include <zypp/base/Sysconfig.h>
21#include <zypp/base/Gettext.h>
22
29#include <zypp/Target.h>
30#include <zypp/ZYppFactory.h>
31#include <zypp/ZConfig.h>
32
33#include <cstdlib>
34#include <sys/types.h>
35#include <sys/stat.h>
36#include <sys/mount.h>
37#include <errno.h>
38#include <dirent.h>
39#include <unistd.h>
40
41using std::endl;
42
43using namespace internal;
44using namespace zypp::base;
45
46namespace zypp {
47
48 namespace media {
49
50 namespace {
51 struct ProgressData
52 {
53 ProgressData( CURL *_curl, time_t _timeout = 0, const Url & _url = Url(),
54 ByteCount expectedFileSize_r = 0,
56 : curl( _curl )
57 , url( _url )
58 , timeout( _timeout )
59 , reached( false )
60 , fileSizeExceeded ( false )
61 , report( _report )
62 , _expectedFileSize( expectedFileSize_r )
63 {}
64
65 CURL *curl;
67 time_t timeout;
68 bool reached;
72
73 time_t _timeStart = 0;
74 time_t _timeLast = 0;
75 time_t _timeRcv = 0;
76 time_t _timeNow = 0;
77
78 double _dnlTotal = 0.0;
79 double _dnlLast = 0.0;
80 double _dnlNow = 0.0;
81
82 int _dnlPercent= 0;
83
84 double _drateTotal= 0.0;
85 double _drateLast = 0.0;
86
87 void updateStats( double dltotal = 0.0, double dlnow = 0.0 )
88 {
89 time_t now = _timeNow = time(0);
90
91 // If called without args (0.0), recompute based on the last values seen
92 if ( dltotal && dltotal != _dnlTotal )
93 _dnlTotal = dltotal;
94
95 if ( dlnow && dlnow != _dnlNow )
96 {
97 _timeRcv = now;
98 _dnlNow = dlnow;
99 }
100 else if ( !_dnlNow && !_dnlTotal )
101 {
102 // Start time counting as soon as first data arrives.
103 // Skip the connection / redirection time at begin.
104 return;
105 }
106
107 // init or reset if time jumps back
108 if ( !_timeStart || _timeStart > now )
109 _timeStart = _timeLast = _timeRcv = now;
110
111 // timeout condition
112 if ( timeout )
113 reached = ( (now - _timeRcv) > timeout );
114
115 // check if the downloaded data is already bigger than what we expected
116 fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
117
118 // percentage:
119 if ( _dnlTotal )
120 _dnlPercent = int(_dnlNow * 100 / _dnlTotal);
121
122 // download rates:
123 _drateTotal = _dnlNow / std::max( int(now - _timeStart), 1 );
124
125 if ( _timeLast < now )
126 {
127 _drateLast = (_dnlNow - _dnlLast) / int(now - _timeLast);
128 // start new period
129 _timeLast = now;
131 }
132 else if ( _timeStart == _timeLast )
134 }
135
136 int reportProgress() const
137 {
138 if ( fileSizeExceeded )
139 return 1;
140 if ( reached )
141 return 1; // no-data timeout
142 if ( report && !(*report)->progress( _dnlPercent, url, _drateTotal, _drateLast ) )
143 return 1; // user requested abort
144 return 0;
145 }
146
147
148 // download rate of the last period (cca 1 sec)
150 // bytes downloaded at the start of the last period
152 // seconds from the start of the download
153 long secs;
154 // average download rate
155 double drate_avg;
156 // last time the progress was reported
157 time_t ltime;
158 // bytes downloaded at the moment the progress was last reported
159 double dload;
160 // bytes uploaded at the moment the progress was last reported
161 double uload;
162 };
163 }
164
165Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
166
167// we use this define to unbloat code as this C setting option
168// and catching exception is done frequently.
170#define SET_OPTION(opt,val) do { \
171 ret = curl_easy_setopt ( _curl, opt, val ); \
172 if ( ret != 0) { \
173 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
174 } \
175 } while ( false )
176
177#define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
178#define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
179#define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
180
182 const Pathname & attach_point_hint_r )
183 : MediaHandler( url_r, attach_point_hint_r,
184 "/", // urlpath at attachpoint
185 true ), // does_download
186 _curl( NULL ),
187 _customHeaders(0L)
188{
189 _curlError[0] = '\0';
190 _curlDebug = 0L;
191
192 MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
193
195
196 if( !attachPoint().empty())
197 {
198 PathInfo ainfo(attachPoint());
199 Pathname apath(attachPoint() + "XXXXXX");
200 char *atemp = ::strdup( apath.asString().c_str());
201 char *atest = NULL;
202 if( !ainfo.isDir() || !ainfo.userMayRWX() ||
203 atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
204 {
205 WAR << "attach point " << ainfo.path()
206 << " is not useable for " << url_r.getScheme() << endl;
207 setAttachPoint("", true);
208 }
209 else if( atest != NULL)
210 ::rmdir(atest);
211
212 if( atemp != NULL)
213 ::free(atemp);
214 }
215}
216
218{
220}
221
223{
224 return _settings;
225}
226
227
228void MediaCurl::setCookieFile( const Pathname &fileName )
229{
230 _cookieFile = fileName;
231}
232
234
236{
237 curl_version_info_data *curl_info = NULL;
238 curl_info = curl_version_info(CURLVERSION_NOW);
239 // curl_info does not need any free (is static)
240 if (curl_info->protocols)
241 {
242 const char * const *proto;
243 std::string scheme( url.getScheme());
244 bool found = false;
245 for(proto=curl_info->protocols; !found && *proto; ++proto)
246 {
247 if( scheme == std::string((const char *)*proto))
248 found = true;
249 }
250 if( !found)
251 {
252 std::string msg("Unsupported protocol '");
253 msg += scheme;
254 msg += "'";
256 }
257 }
258}
259
263 {
264 public:
266 const std::string & err_r,
267 const std::string & msg_r )
268 : media::MediaCurlException( url_r, err_r, msg_r )
269 {}
270 //~MediaCurlExceptionMayRetryInternaly() noexcept {}
271 };
272
274{
275 {
276 char *ptr = getenv("ZYPP_MEDIA_CURL_DEBUG");
277 _curlDebug = (ptr && *ptr) ? str::strtonum<long>( ptr) : 0L;
278 if( _curlDebug > 0)
279 {
280 curl_easy_setopt( _curl, CURLOPT_VERBOSE, 1L);
281 curl_easy_setopt( _curl, CURLOPT_DEBUGFUNCTION, log_curl);
282 curl_easy_setopt( _curl, CURLOPT_DEBUGDATA, &_curlDebug);
283 }
284 }
285
286 curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
287 curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &_lastRedirect);
288 CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
289 if ( ret != 0 ) {
290 ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
291 }
292
293 SET_OPTION(CURLOPT_FAILONERROR, 1L);
294 SET_OPTION(CURLOPT_NOSIGNAL, 1L);
295
296 // create non persistant settings
297 // so that we don't add headers twice
298 TransferSettings vol_settings(_settings);
299
300 // add custom headers for download.opensuse.org (bsc#955801)
301 if ( _url.getHost() == "download.opensuse.org" )
302 {
303 vol_settings.addHeader(anonymousIdHeader());
304 vol_settings.addHeader(distributionFlavorHeader());
305 }
306 vol_settings.addHeader("Pragma:");
307
308 _settings.setTimeout(ZConfig::instance().download_transfer_timeout());
310
312
313 // fill some settings from url query parameters
314 try
315 {
317 }
318 catch ( const MediaException &e )
319 {
321 ZYPP_RETHROW(e);
322 }
323 // if the proxy was not set (or explicitly unset) by url, then look...
324 if ( _settings.proxy().empty() )
325 {
326 // ...at the system proxy settings
328 }
329
332 {
334 {
335 case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
336 case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
337 }
338 }
339
343 SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
344 // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
345 // just in case curl does not trigger its progress callback frequently
346 // enough.
347 if ( _settings.timeout() )
348 {
349 SET_OPTION(CURLOPT_TIMEOUT, 3600L);
350 }
351
352 // follow any Location: header that the server sends as part of
353 // an HTTP header (#113275)
354 SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
355 // 3 redirects seem to be too few in some cases (bnc #465532)
356 SET_OPTION(CURLOPT_MAXREDIRS, 6L);
357
358 if ( _url.getScheme() == "https" )
359 {
360#if CURLVERSION_AT_LEAST(7,19,4)
361 // restrict following of redirections from https to https only
362 SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
363#endif
364
367 {
369 }
370
372 {
373 SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
374 }
375 if( ! _settings.clientKeyPath().empty() )
376 {
377 SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
378 }
379
380#ifdef CURLSSLOPT_ALLOW_BEAST
381 // see bnc#779177
382 ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
383 if ( ret != 0 ) {
386 }
387#endif
388 SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
389 SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
390 // bnc#903405 - POODLE: libzypp should only talk TLS
391 SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
392 }
393
394 SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
395
396 /* Fixes bsc#1174011 "auth=basic ignored in some cases"
397 * We should proactively add the password to the request if basic auth is configured
398 * and a password is available in the credentials but not in the URL.
399 *
400 * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
401 * and ask the server first about the auth method
402 */
403 if ( _settings.authType() == "basic"
404 && _settings.username().size()
405 && !_settings.password().size() ) {
406
408 const auto cred = cm.getCred( _url );
409 if ( cred && cred->valid() ) {
410 if ( !_settings.username().size() )
411 _settings.setUsername(cred->username());
412 _settings.setPassword(cred->password());
413 }
414 }
415
416 /*---------------------------------------------------------------*
417 CURLOPT_USERPWD: [user name]:[password]
418
419 Url::username/password -> CURLOPT_USERPWD
420 If not provided, anonymous FTP identification
421 *---------------------------------------------------------------*/
422
423 if ( _settings.userPassword().size() )
424 {
425 SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
426 std::string use_auth = _settings.authType();
427 if (use_auth.empty())
428 use_auth = "digest,basic"; // our default
429 long auth = CurlAuthData::auth_type_str2long(use_auth);
430 if( auth != CURLAUTH_NONE)
431 {
432 DBG << "Enabling HTTP authentication methods: " << use_auth
433 << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
434 SET_OPTION(CURLOPT_HTTPAUTH, auth);
435 }
436 }
437
438 if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
439 {
440 DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
441 SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
442 SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
443 /*---------------------------------------------------------------*
444 * CURLOPT_PROXYUSERPWD: [user name]:[password]
445 *
446 * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
447 * If not provided, $HOME/.curlrc is evaluated
448 *---------------------------------------------------------------*/
449
450 std::string proxyuserpwd = _settings.proxyUserPassword();
451
452 if ( proxyuserpwd.empty() )
453 {
454 CurlConfig curlconf;
455 CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
456 if ( curlconf.proxyuserpwd.empty() )
457 DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
458 else
459 {
460 proxyuserpwd = curlconf.proxyuserpwd;
461 DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
462 }
463 }
464 else
465 {
466 DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
467 }
468
469 if ( ! proxyuserpwd.empty() )
470 {
471 SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
472 }
473 }
474#if CURLVERSION_AT_LEAST(7,19,4)
475 else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
476 {
477 // Explicitly disabled in URL (see fillSettingsFromUrl()).
478 // This should also prevent libcurl from looking into the environment.
479 DBG << "Proxy: explicitly NOPROXY" << endl;
480 SET_OPTION(CURLOPT_NOPROXY, "*");
481 }
482#endif
483 else
484 {
485 DBG << "Proxy: not explicitly set" << endl;
486 DBG << "Proxy: libcurl may look into the environment" << endl;
487 }
488
490 if ( _settings.minDownloadSpeed() != 0 )
491 {
492 SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
493 // default to 10 seconds at low speed
494 SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
495 }
496
497#if CURLVERSION_AT_LEAST(7,15,5)
498 if ( _settings.maxDownloadSpeed() != 0 )
499 SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
500#endif
501
502 /*---------------------------------------------------------------*
503 *---------------------------------------------------------------*/
504
507 if ( str::strToBool( _url.getQueryParam( "cookies" ), true ) )
508 SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
509 else
510 MIL << "No cookies requested" << endl;
511 SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
512 SET_OPTION(CURLOPT_PROGRESSFUNCTION, &progressCallback );
513 SET_OPTION(CURLOPT_NOPROGRESS, 0L);
514
515#if CURLVERSION_AT_LEAST(7,18,0)
516 // bnc #306272
517 SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
518#endif
519 // append settings custom headers to curl
520 for ( TransferSettings::Headers::const_iterator it = vol_settings.headersBegin();
521 it != vol_settings.headersEnd();
522 ++it )
523 {
524 // MIL << "HEADER " << *it << std::endl;
525
526 _customHeaders = curl_slist_append(_customHeaders, it->c_str());
527 if ( !_customHeaders )
529 }
530
531 SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
532}
533
535
536
537void MediaCurl::attachTo (bool next)
538{
539 if ( next )
541
542 if ( !_url.isValid() )
544
547 {
549 }
550
551 disconnectFrom(); // clean _curl if needed
552 _curl = curl_easy_init();
553 if ( !_curl ) {
555 }
556 try
557 {
558 setupEasy();
559 }
560 catch (Exception & ex)
561 {
563 ZYPP_RETHROW(ex);
564 }
565
566 // FIXME: need a derived class to propelly compare url's
568 setMediaSource(media);
569}
570
571bool
573{
574 return MediaHandler::checkAttachPoint( apoint, true, true);
575}
576
578
580{
581 if ( _customHeaders )
582 {
583 curl_slist_free_all(_customHeaders);
584 _customHeaders = 0L;
585 }
586
587 if ( _curl )
588 {
589 curl_easy_cleanup( _curl );
590 _curl = NULL;
591 }
592}
593
595
596void MediaCurl::releaseFrom( const std::string & ejectDev )
597{
598 disconnect();
599}
600
601Url MediaCurl::getFileUrl( const Pathname & filename_r ) const
602{
603 // Simply extend the URLs pathname. An 'absolute' URL path
604 // is achieved by encoding the leading '/' in an URL path:
605 // URL: ftp://user@server -> ~user
606 // URL: ftp://user@server/ -> ~user
607 // URL: ftp://user@server// -> ~user
608 // URL: ftp://user@server/%2F -> /
609 // ^- this '/' is just a separator
610 Url newurl( _url );
611 newurl.setPathName( ( Pathname("./"+_url.getPathName()) / filename_r ).asString().substr(1) );
612 return newurl;
613}
614
616
617void MediaCurl::getFile(const Pathname & filename , const ByteCount &expectedFileSize_r) const
618{
619 // Use absolute file name to prevent access of files outside of the
620 // hierarchy below the attach point.
621 getFileCopy(filename, localPath(filename).absolutename(), expectedFileSize_r);
622}
623
625
626void MediaCurl::getFileCopy( const Pathname & filename , const Pathname & target, const ByteCount &expectedFileSize_r ) const
627{
629
630 Url fileurl(getFileUrl(filename));
631
632 bool retry = false;
633 unsigned internalTry = 0;
634 static constexpr unsigned maxInternalTry = 3;
635
636 do
637 {
638 retry = false;
639 try
640 {
641 doGetFileCopy(filename, target, report, expectedFileSize_r);
642 }
643 // retry with proper authentication data
644 catch (MediaUnauthorizedException & ex_r)
645 {
646 if(authenticate(ex_r.hint(), !retry))
647 retry = true;
648 else
649 {
651 ZYPP_RETHROW(ex_r);
652 }
653 }
654 // unexpected exception
655 catch (MediaException & excpt_r)
656 {
657 if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
658 ++internalTry;
659 if ( internalTry < maxInternalTry ) {
660 // just report (NO_ERROR); no interactive request to the user
662 retry = true;
663 continue;
664 }
665 excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
666 }
667
669 if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
670 typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
671 {
673 }
674 report->finish(fileurl, reason, excpt_r.asUserHistory());
675 ZYPP_RETHROW(excpt_r);
676 }
677 }
678 while (retry);
679
681}
682
684
685bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
686{
687 bool retry = false;
688
689 do
690 {
691 try
692 {
693 return doGetDoesFileExist( filename );
694 }
695 // authentication problem, retry with proper authentication data
696 catch (MediaUnauthorizedException & ex_r)
697 {
698 if(authenticate(ex_r.hint(), !retry))
699 retry = true;
700 else
701 ZYPP_RETHROW(ex_r);
702 }
703 // unexpected exception
704 catch (MediaException & excpt_r)
705 {
706 ZYPP_RETHROW(excpt_r);
707 }
708 }
709 while (retry);
710
711 return false;
712}
713
715
717 CURLcode code,
718 bool timeout_reached) const
719{
720 if ( code != 0 )
721 {
722 Url url;
723 if (filename.empty())
724 url = _url;
725 else
726 url = getFileUrl(filename);
727
728 std::string err;
729 {
730 switch ( code )
731 {
732 case CURLE_UNSUPPORTED_PROTOCOL:
733 err = " Unsupported protocol";
734 if ( !_lastRedirect.empty() )
735 {
736 err += " or redirect (";
737 err += _lastRedirect;
738 err += ")";
739 }
740 break;
741 case CURLE_URL_MALFORMAT:
742 case CURLE_URL_MALFORMAT_USER:
743 err = " Bad URL";
744 break;
745 case CURLE_LOGIN_DENIED:
747 MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
748 break;
749 case CURLE_HTTP_RETURNED_ERROR:
750 {
751 long httpReturnCode = 0;
752 CURLcode infoRet = curl_easy_getinfo( _curl,
753 CURLINFO_RESPONSE_CODE,
754 &httpReturnCode );
755 if ( infoRet == CURLE_OK )
756 {
757 std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
758 switch ( httpReturnCode )
759 {
760 case 401:
761 {
762 std::string auth_hint = getAuthHint();
763
764 DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
765 DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
766
768 url, "Login failed.", _curlError, auth_hint
769 ));
770 }
771
772 case 502: // bad gateway (bnc #1070851)
773 case 503: // service temporarily unavailable (bnc #462545)
775 case 504: // gateway timeout
777 case 403:
778 {
779 std::string msg403;
780 if ( url.getHost().find(".suse.com") != std::string::npos )
781 msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
782 else if (url.asString().find("novell.com") != std::string::npos)
783 msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
785 }
786 case 404:
787 case 410:
789 }
790
791 DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
793 }
794 else
795 {
796 std::string msg = "Unable to retrieve HTTP response:";
797 DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
799 }
800 }
801 break;
802 case CURLE_FTP_COULDNT_RETR_FILE:
803#if CURLVERSION_AT_LEAST(7,16,0)
804 case CURLE_REMOTE_FILE_NOT_FOUND:
805#endif
806 case CURLE_FTP_ACCESS_DENIED:
807 case CURLE_TFTP_NOTFOUND:
808 err = "File not found";
810 break;
811 case CURLE_BAD_PASSWORD_ENTERED:
812 case CURLE_FTP_USER_PASSWORD_INCORRECT:
813 err = "Login failed";
814 break;
815 case CURLE_COULDNT_RESOLVE_PROXY:
816 case CURLE_COULDNT_RESOLVE_HOST:
817 case CURLE_COULDNT_CONNECT:
818 case CURLE_FTP_CANT_GET_HOST:
819 err = "Connection failed";
820 break;
821 case CURLE_WRITE_ERROR:
822 err = "Write error";
823 break;
824 case CURLE_PARTIAL_FILE:
825 case CURLE_OPERATION_TIMEDOUT:
826 timeout_reached = true; // fall though to TimeoutException
827 // fall though...
828 case CURLE_ABORTED_BY_CALLBACK:
829 if( timeout_reached )
830 {
831 err = "Timeout reached";
833 }
834 else
835 {
836 err = "User abort";
837 }
838 break;
839
840 // Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy
841 case CURLE_HTTP2:
842 case CURLE_HTTP2_STREAM:
843 err = "Curl error " + str::numstring( code );
845 break;
846
847 default:
848 err = "Curl error " + str::numstring( code );
849 break;
850 }
851
852 // uhm, no 0 code but unknown curl exception
854 }
855 }
856 else
857 {
858 // actually the code is 0, nothing happened
859 }
860}
861
863
864bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
865{
866 DBG << filename.asString() << endl;
867
868 if(!_url.isValid())
870
871 if(_url.getHost().empty())
873
874 Url url(getFileUrl(filename));
875
876 DBG << "URL: " << url.asString() << endl;
877 // Use URL without options and without username and passwd
878 // (some proxies dislike them in the URL).
879 // Curl seems to need the just scheme, hostname and a path;
880 // the rest was already passed as curl options (in attachTo).
881 Url curlUrl( clearQueryString(url) );
882
883 //
884 // See also Bug #154197 and ftp url definition in RFC 1738:
885 // The url "ftp://user@host/foo/bar/file" contains a path,
886 // that is relative to the user's home.
887 // The url "ftp://user@host//foo/bar/file" (or also with
888 // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
889 // contains an absolute path.
890 //
891 _lastRedirect.clear();
892 std::string urlBuffer( curlUrl.asString());
893 CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
894 urlBuffer.c_str() );
895 if ( ret != 0 ) {
897 }
898
899 // instead of returning no data with NOBODY, we return
900 // little data, that works with broken servers, and
901 // works for ftp as well, because retrieving only headers
902 // ftp will return always OK code ?
903 // See http://curl.haxx.se/docs/knownbugs.html #58
904 if ( (_url.getScheme() == "http" || _url.getScheme() == "https") &&
906 ret = curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
907 else
908 ret = curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
909
910 if ( ret != 0 ) {
911 curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
912 curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
913 /* yes, this is why we never got to get NOBODY working before,
914 because setting it changes this option too, and we also
915 need to reset it
916 See: http://curl.haxx.se/mail/archive-2005-07/0073.html
917 */
918 curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
920 }
921
922 AutoFILE file { ::fopen( "/dev/null", "w" ) };
923 if ( !file ) {
924 ERR << "fopen failed for /dev/null" << endl;
925 curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
926 curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
927 /* yes, this is why we never got to get NOBODY working before,
928 because setting it changes this option too, and we also
929 need to reset it
930 See: http://curl.haxx.se/mail/archive-2005-07/0073.html
931 */
932 curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
933 if ( ret != 0 ) {
935 }
936 ZYPP_THROW(MediaWriteException("/dev/null"));
937 }
938
939 ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, (*file) );
940 if ( ret != 0 ) {
941 std::string err( _curlError);
942 curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
943 curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
944 /* yes, this is why we never got to get NOBODY working before,
945 because setting it changes this option too, and we also
946 need to reset it
947 See: http://curl.haxx.se/mail/archive-2005-07/0073.html
948 */
949 curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
950 if ( ret != 0 ) {
952 }
954 }
955
956 CURLcode ok = curl_easy_perform( _curl );
957 MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
958
959 // reset curl settings
960 if ( _url.getScheme() == "http" || _url.getScheme() == "https" )
961 {
962 curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
963 if ( ret != 0 ) {
965 }
966
967 /* yes, this is why we never got to get NOBODY working before,
968 because setting it changes this option too, and we also
969 need to reset it
970 See: http://curl.haxx.se/mail/archive-2005-07/0073.html
971 */
972 curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L);
973 if ( ret != 0 ) {
975 }
976
977 }
978 else
979 {
980 // for FTP we set different options
981 curl_easy_setopt( _curl, CURLOPT_RANGE, NULL);
982 if ( ret != 0 ) {
984 }
985 }
986
987 // as we are not having user interaction, the user can't cancel
988 // the file existence checking, a callback or timeout return code
989 // will be always a timeout.
990 try {
991 evaluateCurlCode( filename, ok, true /* timeout */);
992 }
993 catch ( const MediaFileNotFoundException &e ) {
994 // if the file did not exist then we can return false
995 return false;
996 }
997 catch ( const MediaException &e ) {
998 // some error, we are not sure about file existence, rethrw
999 ZYPP_RETHROW(e);
1000 }
1001 // exists
1002 return ( ok == CURLE_OK );
1003}
1004
1006
1007
1008#if DETECT_DIR_INDEX
1009bool MediaCurl::detectDirIndex() const
1010{
1011 if(_url.getScheme() != "http" && _url.getScheme() != "https")
1012 return false;
1013 //
1014 // try to check the effective url and set the not_a_file flag
1015 // if the url path ends with a "/", what usually means, that
1016 // we've received a directory index (index.html content).
1017 //
1018 // Note: This may be dangerous and break file retrieving in
1019 // case of some server redirections ... ?
1020 //
1021 bool not_a_file = false;
1022 char *ptr = NULL;
1023 CURLcode ret = curl_easy_getinfo( _curl,
1024 CURLINFO_EFFECTIVE_URL,
1025 &ptr);
1026 if ( ret == CURLE_OK && ptr != NULL)
1027 {
1028 try
1029 {
1030 Url eurl( ptr);
1031 std::string path( eurl.getPathName());
1032 if( !path.empty() && path != "/" && *path.rbegin() == '/')
1033 {
1034 DBG << "Effective url ("
1035 << eurl
1036 << ") seems to provide the index of a directory"
1037 << endl;
1038 not_a_file = true;
1039 }
1040 }
1041 catch( ... )
1042 {}
1043 }
1044 return not_a_file;
1045}
1046#endif
1047
1049
1050void MediaCurl::doGetFileCopy(const Pathname & filename , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, const ByteCount &expectedFileSize_r, RequestOptions options ) const
1051{
1052 Pathname dest = target.absolutename();
1053 if( assert_dir( dest.dirname() ) )
1054 {
1055 DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1056 ZYPP_THROW( MediaSystemException(getFileUrl(filename), "System error on " + dest.dirname().asString()) );
1057 }
1058
1059 ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1060 AutoFILE file;
1061 {
1062 AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1063 if( ! buf )
1064 {
1065 ERR << "out of memory for temp file name" << endl;
1066 ZYPP_THROW(MediaSystemException(getFileUrl(filename), "out of memory for temp file name"));
1067 }
1068
1069 AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1070 if( tmp_fd == -1 )
1071 {
1072 ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1074 }
1075 destNew = ManagedFile( (*buf), filesystem::unlink );
1076
1077 file = ::fdopen( tmp_fd, "we" );
1078 if ( ! file )
1079 {
1080 ERR << "fopen failed for file '" << destNew << "'" << endl;
1082 }
1083 tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1084 }
1085
1086 DBG << "dest: " << dest << endl;
1087 DBG << "temp: " << destNew << endl;
1088
1089 // set IFMODSINCE time condition (no download if not modified)
1090 if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1091 {
1092 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1093 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1094 }
1095 else
1096 {
1097 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1098 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1099 }
1100 try
1101 {
1102 doGetFileCopyFile(filename, dest, file, report, expectedFileSize_r, options);
1103 }
1104 catch (Exception &e)
1105 {
1106 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1107 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1108 ZYPP_RETHROW(e);
1109 }
1110
1111 long httpReturnCode = 0;
1112 CURLcode infoRet = curl_easy_getinfo(_curl,
1113 CURLINFO_RESPONSE_CODE,
1114 &httpReturnCode);
1115 bool modified = true;
1116 if (infoRet == CURLE_OK)
1117 {
1118 DBG << "HTTP response: " + str::numstring(httpReturnCode);
1119 if ( httpReturnCode == 304
1120 || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1121 {
1122 DBG << " Not modified.";
1123 modified = false;
1124 }
1125 DBG << endl;
1126 }
1127 else
1128 {
1129 WAR << "Could not get the response code." << endl;
1130 }
1131
1132 if (modified || infoRet != CURLE_OK)
1133 {
1134 // apply umask
1135 if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1136 {
1137 ERR << "Failed to chmod file " << destNew << endl;
1138 }
1139
1140 file.resetDispose(); // we're going to close it manually here
1141 if ( ::fclose( file ) )
1142 {
1143 ERR << "Fclose failed for file '" << destNew << "'" << endl;
1145 }
1146
1147 // move the temp file into dest
1148 if ( rename( destNew, dest ) != 0 ) {
1149 ERR << "Rename failed" << endl;
1151 }
1152 destNew.resetDispose(); // no more need to unlink it
1153 }
1154
1155 DBG << "done: " << PathInfo(dest) << endl;
1156}
1157
1159
1160void MediaCurl::doGetFileCopyFile(const Pathname & filename , const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, const ByteCount &expectedFileSize_r, RequestOptions options ) const
1161{
1162 DBG << filename.asString() << endl;
1163
1164 if(!_url.isValid())
1166
1167 if(_url.getHost().empty())
1169
1170 Url url(getFileUrl(filename));
1171
1172 DBG << "URL: " << url.asString() << endl;
1173 // Use URL without options and without username and passwd
1174 // (some proxies dislike them in the URL).
1175 // Curl seems to need the just scheme, hostname and a path;
1176 // the rest was already passed as curl options (in attachTo).
1177 Url curlUrl( clearQueryString(url) );
1178
1179 //
1180 // See also Bug #154197 and ftp url definition in RFC 1738:
1181 // The url "ftp://user@host/foo/bar/file" contains a path,
1182 // that is relative to the user's home.
1183 // The url "ftp://user@host//foo/bar/file" (or also with
1184 // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1185 // contains an absolute path.
1186 //
1187 _lastRedirect.clear();
1188 std::string urlBuffer( curlUrl.asString());
1189 CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1190 urlBuffer.c_str() );
1191 if ( ret != 0 ) {
1193 }
1194
1195 ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1196 if ( ret != 0 ) {
1198 }
1199
1200 // Set callback and perform.
1201 ProgressData progressData(_curl, _settings.timeout(), url, expectedFileSize_r, &report);
1202 if (!(options & OPTION_NO_REPORT_START))
1203 report->start(url, dest);
1204 if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1205 WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1206 }
1207
1208 ret = curl_easy_perform( _curl );
1209#if CURLVERSION_AT_LEAST(7,19,4)
1210 // bnc#692260: If the client sends a request with an If-Modified-Since header
1211 // with a future date for the server, the server may respond 200 sending a
1212 // zero size file.
1213 // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1214 if ( ftell(file) == 0 && ret == 0 )
1215 {
1216 long httpReturnCode = 33;
1217 if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1218 {
1219 long conditionUnmet = 33;
1220 if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1221 {
1222 WAR << "TIMECONDITION unmet - retry without." << endl;
1223 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1224 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1225 ret = curl_easy_perform( _curl );
1226 }
1227 }
1228 }
1229#endif
1230
1231 if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1232 WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1233 }
1234
1235 if ( ret != 0 )
1236 {
1237 ERR << "curl error: " << ret << ": " << _curlError
1238 << ", temp file size " << ftell(file)
1239 << " bytes." << endl;
1240
1241 // the timeout is determined by the progress data object
1242 // which holds whether the timeout was reached or not,
1243 // otherwise it would be a user cancel
1244 try {
1245
1246 if ( progressData.fileSizeExceeded )
1247 ZYPP_THROW(MediaFileSizeExceededException(url, progressData._expectedFileSize));
1248
1249 evaluateCurlCode( filename, ret, progressData.reached );
1250 }
1251 catch ( const MediaException &e ) {
1252 // some error, we are not sure about file existence, rethrw
1253 ZYPP_RETHROW(e);
1254 }
1255 }
1256
1257#if DETECT_DIR_INDEX
1258 if (!ret && detectDirIndex())
1259 {
1261 }
1262#endif // DETECT_DIR_INDEX
1263}
1264
1266
1267void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1268{
1269 filesystem::DirContent content;
1270 getDirInfo( content, dirname, /*dots*/false );
1271
1272 for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1273 Pathname filename = dirname + it->name;
1274 int res = 0;
1275
1276 switch ( it->type ) {
1277 case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1279 getFile( filename, 0 );
1280 break;
1281 case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1282 if ( recurse_r ) {
1283 getDir( filename, recurse_r );
1284 } else {
1285 res = assert_dir( localPath( filename ) );
1286 if ( res ) {
1287 WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1288 }
1289 }
1290 break;
1291 default:
1292 // don't provide devices, sockets, etc.
1293 break;
1294 }
1295 }
1296}
1297
1299
1300void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1301 const Pathname & dirname, bool dots ) const
1302{
1303 getDirectoryYast( retlist, dirname, dots );
1304}
1305
1307
1309 const Pathname & dirname, bool dots ) const
1310{
1311 getDirectoryYast( retlist, dirname, dots );
1312}
1313
1315//
1316int MediaCurl::aliveCallback( void *clientp, double /*dltotal*/, double dlnow, double /*ultotal*/, double /*ulnow*/ )
1317{
1318 ProgressData *pdata = reinterpret_cast<ProgressData *>( clientp );
1319 if( pdata )
1320 {
1321 // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1322 // prevent a percentage raise while downloading a metalink file. Download
1323 // activity however is indicated by propagating the download rate (via dlnow).
1324 pdata->updateStats( 0.0, dlnow );
1325 return pdata->reportProgress();
1326 }
1327 return 0;
1328}
1329
1330int MediaCurl::progressCallback( void *clientp, double dltotal, double dlnow, double ultotal, double ulnow )
1331{
1332 ProgressData *pdata = reinterpret_cast<ProgressData *>( clientp );
1333 if( pdata )
1334 {
1335 // work around curl bug that gives us old data
1336 long httpReturnCode = 0;
1337 if ( curl_easy_getinfo( pdata->curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1338 return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1339
1340 pdata->updateStats( dltotal, dlnow );
1341 return pdata->reportProgress();
1342 }
1343 return 0;
1344}
1345
1347{
1348 ProgressData *pdata = reinterpret_cast<ProgressData *>(clientp);
1349 return pdata ? pdata->curl : 0;
1350}
1351
1353
1354std::string MediaCurl::getAuthHint() const
1355{
1356 long auth_info = CURLAUTH_NONE;
1357
1358 CURLcode infoRet =
1359 curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1360
1361 if(infoRet == CURLE_OK)
1362 {
1363 return CurlAuthData::auth_type_long2str(auth_info);
1364 }
1365
1366 return "";
1367}
1368
1373void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1374{
1375 ProgressData *data = reinterpret_cast<ProgressData *>(clientp);
1376 if ( data ) {
1377 data->_expectedFileSize = expectedFileSize;
1378 }
1379}
1380
1382
1383bool MediaCurl::authenticate(const std::string & availAuthTypes, bool firstTry) const
1384{
1387 CurlAuthData_Ptr credentials;
1388
1389 // get stored credentials
1390 AuthData_Ptr cmcred = cm.getCred(_url);
1391
1392 if (cmcred && firstTry)
1393 {
1394 credentials.reset(new CurlAuthData(*cmcred));
1395 DBG << "got stored credentials:" << endl << *credentials << endl;
1396 }
1397 // if not found, ask user
1398 else
1399 {
1400
1401 CurlAuthData_Ptr curlcred;
1402 curlcred.reset(new CurlAuthData());
1404
1405 // preset the username if present in current url
1406 if (!_url.getUsername().empty() && firstTry)
1407 curlcred->setUsername(_url.getUsername());
1408 // if CM has found some credentials, preset the username from there
1409 else if (cmcred)
1410 curlcred->setUsername(cmcred->username());
1411
1412 // indicate we have no good credentials from CM
1413 cmcred.reset();
1414
1415 std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1416
1417 // set available authentication types from the exception
1418 // might be needed in prompt
1419 curlcred->setAuthType(availAuthTypes);
1420
1421 // ask user
1422 if (auth_report->prompt(_url, prompt_msg, *curlcred))
1423 {
1424 DBG << "callback answer: retry" << endl
1425 << "CurlAuthData: " << *curlcred << endl;
1426
1427 if (curlcred->valid())
1428 {
1429 credentials = curlcred;
1430 // if (credentials->username() != _url.getUsername())
1431 // _url.setUsername(credentials->username());
1439 }
1440 }
1441 else
1442 {
1443 DBG << "callback answer: cancel" << endl;
1444 }
1445 }
1446
1447 // set username and password
1448 if (credentials)
1449 {
1450 // HACK, why is this const?
1451 const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1452 const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1453
1454 // set username and password
1455 CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1457
1458 // set available authentication types from the exception
1459 if (credentials->authType() == CURLAUTH_NONE)
1460 credentials->setAuthType(availAuthTypes);
1461
1462 // set auth type (seems this must be set _after_ setting the userpwd)
1463 if (credentials->authType() != CURLAUTH_NONE)
1464 {
1465 // FIXME: only overwrite if not empty?
1466 const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1467 ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1469 }
1470
1471 if (!cmcred)
1472 {
1473 credentials->setUrl(_url);
1474 cm.addCred(*credentials);
1475 cm.save();
1476 }
1477
1478 return true;
1479 }
1480
1481 return false;
1482}
1483
1484//need a out of line definiton, otherwise vtable is emitted for every translation unit
1486
1487
1488 } // namespace media
1489} // namespace zypp
1490//
#define CONNECT_TIMEOUT
Definition: CurlHelper.h:21
#define EXPLICITLY_NO_PROXY
Definition: CurlHelper.h:25
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:400
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:392
Interface to gettext.
#define _(MSG)
Definition: Gettext.h:37
#define DBG
Definition: Logger.h:78
#define MIL
Definition: Logger.h:79
#define ERR
Definition: Logger.h:81
#define WAR
Definition: Logger.h:80
double _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:78
double uload
Definition: MediaCurl.cc:161
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:74
bool reached
Definition: MediaCurl.cc:68
bool fileSizeExceeded
Definition: MediaCurl.cc:69
ByteCount _expectedFileSize
Definition: MediaCurl.cc:71
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:84
double dload
Definition: MediaCurl.cc:159
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:177
double dload_period
Definition: MediaCurl.cc:151
callback::SendReport< DownloadProgressReport > * report
Definition: MediaCurl.cc:70
long secs
Definition: MediaCurl.cc:153
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:82
CURL * curl
Definition: MediaCurl.cc:65
Url url
Definition: MediaCurl.cc:66
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:85
time_t ltime
Definition: MediaCurl.cc:157
time_t _timeNow
Now.
Definition: MediaCurl.cc:76
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:170
double drate_period
Definition: MediaCurl.cc:149
double drate_avg
Definition: MediaCurl.cc:155
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:73
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:75
double _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:80
double _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:79
time_t timeout
Definition: MediaCurl.cc:67
Convenience interface for handling authentication data of media user.
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
Definition: AutoDispose.h:93
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:162
Store and operate with byte count.
Definition: ByteCount.h:31
Base class for Exception.
Definition: Exception.h:146
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:91
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition: Exception.cc:125
Maintain [min,max] and counter (value) for progress counting.
Definition: ProgressData.h:131
Url manipulation class.
Definition: Url.h:92
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:528
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:492
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:567
std::string getPathName(EEncoding eflag=zypp::url::E_DECODED) const
Returns the path name from the URL.
Definition: Url.cc:599
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:655
void setPathName(const std::string &path, EEncoding eflag=zypp::url::E_DECODED)
Set the path name.
Definition: Url.cc:759
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:583
bool isValid() const
Verifies the Url.
Definition: Url.cc:484
static ZConfig & instance()
Singleton ctor.
Definition: Resolver.cc:126
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:221
bool userMayRWX() const
Definition: PathInfo.h:353
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:246
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:170
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
const char * c_str() const
String representation.
Definition: Pathname.h:110
const std::string & asString() const
String representation.
Definition: Pathname.h:91
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
Pathname absolutename() const
Return this path, adding a leading '/' if relative.
Definition: Pathname.h:139
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods.
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Curl HTTP authentication data.
Definition: MediaUserAuth.h:74
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g.
Definition: MediaCurl.cc:263
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition: MediaCurl.cc:265
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:33
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:273
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
Definition: MediaCurl.cc:601
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:228
virtual bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly,...
Definition: MediaCurl.cc:685
@ OPTION_NO_IFMODSINCE
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:44
@ OPTION_NO_REPORT_START
do not send a start ProgressReport
Definition: MediaCurl.h:46
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1373
std::string _currentCookieFile
Definition: MediaCurl.h:170
static int aliveCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Definition: MediaCurl.cc:1316
static Pathname _cookieFile
Definition: MediaCurl.h:171
static int progressCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1330
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:173
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:217
virtual void attachTo(bool next=false) override
Call concrete handler to attach the media.
Definition: MediaCurl.cc:537
virtual void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1300
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:177
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:235
MediaCurl(const Url &url_r, const Pathname &attach_point_hint_r)
Definition: MediaCurl.cc:181
virtual void getFileCopy(const Pathname &srcFilename, const Pathname &targetFilename, const ByteCount &expectedFileSize_r) const override
Definition: MediaCurl.cc:626
bool detectDirIndex() const
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:716
TransferSettings _settings
Definition: MediaCurl.h:179
virtual void doGetFileCopy(const Pathname &srcFilename, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, const ByteCount &expectedFileSize_r, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1050
virtual bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:572
virtual void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1267
TransferSettings & settings()
Definition: MediaCurl.cc:222
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1383
virtual void getFile(const Pathname &filename, const ByteCount &expectedFileSize_r) const override
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:617
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1346
virtual void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:596
void doGetFileCopyFile(const Pathname &srcFilename, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &_report, const ByteCount &expectedFileSize_r, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1160
virtual void disconnectFrom() override
Definition: MediaCurl.cc:579
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:864
curl_slist * _customHeaders
Definition: MediaCurl.h:178
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1354
Just inherits Exception to separate media exceptions.
Abstract base class for 'physical' MediaHandler like MediaCD, etc.
Definition: MediaHandler.h:45
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
Url url() const
Url used.
Definition: MediaHandler.h:507
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
void disconnect()
Use concrete handler to isconnect media.
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
Pathname localPath(const Pathname &pathname) const
Files provided will be available at 'localPath(filename)'.
const Url _url
Url to handle.
Definition: MediaHandler.h:110
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
Pathname attachPoint() const
Return the currently used attach point.
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:37
const std::string & hint() const
comma separated list of available authentication types
Holds transfer setting.
std::string proxy() const
proxy host
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
long connectTimeout() const
connection timeout
std::string password() const
auth password
long timeout() const
transfer timeout
Headers::const_iterator headersEnd() const
end iterators to additional headers
std::string userPassword() const
returns the user and password as a user:pass string
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
void setAuthType(std::string &&val_r)
set the allowed authentication types
void setUsername(std::string &&val_r)
sets the auth username
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3"
void setConnectTimeout(long t)
set the connect timeout
std::string userAgentString() const
user agent string
void setPassword(std::string &&val_r)
sets the auth password
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar"
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
bool verifyHostEnabled() const
Whether to verify host for ssl.
Pathname clientCertificatePath() const
SSL client certificate file.
Pathname certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
bool headRequestsAllowed() const
whether HEAD requests are allowed
std::string proxyUsername() const
proxy auth username
std::string authType() const
get the allowed authentication types
bool proxyEnabled() const
proxy is enabled
std::string username() const
auth username
Pathname clientKeyPath() const
SSL client key file.
void setTimeout(long t)
set the transfer timeout
Headers::const_iterator headersBegin() const
begin iterators to additional headers
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
int ZYPP_MEDIA_CURL_IPRESOLVE()
Definition: CurlHelper.h:36
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: CurlHelper.cc:110
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: CurlHelper.cc:62
const char * anonymousIdHeader()
initialized only once, this gets the anonymous id from the target, which we pass in the http header
Definition: CurlHelper.cc:294
void globalInitCurlOnce()
Definition: CurlHelper.cc:19
const char * distributionFlavorHeader()
initialized only once, this gets the distribution flavor from the target, which we pass in the http h...
Definition: CurlHelper.cc:308
std::string curlUnEscape(std::string text_r)
Definition: CurlHelper.cc:351
const char * agentString()
initialized only once, this gets the agent string which also includes the curl version
Definition: CurlHelper.cc:322
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: CurlHelper.cc:258
Url clearQueryString(const Url &url)
Definition: CurlHelper.cc:358
int log_curl(CURL *curl, curl_infotype info, char *ptr, size_t len, void *max_lvl)
Definition: CurlHelper.cc:28
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:813
int unlink(const Pathname &path)
Like 'unlink'.
Definition: PathInfo.cc:662
int rename(const Pathname &oldpath, const Pathname &newpath)
Like 'rename'.
Definition: PathInfo.cc:704
int rmdir(const Pathname &path)
Like 'rmdir'.
Definition: PathInfo.cc:367
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Definition: PathInfo.cc:320
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1164
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:547
shared_ptr< CurlAuthData > CurlAuthData_Ptr
shared_ptr< AuthData > AuthData_Ptr
Definition: MediaUserAuth.h:69
std::string numstring(char n, int w=0)
Definition: String.h:286
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:426
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
AutoDispose<int> calling ::close
Definition: AutoDispose.h:281
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:292
Structure holding values of curlrc options.
Definition: CurlConfig.h:17
std::string proxyuserpwd
Definition: CurlConfig.h:39
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: CurlConfig.cc:24
Convenient building of std::string with boost::format.
Definition: String.h:250