libzypp  17.25.7
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <list>
15 
16 #include <zypp/base/Logger.h>
17 #include <zypp/ExternalProgram.h>
18 #include <zypp/base/String.h>
19 #include <zypp/base/Gettext.h>
20 #include <zypp/base/Sysconfig.h>
21 #include <zypp/base/Gettext.h>
22 
23 #include <zypp/media/MediaCurl.h>
24 #include <zypp/media/ProxyInfo.h>
27 #include <zypp/media/CurlConfig.h>
28 #include <zypp/media/CurlHelper.h>
29 #include <zypp/Target.h>
30 #include <zypp/ZYppFactory.h>
31 #include <zypp/ZConfig.h>
32 
33 #include <cstdlib>
34 #include <sys/types.h>
35 #include <sys/stat.h>
36 #include <sys/mount.h>
37 #include <errno.h>
38 #include <dirent.h>
39 #include <unistd.h>
40 
41 using std::endl;
42 
43 using namespace internal;
44 using namespace zypp::base;
45 
46 namespace zypp {
47 
48  namespace media {
49 
50  namespace {
51  struct ProgressData
52  {
53  ProgressData( CURL *_curl, time_t _timeout = 0, const Url & _url = Url(),
54  ByteCount expectedFileSize_r = 0,
55  callback::SendReport<DownloadProgressReport> *_report = nullptr )
56  : curl( _curl )
57  , url( _url )
58  , timeout( _timeout )
59  , reached( false )
60  , fileSizeExceeded ( false )
61  , report( _report )
62  , _expectedFileSize( expectedFileSize_r )
63  {}
64 
65  CURL *curl;
66  Url url;
67  time_t timeout;
68  bool reached;
70  callback::SendReport<DownloadProgressReport> *report;
71  ByteCount _expectedFileSize;
72 
73  time_t _timeStart = 0;
74  time_t _timeLast = 0;
75  time_t _timeRcv = 0;
76  time_t _timeNow = 0;
77 
78  double _dnlTotal = 0.0;
79  double _dnlLast = 0.0;
80  double _dnlNow = 0.0;
81 
82  int _dnlPercent= 0;
83 
84  double _drateTotal= 0.0;
85  double _drateLast = 0.0;
86 
87  void updateStats( double dltotal = 0.0, double dlnow = 0.0 )
88  {
89  time_t now = _timeNow = time(0);
90 
91  // If called without args (0.0), recompute based on the last values seen
92  if ( dltotal && dltotal != _dnlTotal )
93  _dnlTotal = dltotal;
94 
95  if ( dlnow && dlnow != _dnlNow )
96  {
97  _timeRcv = now;
98  _dnlNow = dlnow;
99  }
100  else if ( !_dnlNow && !_dnlTotal )
101  {
102  // Start time counting as soon as first data arrives.
103  // Skip the connection / redirection time at begin.
104  return;
105  }
106 
107  // init or reset if time jumps back
108  if ( !_timeStart || _timeStart > now )
109  _timeStart = _timeLast = _timeRcv = now;
110 
111  // timeout condition
112  if ( timeout )
113  reached = ( (now - _timeRcv) > timeout );
114 
115  // check if the downloaded data is already bigger than what we expected
116  fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
117 
118  // percentage:
119  if ( _dnlTotal )
120  _dnlPercent = int(_dnlNow * 100 / _dnlTotal);
121 
122  // download rates:
123  _drateTotal = _dnlNow / std::max( int(now - _timeStart), 1 );
124 
125  if ( _timeLast < now )
126  {
127  _drateLast = (_dnlNow - _dnlLast) / int(now - _timeLast);
128  // start new period
129  _timeLast = now;
130  _dnlLast = _dnlNow;
131  }
132  else if ( _timeStart == _timeLast )
134  }
135 
136  int reportProgress() const
137  {
138  if ( fileSizeExceeded )
139  return 1;
140  if ( reached )
141  return 1; // no-data timeout
142  if ( report && !(*report)->progress( _dnlPercent, url, _drateTotal, _drateLast ) )
143  return 1; // user requested abort
144  return 0;
145  }
146 
147 
148  // download rate of the last period (cca 1 sec)
149  double drate_period;
150  // bytes downloaded at the start of the last period
151  double dload_period;
152  // seconds from the start of the download
153  long secs;
154  // average download rate
155  double drate_avg;
156  // last time the progress was reported
157  time_t ltime;
158  // bytes downloaded at the moment the progress was last reported
159  double dload;
160  // bytes uploaded at the moment the progress was last reported
161  double uload;
162  };
163  }
164 
165 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
166 
167 // we use this define to unbloat code as this C setting option
168 // and catching exception is done frequently.
170 #define SET_OPTION(opt,val) do { \
171  ret = curl_easy_setopt ( _curl, opt, val ); \
172  if ( ret != 0) { \
173  ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
174  } \
175  } while ( false )
176 
177 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
178 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
179 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
180 
181 MediaCurl::MediaCurl( const Url & url_r,
182  const Pathname & attach_point_hint_r )
183  : MediaHandler( url_r, attach_point_hint_r,
184  "/", // urlpath at attachpoint
185  true ), // does_download
186  _curl( NULL ),
187  _customHeaders(0L)
188 {
189  _curlError[0] = '\0';
190  _curlDebug = 0L;
191 
192  MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
193 
195 
196  if( !attachPoint().empty())
197  {
198  PathInfo ainfo(attachPoint());
199  Pathname apath(attachPoint() + "XXXXXX");
200  char *atemp = ::strdup( apath.asString().c_str());
201  char *atest = NULL;
202  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
203  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
204  {
205  WAR << "attach point " << ainfo.path()
206  << " is not useable for " << url_r.getScheme() << endl;
207  setAttachPoint("", true);
208  }
209  else if( atest != NULL)
210  ::rmdir(atest);
211 
212  if( atemp != NULL)
213  ::free(atemp);
214  }
215 }
216 
218 {
220 }
221 
223 {
224  return _settings;
225 }
226 
227 
228 void MediaCurl::setCookieFile( const Pathname &fileName )
229 {
230  _cookieFile = fileName;
231 }
232 
234 
235 void MediaCurl::checkProtocol(const Url &url) const
236 {
237  curl_version_info_data *curl_info = NULL;
238  curl_info = curl_version_info(CURLVERSION_NOW);
239  // curl_info does not need any free (is static)
240  if (curl_info->protocols)
241  {
242  const char * const *proto;
243  std::string scheme( url.getScheme());
244  bool found = false;
245  for(proto=curl_info->protocols; !found && *proto; ++proto)
246  {
247  if( scheme == std::string((const char *)*proto))
248  found = true;
249  }
250  if( !found)
251  {
252  std::string msg("Unsupported protocol '");
253  msg += scheme;
254  msg += "'";
256  }
257  }
258 }
259 
263  {
264  public:
266  const std::string & err_r,
267  const std::string & msg_r )
268  : media::MediaCurlException( url_r, err_r, msg_r )
269  {}
270  //~MediaCurlExceptionMayRetryInternaly() noexcept {}
271  };
272 
274 {
275  {
276  char *ptr = getenv("ZYPP_MEDIA_CURL_DEBUG");
277  _curlDebug = (ptr && *ptr) ? str::strtonum<long>( ptr) : 0L;
278  if( _curlDebug > 0)
279  {
280  curl_easy_setopt( _curl, CURLOPT_VERBOSE, 1L);
281  curl_easy_setopt( _curl, CURLOPT_DEBUGFUNCTION, log_curl);
282  curl_easy_setopt( _curl, CURLOPT_DEBUGDATA, &_curlDebug);
283  }
284  }
285 
286  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
287  curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &_lastRedirect);
288  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
289  if ( ret != 0 ) {
290  ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
291  }
292 
293  SET_OPTION(CURLOPT_FAILONERROR, 1L);
294  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
295 
296  // create non persistant settings
297  // so that we don't add headers twice
298  TransferSettings vol_settings(_settings);
299 
300  // add custom headers for download.opensuse.org (bsc#955801)
301  if ( _url.getHost() == "download.opensuse.org" )
302  {
303  vol_settings.addHeader(anonymousIdHeader());
304  vol_settings.addHeader(distributionFlavorHeader());
305  }
306  vol_settings.addHeader("Pragma:");
307 
308  _settings.setTimeout(ZConfig::instance().download_transfer_timeout());
310 
312 
313  // fill some settings from url query parameters
314  try
315  {
317  }
318  catch ( const MediaException &e )
319  {
320  disconnectFrom();
321  ZYPP_RETHROW(e);
322  }
323  // if the proxy was not set (or explicitly unset) by url, then look...
324  if ( _settings.proxy().empty() )
325  {
326  // ...at the system proxy settings
328  }
329 
332  {
333  switch ( env::ZYPP_MEDIA_CURL_IPRESOLVE() )
334  {
335  case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
336  case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
337  }
338  }
339 
343  SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
344  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
345  // just in case curl does not trigger its progress callback frequently
346  // enough.
347  if ( _settings.timeout() )
348  {
349  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
350  }
351 
352  // follow any Location: header that the server sends as part of
353  // an HTTP header (#113275)
354  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
355  // 3 redirects seem to be too few in some cases (bnc #465532)
356  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
357 
358  if ( _url.getScheme() == "https" )
359  {
360 #if CURLVERSION_AT_LEAST(7,19,4)
361  // restrict following of redirections from https to https only
362  SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
363 #endif
364 
367  {
369  }
370 
372  {
373  SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
374  }
375  if( ! _settings.clientKeyPath().empty() )
376  {
377  SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
378  }
379 
380 #ifdef CURLSSLOPT_ALLOW_BEAST
381  // see bnc#779177
382  ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
383  if ( ret != 0 ) {
384  disconnectFrom();
386  }
387 #endif
388  SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
389  SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
390  // bnc#903405 - POODLE: libzypp should only talk TLS
391  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
392  }
393 
394  SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
395 
396  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
397  * We should proactively add the password to the request if basic auth is configured
398  * and a password is available in the credentials but not in the URL.
399  *
400  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
401  * and ask the server first about the auth method
402  */
403  if ( _settings.authType() == "basic"
404  && _settings.username().size()
405  && !_settings.password().size() ) {
406 
407  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
408  const auto cred = cm.getCred( _url );
409  if ( cred && cred->valid() ) {
410  if ( !_settings.username().size() )
411  _settings.setUsername(cred->username());
412  _settings.setPassword(cred->password());
413  }
414  }
415 
416  /*---------------------------------------------------------------*
417  CURLOPT_USERPWD: [user name]:[password]
418 
419  Url::username/password -> CURLOPT_USERPWD
420  If not provided, anonymous FTP identification
421  *---------------------------------------------------------------*/
422 
423  if ( _settings.userPassword().size() )
424  {
425  SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
426  std::string use_auth = _settings.authType();
427  if (use_auth.empty())
428  use_auth = "digest,basic"; // our default
429  long auth = CurlAuthData::auth_type_str2long(use_auth);
430  if( auth != CURLAUTH_NONE)
431  {
432  DBG << "Enabling HTTP authentication methods: " << use_auth
433  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
434  SET_OPTION(CURLOPT_HTTPAUTH, auth);
435  }
436  }
437 
438  if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
439  {
440  DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
441  SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
442  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
443  /*---------------------------------------------------------------*
444  * CURLOPT_PROXYUSERPWD: [user name]:[password]
445  *
446  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
447  * If not provided, $HOME/.curlrc is evaluated
448  *---------------------------------------------------------------*/
449 
450  std::string proxyuserpwd = _settings.proxyUserPassword();
451 
452  if ( proxyuserpwd.empty() )
453  {
454  CurlConfig curlconf;
455  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
456  if ( curlconf.proxyuserpwd.empty() )
457  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
458  else
459  {
460  proxyuserpwd = curlconf.proxyuserpwd;
461  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
462  }
463  }
464  else
465  {
466  DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
467  }
468 
469  if ( ! proxyuserpwd.empty() )
470  {
471  SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
472  }
473  }
474 #if CURLVERSION_AT_LEAST(7,19,4)
475  else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
476  {
477  // Explicitly disabled in URL (see fillSettingsFromUrl()).
478  // This should also prevent libcurl from looking into the environment.
479  DBG << "Proxy: explicitly NOPROXY" << endl;
480  SET_OPTION(CURLOPT_NOPROXY, "*");
481  }
482 #endif
483  else
484  {
485  DBG << "Proxy: not explicitly set" << endl;
486  DBG << "Proxy: libcurl may look into the environment" << endl;
487  }
488 
490  if ( _settings.minDownloadSpeed() != 0 )
491  {
492  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
493  // default to 10 seconds at low speed
494  SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
495  }
496 
497 #if CURLVERSION_AT_LEAST(7,15,5)
498  if ( _settings.maxDownloadSpeed() != 0 )
499  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
500 #endif
501 
502  /*---------------------------------------------------------------*
503  *---------------------------------------------------------------*/
504 
507  if ( str::strToBool( _url.getQueryParam( "cookies" ), true ) )
508  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
509  else
510  MIL << "No cookies requested" << endl;
511  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
512  SET_OPTION(CURLOPT_PROGRESSFUNCTION, &progressCallback );
513  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
514 
515 #if CURLVERSION_AT_LEAST(7,18,0)
516  // bnc #306272
517  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
518 #endif
519  // append settings custom headers to curl
520  for ( TransferSettings::Headers::const_iterator it = vol_settings.headersBegin();
521  it != vol_settings.headersEnd();
522  ++it )
523  {
524  // MIL << "HEADER " << *it << std::endl;
525 
526  _customHeaders = curl_slist_append(_customHeaders, it->c_str());
527  if ( !_customHeaders )
529  }
530 
531  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
532 }
533 
535 
536 
537 void MediaCurl::attachTo (bool next)
538 {
539  if ( next )
541 
542  if ( !_url.isValid() )
544 
547  {
549  }
550 
551  disconnectFrom(); // clean _curl if needed
552  _curl = curl_easy_init();
553  if ( !_curl ) {
555  }
556  try
557  {
558  setupEasy();
559  }
560  catch (Exception & ex)
561  {
562  disconnectFrom();
563  ZYPP_RETHROW(ex);
564  }
565 
566  // FIXME: need a derived class to propelly compare url's
568  setMediaSource(media);
569 }
570 
571 bool
573 {
574  return MediaHandler::checkAttachPoint( apoint, true, true);
575 }
576 
578 
580 {
581  if ( _customHeaders )
582  {
583  curl_slist_free_all(_customHeaders);
584  _customHeaders = 0L;
585  }
586 
587  if ( _curl )
588  {
589  curl_easy_cleanup( _curl );
590  _curl = NULL;
591  }
592 }
593 
595 
596 void MediaCurl::releaseFrom( const std::string & ejectDev )
597 {
598  disconnect();
599 }
600 
601 Url MediaCurl::getFileUrl( const Pathname & filename_r ) const
602 {
603  // Simply extend the URLs pathname. An 'absolute' URL path
604  // is achieved by encoding the leading '/' in an URL path:
605  // URL: ftp://user@server -> ~user
606  // URL: ftp://user@server/ -> ~user
607  // URL: ftp://user@server// -> ~user
608  // URL: ftp://user@server/%2F -> /
609  // ^- this '/' is just a separator
610  Url newurl( _url );
611  newurl.setPathName( ( Pathname("./"+_url.getPathName()) / filename_r ).asString().substr(1) );
612  return newurl;
613 }
614 
616 
617 void MediaCurl::getFile(const Pathname & filename , const ByteCount &expectedFileSize_r) const
618 {
619  // Use absolute file name to prevent access of files outside of the
620  // hierarchy below the attach point.
621  getFileCopy(filename, localPath(filename).absolutename(), expectedFileSize_r);
622 }
623 
625 
626 void MediaCurl::getFileCopy( const Pathname & filename , const Pathname & target, const ByteCount &expectedFileSize_r ) const
627 {
629 
630  Url fileurl(getFileUrl(filename));
631 
632  bool retry = false;
633  unsigned internalTry = 0;
634  static constexpr unsigned maxInternalTry = 3;
635 
636  do
637  {
638  retry = false;
639  try
640  {
641  doGetFileCopy(filename, target, report, expectedFileSize_r);
642  }
643  // retry with proper authentication data
644  catch (MediaUnauthorizedException & ex_r)
645  {
646  if(authenticate(ex_r.hint(), !retry))
647  retry = true;
648  else
649  {
651  ZYPP_RETHROW(ex_r);
652  }
653  }
654  // unexpected exception
655  catch (MediaException & excpt_r)
656  {
657  if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
658  ++internalTry;
659  if ( internalTry < maxInternalTry ) {
660  // just report (NO_ERROR); no interactive request to the user
661  report->problem(fileurl, media::DownloadProgressReport::NO_ERROR, excpt_r.asUserHistory());
662  retry = true;
663  continue;
664  }
665  excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
666  }
667 
669  if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
670  typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
671  {
673  }
674  report->finish(fileurl, reason, excpt_r.asUserHistory());
675  ZYPP_RETHROW(excpt_r);
676  }
677  }
678  while (retry);
679 
681 }
682 
684 
685 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
686 {
687  bool retry = false;
688 
689  do
690  {
691  try
692  {
693  return doGetDoesFileExist( filename );
694  }
695  // authentication problem, retry with proper authentication data
696  catch (MediaUnauthorizedException & ex_r)
697  {
698  if(authenticate(ex_r.hint(), !retry))
699  retry = true;
700  else
701  ZYPP_RETHROW(ex_r);
702  }
703  // unexpected exception
704  catch (MediaException & excpt_r)
705  {
706  ZYPP_RETHROW(excpt_r);
707  }
708  }
709  while (retry);
710 
711  return false;
712 }
713 
715 
717  CURLcode code,
718  bool timeout_reached) const
719 {
720  if ( code != 0 )
721  {
722  Url url;
723  if (filename.empty())
724  url = _url;
725  else
726  url = getFileUrl(filename);
727 
728  std::string err;
729  {
730  switch ( code )
731  {
732  case CURLE_UNSUPPORTED_PROTOCOL:
733  err = " Unsupported protocol";
734  if ( !_lastRedirect.empty() )
735  {
736  err += " or redirect (";
737  err += _lastRedirect;
738  err += ")";
739  }
740  break;
741  case CURLE_URL_MALFORMAT:
742  case CURLE_URL_MALFORMAT_USER:
743  err = " Bad URL";
744  break;
745  case CURLE_LOGIN_DENIED:
746  ZYPP_THROW(
747  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
748  break;
749  case CURLE_HTTP_RETURNED_ERROR:
750  {
751  long httpReturnCode = 0;
752  CURLcode infoRet = curl_easy_getinfo( _curl,
753  CURLINFO_RESPONSE_CODE,
754  &httpReturnCode );
755  if ( infoRet == CURLE_OK )
756  {
757  std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
758  switch ( httpReturnCode )
759  {
760  case 401:
761  {
762  std::string auth_hint = getAuthHint();
763 
764  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
765  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
766 
768  url, "Login failed.", _curlError, auth_hint
769  ));
770  }
771 
772  case 502: // bad gateway (bnc #1070851)
773  case 503: // service temporarily unavailable (bnc #462545)
775  case 504: // gateway timeout
777  case 403:
778  {
779  std::string msg403;
780  if ( url.getHost().find(".suse.com") != std::string::npos )
781  msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
782  else if (url.asString().find("novell.com") != std::string::npos)
783  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
785  }
786  case 404:
787  case 410:
789  }
790 
791  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
793  }
794  else
795  {
796  std::string msg = "Unable to retrieve HTTP response:";
797  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
799  }
800  }
801  break;
802  case CURLE_FTP_COULDNT_RETR_FILE:
803 #if CURLVERSION_AT_LEAST(7,16,0)
804  case CURLE_REMOTE_FILE_NOT_FOUND:
805 #endif
806  case CURLE_FTP_ACCESS_DENIED:
807  case CURLE_TFTP_NOTFOUND:
808  err = "File not found";
810  break;
811  case CURLE_BAD_PASSWORD_ENTERED:
812  case CURLE_FTP_USER_PASSWORD_INCORRECT:
813  err = "Login failed";
814  break;
815  case CURLE_COULDNT_RESOLVE_PROXY:
816  case CURLE_COULDNT_RESOLVE_HOST:
817  case CURLE_COULDNT_CONNECT:
818  case CURLE_FTP_CANT_GET_HOST:
819  err = "Connection failed";
820  break;
821  case CURLE_WRITE_ERROR:
822  err = "Write error";
823  break;
824  case CURLE_PARTIAL_FILE:
825  case CURLE_OPERATION_TIMEDOUT:
826  timeout_reached = true; // fall though to TimeoutException
827  // fall though...
828  case CURLE_ABORTED_BY_CALLBACK:
829  if( timeout_reached )
830  {
831  err = "Timeout reached";
833  }
834  else
835  {
836  err = "User abort";
837  }
838  break;
839 
840  // Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy
841  case CURLE_HTTP2:
842  case CURLE_HTTP2_STREAM:
843  err = "Curl error " + str::numstring( code );
845  break;
846 
847  default:
848  err = "Curl error " + str::numstring( code );
849  break;
850  }
851 
852  // uhm, no 0 code but unknown curl exception
854  }
855  }
856  else
857  {
858  // actually the code is 0, nothing happened
859  }
860 }
861 
863 
864 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
865 {
866  DBG << filename.asString() << endl;
867 
868  if(!_url.isValid())
870 
871  if(_url.getHost().empty())
873 
874  Url url(getFileUrl(filename));
875 
876  DBG << "URL: " << url.asString() << endl;
877  // Use URL without options and without username and passwd
878  // (some proxies dislike them in the URL).
879  // Curl seems to need the just scheme, hostname and a path;
880  // the rest was already passed as curl options (in attachTo).
881  Url curlUrl( clearQueryString(url) );
882 
883  //
884  // See also Bug #154197 and ftp url definition in RFC 1738:
885  // The url "ftp://user@host/foo/bar/file" contains a path,
886  // that is relative to the user's home.
887  // The url "ftp://user@host//foo/bar/file" (or also with
888  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
889  // contains an absolute path.
890  //
891  _lastRedirect.clear();
892  std::string urlBuffer( curlUrl.asString());
893  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
894  urlBuffer.c_str() );
895  if ( ret != 0 ) {
897  }
898 
899  // instead of returning no data with NOBODY, we return
900  // little data, that works with broken servers, and
901  // works for ftp as well, because retrieving only headers
902  // ftp will return always OK code ?
903  // See http://curl.haxx.se/docs/knownbugs.html #58
904  if ( (_url.getScheme() == "http" || _url.getScheme() == "https") &&
906  ret = curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
907  else
908  ret = curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
909 
910  if ( ret != 0 ) {
911  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
912  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
913  /* yes, this is why we never got to get NOBODY working before,
914  because setting it changes this option too, and we also
915  need to reset it
916  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
917  */
918  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
920  }
921 
922  AutoFILE file { ::fopen( "/dev/null", "w" ) };
923  if ( !file ) {
924  ERR << "fopen failed for /dev/null" << endl;
925  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
926  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
927  /* yes, this is why we never got to get NOBODY working before,
928  because setting it changes this option too, and we also
929  need to reset it
930  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
931  */
932  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
933  if ( ret != 0 ) {
935  }
936  ZYPP_THROW(MediaWriteException("/dev/null"));
937  }
938 
939  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, (*file) );
940  if ( ret != 0 ) {
941  std::string err( _curlError);
942  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
943  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
944  /* yes, this is why we never got to get NOBODY working before,
945  because setting it changes this option too, and we also
946  need to reset it
947  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
948  */
949  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
950  if ( ret != 0 ) {
952  }
954  }
955 
956  CURLcode ok = curl_easy_perform( _curl );
957  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
958 
959  // reset curl settings
960  if ( _url.getScheme() == "http" || _url.getScheme() == "https" )
961  {
962  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
963  if ( ret != 0 ) {
965  }
966 
967  /* yes, this is why we never got to get NOBODY working before,
968  because setting it changes this option too, and we also
969  need to reset it
970  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
971  */
972  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L);
973  if ( ret != 0 ) {
975  }
976 
977  }
978  else
979  {
980  // for FTP we set different options
981  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL);
982  if ( ret != 0 ) {
984  }
985  }
986 
987  // as we are not having user interaction, the user can't cancel
988  // the file existence checking, a callback or timeout return code
989  // will be always a timeout.
990  try {
991  evaluateCurlCode( filename, ok, true /* timeout */);
992  }
993  catch ( const MediaFileNotFoundException &e ) {
994  // if the file did not exist then we can return false
995  return false;
996  }
997  catch ( const MediaException &e ) {
998  // some error, we are not sure about file existence, rethrw
999  ZYPP_RETHROW(e);
1000  }
1001  // exists
1002  return ( ok == CURLE_OK );
1003 }
1004 
1006 
1007 
1008 #if DETECT_DIR_INDEX
1009 bool MediaCurl::detectDirIndex() const
1010 {
1011  if(_url.getScheme() != "http" && _url.getScheme() != "https")
1012  return false;
1013  //
1014  // try to check the effective url and set the not_a_file flag
1015  // if the url path ends with a "/", what usually means, that
1016  // we've received a directory index (index.html content).
1017  //
1018  // Note: This may be dangerous and break file retrieving in
1019  // case of some server redirections ... ?
1020  //
1021  bool not_a_file = false;
1022  char *ptr = NULL;
1023  CURLcode ret = curl_easy_getinfo( _curl,
1024  CURLINFO_EFFECTIVE_URL,
1025  &ptr);
1026  if ( ret == CURLE_OK && ptr != NULL)
1027  {
1028  try
1029  {
1030  Url eurl( ptr);
1031  std::string path( eurl.getPathName());
1032  if( !path.empty() && path != "/" && *path.rbegin() == '/')
1033  {
1034  DBG << "Effective url ("
1035  << eurl
1036  << ") seems to provide the index of a directory"
1037  << endl;
1038  not_a_file = true;
1039  }
1040  }
1041  catch( ... )
1042  {}
1043  }
1044  return not_a_file;
1045 }
1046 #endif
1047 
1049 
1050 void MediaCurl::doGetFileCopy(const Pathname & filename , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, const ByteCount &expectedFileSize_r, RequestOptions options ) const
1051 {
1052  Pathname dest = target.absolutename();
1053  if( assert_dir( dest.dirname() ) )
1054  {
1055  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1056  ZYPP_THROW( MediaSystemException(getFileUrl(filename), "System error on " + dest.dirname().asString()) );
1057  }
1058 
1059  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1060  AutoFILE file;
1061  {
1062  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1063  if( ! buf )
1064  {
1065  ERR << "out of memory for temp file name" << endl;
1066  ZYPP_THROW(MediaSystemException(getFileUrl(filename), "out of memory for temp file name"));
1067  }
1068 
1069  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1070  if( tmp_fd == -1 )
1071  {
1072  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1073  ZYPP_THROW(MediaWriteException(destNew));
1074  }
1075  destNew = ManagedFile( (*buf), filesystem::unlink );
1076 
1077  file = ::fdopen( tmp_fd, "we" );
1078  if ( ! file )
1079  {
1080  ERR << "fopen failed for file '" << destNew << "'" << endl;
1081  ZYPP_THROW(MediaWriteException(destNew));
1082  }
1083  tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1084  }
1085 
1086  DBG << "dest: " << dest << endl;
1087  DBG << "temp: " << destNew << endl;
1088 
1089  // set IFMODSINCE time condition (no download if not modified)
1090  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1091  {
1092  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1093  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1094  }
1095  else
1096  {
1097  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1098  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1099  }
1100  try
1101  {
1102  doGetFileCopyFile(filename, dest, file, report, expectedFileSize_r, options);
1103  }
1104  catch (Exception &e)
1105  {
1106  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1107  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1108  ZYPP_RETHROW(e);
1109  }
1110 
1111  long httpReturnCode = 0;
1112  CURLcode infoRet = curl_easy_getinfo(_curl,
1113  CURLINFO_RESPONSE_CODE,
1114  &httpReturnCode);
1115  bool modified = true;
1116  if (infoRet == CURLE_OK)
1117  {
1118  DBG << "HTTP response: " + str::numstring(httpReturnCode);
1119  if ( httpReturnCode == 304
1120  || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1121  {
1122  DBG << " Not modified.";
1123  modified = false;
1124  }
1125  DBG << endl;
1126  }
1127  else
1128  {
1129  WAR << "Could not get the response code." << endl;
1130  }
1131 
1132  if (modified || infoRet != CURLE_OK)
1133  {
1134  // apply umask
1135  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1136  {
1137  ERR << "Failed to chmod file " << destNew << endl;
1138  }
1139 
1140  file.resetDispose(); // we're going to close it manually here
1141  if ( ::fclose( file ) )
1142  {
1143  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1144  ZYPP_THROW(MediaWriteException(destNew));
1145  }
1146 
1147  // move the temp file into dest
1148  if ( rename( destNew, dest ) != 0 ) {
1149  ERR << "Rename failed" << endl;
1151  }
1152  destNew.resetDispose(); // no more need to unlink it
1153  }
1154 
1155  DBG << "done: " << PathInfo(dest) << endl;
1156 }
1157 
1159 
1160 void MediaCurl::doGetFileCopyFile(const Pathname & filename , const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, const ByteCount &expectedFileSize_r, RequestOptions options ) const
1161 {
1162  DBG << filename.asString() << endl;
1163 
1164  if(!_url.isValid())
1166 
1167  if(_url.getHost().empty())
1169 
1170  Url url(getFileUrl(filename));
1171 
1172  DBG << "URL: " << url.asString() << endl;
1173  // Use URL without options and without username and passwd
1174  // (some proxies dislike them in the URL).
1175  // Curl seems to need the just scheme, hostname and a path;
1176  // the rest was already passed as curl options (in attachTo).
1177  Url curlUrl( clearQueryString(url) );
1178 
1179  //
1180  // See also Bug #154197 and ftp url definition in RFC 1738:
1181  // The url "ftp://user@host/foo/bar/file" contains a path,
1182  // that is relative to the user's home.
1183  // The url "ftp://user@host//foo/bar/file" (or also with
1184  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1185  // contains an absolute path.
1186  //
1187  _lastRedirect.clear();
1188  std::string urlBuffer( curlUrl.asString());
1189  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1190  urlBuffer.c_str() );
1191  if ( ret != 0 ) {
1193  }
1194 
1195  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1196  if ( ret != 0 ) {
1198  }
1199 
1200  // Set callback and perform.
1201  ProgressData progressData(_curl, _settings.timeout(), url, expectedFileSize_r, &report);
1202  if (!(options & OPTION_NO_REPORT_START))
1203  report->start(url, dest);
1204  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1205  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1206  }
1207 
1208  ret = curl_easy_perform( _curl );
1209 #if CURLVERSION_AT_LEAST(7,19,4)
1210  // bnc#692260: If the client sends a request with an If-Modified-Since header
1211  // with a future date for the server, the server may respond 200 sending a
1212  // zero size file.
1213  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1214  if ( ftell(file) == 0 && ret == 0 )
1215  {
1216  long httpReturnCode = 33;
1217  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1218  {
1219  long conditionUnmet = 33;
1220  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1221  {
1222  WAR << "TIMECONDITION unmet - retry without." << endl;
1223  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1224  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1225  ret = curl_easy_perform( _curl );
1226  }
1227  }
1228  }
1229 #endif
1230 
1231  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1232  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1233  }
1234 
1235  if ( ret != 0 )
1236  {
1237  ERR << "curl error: " << ret << ": " << _curlError
1238  << ", temp file size " << ftell(file)
1239  << " bytes." << endl;
1240 
1241  // the timeout is determined by the progress data object
1242  // which holds whether the timeout was reached or not,
1243  // otherwise it would be a user cancel
1244  try {
1245 
1246  if ( progressData.fileSizeExceeded )
1247  ZYPP_THROW(MediaFileSizeExceededException(url, progressData._expectedFileSize));
1248 
1249  evaluateCurlCode( filename, ret, progressData.reached );
1250  }
1251  catch ( const MediaException &e ) {
1252  // some error, we are not sure about file existence, rethrw
1253  ZYPP_RETHROW(e);
1254  }
1255  }
1256 
1257 #if DETECT_DIR_INDEX
1258  if (!ret && detectDirIndex())
1259  {
1261  }
1262 #endif // DETECT_DIR_INDEX
1263 }
1264 
1266 
1267 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1268 {
1269  filesystem::DirContent content;
1270  getDirInfo( content, dirname, /*dots*/false );
1271 
1272  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1273  Pathname filename = dirname + it->name;
1274  int res = 0;
1275 
1276  switch ( it->type ) {
1277  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1278  case filesystem::FT_FILE:
1279  getFile( filename, 0 );
1280  break;
1281  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1282  if ( recurse_r ) {
1283  getDir( filename, recurse_r );
1284  } else {
1285  res = assert_dir( localPath( filename ) );
1286  if ( res ) {
1287  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1288  }
1289  }
1290  break;
1291  default:
1292  // don't provide devices, sockets, etc.
1293  break;
1294  }
1295  }
1296 }
1297 
1299 
1300 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1301  const Pathname & dirname, bool dots ) const
1302 {
1303  getDirectoryYast( retlist, dirname, dots );
1304 }
1305 
1307 
1309  const Pathname & dirname, bool dots ) const
1310 {
1311  getDirectoryYast( retlist, dirname, dots );
1312 }
1313 
1315 //
1316 int MediaCurl::aliveCallback( void *clientp, double /*dltotal*/, double dlnow, double /*ultotal*/, double /*ulnow*/ )
1317 {
1318  ProgressData *pdata = reinterpret_cast<ProgressData *>( clientp );
1319  if( pdata )
1320  {
1321  // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1322  // prevent a percentage raise while downloading a metalink file. Download
1323  // activity however is indicated by propagating the download rate (via dlnow).
1324  pdata->updateStats( 0.0, dlnow );
1325  return pdata->reportProgress();
1326  }
1327  return 0;
1328 }
1329 
1330 int MediaCurl::progressCallback( void *clientp, double dltotal, double dlnow, double ultotal, double ulnow )
1331 {
1332  ProgressData *pdata = reinterpret_cast<ProgressData *>( clientp );
1333  if( pdata )
1334  {
1335  // work around curl bug that gives us old data
1336  long httpReturnCode = 0;
1337  if ( curl_easy_getinfo( pdata->curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1338  return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1339 
1340  pdata->updateStats( dltotal, dlnow );
1341  return pdata->reportProgress();
1342  }
1343  return 0;
1344 }
1345 
1347 {
1348  ProgressData *pdata = reinterpret_cast<ProgressData *>(clientp);
1349  return pdata ? pdata->curl : 0;
1350 }
1351 
1353 
1354 std::string MediaCurl::getAuthHint() const
1355 {
1356  long auth_info = CURLAUTH_NONE;
1357 
1358  CURLcode infoRet =
1359  curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1360 
1361  if(infoRet == CURLE_OK)
1362  {
1363  return CurlAuthData::auth_type_long2str(auth_info);
1364  }
1365 
1366  return "";
1367 }
1368 
1373 void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1374 {
1375  ProgressData *data = reinterpret_cast<ProgressData *>(clientp);
1376  if ( data ) {
1377  data->_expectedFileSize = expectedFileSize;
1378  }
1379 }
1380 
1382 
1383 bool MediaCurl::authenticate(const std::string & availAuthTypes, bool firstTry) const
1384 {
1386  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
1387  CurlAuthData_Ptr credentials;
1388 
1389  // get stored credentials
1390  AuthData_Ptr cmcred = cm.getCred(_url);
1391 
1392  if (cmcred && firstTry)
1393  {
1394  credentials.reset(new CurlAuthData(*cmcred));
1395  DBG << "got stored credentials:" << endl << *credentials << endl;
1396  }
1397  // if not found, ask user
1398  else
1399  {
1400 
1401  CurlAuthData_Ptr curlcred;
1402  curlcred.reset(new CurlAuthData());
1404 
1405  // preset the username if present in current url
1406  if (!_url.getUsername().empty() && firstTry)
1407  curlcred->setUsername(_url.getUsername());
1408  // if CM has found some credentials, preset the username from there
1409  else if (cmcred)
1410  curlcred->setUsername(cmcred->username());
1411 
1412  // indicate we have no good credentials from CM
1413  cmcred.reset();
1414 
1415  std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1416 
1417  // set available authentication types from the exception
1418  // might be needed in prompt
1419  curlcred->setAuthType(availAuthTypes);
1420 
1421  // ask user
1422  if (auth_report->prompt(_url, prompt_msg, *curlcred))
1423  {
1424  DBG << "callback answer: retry" << endl
1425  << "CurlAuthData: " << *curlcred << endl;
1426 
1427  if (curlcred->valid())
1428  {
1429  credentials = curlcred;
1430  // if (credentials->username() != _url.getUsername())
1431  // _url.setUsername(credentials->username());
1439  }
1440  }
1441  else
1442  {
1443  DBG << "callback answer: cancel" << endl;
1444  }
1445  }
1446 
1447  // set username and password
1448  if (credentials)
1449  {
1450  // HACK, why is this const?
1451  const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1452  const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1453 
1454  // set username and password
1455  CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1457 
1458  // set available authentication types from the exception
1459  if (credentials->authType() == CURLAUTH_NONE)
1460  credentials->setAuthType(availAuthTypes);
1461 
1462  // set auth type (seems this must be set _after_ setting the userpwd)
1463  if (credentials->authType() != CURLAUTH_NONE)
1464  {
1465  // FIXME: only overwrite if not empty?
1466  const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1467  ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1469  }
1470 
1471  if (!cmcred)
1472  {
1473  credentials->setUrl(_url);
1474  cm.addCred(*credentials);
1475  cm.save();
1476  }
1477 
1478  return true;
1479  }
1480 
1481  return false;
1482 }
1483 
1484 //need a out of line definiton, otherwise vtable is emitted for every translation unit
1486 
1487 
1488  } // namespace media
1489 } // namespace zypp
1490 //
#define CONNECT_TIMEOUT
Definition: CurlHelper.h:21
#define EXPLICITLY_NO_PROXY
Definition: CurlHelper.h:25
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:400
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:392
Interface to gettext.
#define _(MSG)
Definition: Gettext.h:37
#define DBG
Definition: Logger.h:78
#define MIL
Definition: Logger.h:79
#define ERR
Definition: Logger.h:81
#define WAR
Definition: Logger.h:80
double _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:78
double uload
Definition: MediaCurl.cc:161
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:74
bool reached
Definition: MediaCurl.cc:68
bool fileSizeExceeded
Definition: MediaCurl.cc:69
ByteCount _expectedFileSize
Definition: MediaCurl.cc:71
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:84
double dload
Definition: MediaCurl.cc:159
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:177
double dload_period
Definition: MediaCurl.cc:151
callback::SendReport< DownloadProgressReport > * report
Definition: MediaCurl.cc:70
long secs
Definition: MediaCurl.cc:153
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:82
CURL * curl
Definition: MediaCurl.cc:65
Url url
Definition: MediaCurl.cc:66
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:85
time_t ltime
Definition: MediaCurl.cc:157
time_t _timeNow
Now.
Definition: MediaCurl.cc:76
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:170
double drate_period
Definition: MediaCurl.cc:149
double drate_avg
Definition: MediaCurl.cc:155
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:73
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:75
double _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:80
double _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:79
time_t timeout
Definition: MediaCurl.cc:67
Convenience interface for handling authentication data of media user.
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
Definition: AutoDispose.h:93
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:162
Store and operate with byte count.
Definition: ByteCount.h:31
Base class for Exception.
Definition: Exception.h:146
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:91
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition: Exception.cc:125
Url manipulation class.
Definition: Url.h:92
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:528
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:492
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:567
std::string getPathName(EEncoding eflag=zypp::url::E_DECODED) const
Returns the path name from the URL.
Definition: Url.cc:599
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:655
void setPathName(const std::string &path, EEncoding eflag=zypp::url::E_DECODED)
Set the path name.
Definition: Url.cc:759
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:583
bool isValid() const
Verifies the Url.
Definition: Url.cc:484
static ZConfig & instance()
Singleton ctor.
Definition: Resolver.cc:126
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:221
bool userMayRWX() const
Definition: PathInfo.h:353
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:246
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:170
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
const std::string & asString() const
String representation.
Definition: Pathname.h:91
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
Pathname absolutename() const
Return this path, adding a leading '/' if relative.
Definition: Pathname.h:139
const char * c_str() const
String representation.
Definition: Pathname.h:110
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods.
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Curl HTTP authentication data.
Definition: MediaUserAuth.h:74
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g.
Definition: MediaCurl.cc:263
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition: MediaCurl.cc:265
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:33
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:273
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
Definition: MediaCurl.cc:601
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:228
virtual bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly,...
Definition: MediaCurl.cc:685
@ OPTION_NO_IFMODSINCE
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:44
@ OPTION_NO_REPORT_START
do not send a start ProgressReport
Definition: MediaCurl.h:46
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1373
std::string _currentCookieFile
Definition: MediaCurl.h:170
static int aliveCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Definition: MediaCurl.cc:1316
static Pathname _cookieFile
Definition: MediaCurl.h:171
static int progressCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1330
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:173
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:217
virtual void attachTo(bool next=false) override
Call concrete handler to attach the media.
Definition: MediaCurl.cc:537
virtual void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1300
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:177
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:235
virtual void getFileCopy(const Pathname &srcFilename, const Pathname &targetFilename, const ByteCount &expectedFileSize_r) const override
Definition: MediaCurl.cc:626
bool detectDirIndex() const
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:716
TransferSettings _settings
Definition: MediaCurl.h:179
virtual void doGetFileCopy(const Pathname &srcFilename, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, const ByteCount &expectedFileSize_r, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1050
virtual bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:572
virtual void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1267
TransferSettings & settings()
Definition: MediaCurl.cc:222
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1383
virtual void getFile(const Pathname &filename, const ByteCount &expectedFileSize_r) const override
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:617
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1346
virtual void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:596
void doGetFileCopyFile(const Pathname &srcFilename, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &_report, const ByteCount &expectedFileSize_r, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1160
virtual void disconnectFrom() override
Definition: MediaCurl.cc:579
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:864
curl_slist * _customHeaders
Definition: MediaCurl.h:178
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1354
Just inherits Exception to separate media exceptions.
Abstract base class for 'physical' MediaHandler like MediaCD, etc.
Definition: MediaHandler.h:45
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
Url url() const
Url used.
Definition: MediaHandler.h:507
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
void disconnect()
Use concrete handler to isconnect media.
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
Pathname localPath(const Pathname &pathname) const
Files provided will be available at 'localPath(filename)'.
const Url _url
Url to handle.
Definition: MediaHandler.h:110
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
Pathname attachPoint() const
Return the currently used attach point.
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:37
const std::string & hint() const
comma separated list of available authentication types
Holds transfer setting.
std::string proxy() const
proxy host
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
long connectTimeout() const
connection timeout
std::string password() const
auth password
long timeout() const
transfer timeout
Headers::const_iterator headersEnd() const
end iterators to additional headers
std::string userPassword() const
returns the user and password as a user:pass string
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
void setAuthType(std::string &&val_r)
set the allowed authentication types
void setUsername(std::string &&val_r)
sets the auth username
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3"
void setConnectTimeout(long t)
set the connect timeout
std::string userAgentString() const
user agent string
void setPassword(std::string &&val_r)
sets the auth password
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar"
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
bool verifyHostEnabled() const
Whether to verify host for ssl.
Pathname clientCertificatePath() const
SSL client certificate file.
Pathname certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
bool headRequestsAllowed() const
whether HEAD requests are allowed
std::string proxyUsername() const
proxy auth username
std::string authType() const
get the allowed authentication types
bool proxyEnabled() const
proxy is enabled
std::string username() const
auth username
Pathname clientKeyPath() const
SSL client key file.
void setTimeout(long t)
set the transfer timeout
Headers::const_iterator headersBegin() const
begin iterators to additional headers
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
int ZYPP_MEDIA_CURL_IPRESOLVE()
Definition: CurlHelper.h:36
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: CurlHelper.cc:110
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: CurlHelper.cc:62
const char * anonymousIdHeader()
initialized only once, this gets the anonymous id from the target, which we pass in the http header
Definition: CurlHelper.cc:294
void globalInitCurlOnce()
Definition: CurlHelper.cc:19
const char * distributionFlavorHeader()
initialized only once, this gets the distribution flavor from the target, which we pass in the http h...
Definition: CurlHelper.cc:308
std::string curlUnEscape(std::string text_r)
Definition: CurlHelper.cc:351
const char * agentString()
initialized only once, this gets the agent string which also includes the curl version
Definition: CurlHelper.cc:322
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: CurlHelper.cc:258
Url clearQueryString(const Url &url)
Definition: CurlHelper.cc:358
int log_curl(CURL *curl, curl_infotype info, char *ptr, size_t len, void *max_lvl)
Definition: CurlHelper.cc:28
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:813
int unlink(const Pathname &path)
Like 'unlink'.
Definition: PathInfo.cc:662
int rename(const Pathname &oldpath, const Pathname &newpath)
Like 'rename'.
Definition: PathInfo.cc:704
int rmdir(const Pathname &path)
Like 'rmdir'.
Definition: PathInfo.cc:367
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Definition: PathInfo.cc:320
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1164
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:547
shared_ptr< CurlAuthData > CurlAuthData_Ptr
shared_ptr< AuthData > AuthData_Ptr
Definition: MediaUserAuth.h:69
std::string numstring(char n, int w=0)
Definition: String.h:286
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:426
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
AutoDispose<int> calling ::close
Definition: AutoDispose.h:281
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:292
Structure holding values of curlrc options.
Definition: CurlConfig.h:17
std::string proxyuserpwd
Definition: CurlConfig.h:39
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: CurlConfig.cc:24
Convenient building of std::string with boost::format.
Definition: String.h:250