libzypp: zypp/target/commitpackagepreloader.cc Source File

libzypp 17.38.11

commitpackagepreloader.cc
Go to the documentation of this file.
1#include "private/commitpackagepreloader_p.h"
2#include "zypp-core/base/Gettext.h"
3#include <zypp-curl/private/curlhelper_p.h>
4#include <zypp-media/auth/credentialmanager.h>
5#include <zypp/media/MediaCurl2.h> // for shared logic like authenticate
6#include <zypp/media/MediaHandlerFactory.h> // to detect the URL type
7#include <zypp-media/mediaconfig.h>
8#include <zypp/media/UrlResolverPlugin.h>
9#include <zypp-core/ng/base/eventloop.h>
10#include <zypp-core/fs/TmpPath.h>
11#include <zypp-curl/transfersettings.h>
12#include <zypp-curl/ng/network/networkrequestdispatcher.h>
13#include <zypp-curl/ng/network/request.h>
14#include <zypp/repo/RepoProvideFile.h>
15#include <zypp/MediaSetAccess.h>
16#include <zypp/Package.h>
17#include <zypp/SrcPackage.h>
18#include <zypp/ZConfig.h>
19#include <zypp-core/base/Env.h>
20
21namespace zypp {
22
23 namespace {
24
25 inline bool preloadEnabled()
26 {
27 TriBool envstate = env::getenvBool( "ZYPP_PCK_PRELOAD" );
28 if ( indeterminate(envstate) ) {
29#if APIConfig(LIBZYPP_CONFIG_USE_SERIAL_PACKAGE_DOWNLOAD_BY_DEFAULT)
30 return false;
31#else
32 return true;
33#endif
34 }
35 return bool(envstate);
36 }
37
38 zypp::Pathname pckCachedLocation ( const PoolItem &pck ) {
39 if ( pck.isKind<Package>() ) {
40 return pck->asKind<Package>()->cachedLocation();
41 } else if ( pck.isKind<SrcPackage>() ) {
42 return pck->asKind<SrcPackage>()->cachedLocation();
43 }
44 return {};
45 }
46
47 }
48
49 struct RepoUrl {
50 zypp::Url baseUrl;
51 media::UrlResolverPlugin::HeaderList headers;
52 };
53
54 class CommitPackagePreloader::PreloadWorker : public zyppng::Base {
55 public:
56 enum State {
57 Pending,
58 SimpleDl,
59 //ZckHead,
60 //ZckData,
61 Finished
62 };
63
64 PreloadWorker( CommitPackagePreloader &parent ) : _parent(parent) {}
65
66 bool finished ( ) const {
67 return (_s == Finished);
68 }
69
70 void nextJob () {
71
72 // clean state vars
73 _started = false;
74 _firstAuth = true;
75 _notFoundRetry = 0;
76 _tmpFile.reset();
77 _lastByteCount = 0;
78 _taintedMirrors.clear();
79
80 if ( _parent._requiredDls.empty() ) {
81
82 if ( _myMirror ) {
83 _myMirror->refs--;
84 _myMirror = nullptr;
85 _currentRepoId = sat::detail::noRepoId;
86 }
87
88 MIL << "No more jobs pending, exiting worker" << std::endl;
89 // exit!
90 _s = Finished;
91 _sigFinished.emit();
92 return;
93 }
94
95 _job = _parent._requiredDls.front();
96 _parent._requiredDls.pop_front();
97
98 auto loc = _job.lookupLocation();
99 const auto repoInfo = _job.repoInfo();
100
101 _targetPath = repoInfo.predownloadPath() / repo::RepoMediaAccess::mapToCachePath( repoInfo, loc );
102
103 // select a mirror we want to use
104 if ( !prepareMirror( ) ) {
105 finishCurrentJob ( _targetPath, {}, media::CommitPreloadReport::ERROR, asString( _("no mirror found") ), true );
106 return nextJob();
107 }
108
109 if ( filesystem::assert_dir( _targetPath.dirname()) != 0 ) {
110 ERR << "Failed to create target dir for file: " << _targetPath << std::endl;
111 finishCurrentJob ( _targetPath, {}, media::CommitPreloadReport::ERROR, asString( _("could not create target file") ), true );
112 return nextJob();
113 }
114
115
116 media::TransferSettings settings;
117 zypp::Url url;
118
119 try {
120 makeJobUrl ( url, settings );
121 } catch ( const zypp::Exception &e ) {
122 ERR << "Failed to create job URL for file: " << _targetPath << " ("<<e<<")" << std::endl;
123 finishCurrentJob ( _targetPath, {}, media::CommitPreloadReport::ERROR, zypp::str::Format(_("Error: Failed to initialize transfer settings (%1%).")) % e.asUserString(), true );
124 return nextJob();
125 }
126
127 // check if the file is there already
128 {
129 PathInfo pathInfo(_targetPath);
130 if ( pathInfo.isExist() ) {
131 // just in case there is something else that is not a file we delete it
132 if ( !pathInfo.isFile() ) {
133 if ( pathInfo.isDir () )
134 filesystem::rmdir( _targetPath );
135 else
136 filesystem::unlink( _targetPath );
137
138 } else if ( is_checksum( _targetPath, loc.checksum() ) ) {
139 // if we have the file already, no need to download again
140 finishCurrentJob ( _targetPath, url, media::CommitPreloadReport::NO_ERROR, asString( _("already in cache") ), false );
141 return nextJob();
142
143 } else {
144 // everything else we delete
145 filesystem::unlink ( _targetPath );
146 }
147 }
148 }
149
150 // we download into a temp file so that we don't leave broken files in case of errors or a crash
151 _tmpFile = filesystem::TmpFile::asManagedFile( _targetPath.dirname(), _targetPath.basename() );
152
153 if ( _s == Pending ) {
154 // init case, set up request
155 _req = std::make_shared<zyppng::NetworkRequest>( url, _tmpFile );
156 _req->connect( &zyppng::NetworkRequest::sigStarted, *this, &PreloadWorker::onRequestStarted );
157 _req->connect( &zyppng::NetworkRequest::sigBytesDownloaded, *this, &PreloadWorker::onRequestProgress );
158 _req->connect( &zyppng::NetworkRequest::sigFinished, *this, &PreloadWorker::onRequestFinished );
159 } else {
160 _req->resetRequestRanges();
161 _req->setUrl( url );
162 _req->setTargetFilePath( _tmpFile );
163 }
164
165 // TODO check for zchunk
166
167 _s = SimpleDl;
168 _req->transferSettings() = settings;
169 _parent._dispatcher->enqueue(_req);
170 }
171
172 zyppng::SignalProxy<void()> sigWorkerFinished() {
173 return _sigFinished;
174 }
175
176 private:
177
178 // TODO some smarter logic that selects mirrors
179 bool prepareMirror( ) {
180
181 const auto &pi = _job;
182
183 if ( _myMirror ) {
184 if ( _currentRepoId == pi.repository().id() ) {
185 return true;
186 }
187 _currentRepoId = sat::detail::noRepoId;
188 _myMirror->refs--;
189 _myMirror = nullptr;
190 }
191
192 _myMirror = findUsableMirror ();
193 if ( !_myMirror )
194 return false;
195
196 _currentRepoId = pi.repository().id();
197 _myMirror->refs++;
198 return true;
199 }
200
204 bool taintCurrentMirror() {
205
206 if ( _myMirror ) {
207 _myMirror->miss++;
208 _taintedMirrors.insert( _myMirror );
209 }
210
211 // try to find another mirror
212 auto mirrPtr = findUsableMirror ( _myMirror, false );
213 if ( mirrPtr ) {
214 if ( _myMirror ) {
215 _myMirror->refs--;
216 }
217 _myMirror = mirrPtr;
218 _myMirror->refs++;
219 return true;
220 }
221 return false;
222 }
223
227 RepoUrl *findUsableMirror( RepoUrl *skip = nullptr, bool allowTainted = true ) {
228 auto &repoDlInfo = _parent._dlRepoInfo.at( _job.repository().id() );
229
230 std::vector<RepoUrl>::iterator curr = repoDlInfo._baseUrls.end();
231 int currentSmallestRef = INT_MAX;
232
233 for ( auto i = repoDlInfo._baseUrls.begin(); i != repoDlInfo._baseUrls.end(); i++ ) {
234 auto mirrorPtr = &(*i);
235
236 if ( skip == mirrorPtr )
237 continue;
238
239 if ( !allowTainted && _taintedMirrors.find(mirrorPtr) != _taintedMirrors.end() )
240 continue;
241
242 // we are adding the file misses on top of the refcount
243 // that way we will use mirrors that often miss a file less
244 if ( ( i->refs + i->miss ) < currentSmallestRef ) {
245 currentSmallestRef = ( i->refs + i->miss );
246 curr = i;
247 }
248 }
249
250 if ( curr == repoDlInfo._baseUrls.end() )
251 return nullptr;
252 return &(*curr);
253 }
254
255 void onRequestStarted ( zyppng::NetworkRequest &req ) {
256 MIL << "Request for " << req.url() << " started" << std::endl;
257 }
258
259 void onRequestProgress( zyppng::NetworkRequest &req, zypp::ByteCount count ) {
260 if ( !_started ) {
261 _started = true;
262
263 callback::UserData userData( "CommitPreloadReport/fileStart" );
264 userData.set( "Url", _req->url() );
265 _parent._report->fileStart( _targetPath, userData );
266 }
267
268 ByteCount downloaded;
269 if ( _lastByteCount == 0 )
270 downloaded = count;
271 else
272 downloaded = count - _lastByteCount;
273 _lastByteCount = count;
274
275 _parent.reportBytesDownloaded( downloaded );
276 }
277
278 void onRequestFinished( zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err ) {
279 MIL << "Request for " << req.url() << " finished. (" << err.toString() << ")" << std::endl;
280 if ( !req.hasError() ) {
281 // apply umask and move the _tmpFile into _targetPath
282 if ( filesystem::chmodApplyUmask( _tmpFile, 0644 ) == 0 && filesystem::rename( _tmpFile, _targetPath ) == 0 ) {
283 _tmpFile.resetDispose(); // rename consumed the file, no need to unlink.
284 finishCurrentJob ( _targetPath, req.url(), media::CommitPreloadReport::NO_ERROR, asString( _("done") ), false );
285 } else {
286 // error
287 finishCurrentJob ( _targetPath, req.url(), media::CommitPreloadReport::ERROR, _("failed to rename temporary file."), true );
288 }
289 } else {
290 // handle errors and auth
291 const auto &error = req.error();
292 switch ( error.type() ) {
293 case zyppng::NetworkRequestError::InternalError:
294 case zyppng::NetworkRequestError::InvalidChecksum:
295 case zyppng::NetworkRequestError::UnsupportedProtocol:
296 case zyppng::NetworkRequestError::MalformedURL:
297 case zyppng::NetworkRequestError::PeerCertificateInvalid:
298 case zyppng::NetworkRequestError::ConnectionFailed:
299 case zyppng::NetworkRequestError::ServerReturnedError:
300 case zyppng::NetworkRequestError::MissingData:
301 case zyppng::NetworkRequestError::RangeFail:
302 case zyppng::NetworkRequestError::ExceededMaxLen:
303 case zyppng::NetworkRequestError::TemporaryProblem:
304 case zyppng::NetworkRequestError::Timeout:
305 case zyppng::NetworkRequestError::Forbidden:
306 case zyppng::NetworkRequestError::Http2Error:
307 case zyppng::NetworkRequestError::Http2StreamError:
308 case zyppng::NetworkRequestError::NotFound: {
309 ERR << "Download from mirror failed for file " << req.url () << " trying to taint mirror and move on" << std::endl;
310
311 std::string lastError = req.extendedErrorString();
312 while ( taintCurrentMirror() ) {
313 _notFoundRetry++;
314
315 const auto str = zypp::str::Format(_("Error: \"%1%\", trying next mirror.")) % lastError;
316 finishCurrentJob ( _targetPath, req.url(), media::CommitPreloadReport::ERROR, str, false );
317
318 try {
319 media::TransferSettings settings;
320 zypp::Url url;
321 makeJobUrl ( url, settings );
322
323 MIL << "Found new mirror: " << url << " recovering, retry count: " << _notFoundRetry << std::endl;
324
325 _req->setUrl( url );
326 _req->transferSettings () = settings;
327
328 _parent._dispatcher->enqueue( _req );
329 return;
330
331 } catch ( const zypp::Exception &e ) {
332 ERR << "Failed to setup mirror: ( " << e << " ), trying next!" << std::endl;
333 lastError = e.asUserString();
334 continue;
335 }
336 break;
337 }
338
339 ERR << "No mirror found, giving up on file: " << req.url() << std::endl;
340 finishCurrentJob ( _targetPath, req.url(), media::CommitPreloadReport::NOT_FOUND, req.extendedErrorString(), true );
341 break;
342 }
343 case zyppng::NetworkRequestError::Unauthorized:
344 case zyppng::NetworkRequestError::AuthFailed: {
345
346 //in case we got a auth hint from the server the error object will contain it
347 std::string authHint = error.extraInfoValue("authHint", std::string());
348
349 media::CredentialManager cm(media::CredManagerOptions(ZConfig::instance().repoManagerRoot()));
350 bool newCreds = media::MediaNetworkCommonHandler::authenticate( _myMirror->baseUrl, cm, req.transferSettings(), authHint, _firstAuth );
351 if ( newCreds) {
352 _firstAuth = false;
353 _parent._dispatcher->enqueue( _req );
354 return;
355 }
356
357 finishCurrentJob ( _targetPath, req.url(), media::CommitPreloadReport::ACCESS_DENIED, req.extendedErrorString(), true );
358 break;
359
360 } case zyppng::NetworkRequestError::Cancelled: {
361 finishCurrentJob ( _targetPath, req.url(), media::CommitPreloadReport::ERROR, req.extendedErrorString(), true );
362 break;
363 }
364 case zyppng::NetworkRequestError::NoError:
365 // should never happen
366 DBG << "BUG: Download error flag is set , but Error code is NoError" << std::endl;
367 break;
368 }
369 }
370 nextJob();
371 }
372
373 void finishCurrentJob( const zypp::Pathname &localPath, const std::optional<zypp::Url> &url, media::CommitPreloadReport::Error e, const std::optional<std::string> &errorMessage, bool fatal ) {
374
375 callback::UserData userData( "CommitPreloadReport/fileDone" );
376 if ( url )
377 userData.set( "Url", *url );
378 if ( errorMessage )
379 userData.set( "description", *errorMessage );
380
381 if ( e != media::CommitPreloadReport::NO_ERROR && fatal )
382 _parent._missedDownloads = true;
383
384 _parent._report->fileDone( localPath, e, userData );
385 }
386
387 void makeJobUrl ( zypp::Url &resultUrl, media::TransferSettings &resultSet ) {
388
389 // rewrite Url
390 zypp::Url url = _myMirror->baseUrl;
391
392 media::TransferSettings settings;
393 ::internal::prepareSettingsAndUrl ( url, settings );
394
395 const auto &loc = _job.lookupLocation();
396
397 // rewrite URL for media handle
398 if ( loc.medianr() > 1 )
399 url = MediaSetAccess::rewriteUrl( url ,loc.medianr() );
400
401 // append path to file
402 url.appendPathName( loc.filename() );
403
404 // add extra headers
405 for ( const auto & el : _myMirror->headers ) {
406 std::string header { el.first };
407 header += ": ";
408 header += el.second;
409 MIL << "Added custom header -> " << header << std::endl;
410 settings.addHeader( std::move(header) );
411 }
412
413 resultUrl = url;
414 resultSet = settings;
415 }
416
417 private:
418 State _s = Pending;
419 CommitPackagePreloader &_parent;
420 zyppng::NetworkRequestRef _req;
421
422 PoolItem _job;
423 ManagedFile _tmpFile;
424 zypp::Pathname _targetPath;
425 bool _started = false;
426 bool _firstAuth = true;
427 RepoUrl *_myMirror = nullptr;
428 ByteCount _lastByteCount = 0;
429 Repository::IdType _currentRepoId = sat::detail::noRepoId;
430
431 // retry handling
432 int _notFoundRetry = 0;
433 std::set<RepoUrl *> _taintedMirrors; //< mirrors that returned 404 for the current request
434
435 zyppng::Signal<void()> _sigFinished;
436
437 };
438
439 CommitPackagePreloader::CommitPackagePreloader()
440 {}
441
442 void CommitPackagePreloader::preloadTransaction( const std::vector<sat::Transaction::Step> &steps)
443 {
444 if ( !preloadEnabled() ) {
445 MIL << "CommitPackagePreloader disabled" << std::endl;
446 return;
447 }
448
449 // preload happens only if someone handles the report
450 if ( !_report->connected() ) {
451 MIL << "No receiver for the CommitPreloadReport, skipping preload phase" << std::endl;
452 return;
453 }
454
455 auto ev = zyppng::EventLoop::create();
456 _dispatcher = std::make_shared<zyppng::NetworkRequestDispatcher>();
457 _dispatcher->setMaximumConcurrentConnections( MediaConfig::instance().download_max_concurrent_connections() );
458 _dispatcher->setAgentString ( str::asString( media::MediaCurl2::agentString () ) );
459 _dispatcher->setHostSpecificHeader ("download.opensuse.org", "X-ZYpp-DistributionFlavor", str::asString(media::MediaCurl2::distributionFlavorHeader()) );
460 _dispatcher->setHostSpecificHeader ("download.opensuse.org", "X-ZYpp-AnonymousId", str::asString(media::MediaCurl2::anonymousIdHeader()) );
461 _dispatcher->setHostSpecificHeader ("cdn.opensuse.org", "X-ZYpp-DistributionFlavor", str::asString(media::MediaCurl2::distributionFlavorHeader()) );
462 _dispatcher->setHostSpecificHeader ("cdn.opensuse.org", "X-ZYpp-AnonymousId", str::asString(media::MediaCurl2::anonymousIdHeader()) );
463 _dispatcher->run();
464
465 _pTracker = std::make_shared<internal::ProgressTracker>();
466 _requiredBytes = 0;
467 _downloadedBytes = 0;
468 _missedDownloads = false;
469 _lastProgressUpdate.reset();
470
471 zypp_defer {
472 _dispatcher.reset();
473 _pTracker.reset();
474 };
475
476 for ( const auto &step : steps ) {
477 switch ( step.stepType() )
478 {
479 case sat::Transaction::TRANSACTION_INSTALL:
480 case sat::Transaction::TRANSACTION_MULTIINSTALL:
481 // proceed: only install actions may require download.
482 break;
483
484 default:
485 // next: no download for non-packages and delete actions.
486 continue;
487 break;
488 }
489
490 PoolItem pi(step.satSolvable());
491
492 if ( !pi->isKind<Package>() && !pi->isKind<SrcPackage>() )
493 continue;
494
495 // no checksum ,no predownload, Fetcher would ignore it
496 if ( pi->lookupLocation().checksum().empty() )
497 continue;
498
499 // check if Package is cached already
500 if( !pckCachedLocation(pi).empty() )
501 continue;
502
503 auto repoDlsIter = _dlRepoInfo.find( pi.repository().id() );
504 if ( repoDlsIter == _dlRepoInfo.end() ) {
505
506 // make sure download path for this repo exists
507 if ( filesystem::assert_dir( pi.repoInfo().predownloadPath() ) != 0 ) {
508 ERR << "Failed to create predownload cache for repo " << pi.repoInfo().alias() << std::endl;
509 return;
510 }
511
512 // filter base URLs that do not download
513 std::vector<RepoUrl> repoUrls;
514 const auto origins = pi.repoInfo().repoOrigins();
515 for ( const auto &origin: origins ) {
516 std::for_each( origin.begin(), origin.end(), [&]( const zypp::OriginEndpoint &u ) {
517 media::UrlResolverPlugin::HeaderList custom_headers;
518 Url url = media::UrlResolverPlugin::resolveUrl(u.url(), custom_headers);
519
520 if ( media::MediaHandlerFactory::handlerType(url) != media::MediaHandlerFactory::MediaCURLType )
521 return;
522
523 // use geo IP if available
524 {
525 const auto rewriteUrl = media::MediaNetworkCommonHandler::findGeoIPRedirect( url );
526 if ( rewriteUrl.isValid () )
527 url = rewriteUrl;
528 }
529
530 if ( !pi.repoInfo().path().emptyOrRoot() )
531 url.appendPathName( pi.repoInfo().path() );
532
533 MIL << "Adding Url: " << url << " to the mirror set" << std::endl;
534
535 repoUrls.push_back( RepoUrl {
536 .baseUrl = std::move(url),
537 .headers = std::move(custom_headers)
538 } );
539 });
540 }
541
542 // skip this solvable if it has no downloading base URLs
543 if( repoUrls.empty() ) {
544 MIL << "Skipping predownload for " << step.satSolvable() << " no downloading URL" << std::endl;
545 continue;
546 }
547
548 // TODO here we could block to fetch mirror informations, either if the RepoInfo has a metalink or mirrorlist entry
549 // or if the hostname of the repo is d.o.o
550 if ( repoUrls.begin()->baseUrl.getHost() == "download.opensuse.org" ){
551 //auto req = std::make_shared<zyppng::NetworkRequest>( );
552 }
553
554 _dlRepoInfo.insert( std::make_pair(
555 pi.repository().id(),
556 RepoDownloadData{
557 ._baseUrls = std::move(repoUrls)
558 }
559 ));
560 }
561
562
563 _requiredBytes += pi.lookupLocation().downloadSize();
564 _requiredDls.push_back( pi );
565 }
566
567 if ( _requiredDls.empty() )
568 return;
569
570 // order by repo
571 std::sort( _requiredDls.begin(), _requiredDls.end(), []( const PoolItem &a , const PoolItem &b ) { return a.repository() < b.repository(); });
572
573 const auto &workerDone = [&, this](){
574 if ( std::all_of( _workers.begin(), _workers.end(), []( const auto &w ) { return w->finished();} ) )
575 ev->quit();
576 };
577
578 _report->start();
579 zypp_defer {
580 _report->finish( _missedDownloads ? media::CommitPreloadReport::MISS : media::CommitPreloadReport::SUCCESS );
581 };
582
583 MIL << "Downloading packages via " << MediaConfig::instance().download_max_concurrent_connections() << " connections." << std::endl;
584
585 // we start a worker for each configured connection
586 for ( int i = 0; i < MediaConfig::instance().download_max_concurrent_connections() ; i++ ) {
587 // if we run out of jobs before we started all workers, stop
588 if (_requiredDls.empty())
589 break;
590 auto worker = std::make_shared<PreloadWorker>(*this);
591 worker->sigWorkerFinished().connect(workerDone);
592 worker->nextJob();
593 _workers.push_back( std::move(worker) );
594 }
595
596 if( std::any_of( _workers.begin(), _workers.end(), []( const auto &w ) { return !w->finished(); } ) ) {
597 MIL << "Running preload event loop!" << std::endl;
598 ev->run();
599 }
600
601 MIL << "Preloading done, mirror stats: " << std::endl;
602 for ( const auto &elem : _dlRepoInfo ) {
603 std::for_each ( elem.second._baseUrls.begin (), elem.second._baseUrls.end(), []( const RepoUrl &repoUrl ){
604 MIL << "url: " << repoUrl.baseUrl << " misses: " << repoUrl.miss << std::endl;
605 });
606 }
607 MIL << "Preloading done, mirror stats end" << std::endl;
608 }
609
610 void CommitPackagePreloader::cleanupCaches()
611 {
612 if ( !preloadEnabled() ) {
613 MIL << "CommitPackagePreloader disabled" << std::endl;
614 return;
615 }
616 std::for_each( _dlRepoInfo.begin (), _dlRepoInfo.end(), []( const auto &elem ){
617 filesystem::clean_dir ( Repository(elem.first).info().predownloadPath() );
618 });
619 }
620
621 bool CommitPackagePreloader::missed() const
622 {
623 return _missedDownloads;
624 }
625
626 void CommitPackagePreloader::reportBytesDownloaded(ByteCount newBytes)
627 {
628 // throttle progress updates to one time per second
629 const auto now = clock::now();
630 bool canUpdate = false;
631 if ( _lastProgressUpdate ) {
632 const auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now - *_lastProgressUpdate);
633 canUpdate = (duration >= std::chrono::milliseconds(500));
634 } else {
635 canUpdate = true;
636 }
637
638 _downloadedBytes += newBytes;
639 _pTracker->updateStats( _requiredBytes, _downloadedBytes );
640
641 // update progress one time per second
642 if( canUpdate ) {
643 _lastProgressUpdate = now;
644 callback::UserData userData( "CommitPreloadReport/progress" );
645 userData.set( "dbps_avg" , static_cast<double>( _pTracker->_drateTotal ) );
646 userData.set( "dbps_current", static_cast<double>( _pTracker->_drateLast ) );
647 userData.set( "bytesReceived", static_cast<double>( _pTracker->_dnlNow ) );
648 userData.set( "bytesRequired", static_cast<double>( _pTracker->_dnlTotal ) );
649 if ( !_report->progress( _pTracker->_dnlPercent, userData ) ) {
650 _missedDownloads = true;
651 _requiredDls.clear();
652 _dispatcher->cancelAll( _("Cancelled by user."));
653 }
654 }
655 }
656
657}
zypp_defer
#define zypp_defer
Definition AutoDispose.h:293
Env.h
Gettext.h
_
#define _(MSG)
Definition Gettext.h:39
DBG
#define DBG
Definition Logger.h:102
MIL
#define MIL
Definition Logger.h:103
ERR
#define ERR
Definition Logger.h:105
MediaCurl2.h
MediaHandlerFactory.h
MediaSetAccess.h
Package.h
RepoProvideFile.h
SrcPackage.h
UrlResolverPlugin.h
ZConfig.h
zypp::ByteCount
Store and operate with byte count.
Definition ByteCount.h:32
zypp::CommitPackagePreloader::PreloadWorker
Definition commitpackagepreloader.cc:54
zypp::CommitPackagePreloader::PreloadWorker::finished
bool finished() const
Definition commitpackagepreloader.cc:66
zypp::CommitPackagePreloader::PreloadWorker::onRequestProgress
void onRequestProgress(zyppng::NetworkRequest &req, zypp::ByteCount count)
Definition commitpackagepreloader.cc:259
zypp::CommitPackagePreloader::PreloadWorker::findUsableMirror
RepoUrl * findUsableMirror(RepoUrl *skip=nullptr, bool allowTainted=true)
Tries to find a usable mirror.
Definition commitpackagepreloader.cc:227
zypp::CommitPackagePreloader::PreloadWorker::_req
zyppng::NetworkRequestRef _req
Definition commitpackagepreloader.cc:420
zypp::CommitPackagePreloader::PreloadWorker::_firstAuth
bool _firstAuth
Definition commitpackagepreloader.cc:426
zypp::CommitPackagePreloader::PreloadWorker::_job
PoolItem _job
Definition commitpackagepreloader.cc:422
zypp::CommitPackagePreloader::PreloadWorker::_currentRepoId
Repository::IdType _currentRepoId
Definition commitpackagepreloader.cc:429
zypp::CommitPackagePreloader::PreloadWorker::makeJobUrl
void makeJobUrl(zypp::Url &resultUrl, media::TransferSettings &resultSet)
Definition commitpackagepreloader.cc:387
zypp::CommitPackagePreloader::PreloadWorker::onRequestStarted
void onRequestStarted(zyppng::NetworkRequest &req)
Definition commitpackagepreloader.cc:255
zypp::CommitPackagePreloader::PreloadWorker::State
State
Definition commitpackagepreloader.cc:56
zypp::CommitPackagePreloader::PreloadWorker::SimpleDl
@ SimpleDl
Definition commitpackagepreloader.cc:58
zypp::CommitPackagePreloader::PreloadWorker::Pending
@ Pending
Definition commitpackagepreloader.cc:57
zypp::CommitPackagePreloader::PreloadWorker::Finished
@ Finished
Definition commitpackagepreloader.cc:61
zypp::CommitPackagePreloader::PreloadWorker::_started
bool _started
Definition commitpackagepreloader.cc:425
zypp::CommitPackagePreloader::PreloadWorker::_notFoundRetry
int _notFoundRetry
Definition commitpackagepreloader.cc:432
zypp::CommitPackagePreloader::PreloadWorker::sigWorkerFinished
zyppng::SignalProxy< void()> sigWorkerFinished()
Definition commitpackagepreloader.cc:172
zypp::CommitPackagePreloader::PreloadWorker::PreloadWorker
PreloadWorker(CommitPackagePreloader &parent)
Definition commitpackagepreloader.cc:64
zypp::CommitPackagePreloader::PreloadWorker::_sigFinished
zyppng::Signal< void()> _sigFinished
Definition commitpackagepreloader.cc:435
zypp::CommitPackagePreloader::PreloadWorker::prepareMirror
bool prepareMirror()
Definition commitpackagepreloader.cc:179
zypp::CommitPackagePreloader::PreloadWorker::_targetPath
zypp::Pathname _targetPath
Definition commitpackagepreloader.cc:424
zypp::CommitPackagePreloader::PreloadWorker::taintCurrentMirror
bool taintCurrentMirror()
Taints the current mirror, returns true if a alternative was found.
Definition commitpackagepreloader.cc:204
zypp::CommitPackagePreloader::PreloadWorker::_taintedMirrors
std::set< RepoUrl * > _taintedMirrors
Definition commitpackagepreloader.cc:433
zypp::CommitPackagePreloader::PreloadWorker::_tmpFile
ManagedFile _tmpFile
Definition commitpackagepreloader.cc:423
zypp::CommitPackagePreloader::PreloadWorker::_parent
CommitPackagePreloader & _parent
Definition commitpackagepreloader.cc:419
zypp::CommitPackagePreloader::PreloadWorker::_myMirror
RepoUrl * _myMirror
Definition commitpackagepreloader.cc:427
zypp::CommitPackagePreloader::PreloadWorker::_lastByteCount
ByteCount _lastByteCount
Definition commitpackagepreloader.cc:428
zypp::CommitPackagePreloader::PreloadWorker::nextJob
void nextJob()
Definition commitpackagepreloader.cc:70
zypp::CommitPackagePreloader::PreloadWorker::_s
State _s
Definition commitpackagepreloader.cc:418
zypp::CommitPackagePreloader::PreloadWorker::onRequestFinished
void onRequestFinished(zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err)
Definition commitpackagepreloader.cc:278
zypp::CommitPackagePreloader::PreloadWorker::finishCurrentJob
void finishCurrentJob(const zypp::Pathname &localPath, const std::optional< zypp::Url > &url, media::CommitPreloadReport::Error e, const std::optional< std::string > &errorMessage, bool fatal)
Definition commitpackagepreloader.cc:373
zypp::CommitPackagePreloader::_report
callback::SendReport< media::CommitPreloadReport > _report
Definition commitpackagepreloader_p.h:63
zypp::CommitPackagePreloader::_downloadedBytes
ByteCount _downloadedBytes
Definition commitpackagepreloader_p.h:60
zypp::CommitPackagePreloader::_lastProgressUpdate
std::optional< clock::time_point > _lastProgressUpdate
Definition commitpackagepreloader_p.h:65
zypp::CommitPackagePreloader::_pTracker
zyppng::Ref< internal::ProgressTracker > _pTracker
Definition commitpackagepreloader_p.h:64
zypp::CommitPackagePreloader::_dlRepoInfo
std::map< Repository::IdType, RepoDownloadData > _dlRepoInfo
Definition commitpackagepreloader_p.h:56
zypp::CommitPackagePreloader::_requiredBytes
ByteCount _requiredBytes
Definition commitpackagepreloader_p.h:59
zypp::CommitPackagePreloader::reportBytesDownloaded
void reportBytesDownloaded(ByteCount newBytes)
Definition commitpackagepreloader.cc:626
zypp::CommitPackagePreloader::preloadTransaction
void preloadTransaction(const std::vector< sat::Transaction::Step > &steps)
Definition commitpackagepreloader.cc:442
zypp::CommitPackagePreloader::CommitPackagePreloader
CommitPackagePreloader()
Definition commitpackagepreloader.cc:439
zypp::CommitPackagePreloader::_dispatcher
zyppng::NetworkRequestDispatcherRef _dispatcher
Definition commitpackagepreloader_p.h:67
zypp::CommitPackagePreloader::_missedDownloads
bool _missedDownloads
Definition commitpackagepreloader_p.h:61
zypp::CommitPackagePreloader::missed
bool missed() const
Definition commitpackagepreloader.cc:621
zypp::CommitPackagePreloader::cleanupCaches
void cleanupCaches()
Definition commitpackagepreloader.cc:610
zypp::CommitPackagePreloader::_requiredDls
std::deque< PoolItem > _requiredDls
Definition commitpackagepreloader_p.h:57
zypp::Exception
Base class for Exception.
Definition Exception.h:153
zypp::Exception::asUserString
std::string asUserString() const
Translated error message as string suitable for the user.
Definition Exception.cc:131
zypp::MediaConfig::download_max_concurrent_connections
long download_max_concurrent_connections() const
Definition mediaconfig.cc:109
zypp::MediaConfig::instance
static MediaConfig & instance()
Definition mediaconfig.cc:46
zypp::MediaSetAccess::rewriteUrl
static Url rewriteUrl(const Url &url_r, const media::MediaNr medianr)
Replaces media number in specified url with given medianr.
Definition MediaSetAccess.cc:500
zypp::OriginEndpoint
Represents a single, configurable network endpoint, combining a URL with specific access settings.
Definition MirroredOrigin.h:41
zypp::Package
Package interface.
Definition Package.h:34
zypp::PoolItem
Combining sat::Solvable and ResStatus.
Definition PoolItem.h:51
zypp::RepoInfo::predownloadPath
Pathname predownloadPath() const
Path where this repo packages are predownloaded.
Definition RepoInfo.cc:816
zypp::RepoInfo::repoOrigins
MirroredOriginSet repoOrigins() const
The repodata origins.
Definition RepoInfo.cc:790
zypp::RepoInfo::path
Pathname path() const
Repository path.
Definition RepoInfo.cc:879
zypp::Repository::id
IdType id() const
Expert backdoor.
Definition Repository.h:321
zypp::Repository::IdType
sat::detail::RepoIdType IdType
Definition Repository.h:44
zypp::SrcPackage
SrcPackage interface.
Definition SrcPackage.h:30
zypp::Url
Url manipulation class.
Definition Url.h:93
zypp::ZConfig::instance
static ZConfig & instance()
Singleton ctor.
Definition ZConfig.cc:788
zypp::callback::UserData
Typesafe passing of user data via callbacks.
Definition UserData.h:40
zypp::callback::UserData::set
bool set(const std::string &key_r, AnyType val_r)
Set the value for key (nonconst version always returns true).
Definition UserData.h:119
zypp::filesystem::PathInfo
Wrapper class for stat/lstat.
Definition PathInfo.h:226
zypp::filesystem::PathInfo::isDir
bool isDir() const
Definition PathInfo.h:296
zypp::filesystem::PathInfo::isFile
bool isFile() const
Definition PathInfo.h:295
zypp::filesystem::PathInfo::isExist
bool isExist() const
Return whether valid stat info exists.
Definition PathInfo.h:286
zypp::filesystem::Pathname
Pathname.
Definition Pathname.h:48
zypp::filesystem::Pathname::emptyOrRoot
bool emptyOrRoot() const
Test for "" or "/".
Definition Pathname.h:127
zypp::filesystem::TmpFile::asManagedFile
static ManagedFile asManagedFile()
Create a temporary file and convert it to a automatically cleaned up ManagedFile.
Definition TmpPath.cc:200
zypp::media::CredentialManager
Definition credentialmanager.h:64
zypp::media::MediaNetworkCommonHandler::authenticate
bool authenticate(const Url &url, TransferSettings &settings, const std::string &availAuthTypes, bool firstTry)
Definition MediaNetworkCommonHandler.cc:322
zypp::media::MediaNetworkCommonHandler::anonymousIdHeader
static const char * anonymousIdHeader()
Definition MediaNetworkCommonHandler.cc:181
zypp::media::MediaNetworkCommonHandler::distributionFlavorHeader
static const char * distributionFlavorHeader()
Definition MediaNetworkCommonHandler.cc:195
zypp::media::MediaNetworkCommonHandler::agentString
static const char * agentString()
Definition MediaNetworkCommonHandler.cc:228
zypp::media::TransferSettings
Holds transfer setting.
Definition transfersettings.h:34
zypp::media::TransferSettings::addHeader
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
Definition transfersettings.cc:138
zypp::media::UrlResolverPlugin::HeaderList
std::multimap< std::string, std::string > HeaderList
Definition UrlResolverPlugin.h:39
zypp::repo::RepoInfoBase::alias
std::string alias() const
unique identifier for this source.
Definition RepoInfoBase.cc:110
zypp::repo::RepoMediaAccess::mapToCachePath
static Pathname mapToCachePath(const RepoInfo &repo_r, const OnMediaLocation &resource_r)
Map a resource filename to a local path below a repositories cache.
Definition RepoProvideFile.cc:385
zypp::sat::Transaction::TRANSACTION_MULTIINSTALL
@ TRANSACTION_MULTIINSTALL
[M] Install(multiversion) item (
Definition Transaction.h:67
zypp::sat::Transaction::TRANSACTION_INSTALL
@ TRANSACTION_INSTALL
[+] Install(update) item
Definition Transaction.h:66
zyppng::Base
Definition base.h:61
zyppng::Base::parent
WeakPtr parent() const
Definition base.cc:26
zyppng::EventLoop::create
static Ptr create(GMainContext *ctx=nullptr)
Definition eventloop_glib.cc:29
zyppng::NetworkRequestError
The NetworkRequestError class Represents a error that occured in.
Definition networkrequesterror.h:24
zyppng::NetworkRequestError::toString
std::string toString() const
toString Returns a string representation of the error
Definition networkrequesterror.cc:235
zyppng::NetworkRequestError::ServerReturnedError
@ ServerReturnedError
Definition networkrequesterror.h:42
zyppng::NetworkRequestError::Forbidden
@ Forbidden
Definition networkrequesterror.h:38
zyppng::NetworkRequestError::PeerCertificateInvalid
@ PeerCertificateInvalid
Definition networkrequesterror.h:30
zyppng::NetworkRequestError::UnsupportedProtocol
@ UnsupportedProtocol
Definition networkrequesterror.h:34
zyppng::NetworkRequestError::InternalError
@ InternalError
Definition networkrequesterror.h:28
zyppng::NetworkRequestError::ExceededMaxLen
@ ExceededMaxLen
Definition networkrequesterror.h:32
zyppng::NetworkRequestError::Unauthorized
@ Unauthorized
Definition networkrequesterror.h:40
zyppng::NetworkRequestError::MalformedURL
@ MalformedURL
Definition networkrequesterror.h:35
zyppng::NetworkRequestError::ConnectionFailed
@ ConnectionFailed
Definition networkrequesterror.h:31
zyppng::NetworkRequestError::RangeFail
@ RangeFail
Definition networkrequesterror.h:44
zyppng::NetworkRequestError::AuthFailed
@ AuthFailed
Definition networkrequesterror.h:41
zyppng::NetworkRequestError::Timeout
@ Timeout
Definition networkrequesterror.h:37
zyppng::NetworkRequestError::Http2Error
@ Http2Error
Definition networkrequesterror.h:45
zyppng::NetworkRequestError::MissingData
@ MissingData
Definition networkrequesterror.h:43
zyppng::NetworkRequestError::NoError
@ NoError
Definition networkrequesterror.h:27
zyppng::NetworkRequestError::NotFound
@ NotFound
Definition networkrequesterror.h:39
zyppng::NetworkRequestError::Cancelled
@ Cancelled
Definition networkrequesterror.h:29
zyppng::NetworkRequestError::Http2StreamError
@ Http2StreamError
Definition networkrequesterror.h:46
zyppng::NetworkRequestError::TemporaryProblem
@ TemporaryProblem
Definition networkrequesterror.h:36
zyppng::NetworkRequestError::InvalidChecksum
@ InvalidChecksum
Definition networkrequesterror.h:33
zyppng::NetworkRequest
Definition request.h:43
zyppng::NetworkRequest::url
Url url() const
Definition request.cc:918
zyppng::NetworkRequest::hasError
bool hasError() const
Checks if there was a error with the request.
Definition request.cc:1035
zyppng::NetworkRequest::sigFinished
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Definition request.cc:1079
zyppng::NetworkRequest::sigBytesDownloaded
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
Definition request.cc:1069
zyppng::NetworkRequest::extendedErrorString
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Definition request.cc:1027
zyppng::NetworkRequest::error
NetworkRequestError error() const
Returns the last set Error.
Definition request.cc:1019
zyppng::NetworkRequest::sigStarted
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
Definition request.cc:1064
zyppng::NetworkRequest::transferSettings
TransferSettings & transferSettings()
Definition request.cc:995
zyppng::SignalProxy
Definition signals.h:325
zyppng::Signal
Definition signals.h:151
commitpackagepreloader_p.h
credentialmanager.h
curlhelper_p.h
eventloop.h
zypp::TriBool
boost::logic::tribool TriBool
3-state boolean logic (true, false and indeterminate).
Definition String.h:31
a
unsigned short a
Definition mediablocklist.cc:0
b
unsigned short b
Definition mediablocklist.cc:1
mediaconfig.h
internal::prepareSettingsAndUrl
void prepareSettingsAndUrl(zypp::Url &url_r, zypp::media::TransferSettings &s)
Definition curlhelper.cc:180
str
String related utilities and Regular expression matching.
zypp::env::getenvBool
TriBool getenvBool(const C_Str &var_r)
If the environment variable var_r is set to a legal true or false string return bool,...
Definition Env.h:32
zypp::filesystem::rmdir
int rmdir(const Pathname &path)
Like 'rmdir'.
Definition PathInfo.cc:385
zypp::filesystem::unlink
int unlink(const Pathname &path)
Like 'unlink'.
Definition PathInfo.cc:719
zypp::filesystem::assert_dir
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Definition PathInfo.cc:338
zypp::filesystem::rename
int rename(const Pathname &oldpath, const Pathname &newpath)
Like 'rename'.
Definition PathInfo.cc:761
zypp::filesystem::chmodApplyUmask
int chmodApplyUmask(const Pathname &path, mode_t mode)
Similar to 'chmod', but mode is modified by the process's umask in the usual way.
Definition PathInfo.cc:1120
zypp::sat::detail::noRepoId
static const RepoIdType noRepoId(0)
Id to denote Repo::noRepository.
zypp::str::asString
const std::string & asString(const std::string &t)
Global asString() that works with std::string too.
Definition String.h:140
zypp::url
Url details namespace.
Definition UrlBase.cc:58
zypp
Easy-to use interface to the ZYPP dependency resolver.
Definition CodePitfalls.doc:2
zypp::ManagedFile
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition ManagedFile.h:27
zypp::asString
std::string asString(const Patch::Category &obj)
relates: Patch::Category string representation.
Definition Patch.cc:122
zyppintern::cachedLocation
Pathname cachedLocation(const OnMediaLocation &loc_r, const RepoInfo &repo_r)
Definition Package.cc:100
networkrequestdispatcher.h
request.h
zypp::CommitPackagePreloader::RepoDownloadData
Definition commitpackagepreloader_p.h:50
zypp::CommitPackagePreloader::RepoUrl
Definition commitpackagepreloader_p.h:43
zypp::RepoUrl
Definition commitpackagepreloader.cc:49
zypp::RepoUrl::headers
media::UrlResolverPlugin::HeaderList headers
Definition commitpackagepreloader.cc:51
zypp::RepoUrl::baseUrl
zypp::Url baseUrl
Definition commitpackagepreloader.cc:50
zypp::media::CommitPreloadReport::Error
Error
Definition ZYppCallbacks.h:409
zypp::media::CommitPreloadReport::ERROR
@ ERROR
Definition ZYppCallbacks.h:414
zypp::media::CommitPreloadReport::NOT_FOUND
@ NOT_FOUND
Definition ZYppCallbacks.h:411
zypp::media::CommitPreloadReport::NO_ERROR
@ NO_ERROR
Definition ZYppCallbacks.h:410
zypp::media::CommitPreloadReport::ACCESS_DENIED
@ ACCESS_DENIED
Definition ZYppCallbacks.h:413
zypp::media::CommitPreloadReport::SUCCESS
@ SUCCESS
Definition ZYppCallbacks.h:463
zypp::media::CommitPreloadReport::MISS
@ MISS
Definition ZYppCallbacks.h:464
zypp::media::CredManagerOptions
Definition credentialmanager.h:40
zypp::sat::SolvableType::repoInfo
RepoInfo repoInfo() const
Definition SolvableType.h:76
zypp::sat::SolvableType::repository
Repository repository() const
Definition SolvableType.h:75
zypp::str::Format
Convenient building of std::string with boost::format.
Definition String.h:254
transfersettings.h
TmpPath.h