From aaaeea297927211fe4e5ea09ce76b324eae24920 Mon Sep 17 00:00:00 2001 From: John Preston Date: Wed, 3 Apr 2024 12:39:08 +0400 Subject: [PATCH] Use plain vector for prepared upload parts. --- Telegram/SourceFiles/storage/file_upload.cpp | 160 ++++++++---------- Telegram/SourceFiles/storage/file_upload.h | 14 +- .../SourceFiles/storage/localimageloader.cpp | 8 +- .../SourceFiles/storage/localimageloader.h | 2 +- 4 files changed, 88 insertions(+), 96 deletions(-) diff --git a/Telegram/SourceFiles/storage/file_upload.cpp b/Telegram/SourceFiles/storage/file_upload.cpp index e0c7f8b57..b19af82df 100644 --- a/Telegram/SourceFiles/storage/file_upload.cpp +++ b/Telegram/SourceFiles/storage/file_upload.cpp @@ -65,7 +65,10 @@ struct Uploader::File { bool setPartSize(uint32 partSize); std::shared_ptr file; - int32 partsCount = 0; + const std::vector &parts; + const uint64 partsOfId = 0; + int partsSent = 0; + mutable int64 fileSentSize = 0; HashMd5 md5Hash; @@ -79,11 +82,15 @@ struct Uploader::File { }; Uploader::File::File(const std::shared_ptr &file) -: file(file) { - partsCount = (file->type == SendMediaType::Photo - || file->type == SendMediaType::Secure) - ? file->fileparts.size() - : file->thumbparts.size(); +: file(file) +, parts((file->type == SendMediaType::Photo + || file->type == SendMediaType::Secure) + ? file->fileparts + : file->thumbparts) +, partsOfId((file->type == SendMediaType::Photo + || file->type == SendMediaType::Secure) + ? file->id + : file->thumbId) { if (file->type == SendMediaType::File || file->type == SendMediaType::ThemeFile || file->type == SendMediaType::Audio) { @@ -174,7 +181,7 @@ Uploader::Uploader(not_null api) _api->instance().nonPremiumDelayedRequests( ) | rpl::start_with_next([=](mtpRequestId id) { - if (dcMap.contains(id)) { + if (_dcIndices.contains(id)) { _nonPremiumDelayed.emplace(id); } }, _lifetime); @@ -299,7 +306,7 @@ void Uploader::upload( } void Uploader::currentFailed() { - auto j = queue.find(uploadingId); + auto j = queue.find(_uploadingId); if (j != queue.end()) { const auto [msgId, file] = std::move(*j); queue.erase(j); @@ -307,11 +314,11 @@ void Uploader::currentFailed() { } cancelRequests(); - dcMap.clear(); - uploadingId = FullMsgId(); - sentSize = 0; + _dcIndices.clear(); + _uploadingId = FullMsgId(); + _sentTotal = 0; for (int i = 0; i < MTP::kUploadSessionsCount; ++i) { - sentSizes[i] = 0; + _sentPerDc[i] = 0; } sendNext(); @@ -343,7 +350,7 @@ void Uploader::stopSessions() { } void Uploader::sendNext() { - if (sentSize >= kMaxUploadFileParallelSize || _pausedId.msg) { + if (_sentTotal >= kMaxUploadFileParallelSize || _pausedId.msg) { return; } @@ -358,33 +365,25 @@ void Uploader::sendNext() { if (stopping) { _stopSessionsTimer.cancel(); } - auto i = uploadingId.msg ? queue.find(uploadingId) : queue.begin(); - if (!uploadingId.msg) { - uploadingId = i->first; + auto i = _uploadingId.msg ? queue.find(_uploadingId) : queue.begin(); + if (!_uploadingId.msg) { + _uploadingId = i->first; } else if (i == queue.end()) { i = queue.begin(); - uploadingId = i->first; + _uploadingId = i->first; } auto &uploadingData = i->second; auto todc = 0; for (auto dc = 1; dc != MTP::kUploadSessionsCount; ++dc) { - if (sentSizes[dc] < sentSizes[todc]) { + if (_sentPerDc[dc] < _sentPerDc[todc]) { todc = dc; } } - auto &parts = (uploadingData.file->type == SendMediaType::Photo - || uploadingData.file->type == SendMediaType::Secure) - ? uploadingData.file->fileparts - : uploadingData.file->thumbparts; - const auto partsOfId = (uploadingData.file->type == SendMediaType::Photo - || uploadingData.file->type == SendMediaType::Secure) - ? uploadingData.file->id - : uploadingData.file->thumbId; - if (parts.isEmpty()) { + if (uploadingData.partsSent >= uploadingData.parts.size()) { if (uploadingData.docSentParts >= uploadingData.docPartsCount) { - if (requestsSent.empty() && docRequestsSent.empty()) { + if (_sentSizes.empty()) { const auto options = uploadingData.file ? uploadingData.file->to.options : Api::SendOptions(); @@ -404,11 +403,11 @@ void Uploader::sendNext() { const auto md5 = uploadingData.file->filemd5; const auto file = MTP_inputFile( MTP_long(uploadingData.file->id), - MTP_int(uploadingData.partsCount), + MTP_int(uploadingData.parts.size()), MTP_string(photoFilename), MTP_bytes(md5)); _photoReady.fire({ - .fullId = uploadingId, + .fullId = _uploadingId, .info = { .file = file, .attachedStickers = attachedStickers, @@ -433,19 +432,19 @@ void Uploader::sendNext() { MTP_string(uploadingData.file->filename), MTP_bytes(docMd5)); const auto thumb = [&]() -> std::optional { - if (!uploadingData.partsCount) { + if (uploadingData.parts.empty()) { return std::nullopt; } const auto thumbFilename = uploadingData.file->thumbname; const auto thumbMd5 = uploadingData.file->thumbmd5; return MTP_inputFile( MTP_long(uploadingData.file->thumbId), - MTP_int(uploadingData.partsCount), + MTP_int(uploadingData.parts.size()), MTP_string(thumbFilename), MTP_bytes(thumbMd5)); }(); _documentReady.fire({ - .fullId = uploadingId, + .fullId = _uploadingId, .info = { .file = file, .thumb = thumb, @@ -456,12 +455,13 @@ void Uploader::sendNext() { }); } else if (uploadingData.file->type == SendMediaType::Secure) { _secureReady.fire({ - uploadingId, + _uploadingId, uploadingData.file->id, - uploadingData.partsCount }); + int(uploadingData.parts.size()), + }); } - queue.erase(uploadingId); - uploadingId = FullMsgId(); + queue.erase(_uploadingId); + _uploadingId = FullMsgId(); sendNext(); } return; @@ -522,36 +522,37 @@ void Uploader::sendNext() { partFailed(error, requestId); }).toDC(MTP::uploadDcId(todc)).send(); } - docRequestsSent.emplace(requestId, uploadingData.docSentParts); - dcMap.emplace(requestId, todc); - sentSize += uploadingData.docPartSize; - sentSizes[todc] += uploadingData.docPartSize; + _sentSizes.emplace(requestId, uploadingData.docPartSize); + _docSentRequests.emplace(requestId); + _dcIndices.emplace(requestId, todc); + _sentTotal += uploadingData.docPartSize; + _sentPerDc[todc] += uploadingData.docPartSize; uploadingData.docSentParts++; } else { - auto part = parts.begin(); + const auto index = uploadingData.partsSent++; + const auto partBytes = uploadingData.parts[index]; + const auto partSize = int(partBytes.size()); const auto requestId = _api->request(MTPupload_SaveFilePart( - MTP_long(partsOfId), - MTP_int(part.key()), - MTP_bytes(part.value()) + MTP_long(uploadingData.partsOfId), + MTP_int(index), + MTP_bytes(partBytes) )).done([=](const MTPBool &result, mtpRequestId requestId) { partLoaded(result, requestId); }).fail([=](const MTP::Error &error, mtpRequestId requestId) { partFailed(error, requestId); }).toDC(MTP::uploadDcId(todc)).send(); - requestsSent.emplace(requestId, part.value()); - dcMap.emplace(requestId, todc); - sentSize += part.value().size(); - sentSizes[todc] += part.value().size(); - - parts.erase(part); + _sentSizes.emplace(requestId, partSize); + _dcIndices.emplace(requestId, todc); + _sentTotal += partSize; + _sentPerDc[todc] += partSize; } _nextTimer.callOnce(kUploadRequestInterval); } void Uploader::cancel(const FullMsgId &msgId) { - if (uploadingId == msgId) { + if (_uploadingId == msgId) { currentFailed(); } else { queue.erase(msgId); @@ -559,12 +560,12 @@ void Uploader::cancel(const FullMsgId &msgId) { } void Uploader::cancelAll() { - const auto single = queue.empty() ? uploadingId : queue.begin()->first; + const auto single = queue.empty() ? _uploadingId : queue.begin()->first; if (!single) { return; } _pausedId = single; - if (uploadingId) { + if (_uploadingId) { currentFailed(); } while (!queue.empty()) { @@ -589,61 +590,49 @@ void Uploader::confirm(const FullMsgId &msgId) { } void Uploader::cancelRequests() { - for (const auto &requestData : requestsSent) { + _docSentRequests.clear(); + for (const auto &requestData : _sentSizes) { _api->request(requestData.first).cancel(); } - requestsSent.clear(); - for (const auto &requestData : docRequestsSent) { - _api->request(requestData.first).cancel(); - } - docRequestsSent.clear(); + _sentSizes.clear(); } void Uploader::clear() { queue.clear(); cancelRequests(); - dcMap.clear(); - sentSize = 0; + _dcIndices.clear(); + _sentTotal = 0; for (int i = 0; i < MTP::kUploadSessionsCount; ++i) { _api->instance().stopSession(MTP::uploadDcId(i)); - sentSizes[i] = 0; + _sentPerDc[i] = 0; } _stopSessionsTimer.cancel(); } void Uploader::partLoaded(const MTPBool &result, mtpRequestId requestId) { - auto j = docRequestsSent.end(); - auto i = requestsSent.find(requestId); - if (i == requestsSent.cend()) { - j = docRequestsSent.find(requestId); - } + _docSentRequests.remove(requestId); + auto i = _sentSizes.find(requestId); const auto wasNonPremiumDelayed = _nonPremiumDelayed.remove(requestId); - if (i != requestsSent.cend() || j != docRequestsSent.cend()) { + if (i != _sentSizes.cend()) { if (mtpIsFalse(result)) { // failed to upload current file currentFailed(); return; } else { - auto dcIt = dcMap.find(requestId); - if (dcIt == dcMap.cend()) { // must not happen + auto dcIt = _dcIndices.find(requestId); + if (dcIt == _dcIndices.cend()) { // must not happen currentFailed(); return; } auto dc = dcIt->second; - dcMap.erase(dcIt); + _dcIndices.erase(dcIt); - int64 sentPartSize = 0; - auto k = queue.find(uploadingId); + int64 sentPartSize = i->second; + auto k = queue.find(_uploadingId); Assert(k != queue.cend()); auto &[fullId, file] = *k; - if (i != requestsSent.cend()) { - sentPartSize = i->second.size(); - requestsSent.erase(i); - } else { - sentPartSize = file.docPartSize; - docRequestsSent.erase(j); - } - sentSize -= sentPartSize; - sentSizes[dc] -= sentPartSize; + _sentSizes.erase(i); + _sentTotal -= sentPartSize; + _sentPerDc[dc] -= sentPartSize; if (file.file->type == SendMediaType::Photo) { file.fileSentSize += sentPartSize; const auto photo = session().data().photo(file.file->id); @@ -658,7 +647,7 @@ void Uploader::partLoaded(const MTPBool &result, mtpRequestId requestId) { const auto document = session().data().document(file.file->id); if (document->uploading()) { const auto doneParts = file.docSentParts - - int(docRequestsSent.size()); + - int(_docSentRequests.size()); document->uploadingData->offset = std::min( document->uploadingData->size, doneParts * file.docPartSize); @@ -683,8 +672,7 @@ void Uploader::partLoaded(const MTPBool &result, mtpRequestId requestId) { void Uploader::partFailed(const MTP::Error &error, mtpRequestId requestId) { // failed to upload current file _nonPremiumDelayed.remove(requestId); - if ((requestsSent.find(requestId) != requestsSent.cend()) - || (docRequestsSent.find(requestId) != docRequestsSent.cend())) { + if (_sentSizes.find(requestId) != _sentSizes.cend()) { currentFailed(); } sendNext(); diff --git a/Telegram/SourceFiles/storage/file_upload.h b/Telegram/SourceFiles/storage/file_upload.h index 4347e5c03..a733775ad 100644 --- a/Telegram/SourceFiles/storage/file_upload.h +++ b/Telegram/SourceFiles/storage/file_upload.h @@ -54,7 +54,7 @@ public: [[nodiscard]] Main::Session &session() const; [[nodiscard]] FullMsgId currentUploadId() const { - return uploadingId; + return _uploadingId; } void upload( @@ -125,14 +125,14 @@ private: int progress = 0); const not_null _api; - base::flat_map requestsSent; - base::flat_map docRequestsSent; - base::flat_map dcMap; + base::flat_map _sentSizes; + base::flat_set _docSentRequests; + base::flat_map _dcIndices; base::flat_set _nonPremiumDelayed; - uint32 sentSize = 0; // FileSize: Right now any file size fits 32 bit. - uint32 sentSizes[MTP::kUploadSessionsCount] = { 0 }; + uint32 _sentTotal = 0; // FileSize: Right now any file size fits 32 bit. + uint32 _sentPerDc[MTP::kUploadSessionsCount] = { 0 }; - FullMsgId uploadingId; + FullMsgId _uploadingId; FullMsgId _pausedId; std::map queue; base::Timer _nextTimer, _stopSessionsTimer; diff --git a/Telegram/SourceFiles/storage/localimageloader.cpp b/Telegram/SourceFiles/storage/localimageloader.cpp index 192c7c112..59f2fb505 100644 --- a/Telegram/SourceFiles/storage/localimageloader.cpp +++ b/Telegram/SourceFiles/storage/localimageloader.cpp @@ -434,8 +434,10 @@ void FilePrepareResult::setFileData(const QByteArray &filedata) { partssize = 0; } else { partssize = filedata.size(); + fileparts.reserve( + (partssize + kPhotoUploadPartSize - 1) / kPhotoUploadPartSize); for (int32 i = 0, part = 0; i < partssize; i += kPhotoUploadPartSize, ++part) { - fileparts.insert(part, filedata.mid(i, kPhotoUploadPartSize)); + fileparts.push_back(filedata.mid(i, kPhotoUploadPartSize)); } filemd5.resize(32); hashMd5Hex(filedata.constData(), filedata.size(), filemd5.data()); @@ -446,8 +448,10 @@ void FilePrepareResult::setThumbData(const QByteArray &thumbdata) { if (!thumbdata.isEmpty()) { thumbbytes = thumbdata; int32 size = thumbdata.size(); + thumbparts.reserve( + (size + kPhotoUploadPartSize - 1) / kPhotoUploadPartSize); for (int32 i = 0, part = 0; i < size; i += kPhotoUploadPartSize, ++part) { - thumbparts.insert(part, thumbdata.mid(i, kPhotoUploadPartSize)); + thumbparts.push_back(thumbdata.mid(i, kPhotoUploadPartSize)); } thumbmd5.resize(32); hashMd5Hex(thumbdata.constData(), thumbdata.size(), thumbmd5.data()); diff --git a/Telegram/SourceFiles/storage/localimageloader.h b/Telegram/SourceFiles/storage/localimageloader.h index 8b3b7bca7..2756ac244 100644 --- a/Telegram/SourceFiles/storage/localimageloader.h +++ b/Telegram/SourceFiles/storage/localimageloader.h @@ -147,7 +147,7 @@ struct FileLoadTo { MsgId replaceMediaOf; }; -using UploadFileParts = QMap; +using UploadFileParts = std::vector; struct FilePrepareDescriptor { TaskId taskId = kEmptyTaskId; base::required id;