From 57f17b7afef8fec0c2e6b9710086803393832363 Mon Sep 17 00:00:00 2001 From: John Preston Date: Sat, 26 Feb 2022 19:08:44 +0300 Subject: [PATCH] Save and restore downloads between launches. --- Telegram/SourceFiles/core/file_location.cpp | 35 +- Telegram/SourceFiles/core/file_location.h | 5 + Telegram/SourceFiles/data/data_document.cpp | 3 +- .../data/data_download_manager.cpp | 322 +++++++++++++++++- .../SourceFiles/data/data_download_manager.h | 32 +- .../SourceFiles/storage/storage_account.cpp | 29 +- .../SourceFiles/storage/storage_account.h | 10 +- 7 files changed, 401 insertions(+), 35 deletions(-) diff --git a/Telegram/SourceFiles/core/file_location.cpp b/Telegram/SourceFiles/core/file_location.cpp index 641bf0ea0..f9c927f7f 100644 --- a/Telegram/SourceFiles/core/file_location.cpp +++ b/Telegram/SourceFiles/core/file_location.cpp @@ -39,23 +39,34 @@ FileLocation::FileLocation(const QString &name) : fname(name) { size = 0; } else { setBookmark(Platform::PathBookmark(name)); + resolveFromInfo(QFileInfo(name)); + } +} - QFileInfo f(name); - if (f.exists()) { - qint64 s = f.size(); - if (s > INT_MAX) { - fname = QString(); - _bookmark = nullptr; - size = 0; - } else { - modified = f.lastModified(); - size = qint32(s); - } - } else { +FileLocation::FileLocation(const QFileInfo &info) : fname(info.filePath()) { + if (fname.isEmpty()) { + size = 0; + } else { + setBookmark(Platform::PathBookmark(fname)); + resolveFromInfo(info); + } +} + +void FileLocation::resolveFromInfo(const QFileInfo &info) { + if (info.exists()) { + const auto s = info.size(); + if (s > INT_MAX) { fname = QString(); _bookmark = nullptr; size = 0; + } else { + modified = info.lastModified(); + size = qint32(s); } + } else { + fname = QString(); + _bookmark = nullptr; + size = 0; } } diff --git a/Telegram/SourceFiles/core/file_location.h b/Telegram/SourceFiles/core/file_location.h index 16db6fce3..f1c62930b 100644 --- a/Telegram/SourceFiles/core/file_location.h +++ b/Telegram/SourceFiles/core/file_location.h @@ -9,6 +9,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include +class QFileInfo; + namespace Platform { class FileBookmark; } // namespace Platform @@ -35,6 +37,7 @@ class FileLocation { public: FileLocation() = default; explicit FileLocation(const QString &name); + explicit FileLocation(const QFileInfo &info); static FileLocation InMediaCacheLocation(); @@ -55,6 +58,8 @@ public: qint32 size; private: + void resolveFromInfo(const QFileInfo &info); + std::shared_ptr _bookmark; }; diff --git a/Telegram/SourceFiles/data/data_document.cpp b/Telegram/SourceFiles/data/data_document.cpp index efb0f0a8f..ef5e08ac8 100644 --- a/Telegram/SourceFiles/data/data_document.cpp +++ b/Telegram/SourceFiles/data/data_document.cpp @@ -1256,7 +1256,8 @@ bool DocumentData::isNull() const { return !hasRemoteLocation() && !hasWebLocation() && _url.isEmpty() - && !uploading(); + && !uploading() + && _location.isEmpty(); } MTPInputDocument DocumentData::mtpInput() const { diff --git a/Telegram/SourceFiles/data/data_download_manager.cpp b/Telegram/SourceFiles/data/data_download_manager.cpp index ad7eb523d..c83b16c1b 100644 --- a/Telegram/SourceFiles/data/data_download_manager.cpp +++ b/Telegram/SourceFiles/data/data_download_manager.cpp @@ -14,18 +14,25 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "data/data_user.h" #include "data/data_channel.h" #include "base/unixtime.h" +#include "base/random.h" #include "main/main_session.h" #include "main/main_account.h" +#include "storage/storage_account.h" #include "history/history.h" #include "history/history_item.h" #include "history/history_message.h" #include "core/application.h" +#include "core/mime_type.h" #include "ui/controls/download_bar.h" +#include "storage/serialize_common.h" +#include "apiwrap.h" namespace Data { namespace { constexpr auto kClearLoadingTimeout = 5 * crl::time(1000); +constexpr auto kMaxFileSize = 2000 * 1024 * 1024; +constexpr auto kMaxResolvePerAttempt = 100; constexpr auto ByItem = [](const auto &entry) { if constexpr (std::is_same_v) { @@ -55,6 +62,8 @@ DownloadManager::~DownloadManager() = default; void DownloadManager::trackSession(not_null session) { auto &data = _sessions.emplace(session, SessionData()).first->second; + data.downloaded = deserialize(session); + data.resolveNeeded = data.downloaded.size(); session->data().itemRepaintRequest( ) | rpl::filter([=](not_null item) { @@ -183,6 +192,11 @@ void DownloadManager::addLoaded( Expects(object.item != nullptr); Expects(object.document || object.photo); + const auto size = QFileInfo(path).size(); + if (size <= 0 || size > kMaxFileSize) { + return; + } + const auto item = object.item; auto &data = sessionData(item); @@ -193,6 +207,7 @@ void DownloadManager::addLoaded( .download = id, .started = started, .path = path, + .size = int32(size), .itemId = item->fullId(), .peerAccessHash = PeerAccessHash(item->history()->peer), .object = std::make_unique(object), @@ -200,6 +215,8 @@ void DownloadManager::addLoaded( _loaded.emplace(item); _loadedAdded.fire(&data.downloaded.back()); + writePostponed(&item->history()->session()); + const auto i = ranges::find(data.downloading, item, ByItem); if (i != end(data.downloading)) { auto &entry = *i; @@ -261,8 +278,11 @@ void DownloadManager::clearLoading() { } } -auto DownloadManager::loadedList() const +auto DownloadManager::loadedList() -> ranges::any_view { + for (auto &[session, data] : _sessions) { + resolve(session, data); + } return ranges::views::all( _sessions ) | ranges::views::transform([=](const auto &pair) { @@ -276,6 +296,144 @@ auto DownloadManager::loadedList() const }) | ranges::views::join; } +void DownloadManager::resolve( + not_null session, + SessionData &data) { + if (data.resolveSentTotal >= data.resolveNeeded + || data.resolveSentTotal >= kMaxResolvePerAttempt) { + return; + } + struct Prepared { + uint64 peerAccessHash = 0; + QVector ids; + }; + auto &owner = session->data(); + auto prepared = base::flat_map(); + auto last = begin(data.downloaded); + auto from = last + (data.resolveNeeded - data.resolveSentTotal); + for (auto i = from; i != last;) { + auto &id = *--i; + const auto msgId = id.itemId.msg; + const auto info = QFileInfo(id.path); + if (!info.exists() || info.size() != id.size) { + // Mark as deleted. + id.path = QString(); + } else if (!owner.message(id.itemId) && IsServerMsgId(msgId)) { + const auto groupByPeer = peerIsChannel(id.itemId.peer) + ? id.itemId.peer + : session->userPeerId(); + auto &perPeer = prepared[groupByPeer]; + if (peerIsChannel(id.itemId.peer) && !perPeer.peerAccessHash) { + perPeer.peerAccessHash = id.peerAccessHash; + } + perPeer.ids.push_back(MTP_inputMessageID(MTP_int(msgId.bare))); + } + if (++data.resolveSentTotal >= kMaxResolvePerAttempt) { + break; + } + } + const auto check = [=] { + auto &data = sessionData(session); + if (!data.resolveSentRequests) { + resolveRequestsFinished(session, data); + } + }; + const auto requestFinished = [=] { + --sessionData(session).resolveSentRequests; + check(); + }; + for (auto &[peer, perPeer] : prepared) { + if (const auto channelId = peerToChannel(peer)) { + session->api().request(MTPchannels_GetMessages( + MTP_inputChannel( + MTP_long(channelId.bare), + MTP_long(perPeer.peerAccessHash)), + MTP_vector(perPeer.ids) + )).done([=](const MTPmessages_Messages &result) { + session->data().processExistingMessages( + session->data().channelLoaded(channelId), + result); + requestFinished(); + }).fail(requestFinished).send(); + } else { + session->api().request(MTPmessages_GetMessages( + MTP_vector(perPeer.ids) + )).done([=](const MTPmessages_Messages &result) { + session->data().processExistingMessages(nullptr, result); + requestFinished(); + }).fail(requestFinished).send(); + } + } + data.resolveSentRequests += prepared.size(); + check(); +} + +void DownloadManager::resolveRequestsFinished( + not_null session, + SessionData &data) { + auto &owner = session->data(); + for (; data.resolveSentTotal > 0; --data.resolveSentTotal) { + const auto i = begin(data.downloaded) + (--data.resolveNeeded); + if (i->path.isEmpty()) { + data.downloaded.erase(i); + continue; + } + const auto item = owner.message(i->itemId); + const auto media = item ? item->media() : nullptr; + const auto document = media ? media->document() : nullptr; + const auto photo = media ? media->photo() : nullptr; + if (i->download.type == DownloadType::Document + && (!document || document->id != i->download.objectId)) { + generateEntry(session, *i); + } else if (i->download.type == DownloadType::Photo + && (!photo || photo->id != i->download.objectId)) { + generateEntry(session, *i); + } else { + i->object = std::make_unique(DownloadObject{ + .item = item, + .document = document, + .photo = photo, + }); + _loaded.emplace(item); + } + _loadedAdded.fire(&*i); + } + crl::on_main(session, [=] { + resolve(session, sessionData(session)); + }); +} + +void DownloadManager::generateEntry( + not_null session, + DownloadedId &id) { + Expects(!id.object); + + const auto info = QFileInfo(id.path); + const auto document = session->data().document( + base::RandomValue(), + 0, // accessHash + QByteArray(), // fileReference + TimeId(id.started / 1000), + QVector( + 1, + MTP_documentAttributeFilename( + MTP_string(info.fileName()))), + Core::MimeTypeForFile(info).name(), + InlineImageLocation(), // inlineThumbnail + ImageWithLocation(), // thumbnail + ImageWithLocation(), // videoThumbnail + 0, // dc + id.size); + document->setLocation(Core::FileLocation(info)); + _generatedDocuments.emplace(document); + + id.object = std::make_unique(DownloadObject{ + .item = generateFakeItem(document), + .document = document, + }); + _loaded.emplace(id.object->item); +} + auto DownloadManager::loadedAdded() const -> rpl::producer> { return _loadedAdded.events(); @@ -353,18 +511,30 @@ void DownloadManager::removed(not_null item) { } } -not_null DownloadManager::generateItem( - const DownloadObject &object) { - Expects(object.document || object.photo); +not_null DownloadManager::regenerateItem( + const DownloadObject &previous) { + return generateItem(previous.item, previous.document, previous.photo); +} - const auto session = object.document - ? &object.document->session() - : &object.photo->session(); - const auto fromId = object.item - ? object.item->from()->id +not_null DownloadManager::generateFakeItem( + not_null document) { + return generateItem(nullptr, document, nullptr); +} + +not_null DownloadManager::generateItem( + HistoryItem *previousItem, + DocumentData *document, + PhotoData *photo) { + Expects(document || photo); + + const auto session = document + ? &document->session() + : &photo->session(); + const auto fromId = previousItem + ? previousItem->from()->id : session->userPeerId(); - const auto history = object.item - ? object.item->history() + const auto history = previousItem + ? previousItem->history() : session->data().history(session->user()); const auto flags = MessageFlag::FakeHistoryItem; const auto replyTo = MsgId(); @@ -386,9 +556,7 @@ not_null DownloadManager::generateItem( caption, HistoryMessageMarkupData()); }; - const auto result = object.document - ? make(object.document) - : make(object.photo); + const auto result = document ? make(document) : make(photo); _generated.emplace(result); return result; } @@ -400,7 +568,7 @@ void DownloadManager::detach(DownloadedId &id) { // Maybe generate new document? const auto was = id.object->item; - const auto now = generateItem(*id.object); + const auto now = regenerateItem(*id.object); _loaded.remove(was); _loaded.emplace(now); id.object->item = now; @@ -416,19 +584,139 @@ DownloadManager::SessionData &DownloadManager::sessionData( return i->second; } +const DownloadManager::SessionData &DownloadManager::sessionData( + not_null session) const { + const auto i = _sessions.find(session); + Assert(i != end(_sessions)); + return i->second; +} + DownloadManager::SessionData &DownloadManager::sessionData( not_null item) { return sessionData(&item->history()->session()); } +void DownloadManager::writePostponed(not_null session) { + session->account().local().updateDownloads(serializator(session)); +} + +Fn()> DownloadManager::serializator( + not_null session) const { + return [this, weak = base::make_weak(session.get())]() + -> std::optional { + const auto strong = weak.get(); + if (!strong) { + return std::nullopt; + } else if (!_sessions.contains(strong)) { + return QByteArray(); + } + auto result = QByteArray(); + const auto &data = sessionData(strong); + const auto count = data.downloaded.size(); + const auto constant = sizeof(quint64) // download.objectId + + sizeof(qint32) // download.type + + sizeof(qint64) // started + + sizeof(qint32) // size + + sizeof(quint64) // itemId.peer + + sizeof(qint64) // itemId.msg + + sizeof(quint64); // peerAccessHash + auto size = sizeof(qint32) // count + + count * constant; + for (const auto &id : data.downloaded) { + size += Serialize::stringSize(id.path); + } + result.reserve(size); + + auto stream = QDataStream(&result, QIODevice::WriteOnly); + stream.setVersion(QDataStream::Qt_5_1); + stream << qint32(count); + for (const auto &id : data.downloaded) { + stream + << quint64(id.download.objectId) + << qint32(id.download.type) + << qint64(id.started) + << qint32(id.size) + << quint64(id.itemId.peer.value) + << qint64(id.itemId.msg.bare) + << quint64(id.peerAccessHash) + << id.path; + } + stream.device()->close(); + + return result; + }; +} + +std::vector DownloadManager::deserialize( + not_null session) const { + const auto serialized = session->account().local().downloadsSerialized(); + if (serialized.isEmpty()) { + return {}; + } + + QDataStream stream(serialized); + stream.setVersion(QDataStream::Qt_5_1); + + auto count = qint32(); + stream >> count; + if (stream.status() != QDataStream::Ok || count <= 0 || count > 99'999) { + return {}; + } + auto result = std::vector(); + result.reserve(count); + for (auto i = 0; i != count; ++i) { + auto downloadObjectId = quint64(); + auto uncheckedDownloadType = qint32(); + auto started = qint64(); + auto size = qint32(); + auto itemIdPeer = quint64(); + auto itemIdMsg = qint64(); + auto peerAccessHash = quint64(); + auto path = QString(); + stream + >> downloadObjectId + >> uncheckedDownloadType + >> started + >> size + >> itemIdPeer + >> itemIdMsg + >> peerAccessHash + >> path; + const auto downloadType = DownloadType(uncheckedDownloadType); + if (stream.status() != QDataStream::Ok + || path.isEmpty() + || size <= 0 + || size > kMaxFileSize + || (downloadType != DownloadType::Document + && downloadType != DownloadType::Photo)) { + return {}; + } + result.push_back({ + .download = { + .objectId = downloadObjectId, + .type = downloadType, + }, + .started = started, + .path = path, + .size = size, + .itemId = { PeerId(itemIdPeer), MsgId(itemIdMsg) }, + .peerAccessHash = peerAccessHash, + }); + } + return result; +} + void DownloadManager::untrack(not_null session) { const auto i = _sessions.find(session); Assert(i != end(_sessions)); for (const auto &entry : i->second.downloaded) { if (const auto resolved = entry.object.get()) { - if (const auto item = resolved->item) { - _loaded.remove(item); + const auto item = resolved->item; + _loaded.remove(item); + _generated.remove(item); + if (const auto document = resolved->document) { + _generatedDocuments.remove(document); } } } diff --git a/Telegram/SourceFiles/data/data_download_manager.h b/Telegram/SourceFiles/data/data_download_manager.h index 04b0e8e5d..6760d14aa 100644 --- a/Telegram/SourceFiles/data/data_download_manager.h +++ b/Telegram/SourceFiles/data/data_download_manager.h @@ -20,6 +20,7 @@ class Session; namespace Data { +// Used in serialization! enum class DownloadType { Document, Photo, @@ -53,6 +54,7 @@ struct DownloadedId { DownloadId download; DownloadDate started = 0; QString path; + int32 size = 0; FullMsgId itemId; uint64 peerAccessHash = 0; @@ -88,7 +90,7 @@ public: [[nodiscard]] auto loadingProgressValue() const -> rpl::producer; - [[nodiscard]] auto loadedList() const + [[nodiscard]] auto loadedList() -> ranges::any_view; [[nodiscard]] auto loadedAdded() const -> rpl::producer>; @@ -99,6 +101,9 @@ private: struct SessionData { std::vector downloaded; std::vector downloading; + int resolveNeeded = 0; + int resolveSentRequests = 0; + int resolveSentTotal = 0; rpl::lifetime lifetime; }; @@ -116,18 +121,39 @@ private: void clearLoading(); [[nodiscard]] int64 computeNextStarted(); - [[nodiscard]] not_null generateItem( - const DownloadObject &object); [[nodiscard]] SessionData &sessionData(not_null session); + [[nodiscard]] const SessionData &sessionData( + not_null session) const; [[nodiscard]] SessionData &sessionData( not_null item); + void resolve(not_null session, SessionData &data); + void resolveRequestsFinished( + not_null session, + SessionData &data); + [[nodiscard]] not_null regenerateItem( + const DownloadObject &previous); + [[nodiscard]] not_null generateFakeItem( + not_null document); + [[nodiscard]] not_null generateItem( + HistoryItem *previousItem, + DocumentData *document, + PhotoData *photo); + void generateEntry(not_null session, DownloadedId &id); + + void writePostponed(not_null session); + [[nodiscard]] Fn()> serializator( + not_null session) const; + [[nodiscard]] std::vector deserialize( + not_null session) const; + base::flat_map, SessionData> _sessions; base::flat_set> _loading; base::flat_set> _loadingDone; base::flat_set> _loaded; base::flat_set> _generated; + base::flat_set> _generatedDocuments; TimeId _lastStartedBase = 0; int _lastStartedAdded = 0; diff --git a/Telegram/SourceFiles/storage/storage_account.cpp b/Telegram/SourceFiles/storage/storage_account.cpp index e9c2616d7..60e08be7a 100644 --- a/Telegram/SourceFiles/storage/storage_account.cpp +++ b/Telegram/SourceFiles/storage/storage_account.cpp @@ -286,6 +286,7 @@ Account::ReadMapResult Account::readMapWith( quint64 savedGifsKey = 0; quint64 legacyBackgroundKeyDay = 0, legacyBackgroundKeyNight = 0; quint64 userSettingsKey = 0, recentHashtagsAndBotsKey = 0, exportSettingsKey = 0; + quint64 downloadsKey = 0; while (!map.stream.atEnd()) { quint32 keyType; map.stream >> keyType; @@ -598,6 +599,8 @@ void Account::reset() { _fileLocations.clear(); _fileLocationPairs.clear(); _fileLocationAliases.clear(); + _downloadsSerialize = nullptr; + _downloadsSerialized = QByteArray(); _cacheTotalSizeLimit = Database::Settings().totalSizeLimit; _cacheTotalTimeLimit = Database::Settings().totalTimeLimit; _cacheBigFileTotalSizeLimit = Database::Settings().totalSizeLimit; @@ -629,7 +632,12 @@ void Account::writeLocations() { } _locationsChanged = false; - if (_fileLocations.isEmpty()) { + if (_downloadsSerialize) { + if (auto serialized = _downloadsSerialize()) { + _downloadsSerialized = std::move(*serialized); + } + } + if (_fileLocations.isEmpty() && _downloadsSerialized.isEmpty()) { if (_locationsKey) { ClearKey(_locationsKey, _basePath); _locationsKey = 0; @@ -665,6 +673,9 @@ void Account::writeLocations() { size += sizeof(quint64) * 2 + sizeof(quint64) * 2; } + size += sizeof(quint32); // legacy webLocationsCount + size += Serialize::bytearraySize(_downloadsSerialized); + EncryptedDescriptor data(size); auto legacyTypeField = 0; for (auto i = _fileLocations.cbegin(); i != _fileLocations.cend(); ++i) { @@ -686,6 +697,8 @@ void Account::writeLocations() { data.stream << quint64(i.key().first) << quint64(i.key().second) << quint64(i.value().first) << quint64(i.value().second); } + data.stream << quint32(0) << _downloadsSerialized; + FileWriteDescriptor file(_locationsKey, _basePath); file.writeEncrypted(data, _localKey); } @@ -757,10 +770,24 @@ void Account::readLocations() { locations.stream >> url >> key >> size; ClearKey(key, _basePath); } + + if (!locations.stream.atEnd()) { + locations.stream >> _downloadsSerialized; + } } } } +void Account::updateDownloads( + Fn()> downloadsSerialize) { + _downloadsSerialize = std::move(downloadsSerialize); + writeLocationsDelayed(); +} + +QByteArray Account::downloadsSerialized() const { + return _downloadsSerialized; +} + void Account::writeSessionSettings() { writeSessionSettings(nullptr); } diff --git a/Telegram/SourceFiles/storage/storage_account.h b/Telegram/SourceFiles/storage/storage_account.h index 7b7a5609e..c2d0f6104 100644 --- a/Telegram/SourceFiles/storage/storage_account.h +++ b/Telegram/SourceFiles/storage/storage_account.h @@ -97,10 +97,15 @@ public: [[nodiscard]] bool hasDraftCursors(PeerId peerId); [[nodiscard]] bool hasDraft(PeerId peerId); - void writeFileLocation(MediaKey location, const Core::FileLocation &local); + void writeFileLocation( + MediaKey location, + const Core::FileLocation &local); [[nodiscard]] Core::FileLocation readFileLocation(MediaKey location); void removeFileLocation(MediaKey location); + void updateDownloads(Fn()> downloadsSerialize); + [[nodiscard]] QByteArray downloadsSerialized() const; + [[nodiscard]] EncryptionKey cacheKey() const; [[nodiscard]] QString cachePath() const; [[nodiscard]] Cache::Database::Settings cacheSettings() const; @@ -256,6 +261,9 @@ private: QMap> _fileLocationPairs; QMap _fileLocationAliases; + QByteArray _downloadsSerialized; + Fn()> _downloadsSerialize; + FileKey _locationsKey = 0; FileKey _trustedBotsKey = 0; FileKey _installedStickersKey = 0;