Save and restore downloads between launches.

This commit is contained in:
John Preston 2022-02-26 19:08:44 +03:00
parent 3425dc027c
commit 57f17b7afe
7 changed files with 401 additions and 35 deletions

View file

@ -39,23 +39,34 @@ FileLocation::FileLocation(const QString &name) : fname(name) {
size = 0;
} else {
setBookmark(Platform::PathBookmark(name));
resolveFromInfo(QFileInfo(name));
}
}
QFileInfo f(name);
if (f.exists()) {
qint64 s = f.size();
if (s > INT_MAX) {
fname = QString();
_bookmark = nullptr;
size = 0;
} else {
modified = f.lastModified();
size = qint32(s);
}
} else {
FileLocation::FileLocation(const QFileInfo &info) : fname(info.filePath()) {
if (fname.isEmpty()) {
size = 0;
} else {
setBookmark(Platform::PathBookmark(fname));
resolveFromInfo(info);
}
}
void FileLocation::resolveFromInfo(const QFileInfo &info) {
if (info.exists()) {
const auto s = info.size();
if (s > INT_MAX) {
fname = QString();
_bookmark = nullptr;
size = 0;
} else {
modified = info.lastModified();
size = qint32(s);
}
} else {
fname = QString();
_bookmark = nullptr;
size = 0;
}
}

View file

@ -9,6 +9,8 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include <QtCore/QDateTime>
class QFileInfo;
namespace Platform {
class FileBookmark;
} // namespace Platform
@ -35,6 +37,7 @@ class FileLocation {
public:
FileLocation() = default;
explicit FileLocation(const QString &name);
explicit FileLocation(const QFileInfo &info);
static FileLocation InMediaCacheLocation();
@ -55,6 +58,8 @@ public:
qint32 size;
private:
void resolveFromInfo(const QFileInfo &info);
std::shared_ptr<Platform::FileBookmark> _bookmark;
};

View file

@ -1256,7 +1256,8 @@ bool DocumentData::isNull() const {
return !hasRemoteLocation()
&& !hasWebLocation()
&& _url.isEmpty()
&& !uploading();
&& !uploading()
&& _location.isEmpty();
}
MTPInputDocument DocumentData::mtpInput() const {

View file

@ -14,18 +14,25 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
#include "data/data_user.h"
#include "data/data_channel.h"
#include "base/unixtime.h"
#include "base/random.h"
#include "main/main_session.h"
#include "main/main_account.h"
#include "storage/storage_account.h"
#include "history/history.h"
#include "history/history_item.h"
#include "history/history_message.h"
#include "core/application.h"
#include "core/mime_type.h"
#include "ui/controls/download_bar.h"
#include "storage/serialize_common.h"
#include "apiwrap.h"
namespace Data {
namespace {
constexpr auto kClearLoadingTimeout = 5 * crl::time(1000);
constexpr auto kMaxFileSize = 2000 * 1024 * 1024;
constexpr auto kMaxResolvePerAttempt = 100;
constexpr auto ByItem = [](const auto &entry) {
if constexpr (std::is_same_v<decltype(entry), const DownloadingId&>) {
@ -55,6 +62,8 @@ DownloadManager::~DownloadManager() = default;
void DownloadManager::trackSession(not_null<Main::Session*> session) {
auto &data = _sessions.emplace(session, SessionData()).first->second;
data.downloaded = deserialize(session);
data.resolveNeeded = data.downloaded.size();
session->data().itemRepaintRequest(
) | rpl::filter([=](not_null<const HistoryItem*> item) {
@ -183,6 +192,11 @@ void DownloadManager::addLoaded(
Expects(object.item != nullptr);
Expects(object.document || object.photo);
const auto size = QFileInfo(path).size();
if (size <= 0 || size > kMaxFileSize) {
return;
}
const auto item = object.item;
auto &data = sessionData(item);
@ -193,6 +207,7 @@ void DownloadManager::addLoaded(
.download = id,
.started = started,
.path = path,
.size = int32(size),
.itemId = item->fullId(),
.peerAccessHash = PeerAccessHash(item->history()->peer),
.object = std::make_unique<DownloadObject>(object),
@ -200,6 +215,8 @@ void DownloadManager::addLoaded(
_loaded.emplace(item);
_loadedAdded.fire(&data.downloaded.back());
writePostponed(&item->history()->session());
const auto i = ranges::find(data.downloading, item, ByItem);
if (i != end(data.downloading)) {
auto &entry = *i;
@ -261,8 +278,11 @@ void DownloadManager::clearLoading() {
}
}
auto DownloadManager::loadedList() const
auto DownloadManager::loadedList()
-> ranges::any_view<const DownloadedId*, ranges::category::input> {
for (auto &[session, data] : _sessions) {
resolve(session, data);
}
return ranges::views::all(
_sessions
) | ranges::views::transform([=](const auto &pair) {
@ -276,6 +296,144 @@ auto DownloadManager::loadedList() const
}) | ranges::views::join;
}
void DownloadManager::resolve(
not_null<Main::Session*> session,
SessionData &data) {
if (data.resolveSentTotal >= data.resolveNeeded
|| data.resolveSentTotal >= kMaxResolvePerAttempt) {
return;
}
struct Prepared {
uint64 peerAccessHash = 0;
QVector<MTPInputMessage> ids;
};
auto &owner = session->data();
auto prepared = base::flat_map<PeerId, Prepared>();
auto last = begin(data.downloaded);
auto from = last + (data.resolveNeeded - data.resolveSentTotal);
for (auto i = from; i != last;) {
auto &id = *--i;
const auto msgId = id.itemId.msg;
const auto info = QFileInfo(id.path);
if (!info.exists() || info.size() != id.size) {
// Mark as deleted.
id.path = QString();
} else if (!owner.message(id.itemId) && IsServerMsgId(msgId)) {
const auto groupByPeer = peerIsChannel(id.itemId.peer)
? id.itemId.peer
: session->userPeerId();
auto &perPeer = prepared[groupByPeer];
if (peerIsChannel(id.itemId.peer) && !perPeer.peerAccessHash) {
perPeer.peerAccessHash = id.peerAccessHash;
}
perPeer.ids.push_back(MTP_inputMessageID(MTP_int(msgId.bare)));
}
if (++data.resolveSentTotal >= kMaxResolvePerAttempt) {
break;
}
}
const auto check = [=] {
auto &data = sessionData(session);
if (!data.resolveSentRequests) {
resolveRequestsFinished(session, data);
}
};
const auto requestFinished = [=] {
--sessionData(session).resolveSentRequests;
check();
};
for (auto &[peer, perPeer] : prepared) {
if (const auto channelId = peerToChannel(peer)) {
session->api().request(MTPchannels_GetMessages(
MTP_inputChannel(
MTP_long(channelId.bare),
MTP_long(perPeer.peerAccessHash)),
MTP_vector<MTPInputMessage>(perPeer.ids)
)).done([=](const MTPmessages_Messages &result) {
session->data().processExistingMessages(
session->data().channelLoaded(channelId),
result);
requestFinished();
}).fail(requestFinished).send();
} else {
session->api().request(MTPmessages_GetMessages(
MTP_vector<MTPInputMessage>(perPeer.ids)
)).done([=](const MTPmessages_Messages &result) {
session->data().processExistingMessages(nullptr, result);
requestFinished();
}).fail(requestFinished).send();
}
}
data.resolveSentRequests += prepared.size();
check();
}
void DownloadManager::resolveRequestsFinished(
not_null<Main::Session*> session,
SessionData &data) {
auto &owner = session->data();
for (; data.resolveSentTotal > 0; --data.resolveSentTotal) {
const auto i = begin(data.downloaded) + (--data.resolveNeeded);
if (i->path.isEmpty()) {
data.downloaded.erase(i);
continue;
}
const auto item = owner.message(i->itemId);
const auto media = item ? item->media() : nullptr;
const auto document = media ? media->document() : nullptr;
const auto photo = media ? media->photo() : nullptr;
if (i->download.type == DownloadType::Document
&& (!document || document->id != i->download.objectId)) {
generateEntry(session, *i);
} else if (i->download.type == DownloadType::Photo
&& (!photo || photo->id != i->download.objectId)) {
generateEntry(session, *i);
} else {
i->object = std::make_unique<DownloadObject>(DownloadObject{
.item = item,
.document = document,
.photo = photo,
});
_loaded.emplace(item);
}
_loadedAdded.fire(&*i);
}
crl::on_main(session, [=] {
resolve(session, sessionData(session));
});
}
void DownloadManager::generateEntry(
not_null<Main::Session*> session,
DownloadedId &id) {
Expects(!id.object);
const auto info = QFileInfo(id.path);
const auto document = session->data().document(
base::RandomValue<DocumentId>(),
0, // accessHash
QByteArray(), // fileReference
TimeId(id.started / 1000),
QVector<MTPDocumentAttribute>(
1,
MTP_documentAttributeFilename(
MTP_string(info.fileName()))),
Core::MimeTypeForFile(info).name(),
InlineImageLocation(), // inlineThumbnail
ImageWithLocation(), // thumbnail
ImageWithLocation(), // videoThumbnail
0, // dc
id.size);
document->setLocation(Core::FileLocation(info));
_generatedDocuments.emplace(document);
id.object = std::make_unique<DownloadObject>(DownloadObject{
.item = generateFakeItem(document),
.document = document,
});
_loaded.emplace(id.object->item);
}
auto DownloadManager::loadedAdded() const
-> rpl::producer<not_null<const DownloadedId*>> {
return _loadedAdded.events();
@ -353,18 +511,30 @@ void DownloadManager::removed(not_null<const HistoryItem*> item) {
}
}
not_null<HistoryItem*> DownloadManager::generateItem(
const DownloadObject &object) {
Expects(object.document || object.photo);
not_null<HistoryItem*> DownloadManager::regenerateItem(
const DownloadObject &previous) {
return generateItem(previous.item, previous.document, previous.photo);
}
const auto session = object.document
? &object.document->session()
: &object.photo->session();
const auto fromId = object.item
? object.item->from()->id
not_null<HistoryItem*> DownloadManager::generateFakeItem(
not_null<DocumentData*> document) {
return generateItem(nullptr, document, nullptr);
}
not_null<HistoryItem*> DownloadManager::generateItem(
HistoryItem *previousItem,
DocumentData *document,
PhotoData *photo) {
Expects(document || photo);
const auto session = document
? &document->session()
: &photo->session();
const auto fromId = previousItem
? previousItem->from()->id
: session->userPeerId();
const auto history = object.item
? object.item->history()
const auto history = previousItem
? previousItem->history()
: session->data().history(session->user());
const auto flags = MessageFlag::FakeHistoryItem;
const auto replyTo = MsgId();
@ -386,9 +556,7 @@ not_null<HistoryItem*> DownloadManager::generateItem(
caption,
HistoryMessageMarkupData());
};
const auto result = object.document
? make(object.document)
: make(object.photo);
const auto result = document ? make(document) : make(photo);
_generated.emplace(result);
return result;
}
@ -400,7 +568,7 @@ void DownloadManager::detach(DownloadedId &id) {
// Maybe generate new document?
const auto was = id.object->item;
const auto now = generateItem(*id.object);
const auto now = regenerateItem(*id.object);
_loaded.remove(was);
_loaded.emplace(now);
id.object->item = now;
@ -416,19 +584,139 @@ DownloadManager::SessionData &DownloadManager::sessionData(
return i->second;
}
const DownloadManager::SessionData &DownloadManager::sessionData(
not_null<Main::Session*> session) const {
const auto i = _sessions.find(session);
Assert(i != end(_sessions));
return i->second;
}
DownloadManager::SessionData &DownloadManager::sessionData(
not_null<const HistoryItem*> item) {
return sessionData(&item->history()->session());
}
void DownloadManager::writePostponed(not_null<Main::Session*> session) {
session->account().local().updateDownloads(serializator(session));
}
Fn<std::optional<QByteArray>()> DownloadManager::serializator(
not_null<Main::Session*> session) const {
return [this, weak = base::make_weak(session.get())]()
-> std::optional<QByteArray> {
const auto strong = weak.get();
if (!strong) {
return std::nullopt;
} else if (!_sessions.contains(strong)) {
return QByteArray();
}
auto result = QByteArray();
const auto &data = sessionData(strong);
const auto count = data.downloaded.size();
const auto constant = sizeof(quint64) // download.objectId
+ sizeof(qint32) // download.type
+ sizeof(qint64) // started
+ sizeof(qint32) // size
+ sizeof(quint64) // itemId.peer
+ sizeof(qint64) // itemId.msg
+ sizeof(quint64); // peerAccessHash
auto size = sizeof(qint32) // count
+ count * constant;
for (const auto &id : data.downloaded) {
size += Serialize::stringSize(id.path);
}
result.reserve(size);
auto stream = QDataStream(&result, QIODevice::WriteOnly);
stream.setVersion(QDataStream::Qt_5_1);
stream << qint32(count);
for (const auto &id : data.downloaded) {
stream
<< quint64(id.download.objectId)
<< qint32(id.download.type)
<< qint64(id.started)
<< qint32(id.size)
<< quint64(id.itemId.peer.value)
<< qint64(id.itemId.msg.bare)
<< quint64(id.peerAccessHash)
<< id.path;
}
stream.device()->close();
return result;
};
}
std::vector<DownloadedId> DownloadManager::deserialize(
not_null<Main::Session*> session) const {
const auto serialized = session->account().local().downloadsSerialized();
if (serialized.isEmpty()) {
return {};
}
QDataStream stream(serialized);
stream.setVersion(QDataStream::Qt_5_1);
auto count = qint32();
stream >> count;
if (stream.status() != QDataStream::Ok || count <= 0 || count > 99'999) {
return {};
}
auto result = std::vector<DownloadedId>();
result.reserve(count);
for (auto i = 0; i != count; ++i) {
auto downloadObjectId = quint64();
auto uncheckedDownloadType = qint32();
auto started = qint64();
auto size = qint32();
auto itemIdPeer = quint64();
auto itemIdMsg = qint64();
auto peerAccessHash = quint64();
auto path = QString();
stream
>> downloadObjectId
>> uncheckedDownloadType
>> started
>> size
>> itemIdPeer
>> itemIdMsg
>> peerAccessHash
>> path;
const auto downloadType = DownloadType(uncheckedDownloadType);
if (stream.status() != QDataStream::Ok
|| path.isEmpty()
|| size <= 0
|| size > kMaxFileSize
|| (downloadType != DownloadType::Document
&& downloadType != DownloadType::Photo)) {
return {};
}
result.push_back({
.download = {
.objectId = downloadObjectId,
.type = downloadType,
},
.started = started,
.path = path,
.size = size,
.itemId = { PeerId(itemIdPeer), MsgId(itemIdMsg) },
.peerAccessHash = peerAccessHash,
});
}
return result;
}
void DownloadManager::untrack(not_null<Main::Session*> session) {
const auto i = _sessions.find(session);
Assert(i != end(_sessions));
for (const auto &entry : i->second.downloaded) {
if (const auto resolved = entry.object.get()) {
if (const auto item = resolved->item) {
_loaded.remove(item);
const auto item = resolved->item;
_loaded.remove(item);
_generated.remove(item);
if (const auto document = resolved->document) {
_generatedDocuments.remove(document);
}
}
}

View file

@ -20,6 +20,7 @@ class Session;
namespace Data {
// Used in serialization!
enum class DownloadType {
Document,
Photo,
@ -53,6 +54,7 @@ struct DownloadedId {
DownloadId download;
DownloadDate started = 0;
QString path;
int32 size = 0;
FullMsgId itemId;
uint64 peerAccessHash = 0;
@ -88,7 +90,7 @@ public:
[[nodiscard]] auto loadingProgressValue() const
-> rpl::producer<DownloadProgress>;
[[nodiscard]] auto loadedList() const
[[nodiscard]] auto loadedList()
-> ranges::any_view<const DownloadedId*, ranges::category::input>;
[[nodiscard]] auto loadedAdded() const
-> rpl::producer<not_null<const DownloadedId*>>;
@ -99,6 +101,9 @@ private:
struct SessionData {
std::vector<DownloadedId> downloaded;
std::vector<DownloadingId> downloading;
int resolveNeeded = 0;
int resolveSentRequests = 0;
int resolveSentTotal = 0;
rpl::lifetime lifetime;
};
@ -116,18 +121,39 @@ private:
void clearLoading();
[[nodiscard]] int64 computeNextStarted();
[[nodiscard]] not_null<HistoryItem*> generateItem(
const DownloadObject &object);
[[nodiscard]] SessionData &sessionData(not_null<Main::Session*> session);
[[nodiscard]] const SessionData &sessionData(
not_null<Main::Session*> session) const;
[[nodiscard]] SessionData &sessionData(
not_null<const HistoryItem*> item);
void resolve(not_null<Main::Session*> session, SessionData &data);
void resolveRequestsFinished(
not_null<Main::Session*> session,
SessionData &data);
[[nodiscard]] not_null<HistoryItem*> regenerateItem(
const DownloadObject &previous);
[[nodiscard]] not_null<HistoryItem*> generateFakeItem(
not_null<DocumentData*> document);
[[nodiscard]] not_null<HistoryItem*> generateItem(
HistoryItem *previousItem,
DocumentData *document,
PhotoData *photo);
void generateEntry(not_null<Main::Session*> session, DownloadedId &id);
void writePostponed(not_null<Main::Session*> session);
[[nodiscard]] Fn<std::optional<QByteArray>()> serializator(
not_null<Main::Session*> session) const;
[[nodiscard]] std::vector<DownloadedId> deserialize(
not_null<Main::Session*> session) const;
base::flat_map<not_null<Main::Session*>, SessionData> _sessions;
base::flat_set<not_null<const HistoryItem*>> _loading;
base::flat_set<not_null<const HistoryItem*>> _loadingDone;
base::flat_set<not_null<const HistoryItem*>> _loaded;
base::flat_set<not_null<HistoryItem*>> _generated;
base::flat_set<not_null<DocumentData*>> _generatedDocuments;
TimeId _lastStartedBase = 0;
int _lastStartedAdded = 0;

View file

@ -286,6 +286,7 @@ Account::ReadMapResult Account::readMapWith(
quint64 savedGifsKey = 0;
quint64 legacyBackgroundKeyDay = 0, legacyBackgroundKeyNight = 0;
quint64 userSettingsKey = 0, recentHashtagsAndBotsKey = 0, exportSettingsKey = 0;
quint64 downloadsKey = 0;
while (!map.stream.atEnd()) {
quint32 keyType;
map.stream >> keyType;
@ -598,6 +599,8 @@ void Account::reset() {
_fileLocations.clear();
_fileLocationPairs.clear();
_fileLocationAliases.clear();
_downloadsSerialize = nullptr;
_downloadsSerialized = QByteArray();
_cacheTotalSizeLimit = Database::Settings().totalSizeLimit;
_cacheTotalTimeLimit = Database::Settings().totalTimeLimit;
_cacheBigFileTotalSizeLimit = Database::Settings().totalSizeLimit;
@ -629,7 +632,12 @@ void Account::writeLocations() {
}
_locationsChanged = false;
if (_fileLocations.isEmpty()) {
if (_downloadsSerialize) {
if (auto serialized = _downloadsSerialize()) {
_downloadsSerialized = std::move(*serialized);
}
}
if (_fileLocations.isEmpty() && _downloadsSerialized.isEmpty()) {
if (_locationsKey) {
ClearKey(_locationsKey, _basePath);
_locationsKey = 0;
@ -665,6 +673,9 @@ void Account::writeLocations() {
size += sizeof(quint64) * 2 + sizeof(quint64) * 2;
}
size += sizeof(quint32); // legacy webLocationsCount
size += Serialize::bytearraySize(_downloadsSerialized);
EncryptedDescriptor data(size);
auto legacyTypeField = 0;
for (auto i = _fileLocations.cbegin(); i != _fileLocations.cend(); ++i) {
@ -686,6 +697,8 @@ void Account::writeLocations() {
data.stream << quint64(i.key().first) << quint64(i.key().second) << quint64(i.value().first) << quint64(i.value().second);
}
data.stream << quint32(0) << _downloadsSerialized;
FileWriteDescriptor file(_locationsKey, _basePath);
file.writeEncrypted(data, _localKey);
}
@ -757,10 +770,24 @@ void Account::readLocations() {
locations.stream >> url >> key >> size;
ClearKey(key, _basePath);
}
if (!locations.stream.atEnd()) {
locations.stream >> _downloadsSerialized;
}
}
}
}
void Account::updateDownloads(
Fn<std::optional<QByteArray>()> downloadsSerialize) {
_downloadsSerialize = std::move(downloadsSerialize);
writeLocationsDelayed();
}
QByteArray Account::downloadsSerialized() const {
return _downloadsSerialized;
}
void Account::writeSessionSettings() {
writeSessionSettings(nullptr);
}

View file

@ -97,10 +97,15 @@ public:
[[nodiscard]] bool hasDraftCursors(PeerId peerId);
[[nodiscard]] bool hasDraft(PeerId peerId);
void writeFileLocation(MediaKey location, const Core::FileLocation &local);
void writeFileLocation(
MediaKey location,
const Core::FileLocation &local);
[[nodiscard]] Core::FileLocation readFileLocation(MediaKey location);
void removeFileLocation(MediaKey location);
void updateDownloads(Fn<std::optional<QByteArray>()> downloadsSerialize);
[[nodiscard]] QByteArray downloadsSerialized() const;
[[nodiscard]] EncryptionKey cacheKey() const;
[[nodiscard]] QString cachePath() const;
[[nodiscard]] Cache::Database::Settings cacheSettings() const;
@ -256,6 +261,9 @@ private:
QMap<QString, QPair<MediaKey, Core::FileLocation>> _fileLocationPairs;
QMap<MediaKey, MediaKey> _fileLocationAliases;
QByteArray _downloadsSerialized;
Fn<std::optional<QByteArray>()> _downloadsSerialize;
FileKey _locationsKey = 0;
FileKey _trustedBotsKey = 0;
FileKey _installedStickersKey = 0;