Improve streaming of large files.

This commit is contained in:
John Preston 2024-12-20 10:36:26 +04:00
parent 4569f93e70
commit 188d65d700

View file

@ -536,12 +536,12 @@ void Reader::Slices::processPart(
_header.addPart(offset, bytes); _header.addPart(offset, bytes);
checkSliceFullLoaded(0); checkSliceFullLoaded(0);
return; return;
} else if (_headerMode == HeaderMode::Unknown) { //} else if (_headerMode == HeaderMode::Unknown) {
if (_header.parts.contains(offset)) { // if (_header.parts.contains(offset)) {
return; // return;
} else if (_header.parts.size() < kMaxPartsInHeader) { // } else if (_header.parts.size() < kMaxPartsInHeader) {
_header.addPart(offset, bytes); // _header.addPart(offset, bytes);
} // }
} }
const auto index = offset / kInSlice; const auto index = offset / kInSlice;
_data[index].addPart(offset - index * kInSlice, std::move(bytes)); _data[index].addPart(offset - index * kInSlice, std::move(bytes));
@ -599,6 +599,17 @@ auto Reader::Slices::fill(uint32 offset, bytes::span buffer) -> FillResult {
result.state = FillState::WaitingCache; result.state = FillState::WaitingCache;
} }
}; };
const auto addToHeader = [&](int slice, auto parts) {
if (_headerMode == HeaderMode::Unknown) {
for (const auto &part : parts) {
const auto totalOffset = slice * kInSlice + part.first;
if (!_header.parts.contains(totalOffset)
&& _header.parts.size() < kMaxPartsInHeader) {
_header.addPart(totalOffset, part.second);
}
}
}
};
const auto firstFrom = offset - fromSlice * kInSlice; const auto firstFrom = offset - fromSlice * kInSlice;
const auto firstTill = std::min(kInSlice, till - fromSlice * kInSlice); const auto firstTill = std::min(kInSlice, till - fromSlice * kInSlice);
const auto secondFrom = 0; const auto secondFrom = 0;
@ -615,18 +626,18 @@ auto Reader::Slices::fill(uint32 offset, bytes::span buffer) -> FillResult {
} }
if (first.ready && second.ready) { if (first.ready && second.ready) {
markSliceUsed(fromSlice); markSliceUsed(fromSlice);
CopyLoaded( auto &&list = ranges::make_subrange(first.start, first.finish);
buffer, CopyLoaded(buffer, list, firstFrom, firstTill);
ranges::make_subrange(first.start, first.finish), addToHeader(fromSlice, list);
firstFrom,
firstTill);
if (fromSlice + 1 < tillSlice) { if (fromSlice + 1 < tillSlice) {
markSliceUsed(fromSlice + 1); markSliceUsed(fromSlice + 1);
auto &&list = ranges::make_subrange(second.start, second.finish);
CopyLoaded( CopyLoaded(
buffer.subspan(firstTill - firstFrom), buffer.subspan(firstTill - firstFrom),
ranges::make_subrange(second.start, second.finish), list,
secondFrom, secondFrom,
secondTill); secondTill);
addToHeader(fromSlice + 1, list);
} }
result.toCache = serializeAndUnloadUnused(); result.toCache = serializeAndUnloadUnused();
result.state = FillState::Success; result.state = FillState::Success;