Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
ChunkingNG: add '0' padding on the filename
The server sorts the chunk by name alphabetically. So if we want to keep
the chunk in order, we need to add a few '0' in front of the chunk name
  • Loading branch information
ogoffart committed Jan 19, 2017
commit 403553a3d6c096b18a81b7c524d1304877ff4ea0
3 changes: 2 additions & 1 deletion src/libsync/propagateupload.h
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,8 @@ class PropagateUploadFileNG : public PropagateUploadFileCommon {

// Map chunk number with its size from the PROPFIND on resume.
// (Only used from slotPropfindIterate/slotPropfindFinished because the LsColJob use signals to report data.)
QMap<int, quint64> _serverChunks;
struct ServerChunkInfo { quint64 size; QString originalName; };
QMap<int, ServerChunkInfo> _serverChunks;

quint64 chunkSize() const { return propagator()->chunkSize(); }
/**
Expand Down
15 changes: 9 additions & 6 deletions src/libsync/propagateuploadng.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ QUrl PropagateUploadFileNG::chunkUrl(int chunk)
+ propagator()->account()->davUser()
+ QLatin1Char('/') + QString::number(_transferId);
if (chunk >= 0) {
path += QLatin1Char('/') + QString::number(chunk);
// We need to do add leading 0 because the server orders the chunk alphabetically
path += QLatin1Char('/') + QString::number(chunk).rightJustified(8, '0');
}
return Utility::concatUrlPath(propagator()->account()->url(), path);
}
Expand Down Expand Up @@ -108,9 +109,11 @@ void PropagateUploadFileNG::slotPropfindIterate(const QString &name, const QMap<
return; // skip the info about the path itself
}
bool ok = false;
auto chunkId = name.mid(name.lastIndexOf('/')+1).toUInt(&ok);
QString chunkName = name.mid(name.lastIndexOf('/')+1);
auto chunkId = chunkName.toUInt(&ok);
if (ok) {
_serverChunks[chunkId] = properties["getcontentlength"].toULongLong();
ServerChunkInfo chunkinfo = { properties["getcontentlength"].toULongLong(), chunkName };
_serverChunks[chunkId] = chunkinfo;
}
}

Expand All @@ -123,7 +126,7 @@ void PropagateUploadFileNG::slotPropfindFinished()
_currentChunk = 0;
_sent = 0;
while (_serverChunks.contains(_currentChunk)) {
_sent += _serverChunks[_currentChunk];
_sent += _serverChunks[_currentChunk].size;
_serverChunks.remove(_currentChunk);
++_currentChunk;
}
Expand All @@ -141,15 +144,15 @@ void PropagateUploadFileNG::slotPropfindFinished()
qDebug() << "Resuming "<< _item->_file << " from chunk " << _currentChunk << "; sent ="<< _sent;

if (!_serverChunks.isEmpty()) {
qDebug() << "To Delete" << _serverChunks;
qDebug() << "To Delete" << _serverChunks.keys();
propagator()->_activeJobList.append(this);
_removeJobError = false;

// Make sure that if there is a "hole" and then a few more chunks, on the server
// we should remove the later chunks. Otherwise when we do dynamic chunk sizing, we may end up
// with corruptions if there are too many chunks, or if we abort and there are still stale chunks.
for (auto it = _serverChunks.begin(); it != _serverChunks.end(); ++it) {
auto job = new DeleteJob(propagator()->account(), Utility::concatUrlPath(chunkUrl(), QString::number(it.key())), this);
auto job = new DeleteJob(propagator()->account(), Utility::concatUrlPath(chunkUrl(), it->originalName), this);
QObject::connect(job, SIGNAL(finishedSignal()), this, SLOT(slotDeleteJobFinished()));
_jobs.append(job);
job->start();
Expand Down
5 changes: 3 additions & 2 deletions test/syncenginetestutils.h
Original file line number Diff line number Diff line change
Expand Up @@ -589,9 +589,10 @@ class FakeChunkMoveReply : public QNetworkReply
char payload = '*';

do {
if (!sourceFolder->children.contains(QString::number(count)))
QString chunkName = QString::number(count).rightJustified(8, '0');
if (!sourceFolder->children.contains(chunkName))
break;
auto &x = sourceFolder->children[QString::number(count)];
auto &x = sourceFolder->children[chunkName];
Q_ASSERT(!x.isDir);
Q_ASSERT(x.size > 0); // There should not be empty chunks
size += x.size;
Expand Down