Index: src/libchcore/TSubTaskCopyMove.cpp =================================================================== diff -u -r08cb83a990d701a1e962630efeb6640a8c2d3dcd -rdb818324a61602e118506ca8b3973e44d997c8de --- src/libchcore/TSubTaskCopyMove.cpp (.../TSubTaskCopyMove.cpp) (revision 08cb83a990d701a1e962630efeb6640a8c2d3dcd) +++ src/libchcore/TSubTaskCopyMove.cpp (.../TSubTaskCopyMove.cpp) (revision db818324a61602e118506ca8b3973e44d997c8de) @@ -155,9 +155,8 @@ m_tSubTaskStats.SetCurrentIndex(fcIndex); m_tSubTaskStats.SetProcessedCount(fcIndex); m_tSubTaskStats.SetCurrentPath(pathCurrent.ToString()); - m_tSubTaskStats.SetCurrentItemProcessedSize(ullCurrentItemProcessedSize); // preserve the processed size for the first item - ullCurrentItemProcessedSize = 0; - m_tSubTaskStats.SetCurrentItemTotalSize(spFileInfo->GetLength64()); + m_tSubTaskStats.SetCurrentItemSizes(ullCurrentItemProcessedSize, spFileInfo->GetLength64()); // preserve the processed size for the first item + ullCurrentItemProcessedSize = 0; // in next iteration we're not resuming anymore m_tSubTaskStats.SetCurrentItemSilentResume(bCurrentFileSilentResume); bCurrentFileSilentResume = false; @@ -175,8 +174,7 @@ return eResult; // new stats - m_tSubTaskStats.IncreaseProcessedSize(spFileInfo->GetLength64()); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(spFileInfo->GetLength64()); + AdjustProcessedSizeForSkip(spFileInfo); spFileInfo->MarkAsProcessed(true); } @@ -362,7 +360,7 @@ else if(bSkip) { // new stats - AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); + AdjustProcessedSizeForSkip(pData->spSrcFile); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -388,8 +386,7 @@ return eResult; else if(bSkip) { - // new stats - AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); + AdjustProcessedSizeForSkip(pData->spSrcFile); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -399,8 +396,7 @@ return eResult; else if(bSkip) { - // new stats - AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); + AdjustProcessedSizeForSkip(pData->spSrcFile); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -415,8 +411,7 @@ return eResult; else if(bSkip) { - // new stats - AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); + AdjustProcessedSizeForSkip(pData->spSrcFile); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -442,8 +437,7 @@ return eResult; else if(bSkip) { - // new stats - AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); + AdjustProcessedSizeForSkip(pData->spSrcFile); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -453,8 +447,7 @@ return eResult; else if(bSkip) { - // new stats - AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); + AdjustProcessedSizeForSkip(pData->spSrcFile); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -467,27 +460,17 @@ return eResult; else if (bSkip) { - // new stats - AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); + AdjustProcessedSizeForSkip(pData->spSrcFile); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; } - unsigned long long ullCITotalSize = m_tSubTaskStats.GetCurrentItemTotalSize(); - unsigned long long ullCIProcessedSize = m_tSubTaskStats.GetCurrentItemProcessedSize(); + file_size_t fsWritten = pBuffer->GetRealDataSize(); - unsigned long long ullWritten = pBuffer->GetBytesTransferred(); - if(ullCIProcessedSize + ullWritten > ullCITotalSize) - { - // total size changed - pData->spSrcFile->SetLength64(ullCIProcessedSize + ullWritten); - AdjustTotalSize(ullCITotalSize, ullCIProcessedSize + ullWritten); - } + // in case we read past the original eof, try to get new file size from filesystem + AdjustProcessedSize(fsWritten, pData->spSrcFile, fileSrc); - // new stats - AdjustProcessedSize(0, ullWritten); - // stop iterating through file bStopProcessing = pBuffer->IsLastPart(); pBuffer->RequeueAsEmpty(); @@ -502,23 +485,55 @@ } // this is the end of copying of src file - in case it is smaller than expected fix the stats so that difference is accounted for - unsigned long long ullCITotalSize = m_tSubTaskStats.GetCurrentItemTotalSize(); - unsigned long long ullCIProcessedSize = m_tSubTaskStats.GetCurrentItemProcessedSize(); - if(ullCIProcessedSize < ullCITotalSize) - { - pData->spSrcFile->SetLength64(ullCIProcessedSize); + AdjustFinalSize(pData->spSrcFile, fileSrc); - AdjustTotalSize(ullCITotalSize, ullCIProcessedSize); - } - pData->bProcessed = true; - m_tSubTaskStats.SetCurrentItemProcessedSize(0); + m_tSubTaskStats.ResetCurrentItemProcessedSize(); pData->dbBuffer.WaitForMissingBuffers(rThreadController.GetKillThreadHandle()); return TSubTaskBase::eSubResult_Continue; } + void TSubTaskCopyMove::AdjustProcessedSize(file_size_t fsWritten, const TFileInfoPtr& spSrcFileInfo, const IFilesystemFilePtr& spSrcFile) + { + // in case we read past the original eof, try to get new file size from filesystem + if (m_tSubTaskStats.WillAdjustProcessedSizeExceedTotalSize(0, fsWritten)) + { + file_size_t fsNewSize = spSrcFile->GetFileSize(); + if (fsNewSize == spSrcFileInfo->GetLength64()) + THROW_CORE_EXCEPTION_MSG(eErr_InternalProblem, L"Read more data from file than it really contained. Possible destination file corruption."); + + m_tSubTaskStats.AdjustTotalSize(spSrcFileInfo->GetLength64(), fsNewSize); + spSrcFileInfo->SetLength64(m_tSubTaskStats.GetCurrentItemTotalSize()); + } + + m_tSubTaskStats.AdjustProcessedSize(0, fsWritten); + } + + void TSubTaskCopyMove::AdjustFinalSize(const TFileInfoPtr& spSrcFileInfo, const IFilesystemFilePtr& spSrcFile) + { + unsigned long long ullCITotalSize = m_tSubTaskStats.GetCurrentItemTotalSize(); + unsigned long long ullCIProcessedSize = m_tSubTaskStats.GetCurrentItemProcessedSize(); + if (ullCIProcessedSize < ullCITotalSize) + { + file_size_t fsNewSize = spSrcFile->GetFileSize(); + if (fsNewSize == spSrcFileInfo->GetLength64()) + THROW_CORE_EXCEPTION_MSG(eErr_InternalProblem, L"Read less data from file than it really contained. Possible destination file corruption."); + + if (fsNewSize != ullCIProcessedSize) + THROW_CORE_EXCEPTION_MSG(eErr_InternalProblem, L"Updated file size still does not match the count of data read. Possible destination file corruption."); + + m_tSubTaskStats.AdjustTotalSize(ullCITotalSize, fsNewSize); + spSrcFileInfo->SetLength64(fsNewSize); + } + } + + void TSubTaskCopyMove::AdjustProcessedSizeForSkip(const TFileInfoPtr& spSrcFileInfo) + { + m_tSubTaskStats.AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), spSrcFileInfo->GetLength64()); + } + TSubTaskCopyMove::ESubOperationResult TSubTaskCopyMove::OpenSrcAndDstFilesFB(const IFeedbackHandlerPtr& spFeedbackHandler, CUSTOM_COPY_PARAMS* pData, const IFilesystemFilePtr& spFileSrc, const IFilesystemFilePtr& spFileDst, bool& bSkip) { @@ -536,7 +551,7 @@ else if(!spFileSrc->IsOpen()) { // invalid handle = operation skipped by user - AdjustProcessedSize(ullProcessedSize, pData->spSrcFile->GetLength64()); + AdjustProcessedSizeForSkip(pData->spSrcFile); pData->bProcessed = false; bSkip = true; @@ -552,8 +567,7 @@ file_size_t fsOldSize = pData->spSrcFile->GetLength64(); if(fsNewSize != fsOldSize) { - AdjustTotalSize(fsOldSize, fsNewSize); - + m_tSubTaskStats.AdjustTotalSize(fsOldSize, fsNewSize); pData->spSrcFile->SetLength64(fsNewSize); } @@ -594,7 +608,7 @@ return eResult; else if (!spFileDst->IsOpen()) { - AdjustProcessedSize(ullProcessedSize, pData->spSrcFile->GetLength64()); + AdjustProcessedSizeForSkip(pData->spSrcFile); pData->bProcessed = false; bSkip = true; @@ -614,7 +628,7 @@ return eResult; else if(!spFileDst->IsOpen()) { - AdjustProcessedSize(ullProcessedSize, pData->spSrcFile->GetLength64()); + AdjustProcessedSizeForSkip(pData->spSrcFile); pData->bProcessed = false; bSkip = true; @@ -625,7 +639,7 @@ if(pData->bOnlyCreate) { // we don't copy contents, but need to increase processed size - AdjustProcessedSize(ullProcessedSize, pData->spSrcFile->GetLength64()); + AdjustProcessedSizeForSkip(pData->spSrcFile); return TSubTaskBase::eSubResult_Continue; } @@ -640,7 +654,7 @@ THROW_CORE_EXCEPTION(eErr_InternalProblem); // adjust the stats for the difference between what was already processed and what will now be considered processed - AdjustProcessedSize(ullProcessedSize, fsMoveTo); + m_tSubTaskStats.AdjustProcessedSize(ullProcessedSize, fsMoveTo); // if the destination file already exists - truncate it to the current file position if(!bDstFileFreshlyCreated) @@ -663,36 +677,6 @@ return eResult; } - void TSubTaskCopyMove::AdjustTotalSize(file_size_t fsIncludedSize, file_size_t fsNewSize) - { - if (fsNewSize > fsIncludedSize) - { - m_tSubTaskStats.IncreaseTotalSize(fsNewSize - fsIncludedSize); - m_tSubTaskStats.IncreaseCurrentItemTotalSize(fsNewSize - fsIncludedSize); - } - else if(fsNewSize < fsIncludedSize) - { - m_tSubTaskStats.DecreaseTotalSize(fsIncludedSize - fsNewSize); - m_tSubTaskStats.DecreaseCurrentItemTotalSize(fsIncludedSize - fsNewSize); - } - } - - void TSubTaskCopyMove::AdjustProcessedSize(file_size_t fsIncludedProcessedSize, file_size_t fsNewProcessedSize) - { - if(fsNewProcessedSize > fsIncludedProcessedSize) - { - file_size_t fsDiff = fsNewProcessedSize - fsIncludedProcessedSize; - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(fsDiff); - m_tSubTaskStats.IncreaseProcessedSize(fsDiff); - } - else if (fsNewProcessedSize < fsIncludedProcessedSize) - { - file_size_t fsDiff = fsIncludedProcessedSize - fsNewProcessedSize; - m_tSubTaskStats.DecreaseCurrentItemProcessedSize(fsDiff); - m_tSubTaskStats.DecreaseProcessedSize(fsDiff); - } - } - bool TSubTaskCopyMove::AdjustBufferIfNeeded(TOverlappedDataBufferQueue& rBuffer, TBufferSizes& rBufferSizes, bool bForce) { const TConfig& rConfig = GetContext().GetConfig(); Index: src/libchcore/TSubTaskCopyMove.h =================================================================== diff -u -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc -rdb818324a61602e118506ca8b3973e44d997c8de --- src/libchcore/TSubTaskCopyMove.h (.../TSubTaskCopyMove.h) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) +++ src/libchcore/TSubTaskCopyMove.h (.../TSubTaskCopyMove.h) (revision db818324a61602e118506ca8b3973e44d997c8de) @@ -67,8 +67,6 @@ ESubOperationResult OpenSrcAndDstFilesFB(const IFeedbackHandlerPtr& spFeedbackHandler, CUSTOM_COPY_PARAMS* pData, const IFilesystemFilePtr& spFileSrc, const IFilesystemFilePtr& spFileDst, bool& bSkip); - void AdjustTotalSize(unsigned long long ullNewSize, unsigned long long ullOldSize); - ESubOperationResult OpenSourceFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileSrc); ESubOperationResult OpenDestinationFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileDst, const TFileInfoPtr& spSrcFileInfo, unsigned long long& ullSeekTo, bool& bFreshlyCreated); @@ -94,7 +92,9 @@ ESubOperationResult CheckForFreeSpaceFB(const IFeedbackHandlerPtr& spFeedbackHandler); - void AdjustProcessedSize(file_size_t fsIncludedProcessedSize, file_size_t fsNewProcessedSize); + void AdjustProcessedSize(file_size_t fsWritten, const TFileInfoPtr& spSrcFileInfo, const IFilesystemFilePtr& spSrcFile); + void AdjustFinalSize(const TFileInfoPtr& spSrcFileInfo, const IFilesystemFilePtr& spSrcFile); + void AdjustProcessedSizeForSkip(const TFileInfoPtr& spSrcFileInfo); private: #pragma warning(push) Index: src/libchcore/TSubTaskStatsInfo.cpp =================================================================== diff -u -re96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3 -rdb818324a61602e118506ca8b3973e44d997c8de --- src/libchcore/TSubTaskStatsInfo.cpp (.../TSubTaskStatsInfo.cpp) (revision e96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3) +++ src/libchcore/TSubTaskStatsInfo.cpp (.../TSubTaskStatsInfo.cpp) (revision db818324a61602e118506ca8b3973e44d997c8de) @@ -200,43 +200,86 @@ } // current item - void TSubTaskStatsInfo::IncreaseCurrentItemProcessedSize(unsigned long long ullIncreaseBy) + void TSubTaskStatsInfo::ResetCurrentItemProcessedSize() { boost::unique_lock lock(m_lock); - m_ullCurrentItemProcessedSize.Modify() += ullIncreaseBy; - - _ASSERTE(m_ullCurrentItemProcessedSize <= m_ullCurrentItemTotalSize); - if (m_ullCurrentItemProcessedSize > m_ullCurrentItemTotalSize) - THROW_CORE_EXCEPTION(eErr_InternalProblem); + m_ullCurrentItemProcessedSize = 0; } - void TSubTaskStatsInfo::DecreaseCurrentItemProcessedSize(unsigned long long ullDecreaseBy) + void TSubTaskStatsInfo::SetCurrentItemSizes(unsigned long long ullProcessedSize, unsigned long long ullTotalSize) { boost::unique_lock lock(m_lock); - m_ullCurrentItemProcessedSize.Modify() -= ullDecreaseBy; + m_ullCurrentItemProcessedSize = ullProcessedSize; + m_ullCurrentItemTotalSize = ullTotalSize; + _ASSERTE(m_ullCurrentItemProcessedSize <= m_ullCurrentItemTotalSize); if (m_ullCurrentItemProcessedSize > m_ullCurrentItemTotalSize) THROW_CORE_EXCEPTION(eErr_InternalProblem); } - void TSubTaskStatsInfo::SetCurrentItemProcessedSize(unsigned long long ullProcessedSize) + bool TSubTaskStatsInfo::WillAdjustProcessedSizeExceedTotalSize(file_size_t fsIncludedProcessedSize, file_size_t fsNewProcessedSize) { + if (fsNewProcessedSize <= fsIncludedProcessedSize) + return false; + + boost::shared_lock lock(m_lock); + + file_size_t fsDiff = fsNewProcessedSize - fsIncludedProcessedSize; + if (m_ullCurrentItemProcessedSize.Get() + fsDiff > m_ullCurrentItemTotalSize.Get()) + return true; + + return false; + } + + void TSubTaskStatsInfo::AdjustProcessedSize(file_size_t fsIncludedProcessedSize, file_size_t fsNewProcessedSize) + { + if (fsNewProcessedSize == fsIncludedProcessedSize) + return; + + file_size_t fsDiff = 0; + boost::unique_lock lock(m_lock); + if (fsNewProcessedSize > fsIncludedProcessedSize) + { + fsDiff = fsNewProcessedSize - fsIncludedProcessedSize; - m_ullCurrentItemProcessedSize = ullProcessedSize; - _ASSERTE(m_ullCurrentItemProcessedSize <= m_ullCurrentItemTotalSize); - if (m_ullCurrentItemProcessedSize > m_ullCurrentItemTotalSize) - THROW_CORE_EXCEPTION(eErr_InternalProblem); + m_ullCurrentItemProcessedSize.Modify() += fsDiff; + m_ullProcessedSize.Modify() += fsDiff; + + m_tSizeSpeed.Modify().AddSample(fsDiff, m_tTimer.Modify().Tick()); + } + else + { + fsDiff = fsIncludedProcessedSize - fsNewProcessedSize; + + m_ullCurrentItemProcessedSize.Modify() -= fsDiff; + m_ullProcessedSize.Modify() -= fsDiff; + + m_tSizeSpeed.Modify().AddSample(0, m_tTimer.Modify().Tick()); + } + + VerifyProcessedVsTotal(); } - void TSubTaskStatsInfo::SetCurrentItemTotalSize(unsigned long long ullTotalSize) + void TSubTaskStatsInfo::AdjustTotalSize(file_size_t fsIncludedSize, file_size_t fsNewSize) { + if (fsIncludedSize == fsNewSize) + return; + boost::unique_lock lock(m_lock); - m_ullCurrentItemTotalSize = ullTotalSize; - _ASSERTE(m_ullCurrentItemProcessedSize <= m_ullCurrentItemTotalSize); - if (m_ullCurrentItemProcessedSize > m_ullCurrentItemTotalSize) - THROW_CORE_EXCEPTION(eErr_InternalProblem); + if (fsNewSize > fsIncludedSize) + { + m_ullTotalSize.Modify() += fsNewSize - fsIncludedSize; + m_ullCurrentItemTotalSize.Modify() += fsNewSize - fsIncludedSize; + } + else + { + m_ullTotalSize.Modify() -= fsIncludedSize - fsNewSize; + m_ullCurrentItemTotalSize.Modify() -= fsIncludedSize - fsNewSize; + } + + VerifyProcessedVsTotal(); } // buffer index @@ -467,15 +510,14 @@ m_ullTotalSize = m_ullTotalSize - ullDecreaseBy; } - void TSubTaskStatsInfo::IncreaseCurrentItemTotalSize(unsigned long long ullIncreaseBy) + void TSubTaskStatsInfo::VerifyProcessedVsTotal() { - boost::unique_lock lock(m_lock); - m_ullCurrentItemTotalSize = m_ullCurrentItemTotalSize + ullIncreaseBy; - } + _ASSERTE(m_ullCurrentItemProcessedSize <= m_ullCurrentItemTotalSize); + _ASSERTE(m_ullProcessedSize <= m_ullTotalSize); - void TSubTaskStatsInfo::DecreaseCurrentItemTotalSize(unsigned long long ullDecreaseBy) - { - boost::unique_lock lock(m_lock); - m_ullCurrentItemTotalSize = m_ullCurrentItemTotalSize - ullDecreaseBy; + if (m_ullCurrentItemProcessedSize > m_ullCurrentItemTotalSize) + THROW_CORE_EXCEPTION(eErr_InternalProblem); + if (m_ullProcessedSize > m_ullTotalSize) + THROW_CORE_EXCEPTION(eErr_InternalProblem); } } Index: src/libchcore/TSubTaskStatsInfo.h =================================================================== diff -u -re96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3 -rdb818324a61602e118506ca8b3973e44d997c8de --- src/libchcore/TSubTaskStatsInfo.h (.../TSubTaskStatsInfo.h) (revision e96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3) +++ src/libchcore/TSubTaskStatsInfo.h (.../TSubTaskStatsInfo.h) (revision db818324a61602e118506ca8b3973e44d997c8de) @@ -74,17 +74,16 @@ void SetTotalSize(unsigned long long ullTotalSize); // current item - void IncreaseCurrentItemProcessedSize(unsigned long long ullIncreaseBy); - void DecreaseCurrentItemProcessedSize(unsigned long long ullDecreaseBy); - void SetCurrentItemProcessedSize(unsigned long long ullProcessedSize); - - void IncreaseCurrentItemTotalSize(unsigned long long ullIncreaseBy); - void DecreaseCurrentItemTotalSize(unsigned long long ullDecreaseBy); - void SetCurrentItemTotalSize(unsigned long long ullTotalSize); - + void SetCurrentItemSizes(unsigned long long ullProcessedSize, unsigned long long ullTotalSize); + void ResetCurrentItemProcessedSize(); unsigned long long GetCurrentItemProcessedSize() const; unsigned long long GetCurrentItemTotalSize() const; + // current item processed and overall processed + bool WillAdjustProcessedSizeExceedTotalSize(file_size_t fsIncludedProcessedSize, file_size_t fsNewProcessedSize); + void AdjustProcessedSize(file_size_t fsIncludedProcessedSize, file_size_t fsNewProcessedSize); + void AdjustTotalSize(file_size_t fsIncludedSize, file_size_t fsNewSize); + bool CanCurrentItemSilentResume() const; void SetCurrentItemSilentResume(bool bEnableSilentResume); @@ -119,6 +118,8 @@ void UpdateTime(boost::upgrade_lock& lock) const; + void VerifyProcessedVsTotal(); + private: enum EModifications {