Index: src/libchcore/IFilesystem.h =================================================================== diff -u -N -re96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3 -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc --- src/libchcore/IFilesystem.h (.../IFilesystem.h) (revision e96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3) +++ src/libchcore/IFilesystem.h (.../IFilesystem.h) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) @@ -57,7 +57,7 @@ virtual bool FastMove(const TSmartPath& pathSource, const TSmartPath& pathDestination) = 0; virtual IFilesystemFindPtr CreateFinderObject(const TSmartPath& pathDir, const TSmartPath& pathMask) = 0; - virtual IFilesystemFilePtr CreateFileObject(const TSmartPath& pathFile) = 0; + virtual IFilesystemFilePtr CreateFileObject(const TSmartPath& pathFile, bool bNoBuffering) = 0; virtual EPathsRelation GetPathsRelation(const TSmartPath& pathFirst, const TSmartPath& pathSecond) = 0; Index: src/libchcore/IFilesystemFile.h =================================================================== diff -u -N -r27c262eb9cae55720e10f4886af6b5a82cb94fe9 -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc --- src/libchcore/IFilesystemFile.h (.../IFilesystemFile.h) (revision 27c262eb9cae55720e10f4886af6b5a82cb94fe9) +++ src/libchcore/IFilesystemFile.h (.../IFilesystemFile.h) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) @@ -22,6 +22,7 @@ #include "libchcore.h" #include "TPath.h" #include "TOverlappedDataBuffer.h" +#include "CommonDataTypes.h" namespace chcore { @@ -33,12 +34,14 @@ public: virtual ~IFilesystemFile(); - virtual bool OpenExistingForReading(bool bNoBuffering) = 0; - virtual bool CreateNewForWriting(bool bNoBuffering) = 0; - virtual bool OpenExistingForWriting(bool bNoBuffering) = 0; + virtual bool OpenExistingForReading() = 0; + virtual bool CreateNewForWriting() = 0; + virtual bool OpenExistingForWriting() = 0; - virtual bool Truncate(long long ullNewSize) = 0; + virtual file_size_t GetSeekPositionForResume(file_size_t fsLastAvailablePosition) = 0; + virtual bool Truncate(file_size_t fsNewSize) = 0; + virtual bool ReadFile(TOverlappedDataBuffer& rBuffer) = 0; virtual bool WriteFile(TOverlappedDataBuffer& rBuffer) = 0; virtual bool FinalizeFile(TOverlappedDataBuffer& rBuffer) = 0; Index: src/libchcore/TFakeFilesystem.cpp =================================================================== diff -u -N -r27c262eb9cae55720e10f4886af6b5a82cb94fe9 -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc --- src/libchcore/TFakeFilesystem.cpp (.../TFakeFilesystem.cpp) (revision 27c262eb9cae55720e10f4886af6b5a82cb94fe9) +++ src/libchcore/TFakeFilesystem.cpp (.../TFakeFilesystem.cpp) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) @@ -126,9 +126,9 @@ return eRelation; } - IFilesystemFilePtr TFakeFilesystem::CreateFileObject(const TSmartPath& spFilename) + IFilesystemFilePtr TFakeFilesystem::CreateFileObject(const TSmartPath& spFilename, bool bNoBuffering) { - IFilesystemFilePtr spFile = std::make_shared(spFilename, this); + IFilesystemFilePtr spFile = std::make_shared(spFilename, bNoBuffering, this); return spFile; } Index: src/libchcore/TFakeFilesystem.h =================================================================== diff -u -N -r27c262eb9cae55720e10f4886af6b5a82cb94fe9 -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc --- src/libchcore/TFakeFilesystem.h (.../TFakeFilesystem.h) (revision 27c262eb9cae55720e10f4886af6b5a82cb94fe9) +++ src/libchcore/TFakeFilesystem.h (.../TFakeFilesystem.h) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) @@ -42,7 +42,7 @@ virtual bool GetFileInfo(const TSmartPath& pathFile, TFileInfoPtr& rFileInfo, const TBasePathDataPtr& spBasePathData = TBasePathDataPtr()) override; virtual bool FastMove(const TSmartPath& pathSource, const TSmartPath& pathDestination) override; virtual IFilesystemFindPtr CreateFinderObject(const TSmartPath& pathDir, const TSmartPath& pathMask) override; - virtual IFilesystemFilePtr CreateFileObject(const TSmartPath& spFilename) override; + virtual IFilesystemFilePtr CreateFileObject(const TSmartPath& spFilename, bool bNoBuffering) override; virtual EPathsRelation GetPathsRelation(const TSmartPath& pathFirst, const TSmartPath& pathSecond) override; virtual bool GetDynamicFreeSpace(const TSmartPath& path, unsigned long long& rullFree) override; Index: src/libchcore/TFakeFilesystemFile.cpp =================================================================== diff -u -N -r27c262eb9cae55720e10f4886af6b5a82cb94fe9 -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc --- src/libchcore/TFakeFilesystemFile.cpp (.../TFakeFilesystemFile.cpp) (revision 27c262eb9cae55720e10f4886af6b5a82cb94fe9) +++ src/libchcore/TFakeFilesystemFile.cpp (.../TFakeFilesystemFile.cpp) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) @@ -22,6 +22,7 @@ #include "ErrorCodes.h" #include "TFakeFilesystem.h" #include +#include "RoundingFunctions.h" namespace { @@ -57,11 +58,11 @@ delete pApcInfo; } - TFakeFilesystemFile::TFakeFilesystemFile(const TSmartPath& pathFile, TFakeFilesystem* pFilesystem) : + TFakeFilesystemFile::TFakeFilesystemFile(const TSmartPath& pathFile, bool bNoBuffering, TFakeFilesystem* pFilesystem) : m_pathFile(pathFile), m_pFilesystem(pFilesystem), m_bIsOpen(false), - m_bNoBuffering(false), + m_bNoBuffering(bNoBuffering), m_bModeReading(true) { if (!pFilesystem || pathFile.IsEmpty()) @@ -201,8 +202,14 @@ return true; } - bool TFakeFilesystemFile::Truncate(long long llNewSize) + file_size_t TFakeFilesystemFile::GetSeekPositionForResume(file_size_t fsLastAvailablePosition) { + file_size_t fsMove = (m_bNoBuffering ? RoundDown(fsLastAvailablePosition, MaxSectorSize) : fsLastAvailablePosition); + return fsMove; + } + + bool TFakeFilesystemFile::Truncate(file_size_t fsNewSize) + { if (!IsOpen()) THROW_CORE_EXCEPTION(eErr_InternalProblem); @@ -211,11 +218,11 @@ if (!spFileDesc) return false; - spFileDesc->GetFileInfo().SetLength64(llNewSize); + spFileDesc->GetFileInfo().SetLength64(fsNewSize); return true; } - bool TFakeFilesystemFile::OpenExistingForWriting(bool bNoBuffering) + bool TFakeFilesystemFile::OpenExistingForWriting() { TFakeFileDescriptionPtr spFileDesc = m_pFilesystem->FindFileByLocation(m_pathFile); if (!spFileDesc) @@ -224,13 +231,12 @@ Close(); m_bIsOpen = true; - m_bNoBuffering = bNoBuffering; m_bModeReading = false; return true; } - bool TFakeFilesystemFile::CreateNewForWriting(bool bNoBuffering) + bool TFakeFilesystemFile::CreateNewForWriting() { TFakeFileDescriptionPtr spFileDesc = m_pFilesystem->FindFileByLocation(m_pathFile); if(!spFileDesc) @@ -251,13 +257,12 @@ Close(); m_bIsOpen = true; - m_bNoBuffering = bNoBuffering; m_bModeReading = false; return true; } - bool TFakeFilesystemFile::OpenExistingForReading(bool bNoBuffering) + bool TFakeFilesystemFile::OpenExistingForReading() { TFakeFileDescriptionPtr spFileDesc = m_pFilesystem->FindFileByLocation(m_pathFile); if (!spFileDesc) @@ -266,7 +271,6 @@ Close(); m_bIsOpen = true; - m_bNoBuffering = bNoBuffering; m_bModeReading = true; return true; Index: src/libchcore/TFakeFilesystemFile.h =================================================================== diff -u -N -r27c262eb9cae55720e10f4886af6b5a82cb94fe9 -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc --- src/libchcore/TFakeFilesystemFile.h (.../TFakeFilesystemFile.h) (revision 27c262eb9cae55720e10f4886af6b5a82cb94fe9) +++ src/libchcore/TFakeFilesystemFile.h (.../TFakeFilesystemFile.h) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) @@ -30,16 +30,16 @@ class LIBCHCORE_API TFakeFilesystemFile : public IFilesystemFile { public: - TFakeFilesystemFile(const TSmartPath& pathFile, TFakeFilesystem* pFilesystem); + TFakeFilesystemFile(const TSmartPath& pathFile, bool bNoBuffering, TFakeFilesystem* pFilesystem); ~TFakeFilesystemFile(); - virtual bool OpenExistingForReading(bool bNoBuffering) override; - virtual bool CreateNewForWriting(bool bNoBuffering) override; - virtual bool OpenExistingForWriting(bool bNoBuffering) override; - virtual bool Truncate(long long ullNewSize) override; + virtual bool OpenExistingForReading() override; + virtual bool CreateNewForWriting() override; + virtual bool OpenExistingForWriting() override; + virtual bool Truncate(file_size_t fsNewSize) override; virtual bool ReadFile(TOverlappedDataBuffer& rBuffer) override; - void GenerateBufferContent(TOverlappedDataBuffer &rBuffer); + file_size_t GetSeekPositionForResume(file_size_t fsLastAvailablePosition) override; virtual bool WriteFile(TOverlappedDataBuffer& rBuffer) override; virtual bool FinalizeFile(TOverlappedDataBuffer& rBuffer) override; @@ -50,6 +50,9 @@ virtual TSmartPath GetFilePath() const override; private: + void GenerateBufferContent(TOverlappedDataBuffer &rBuffer); + + private: #pragma warning(push) #pragma warning(disable: 4251) TSmartPath m_pathFile; Index: src/libchcore/TLocalFilesystem.cpp =================================================================== diff -u -N -re96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3 -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc --- src/libchcore/TLocalFilesystem.cpp (.../TLocalFilesystem.cpp) (revision e96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3) +++ src/libchcore/TLocalFilesystem.cpp (.../TLocalFilesystem.cpp) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) @@ -203,9 +203,9 @@ return std::shared_ptr(new TLocalFilesystemFind(pathDir, pathMask)); } - IFilesystemFilePtr TLocalFilesystem::CreateFileObject(const TSmartPath& pathFile) + IFilesystemFilePtr TLocalFilesystem::CreateFileObject(const TSmartPath& pathFile, bool bNoBuffering) { - return std::shared_ptr(new TLocalFilesystemFile(pathFile)); + return std::shared_ptr(new TLocalFilesystemFile(pathFile, bNoBuffering)); } TSmartPath TLocalFilesystem::PrependPathExtensionIfNeeded(const TSmartPath& pathInput) Index: src/libchcore/TLocalFilesystem.h =================================================================== diff -u -N -re96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3 -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc --- src/libchcore/TLocalFilesystem.h (.../TLocalFilesystem.h) (revision e96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3) +++ src/libchcore/TLocalFilesystem.h (.../TLocalFilesystem.h) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) @@ -57,7 +57,7 @@ virtual bool FastMove(const TSmartPath& pathSource, const TSmartPath& pathDestination) override; virtual IFilesystemFindPtr CreateFinderObject(const TSmartPath& pathDir, const TSmartPath& pathMask) override; - virtual IFilesystemFilePtr CreateFileObject(const TSmartPath& pathFile) override; + virtual IFilesystemFilePtr CreateFileObject(const TSmartPath& pathFile, bool bNoBuffering) override; virtual EPathsRelation GetPathsRelation(const TSmartPath& pathFirst, const TSmartPath& pathSecond) override; Index: src/libchcore/TLocalFilesystemFile.cpp =================================================================== diff -u -N -r27c262eb9cae55720e10f4886af6b5a82cb94fe9 -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc --- src/libchcore/TLocalFilesystemFile.cpp (.../TLocalFilesystemFile.cpp) (revision 27c262eb9cae55720e10f4886af6b5a82cb94fe9) +++ src/libchcore/TLocalFilesystemFile.cpp (.../TLocalFilesystemFile.cpp) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) @@ -31,10 +31,10 @@ // compile-time check - ensure the buffer granularity used for transfers are bigger than expected sector size static_assert(TLocalFilesystemFile::MaxSectorSize <= TBufferSizes::BufferGranularity, "Buffer granularity must be equal to or bigger than the max sector size"); - TLocalFilesystemFile::TLocalFilesystemFile(const TSmartPath& pathFile) : + TLocalFilesystemFile::TLocalFilesystemFile(const TSmartPath& pathFile, bool bNoBuffering) : m_pathFile(TLocalFilesystem::PrependPathExtensionIfNeeded(pathFile)), m_hFile(INVALID_HANDLE_VALUE), - m_bNoBuffering(false) + m_bNoBuffering(bNoBuffering) { if (pathFile.IsEmpty()) THROW_CORE_EXCEPTION(eErr_InvalidArgument); @@ -45,35 +45,38 @@ Close(); } - DWORD TLocalFilesystemFile::GetFlagsAndAttributes(bool bNoBuffering) const + constexpr DWORD TLocalFilesystemFile::GetFlagsAndAttributes(bool bNoBuffering) const { - return FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED | FILE_FLAG_SEQUENTIAL_SCAN | (bNoBuffering ? FILE_FLAG_NO_BUFFERING /*| FILE_FLAG_WRITE_THROUGH*/ : 0); + return FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED | FILE_FLAG_SEQUENTIAL_SCAN | (bNoBuffering ? FILE_FLAG_NO_BUFFERING : 0); } - bool TLocalFilesystemFile::OpenExistingForReading(bool bNoBuffering) + bool TLocalFilesystemFile::OpenExistingForReading() { Close(); - m_hFile = ::CreateFile(m_pathFile.ToString(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, GetFlagsAndAttributes(bNoBuffering), NULL); + m_hFile = ::CreateFile(m_pathFile.ToString(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, GetFlagsAndAttributes(m_bNoBuffering), NULL); if (m_hFile == INVALID_HANDLE_VALUE) return false; - m_bNoBuffering = bNoBuffering; return true; } - bool TLocalFilesystemFile::CreateNewForWriting(bool bNoBuffering) + bool TLocalFilesystemFile::CreateNewForWriting() { Close(); - m_hFile = ::CreateFile(m_pathFile.ToString(), GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_NEW, GetFlagsAndAttributes(bNoBuffering), NULL); + m_hFile = ::CreateFile(m_pathFile.ToString(), GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_NEW, GetFlagsAndAttributes(m_bNoBuffering), NULL); if (m_hFile == INVALID_HANDLE_VALUE) return false; - m_bNoBuffering = bNoBuffering; return true; } + bool TLocalFilesystemFile::OpenExistingForWriting() + { + return OpenExistingForWriting(m_bNoBuffering); + } + bool TLocalFilesystemFile::OpenExistingForWriting(bool bNoBuffering) { Close(); @@ -82,24 +85,52 @@ if (m_hFile == INVALID_HANDLE_VALUE) return false; - m_bNoBuffering = bNoBuffering; return true; } - bool TLocalFilesystemFile::Truncate(long long llNewSize) + file_size_t TLocalFilesystemFile::GetSeekPositionForResume(file_size_t fsLastAvailablePosition) { + file_size_t fsMove = (m_bNoBuffering ? RoundDown(fsLastAvailablePosition, MaxSectorSize) : fsLastAvailablePosition); + return fsMove; + } + + bool TLocalFilesystemFile::Truncate(file_size_t fsNewSize) + { if (!IsOpen()) return false; + // when no-buffering is used, there are cases where we'd need to switch to buffered ops + // to adjust file size + bool bFileSettingsChanged = false; + if (m_bNoBuffering) + { + file_size_t fsNewAlignedSize = RoundUp(fsNewSize, MaxSectorSize); + if (fsNewAlignedSize != fsNewSize) + { + Close(); + + if (!OpenExistingForWriting(false)) + return false; + + bFileSettingsChanged = true; + } + } + LARGE_INTEGER li = { 0, 0 }; LARGE_INTEGER liNew = { 0, 0 }; - li.QuadPart = llNewSize; + li.QuadPart = fsNewSize; if (!SetFilePointerEx(m_hFile, li, &liNew, FILE_BEGIN)) return false; - return ::SetEndOfFile(m_hFile) != FALSE; + bool bResult = ::SetEndOfFile(m_hFile) != FALSE; + + // close the file that was open in inappropriate mode + if(bFileSettingsChanged) + Close(); + + return bResult; } bool TLocalFilesystemFile::ReadFile(TOverlappedDataBuffer& rBuffer) @@ -169,14 +200,11 @@ if (dwToWrite != dwReallyWritten) { - unsigned long long ullNewFileSize = rBuffer.GetFilePosition() + dwToWrite; // new size + file_size_t fsNewFileSize = rBuffer.GetFilePosition() + dwToWrite; // new size - if (!OpenExistingForWriting(false)) - return false; - //seek - ATLTRACE(_T("Truncating file to %I64u bytes\n"), ullNewFileSize); - if (!Truncate(ullNewFileSize)) + ATLTRACE(_T("Truncating file to %I64u bytes\n"), fsNewFileSize); + if (!Truncate(fsNewFileSize)) return false; } } Index: src/libchcore/TLocalFilesystemFile.h =================================================================== diff -u -N -r27c262eb9cae55720e10f4886af6b5a82cb94fe9 -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc --- src/libchcore/TLocalFilesystemFile.h (.../TLocalFilesystemFile.h) (revision 27c262eb9cae55720e10f4886af6b5a82cb94fe9) +++ src/libchcore/TLocalFilesystemFile.h (.../TLocalFilesystemFile.h) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) @@ -31,12 +31,14 @@ public: virtual ~TLocalFilesystemFile(); - virtual bool OpenExistingForReading(bool bNoBuffering) override; - virtual bool CreateNewForWriting(bool bNoBuffering) override; - virtual bool OpenExistingForWriting(bool bNoBuffering) override; + virtual bool OpenExistingForReading() override; + virtual bool CreateNewForWriting() override; + virtual bool OpenExistingForWriting() override; - virtual bool Truncate(long long llNewSize) override; + virtual file_size_t GetSeekPositionForResume(file_size_t fsLastAvailablePosition); + virtual bool Truncate(file_size_t fsNewSize) override; + virtual bool ReadFile(TOverlappedDataBuffer& rBuffer) override; virtual bool WriteFile(TOverlappedDataBuffer& rBuffer) override; virtual bool FinalizeFile(TOverlappedDataBuffer& rBuffer) override; @@ -48,9 +50,12 @@ virtual void Close() override; private: - TLocalFilesystemFile(const TSmartPath& pathFile); - DWORD GetFlagsAndAttributes(bool bNoBuffering) const; + TLocalFilesystemFile(const TSmartPath& pathFile, bool bNoBuffering); + constexpr DWORD GetFlagsAndAttributes(bool bNoBuffering) const; + + bool OpenExistingForWriting(bool bNoBuffering); + private: TSmartPath m_pathFile; HANDLE m_hFile; Index: src/libchcore/TSubTaskCopyMove.cpp =================================================================== diff -u -N -re96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3 -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc --- src/libchcore/TSubTaskCopyMove.cpp (.../TSubTaskCopyMove.cpp) (revision e96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3) +++ src/libchcore/TSubTaskCopyMove.cpp (.../TSubTaskCopyMove.cpp) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) @@ -228,8 +228,9 @@ // if this subtask is not started yet, try to get the most fresh information for processing if(!spStats->IsRunning() && spStats->GetTotalCount() == 0 && spStats->GetTotalSize() == 0) { - spStats->SetTotalCount(GetContext().GetFilesCache().GetSize()); - spStats->SetTotalSize(GetContext().GetFilesCache().CalculateTotalSize()); + const auto& rCache = GetContext().GetFilesCache(); + spStats->SetTotalCount(rCache.GetSize()); + spStats->SetTotalSize(rCache.CalculateTotalSize()); } } @@ -272,9 +273,6 @@ const TConfig& rConfig = GetContext().GetConfig(); IFilesystemPtr spFilesystem = GetContext().GetLocalFilesystem(); - IFilesystemFilePtr fileSrc = spFilesystem->CreateFileObject(pData->spSrcFile->GetFullFilePath()); - IFilesystemFilePtr fileDst = spFilesystem->CreateFileObject(pData->pathDstFile); - TString strFormat; TSubTaskBase::ESubOperationResult eResult = TSubTaskBase::eSubResult_Continue; @@ -285,8 +283,11 @@ bool bNoBuffer = (GetTaskPropValue(rConfig) && pData->spSrcFile->GetLength64() >= GetTaskPropValue(rConfig)); + IFilesystemFilePtr fileSrc = spFilesystem->CreateFileObject(pData->spSrcFile->GetFullFilePath(), bNoBuffer); + IFilesystemFilePtr fileDst = spFilesystem->CreateFileObject(pData->pathDstFile, bNoBuffer); + bool bSkip = false; - eResult = OpenSrcAndDstFilesFB(spFeedbackHandler, pData, fileSrc, fileDst, bNoBuffer, bSkip); + eResult = OpenSrcAndDstFilesFB(spFeedbackHandler, pData, fileSrc, fileDst, bSkip); if(eResult != TSubTaskBase::eSubResult_Continue) return eResult; else if(bSkip) @@ -352,8 +353,7 @@ else if(bSkip) { // new stats - m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -380,8 +380,7 @@ else if(bSkip) { // new stats - m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -392,8 +391,7 @@ else if(bSkip) { // new stats - m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -409,8 +407,7 @@ else if(bSkip) { // new stats - m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -437,8 +434,7 @@ else if(bSkip) { // new stats - m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -449,8 +445,7 @@ else if(bSkip) { // new stats - m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -464,8 +459,7 @@ else if (bSkip) { // new stats - m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + AdjustProcessedSize(m_tSubTaskStats.GetCurrentItemProcessedSize(), pData->spSrcFile->GetLength64()); pData->bProcessed = false; return TSubTaskBase::eSubResult_Continue; @@ -479,13 +473,11 @@ { // total size changed pData->spSrcFile->SetLength64(ullCIProcessedSize + ullWritten); - m_tSubTaskStats.IncreaseCurrentItemTotalSize(ullCIProcessedSize + ullWritten - ullCITotalSize); - m_tSubTaskStats.IncreaseTotalSize(ullCIProcessedSize + ullWritten - ullCITotalSize); + AdjustTotalSize(ullCITotalSize, ullCIProcessedSize + ullWritten); } // new stats - m_tSubTaskStats.IncreaseProcessedSize(ullWritten); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(ullWritten); + AdjustProcessedSize(0, ullWritten); // stop iterating through file bStopProcessing = pBuffer->IsLastPart(); @@ -500,14 +492,14 @@ } } - // fix the stats for files shorter than expected + // this is the end of copying of src file - in case it is smaller than expected fix the stats so that difference is accounted for unsigned long long ullCITotalSize = m_tSubTaskStats.GetCurrentItemTotalSize(); unsigned long long ullCIProcessedSize = m_tSubTaskStats.GetCurrentItemProcessedSize(); if(ullCIProcessedSize < ullCITotalSize) { pData->spSrcFile->SetLength64(ullCIProcessedSize); - m_tSubTaskStats.DecreaseCurrentItemTotalSize(ullCITotalSize - ullCIProcessedSize); - m_tSubTaskStats.DecreaseTotalSize(ullCITotalSize - ullCIProcessedSize); + + AdjustTotalSize(ullCITotalSize, ullCIProcessedSize); } pData->bProcessed = true; @@ -519,24 +511,23 @@ } TSubTaskCopyMove::ESubOperationResult TSubTaskCopyMove::OpenSrcAndDstFilesFB(const IFeedbackHandlerPtr& spFeedbackHandler, CUSTOM_COPY_PARAMS* pData, - const IFilesystemFilePtr& spFileSrc, const IFilesystemFilePtr& spFileDst, bool bNoBuffer, bool& bSkip) + const IFilesystemFilePtr& spFileSrc, const IFilesystemFilePtr& spFileDst, bool& bSkip) { const TConfig& rConfig = GetContext().GetConfig(); bSkip = false; + unsigned long long ullProcessedSize = m_tSubTaskStats.GetCurrentItemProcessedSize(); + // first open the source file and handle any failures - TSubTaskCopyMove::ESubOperationResult eResult = OpenSourceFileFB(spFeedbackHandler, spFileSrc, bNoBuffer); + TSubTaskCopyMove::ESubOperationResult eResult = OpenSourceFileFB(spFeedbackHandler, spFileSrc); if(eResult != TSubTaskBase::eSubResult_Continue) return eResult; else if(!spFileSrc->IsOpen()) { // invalid handle = operation skipped by user - unsigned long long ullDiff = pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize(); + AdjustProcessedSize(ullProcessedSize, pData->spSrcFile->GetLength64()); - m_tSubTaskStats.IncreaseProcessedSize(ullDiff); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(ullDiff); - pData->bProcessed = false; bSkip = true; return TSubTaskBase::eSubResult_Continue; @@ -547,22 +538,13 @@ // but it would require frequent total size updates and thus - serializations). // NOTE2: the by-chunk corrections of stats are still applied when copying to ensure even further size // matching; this update however still allows for better serialization management. - unsigned long long ullNewSize = spFileSrc->GetFileSize(); - unsigned long long ullOldSize = pData->spSrcFile->GetLength64(); - if(ullNewSize != ullOldSize) + file_size_t fsNewSize = spFileSrc->GetFileSize(); + file_size_t fsOldSize = pData->spSrcFile->GetLength64(); + if(fsNewSize != fsOldSize) { - if(ullNewSize > ullOldSize) - { - m_tSubTaskStats.IncreaseTotalSize(ullNewSize - ullOldSize); - m_tSubTaskStats.IncreaseCurrentItemTotalSize(ullNewSize - ullOldSize); - } - else - { - m_tSubTaskStats.DecreaseTotalSize(ullOldSize - ullNewSize); - m_tSubTaskStats.DecreaseCurrentItemTotalSize(ullOldSize - ullNewSize); - } + AdjustTotalSize(fsOldSize, fsNewSize); - pData->spSrcFile->SetLength64(ullNewSize); + pData->spSrcFile->SetLength64(fsNewSize); } // change attributes of a dest file @@ -572,24 +554,20 @@ SetFileAttributes(pData->pathDstFile.ToString(), FILE_ATTRIBUTE_NORMAL); // open destination file, handle the failures and possibly existence of the destination file - unsigned long long ullProcessedSize = m_tSubTaskStats.GetCurrentItemProcessedSize(); unsigned long long ullSeekTo = 0; bool bDstFileFreshlyCreated = false; if (!m_tSubTaskStats.CanCurrentItemSilentResume()) { // open destination file for case, when we start operation on this file (i.e. it is not resume of the // old operation) - eResult = OpenDestinationFileFB(spFeedbackHandler, spFileDst, bNoBuffer, pData->spSrcFile, ullSeekTo, bDstFileFreshlyCreated); + eResult = OpenDestinationFileFB(spFeedbackHandler, spFileDst, pData->spSrcFile, ullSeekTo, bDstFileFreshlyCreated); if(eResult != TSubTaskBase::eSubResult_Continue) return eResult; else if(!spFileDst->IsOpen()) { - unsigned long long ullDiff = pData->spSrcFile->GetLength64() - ullProcessedSize; + AdjustProcessedSize(ullProcessedSize, pData->spSrcFile->GetLength64()); - m_tSubTaskStats.IncreaseProcessedSize(ullDiff); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(ullDiff); - pData->bProcessed = false; bSkip = true; return TSubTaskBase::eSubResult_Continue; @@ -598,16 +576,13 @@ else { // we are resuming previous operation - eResult = OpenExistingDestinationFileFB(spFeedbackHandler, spFileDst, bNoBuffer); + eResult = OpenExistingDestinationFileFB(spFeedbackHandler, spFileDst); if(eResult != TSubTaskBase::eSubResult_Continue) return eResult; else if(!spFileDst->IsOpen()) { - unsigned long long ullDiff = pData->spSrcFile->GetLength64() - ullProcessedSize; + AdjustProcessedSize(ullProcessedSize, pData->spSrcFile->GetLength64()); - m_tSubTaskStats.IncreaseProcessedSize(ullDiff); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(ullDiff); - pData->bProcessed = false; bSkip = true; return TSubTaskBase::eSubResult_Continue; @@ -617,38 +592,28 @@ if(pData->bOnlyCreate) { // we don't copy contents, but need to increase processed size - unsigned long long ullDiff = pData->spSrcFile->GetLength64() - ullProcessedSize; + AdjustProcessedSize(ullProcessedSize, pData->spSrcFile->GetLength64()); - m_tSubTaskStats.IncreaseProcessedSize(ullDiff); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(ullDiff); - return TSubTaskBase::eSubResult_Continue; } // seek to the position where copying will start - ULONGLONG ullMove = (bNoBuffer ? RoundDown(ullSeekTo, IFilesystemFile::MaxSectorSize) : ullSeekTo);; - if(ullMove != 0) // src and dst files exists, requested resume at the specified index - { - // adjust the stats for the difference between what was already processed and what will now be considered processed - if (ullMove > ullProcessedSize) - { - unsigned long long ullDiff = ullMove - ullProcessedSize; - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(ullDiff); - m_tSubTaskStats.IncreaseProcessedSize(ullDiff); - } - else if (ullMove < ullProcessedSize) - { - unsigned long long ullDiff = ullProcessedSize - ullMove; - m_tSubTaskStats.DecreaseCurrentItemProcessedSize(ullDiff); - m_tSubTaskStats.DecreaseProcessedSize(ullDiff); - } - } + file_size_t fsMoveTo = spFileDst->GetSeekPositionForResume(ullSeekTo); + // sanity check + if (bDstFileFreshlyCreated && ullSeekTo != 0) + THROW_CORE_EXCEPTION(eErr_InternalProblem); + if(fsMoveTo > ullSeekTo) + THROW_CORE_EXCEPTION(eErr_InternalProblem); + + // adjust the stats for the difference between what was already processed and what will now be considered processed + AdjustProcessedSize(ullProcessedSize, fsMoveTo); + // if the destination file already exists - truncate it to the current file position if(!bDstFileFreshlyCreated) { // if destination file was opened (as opposed to newly created) - eResult = TruncateFileFB(spFeedbackHandler, spFileDst, ullMove, pData->pathDstFile, bSkip); + eResult = TruncateFileFB(spFeedbackHandler, spFileDst, fsMoveTo, pData->pathDstFile, bSkip); if(eResult != TSubTaskBase::eSubResult_Continue) return eResult; else if(bSkip) @@ -665,6 +630,36 @@ return eResult; } + void TSubTaskCopyMove::AdjustTotalSize(file_size_t fsIncludedSize, file_size_t fsNewSize) + { + if (fsNewSize > fsIncludedSize) + { + m_tSubTaskStats.IncreaseTotalSize(fsNewSize - fsIncludedSize); + m_tSubTaskStats.IncreaseCurrentItemTotalSize(fsNewSize - fsIncludedSize); + } + else if(fsNewSize < fsIncludedSize) + { + m_tSubTaskStats.DecreaseTotalSize(fsIncludedSize - fsNewSize); + m_tSubTaskStats.DecreaseCurrentItemTotalSize(fsIncludedSize - fsNewSize); + } + } + + void TSubTaskCopyMove::AdjustProcessedSize(file_size_t fsIncludedProcessedSize, file_size_t fsNewProcessedSize) + { + if(fsNewProcessedSize > fsIncludedProcessedSize) + { + file_size_t fsDiff = fsNewProcessedSize - fsIncludedProcessedSize; + m_tSubTaskStats.IncreaseCurrentItemProcessedSize(fsDiff); + m_tSubTaskStats.IncreaseProcessedSize(fsDiff); + } + else if (fsNewProcessedSize < fsIncludedProcessedSize) + { + file_size_t fsDiff = fsIncludedProcessedSize - fsNewProcessedSize; + m_tSubTaskStats.DecreaseCurrentItemProcessedSize(fsDiff); + m_tSubTaskStats.DecreaseProcessedSize(fsDiff); + } + } + bool TSubTaskCopyMove::AdjustBufferIfNeeded(TOverlappedDataBufferQueue& rBuffer, TBufferSizes& rBufferSizes, bool bForce) { const TConfig& rConfig = GetContext().GetConfig(); @@ -696,7 +691,7 @@ return false; // buffer did not need adjusting } - TSubTaskBase::ESubOperationResult TSubTaskCopyMove::OpenSourceFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileSrc, bool bNoBuffering) + TSubTaskBase::ESubOperationResult TSubTaskCopyMove::OpenSourceFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileSrc) { icpf::log_file& rLog = GetContext().GetLog(); @@ -708,7 +703,7 @@ { bRetry = false; - if(!fileSrc->OpenExistingForReading(bNoBuffering)) + if(!fileSrc->OpenExistingForReading()) { DWORD dwLastError = GetLastError(); @@ -755,7 +750,8 @@ return TSubTaskBase::eSubResult_Continue; } - TSubTaskBase::ESubOperationResult TSubTaskCopyMove::OpenDestinationFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileDst, bool bNoBuffering, const TFileInfoPtr& spSrcFileInfo, unsigned long long& ullSeekTo, bool& bFreshlyCreated) + TSubTaskBase::ESubOperationResult TSubTaskCopyMove::OpenDestinationFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileDst, + const TFileInfoPtr& spSrcFileInfo, unsigned long long& ullSeekTo, bool& bFreshlyCreated) { icpf::log_file& rLog = GetContext().GetLog(); IFilesystemPtr spFilesystem = GetContext().GetLocalFilesystem(); @@ -770,15 +766,15 @@ { bRetry = false; - if(!fileDst->CreateNewForWriting(bNoBuffering)) + if(!fileDst->CreateNewForWriting()) { DWORD dwLastError = GetLastError(); if(dwLastError == ERROR_FILE_EXISTS) { bFreshlyCreated = false; // pass it to the specialized method - TSubTaskBase::ESubOperationResult eResult = OpenExistingDestinationFileFB(spFeedbackHandler, fileDst, bNoBuffering); + TSubTaskBase::ESubOperationResult eResult = OpenExistingDestinationFileFB(spFeedbackHandler, fileDst); if(eResult != TSubTaskBase::eSubResult_Continue) return eResult; else if(!fileDst->IsOpen()) @@ -871,7 +867,7 @@ return TSubTaskBase::eSubResult_Continue; } - TSubTaskBase::ESubOperationResult TSubTaskCopyMove::OpenExistingDestinationFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileDst, bool bNoBuffering) + TSubTaskBase::ESubOperationResult TSubTaskCopyMove::OpenExistingDestinationFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileDst) { icpf::log_file& rLog = GetContext().GetLog(); @@ -883,7 +879,7 @@ { bRetry = false; - if(!fileDst->OpenExistingForWriting(bNoBuffering)) + if(!fileDst->OpenExistingForWriting()) { DWORD dwLastError = GetLastError(); @@ -930,7 +926,7 @@ return TSubTaskBase::eSubResult_Continue; } - TSubTaskBase::ESubOperationResult TSubTaskCopyMove::TruncateFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& spFile, long long llNewSize, + TSubTaskBase::ESubOperationResult TSubTaskCopyMove::TruncateFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& spFile, file_size_t fsNewSize, const TSmartPath& pathFile, bool& bSkip) { icpf::log_file& rLog = GetContext().GetLog(); @@ -940,7 +936,7 @@ bool bRetry = false; do { - if(!spFile->Truncate(llNewSize)) + if(!spFile->Truncate(fsNewSize)) { // log DWORD dwLastError = GetLastError(); @@ -1262,7 +1258,8 @@ rLog.logi(_T("Checking for free space on destination disk...")); - ullNeededSize = rFilesCache.CalculateTotalSize() - rFilesCache.CalculatePartialSize(m_tSubTaskStats.GetCurrentIndex()); // it'd be nice to round up to take cluster size into consideration + // #nicetohave it'd be nice to round up to take cluster size into consideration + ullNeededSize = rFilesCache.CalculateTotalSize() - rFilesCache.CalculatePartialSize(m_tSubTaskStats.GetCurrentIndex()); // get free space bool bResult = spFilesystem->GetDynamicFreeSpace(pathDestination, ullAvailableSize); Index: src/libchcore/TSubTaskCopyMove.h =================================================================== diff -u -N -re96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3 -r479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc --- src/libchcore/TSubTaskCopyMove.h (.../TSubTaskCopyMove.h) (revision e96806b7f8ff7ca7e9f4afbea603e6351a3dc3e3) +++ src/libchcore/TSubTaskCopyMove.h (.../TSubTaskCopyMove.h) (revision 479ad4e8f81a68cbf6d1623cd9b1f3342d8cfdcc) @@ -65,14 +65,16 @@ ESubOperationResult CustomCopyFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, CUSTOM_COPY_PARAMS* pData); ESubOperationResult OpenSrcAndDstFilesFB(const IFeedbackHandlerPtr& spFeedbackHandler, CUSTOM_COPY_PARAMS* pData, - const IFilesystemFilePtr& spFileSrc, const IFilesystemFilePtr& spFileDst, bool bNoBuffer, bool& bSkip); + const IFilesystemFilePtr& spFileSrc, const IFilesystemFilePtr& spFileDst, bool& bSkip); - ESubOperationResult OpenSourceFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileSrc, bool bNoBuffering); - ESubOperationResult OpenDestinationFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileDst, bool bNoBuffering, const TFileInfoPtr& spSrcFileInfo, + void AdjustTotalSize(unsigned long long ullNewSize, unsigned long long ullOldSize); + + ESubOperationResult OpenSourceFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileSrc); + ESubOperationResult OpenDestinationFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileDst, const TFileInfoPtr& spSrcFileInfo, unsigned long long& ullSeekTo, bool& bFreshlyCreated); - ESubOperationResult OpenExistingDestinationFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileDst, bool bNoBuffering); + ESubOperationResult OpenExistingDestinationFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& fileDst); - ESubOperationResult TruncateFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& file, long long llNewSize, + ESubOperationResult TruncateFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& file, file_size_t fsNewSize, const TSmartPath& pathFile, bool& bSkip); ESubOperationResult ReadFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, const IFilesystemFilePtr& file, @@ -92,6 +94,8 @@ ESubOperationResult CheckForFreeSpaceFB(const IFeedbackHandlerPtr& spFeedbackHandler); + void AdjustProcessedSize(file_size_t fsIncludedProcessedSize, file_size_t fsNewProcessedSize); + private: #pragma warning(push) #pragma warning(disable: 4251)