Index: src/ch/CfgProperties.h =================================================================== diff -u -N -rfb4c4006dee5aaf815d08bc3e89312445b994307 -rfdf4929dc7df1376ed439b7271765f1a4ca31de6 --- src/ch/CfgProperties.h (.../CfgProperties.h) (revision fb4c4006dee5aaf815d08bc3e89312445b994307) +++ src/ch/CfgProperties.h (.../CfgProperties.h) (revision fdf4929dc7df1376ed439b7271765f1a4ca31de6) @@ -89,6 +89,8 @@ PP_BFLAN, PP_BFUSENOBUFFERING, PP_BFBOUNDARYLIMIT, + PP_BUFFERCHUNKSIZE, + PP_BUFFERPAGESIZE, PP_LOGENABLELOGGING, PP_LOGMAXSIZE, @@ -219,6 +221,8 @@ ADAPT_TASK_PROPERTY(PP_BFLAN, chcore::eTO_LANBufferSize); ADAPT_TASK_PROPERTY(PP_BFUSENOBUFFERING, chcore::eTO_DisableBuffering); ADAPT_TASK_PROPERTY(PP_BFBOUNDARYLIMIT, chcore::eTO_DisableBufferingMinSize); +ADAPT_TASK_PROPERTY(PP_BUFFERCHUNKSIZE, chcore::eTO_BufferChunkSize); +ADAPT_TASK_PROPERTY(PP_BUFFERPAGESIZE, chcore::eTO_BufferPageSize); ADAPT_TASK_PROPERTY(PP_CMSETDESTATTRIBUTES, chcore::eTO_SetDestinationAttributes); ADAPT_TASK_PROPERTY(PP_CMSETDESTDATE, chcore::eTO_SetDestinationDateTime); Index: src/libchcore/DataBuffer.cpp =================================================================== diff -u -N -r6df9b4cff81dedf8c5571a98702b584b061eac09 -rfdf4929dc7df1376ed439b7271765f1a4ca31de6 --- src/libchcore/DataBuffer.cpp (.../DataBuffer.cpp) (revision 6df9b4cff81dedf8c5571a98702b584b061eac09) +++ src/libchcore/DataBuffer.cpp (.../DataBuffer.cpp) (revision fdf4929dc7df1376ed439b7271765f1a4ca31de6) @@ -86,117 +86,20 @@ m_auiSizes[eType] = uiSize; } -TDataBuffer::TDataBuffer() : - m_pBuffer(NULL), - m_uiRealSize(0) +UINT TBufferSizes::GetMaxSize() const { -} - -TDataBuffer::~TDataBuffer() -{ - Delete(); -} - -const TBufferSizes& TDataBuffer::Create(const TBufferSizes& rbsSizes) -{ - // if trying to set 0-size buffer - TBufferSizes bsSizes = rbsSizes; // copy - not to mix in the def. param - - for(int iIndex = TBufferSizes::eBuffer_Default; iIndex < TBufferSizes::eBuffer_Last; ++iIndex) - { - TBufferSizes::EBufferType eType = (TBufferSizes::EBufferType)iIndex; - if(bsSizes.GetSizeByType(eType) == 0) - bsSizes.SetSizeByType(eType, DEFAULT_SIZE); - } - - // max value from the all - UINT uiLargest = 0; - if(bsSizes.IsOnlyDefault()) - uiLargest = bsSizes.GetDefaultSize(); + if(m_bOnlyDefault) + return m_auiSizes[eBuffer_Default]; else { - for(int iIndex = TBufferSizes::eBuffer_Default; iIndex < TBufferSizes::eBuffer_Last; ++iIndex) + UINT uiMaxSize = 0; + for(size_t stIndex = 0; stIndex < eBuffer_Last; ++stIndex) { - TBufferSizes::EBufferType eType = (TBufferSizes::EBufferType)iIndex; - - if(uiLargest < bsSizes.GetSizeByType(eType)) - uiLargest = bsSizes.GetSizeByType(eType); + if(m_auiSizes[stIndex] > uiMaxSize) + uiMaxSize = m_auiSizes[stIndex]; } + return uiMaxSize; } - - // modify buffer size to the next 64k boundary - UINT uiRealSize = ROUNDTODS(uiLargest); - if(m_uiRealSize == uiRealSize) - { - // real buffersize hasn't changed - m_bsSizes = bsSizes; - - return m_bsSizes; - } - - // try to allocate buffer - LPVOID pBuffer = VirtualAlloc(NULL, uiRealSize, MEM_COMMIT, PAGE_READWRITE); - if(pBuffer == NULL) - { - if(m_pBuffer == NULL) - { - // try safe buffesize - pBuffer = VirtualAlloc(NULL, DEFAULT_SIZE, MEM_COMMIT, PAGE_READWRITE); - if(pBuffer == NULL) - return m_bsSizes; // do not change anything - - // delete old buffer - Delete(); - - // store data - m_pBuffer = static_cast(pBuffer); - m_uiRealSize = DEFAULT_SIZE; - m_bsSizes.SetOnlyDefault(bsSizes.IsOnlyDefault()); - m_bsSizes.SetDefaultSize(DEFAULT_SIZE); - m_bsSizes.SetOneDiskSize(DEFAULT_SIZE); - m_bsSizes.SetTwoDisksSize(DEFAULT_SIZE); - m_bsSizes.SetCDSize(DEFAULT_SIZE); - m_bsSizes.SetLANSize(DEFAULT_SIZE); - - return m_bsSizes; - } - else - { - // no new buffer could be created - leave the old one - return m_bsSizes; - } - } - else - { - // succeeded - Delete(); // get rid of old buffer - - // store data - m_pBuffer = static_cast(pBuffer); - m_uiRealSize = uiRealSize; - m_bsSizes = bsSizes; - - return m_bsSizes; - } } -void TDataBuffer::Delete() -{ - if(m_pBuffer != NULL) - { - VirtualFree(static_cast(m_pBuffer), 0, MEM_RELEASE); - m_pBuffer = NULL; - m_uiRealSize = 0; - m_bsSizes.Clear(); - } -} - -void TDataBuffer::CutDataFromBuffer(UINT uiCount) -{ - if(uiCount >= m_uiRealSize || !m_pBuffer) - return; // nothing to do - - memmove(m_pBuffer, m_pBuffer + uiCount, m_uiRealSize - uiCount); -} - END_CHCORE_NAMESPACE Index: src/libchcore/DataBuffer.h =================================================================== diff -u -N -r6df9b4cff81dedf8c5571a98702b584b061eac09 -rfdf4929dc7df1376ed439b7271765f1a4ca31de6 --- src/libchcore/DataBuffer.h (.../DataBuffer.h) (revision 6df9b4cff81dedf8c5571a98702b584b061eac09) +++ src/libchcore/DataBuffer.h (.../DataBuffer.h) (revision fdf4929dc7df1376ed439b7271765f1a4ca31de6) @@ -79,41 +79,13 @@ void SetLANSize(UINT uiSize) { m_auiSizes[eBuffer_LAN] = uiSize; } void SetSizeByType(EBufferType eType, UINT uiSize); + UINT GetMaxSize() const; + private: UINT m_auiSizes[eBuffer_Last]; bool m_bOnlyDefault; }; -//#pragma warning (default: 4201) -class LIBCHCORE_API TDataBuffer -{ -public: - TDataBuffer(); - ~TDataBuffer(); - - const TBufferSizes& Create(const TBufferSizes& rbsSizes); // (re)allocates the buffer; if there's an error - restores previous buffer size - void Delete(); // deletes buffer - - UINT GetRealSize() { return m_uiRealSize; } - UINT GetDefaultSize() { return m_bsSizes.GetDefaultSize(); } - UINT GetOneDiskSize() { return m_bsSizes.GetOneDiskSize(); } - UINT GetTwoDisksSize() { return m_bsSizes.GetTwoDisksSize(); } - UINT GetCDSize() { return m_bsSizes.GetCDSize(); } - UINT GetLANSize() { return m_bsSizes.GetLANSize(); } - const TBufferSizes& GetSizes() { return m_bsSizes; } - - // shifts data from buffer from position uiCount to 0 (effectively cuts uiCount bytes of data at the beginning of buffer) - void CutDataFromBuffer(UINT uiCount); - - // operators - operator unsigned char*() { return m_pBuffer; } - -protected: - unsigned char *m_pBuffer; // buffer address - UINT m_uiRealSize; // real buffer size - TBufferSizes m_bsSizes; -}; - END_CHCORE_NAMESPACE #endif \ No newline at end of file Index: src/libchcore/TDataBuffer.cpp =================================================================== diff -u -N -r2c34ba2265e64a9547ace8cf1d29b1def1a552f4 -rfdf4929dc7df1376ed439b7271765f1a4ca31de6 --- src/libchcore/TDataBuffer.cpp (.../TDataBuffer.cpp) (revision 2c34ba2265e64a9547ace8cf1d29b1def1a552f4) +++ src/libchcore/TDataBuffer.cpp (.../TDataBuffer.cpp) (revision fdf4929dc7df1376ed439b7271765f1a4ca31de6) @@ -27,12 +27,9 @@ namespace { - const size_t c_DefaultAllocGranularity = 4096; const size_t c_DefaultBufferSize = 65536; const size_t c_DefaultPageSize = 1024*1024; const size_t c_DefaultMaxMemory = 1024*1024; - - template T RoundUp(T number, T roundValue) { return ((number + roundValue - 1) & ~(roundValue - 1)); } } namespace details @@ -64,16 +61,24 @@ void TVirtualAllocMemoryBlock::ReleaseChunks(std::list& rListChunks) { - for(std::list::iterator iterList = rListChunks.begin(); iterList != rListChunks.end(); ++iterList) + std::list::iterator iterList = rListChunks.begin(); + while(iterList != rListChunks.end()) { - ReleaseChunk(*iterList); + if(ReleaseChunk(*iterList)) + iterList = rListChunks.erase(iterList); + else + ++iterList; } } - void TVirtualAllocMemoryBlock::ReleaseChunk(LPVOID pChunk) + bool TVirtualAllocMemoryBlock::ReleaseChunk(LPVOID pChunk) { if(IsValidChunk(pChunk)) + { m_setFreeChunks.insert(pChunk); + return true; + } + return false; } size_t TVirtualAllocMemoryBlock::CountOwnChunks(const std::list& rListChunks) @@ -141,7 +146,7 @@ { if(IsChunkOwner(pChunk)) { - bool bValidPtr = (((BYTE*)pChunk - (BYTE*)m_pMemory) % m_stChunkSize) != 0; + bool bValidPtr = (((BYTE*)pChunk - (BYTE*)m_pMemory) % m_stChunkSize) == 0; _ASSERTE(bValidPtr); return bValidPtr; } @@ -156,7 +161,8 @@ TSimpleDataBuffer::TSimpleDataBuffer() : m_pBuffer(NULL), m_pBufferManager(NULL), - m_stBufferSize(0) + m_stBufferSize(0), + m_stDataSize(0) { } @@ -185,6 +191,22 @@ m_stBufferSize = stBufferSize; } +void TSimpleDataBuffer::SetDataSize(size_t stDataSize) +{ + if(stDataSize > m_stBufferSize) + THROW_CORE_EXCEPTION(eErr_InvalidArgument); + + m_stDataSize = stDataSize; +} + +void TSimpleDataBuffer::CutDataFromBuffer(size_t stCount) +{ + if(stCount >= m_stBufferSize || !m_pBuffer) + return; // nothing to do + + memmove(m_pBuffer, (BYTE*)m_pBuffer + stCount, m_stBufferSize - stCount); +} + /////////////////////////////////////////////////////////////////////////////////// // class TDataBufferManager @@ -218,7 +240,7 @@ } else { - size_t stNewSize = RoundUp(stBufferSize, c_DefaultAllocGranularity); + size_t stNewSize = RoundUp(stBufferSize, DefaultAllocGranularity); if(stBufferSize != stNewSize) { stBufferSize = stNewSize; Index: src/libchcore/TDataBuffer.h =================================================================== diff -u -N -r2c34ba2265e64a9547ace8cf1d29b1def1a552f4 -rfdf4929dc7df1376ed439b7271765f1a4ca31de6 --- src/libchcore/TDataBuffer.h (.../TDataBuffer.h) (revision 2c34ba2265e64a9547ace8cf1d29b1def1a552f4) +++ src/libchcore/TDataBuffer.h (.../TDataBuffer.h) (revision fdf4929dc7df1376ed439b7271765f1a4ca31de6) @@ -37,7 +37,7 @@ void GetFreeChunks(std::list& rListChunks); void ReleaseChunks(std::list& rListChunks); - void ReleaseChunk(LPVOID pChunk); + bool ReleaseChunk(LPVOID pChunk); size_t CountOwnChunks(const std::list& rListChunks); @@ -64,6 +64,8 @@ typedef boost::shared_ptr TVirtualAllocMemoryBlockPtr; } +template T RoundUp(T number, T roundValue) { return ((number + roundValue - 1) & ~(roundValue - 1)); } + class TDataBufferManager; class LIBCHCORE_API TSimpleDataBuffer @@ -75,6 +77,11 @@ LPVOID GetBufferPtr(); void ReleaseBuffer(); + void SetDataSize(size_t stDataSize); + size_t GetDataSize() const { return m_stDataSize; } + + void CutDataFromBuffer(size_t stCount); + private: TSimpleDataBuffer(const TSimpleDataBuffer&); TSimpleDataBuffer& operator=(const TSimpleDataBuffer&); @@ -85,13 +92,19 @@ LPVOID m_pBuffer; TDataBufferManager* m_pBufferManager; size_t m_stBufferSize; + size_t m_stDataSize; friend class TDataBufferManager; }; +typedef boost::shared_ptr TSimpleDataBufferPtr; + class LIBCHCORE_API TDataBufferManager { public: + static const size_t DefaultAllocGranularity = 4096; + +public: TDataBufferManager(); ~TDataBufferManager(); @@ -109,7 +122,7 @@ // current settings size_t GetMaxMemorySize() const { return m_stMaxMemory; } size_t GetPageSize() const { return m_stPageSize; } - size_t GetBufferSize() const { return m_stBufferSize; } + size_t GetSimpleBufferSize() const { return m_stBufferSize; } // buffer retrieval bool HasFreeBuffer() const; // checks if a buffer is available without allocating any new memory Index: src/libchcore/TLocalFilesystem.cpp =================================================================== diff -u -N -r0b69828c1b85f446e1e015d8feeed551316bf138 -rfdf4929dc7df1376ed439b7271765f1a4ca31de6 --- src/libchcore/TLocalFilesystem.cpp (.../TLocalFilesystem.cpp) (revision 0b69828c1b85f446e1e015d8feeed551316bf138) +++ src/libchcore/TLocalFilesystem.cpp (.../TLocalFilesystem.cpp) (revision fdf4929dc7df1376ed439b7271765f1a4ca31de6) @@ -33,6 +33,7 @@ #pragma warning(disable: 4201) #include #pragma warning(pop) +#include "TDataBuffer.h" BEGIN_CHCORE_NAMESPACE @@ -431,20 +432,20 @@ return ::SetEndOfFile(m_hFile) != FALSE; } -bool TLocalFilesystemFile::ReadFile(TDataBuffer& rBuffer, DWORD dwToRead, DWORD& rdwBytesRead) +bool TLocalFilesystemFile::ReadFile(TSimpleDataBuffer& rBuffer, DWORD dwToRead, DWORD& rdwBytesRead) { if(!IsOpen()) return false; - return ::ReadFile(m_hFile, rBuffer, dwToRead, &rdwBytesRead, NULL) != FALSE; + return ::ReadFile(m_hFile, rBuffer.GetBufferPtr(), dwToRead, &rdwBytesRead, NULL) != FALSE; } -bool TLocalFilesystemFile::WriteFile(TDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten) +bool TLocalFilesystemFile::WriteFile(TSimpleDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten) { if(!IsOpen()) return false; - return ::WriteFile(m_hFile, rBuffer, dwToWrite, &rdwBytesWritten, NULL) != NULL && dwToWrite == rdwBytesWritten; + return ::WriteFile(m_hFile, rBuffer.GetBufferPtr(), dwToWrite, &rdwBytesWritten, NULL) != NULL && dwToWrite == rdwBytesWritten; } void TLocalFilesystemFile::Close() Index: src/libchcore/TLocalFilesystem.h =================================================================== diff -u -N -rfb4c4006dee5aaf815d08bc3e89312445b994307 -rfdf4929dc7df1376ed439b7271765f1a4ca31de6 --- src/libchcore/TLocalFilesystem.h (.../TLocalFilesystem.h) (revision fb4c4006dee5aaf815d08bc3e89312445b994307) +++ src/libchcore/TLocalFilesystem.h (.../TLocalFilesystem.h) (revision fdf4929dc7df1376ed439b7271765f1a4ca31de6) @@ -35,7 +35,7 @@ class TAutoFileHandle; class TLocalFilesystemFind; class TLocalFilesystemFile; -class TDataBuffer; +class TSimpleDataBuffer; class LIBCHCORE_API TLocalFilesystem { @@ -116,8 +116,8 @@ bool SetFilePointer(long long llNewPos, DWORD dwMoveMethod); bool SetEndOfFile(); - bool ReadFile(TDataBuffer& rBuffer, DWORD dwToRead, DWORD& rdwBytesRead); - bool WriteFile(TDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten); + bool ReadFile(TSimpleDataBuffer& rBuffer, DWORD dwToRead, DWORD& rdwBytesRead); + bool WriteFile(TSimpleDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten); bool IsOpen() const { return m_hFile != INVALID_HANDLE_VALUE; } Index: src/libchcore/TSubTaskCopyMove.cpp =================================================================== diff -u -N -rdc1988138aa8e37ce585fed1caa25294781578ac -rfdf4929dc7df1376ed439b7271765f1a4ca31de6 --- src/libchcore/TSubTaskCopyMove.cpp (.../TSubTaskCopyMove.cpp) (revision dc1988138aa8e37ce585fed1caa25294781578ac) +++ src/libchcore/TSubTaskCopyMove.cpp (.../TSubTaskCopyMove.cpp) (revision fdf4929dc7df1376ed439b7271765f1a4ca31de6) @@ -39,6 +39,7 @@ #include "TFileInfoArray.h" #include "SerializationHelpers.h" #include "TBinarySerializer.h" +#include "TDataBuffer.h" BEGIN_CHCORE_NAMESPACE @@ -136,7 +137,8 @@ TFileInfoPtr spSrcFile; // CFileInfo - src file TSmartPath pathDstFile; // dest path with filename - TDataBuffer dbBuffer; // buffer handling + TBufferSizes tBufferSizes; + TDataBufferManager dbBuffer; // buffer handling bool bOnlyCreate; // flag from configuration - skips real copying - only create bool bProcessed; // has the element been processed ? (false if skipped) }; @@ -199,7 +201,7 @@ // remove changes in buffer sizes to avoid re-creation later rCfgTracker.RemoveModificationSet(TOptionsSet() % eTO_DefaultBufferSize % eTO_OneDiskBufferSize % eTO_TwoDisksBufferSize % eTO_CDBufferSize % eTO_LANBufferSize % eTO_UseOnlyDefaultBuffer); - RecreateBufferIfNeeded(ccp.dbBuffer, true); + AdjustBufferIfNeeded(ccp.dbBuffer, ccp.tBufferSizes); // log TString strFormat; @@ -289,9 +291,6 @@ m_tSubTaskStats.SetProcessedCount(stIndex); m_tSubTaskStats.SetCurrentPath(TString()); - // delete buffer - it's not needed - ccp.dbBuffer.Delete(); - // log rLog.logi(_T("Finished processing in ProcessFiles")); @@ -303,8 +302,11 @@ m_tSubTaskStats.GetSnapshot(rStats); } -int TSubTaskCopyMove::GetBufferIndex(const TFileInfoPtr& spFileInfo) +TBufferSizes::EBufferType TSubTaskCopyMove::GetBufferIndex(const TBufferSizes& rBufferSizes, const TFileInfoPtr& spFileInfo) { + if(rBufferSizes.IsOnlyDefault()) + return TBufferSizes::eBuffer_Default; + if(!spFileInfo) THROW_CORE_EXCEPTION(eErr_InvalidArgument); @@ -359,10 +361,13 @@ return TSubTaskBase::eSubResult_Continue; // copying - unsigned long ulToRead = 0; + std::list listDataBuffers; + std::list listEmptyBuffers; + + size_t stToRead = 0; unsigned long ulRead = 0; unsigned long ulWritten = 0; - int iBufferIndex = 0; + TBufferSizes::EBufferType eBufferIndex = TBufferSizes::eBuffer_Default; bool bLastPart = false; do @@ -379,36 +384,66 @@ } // recreate buffer if needed - RecreateBufferIfNeeded(pData->dbBuffer, false); + AdjustBufferIfNeeded(pData->dbBuffer, pData->tBufferSizes); // establish count of data to read - if(GetTaskPropValue(rTaskDefinition.GetConfiguration())) - iBufferIndex = TBufferSizes::eBuffer_Default; - else - iBufferIndex = GetBufferIndex(pData->spSrcFile); - // new stats - m_tSubTaskStats.SetCurrentBufferIndex(iBufferIndex); + eBufferIndex = GetBufferIndex(pData->tBufferSizes, pData->spSrcFile); + m_tSubTaskStats.SetCurrentBufferIndex(eBufferIndex); - ulToRead = bNoBuffer ? ROUNDUP(pData->dbBuffer.GetSizes().GetSizeByType((TBufferSizes::EBufferType)iBufferIndex), MAXSECTORSIZE) : pData->dbBuffer.GetSizes().GetSizeByType((TBufferSizes::EBufferType)iBufferIndex); + stToRead = RoundUp((size_t)pData->tBufferSizes.GetSizeByType(eBufferIndex), pData->dbBuffer.GetSimpleBufferSize()); + size_t stBuffersToRead = stToRead / pData->dbBuffer.GetSimpleBufferSize(); // read data from file to buffer - eResult = ReadFileFB(fileSrc, pData->dbBuffer, ulToRead, ulRead, pData->spSrcFile->GetFullFilePath(), bSkip); - if(eResult != TSubTaskBase::eSubResult_Continue) - return eResult; - else if(bSkip) + for(size_t stIndex = 0; stIndex < stBuffersToRead; ++stIndex) { - // new stats - m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tProgressInfo.GetCurrentFileProcessedSize()); + // get new simple buffer + TSimpleDataBufferPtr spBuffer; + if(listEmptyBuffers.empty()) + { + spBuffer.reset(new TSimpleDataBuffer); + if(pData->dbBuffer.GetFreeBuffer(*spBuffer.get())) + listEmptyBuffers.push_back(spBuffer); + else + { + if(listDataBuffers.empty()) + THROW_CORE_EXCEPTION(eErr_InternalProblem); + break; + } + } - pData->bProcessed = false; - return TSubTaskBase::eSubResult_Continue; - } + spBuffer = listEmptyBuffers.back(); + listEmptyBuffers.pop_back(); - bLastPart = (ulToRead != ulRead); + eResult = ReadFileFB(fileSrc, *spBuffer.get(), boost::numeric_cast(pData->dbBuffer.GetSimpleBufferSize()), ulRead, pData->spSrcFile->GetFullFilePath(), bSkip); + if(eResult != TSubTaskBase::eSubResult_Continue) + return eResult; + else if(bSkip) + { + // new stats + m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tProgressInfo.GetCurrentFileProcessedSize()); - if(ulRead > 0) + pData->bProcessed = false; + return TSubTaskBase::eSubResult_Continue; + } + + spBuffer->SetDataSize(ulRead); + + if(ulRead > 0) + listDataBuffers.push_back(spBuffer); + else + listEmptyBuffers.push_back(spBuffer); + + bLastPart = (pData->dbBuffer.GetSimpleBufferSize() != ulRead); + if(bLastPart) + break; + } + + while(!listDataBuffers.empty()) { - eResult = WriteFileExFB(fileDst, pData->dbBuffer, ulRead, ulWritten, pData->pathDstFile, bSkip, bNoBuffer); + TSimpleDataBufferPtr spBuffer = listDataBuffers.front(); + listDataBuffers.pop_front(); + + eResult = WriteFileExFB(fileDst, *spBuffer.get(), boost::numeric_cast(spBuffer->GetDataSize()), ulWritten, pData->pathDstFile, bSkip, bNoBuffer); if(eResult != TSubTaskBase::eSubResult_Continue) return eResult; else if(bSkip) @@ -420,13 +455,15 @@ return TSubTaskBase::eSubResult_Continue; } + listEmptyBuffers.push_back(spBuffer); + // increase count of processed data m_tProgressInfo.IncreaseCurrentFileProcessedSize(ulWritten); // new stats m_tSubTaskStats.IncreaseProcessedSize(ulWritten); } } - while(ulRead != 0 && !bLastPart); + while(!bLastPart); pData->bProcessed = true; m_tProgressInfo.SetCurrentFileProcessedSize(0); @@ -562,42 +599,55 @@ return eResult; } -void TSubTaskCopyMove::RecreateBufferIfNeeded(TDataBuffer& rBuffer, bool bInitialCreate) +bool TSubTaskCopyMove::AdjustBufferIfNeeded(chcore::TDataBufferManager& rBuffer, TBufferSizes& rBufferSizes) { TTaskConfigTracker& rCfgTracker = GetContext().GetCfgTracker(); TTaskDefinition& rTaskDefinition = GetContext().GetTaskDefinition(); icpf::log_file& rLog = GetContext().GetLog(); - if(bInitialCreate || (rCfgTracker.IsModified() && rCfgTracker.IsModified(TOptionsSet() % eTO_DefaultBufferSize % eTO_OneDiskBufferSize % eTO_TwoDisksBufferSize % eTO_CDBufferSize % eTO_LANBufferSize % eTO_UseOnlyDefaultBuffer, true))) + if(!rBuffer.IsInitialized() || (rCfgTracker.IsModified() && rCfgTracker.IsModified(TOptionsSet() % eTO_DefaultBufferSize % eTO_OneDiskBufferSize % eTO_TwoDisksBufferSize % eTO_CDBufferSize % eTO_LANBufferSize % eTO_UseOnlyDefaultBuffer, true))) { - TBufferSizes bs; - bs.SetOnlyDefault(GetTaskPropValue(rTaskDefinition.GetConfiguration())); - bs.SetDefaultSize(GetTaskPropValue(rTaskDefinition.GetConfiguration())); - bs.SetOneDiskSize(GetTaskPropValue(rTaskDefinition.GetConfiguration())); - bs.SetTwoDisksSize(GetTaskPropValue(rTaskDefinition.GetConfiguration())); - bs.SetCDSize(GetTaskPropValue(rTaskDefinition.GetConfiguration())); - bs.SetLANSize(GetTaskPropValue(rTaskDefinition.GetConfiguration())); + rBufferSizes.SetOnlyDefault(GetTaskPropValue(rTaskDefinition.GetConfiguration())); + rBufferSizes.SetDefaultSize(GetTaskPropValue(rTaskDefinition.GetConfiguration())); + rBufferSizes.SetOneDiskSize(GetTaskPropValue(rTaskDefinition.GetConfiguration())); + rBufferSizes.SetTwoDisksSize(GetTaskPropValue(rTaskDefinition.GetConfiguration())); + rBufferSizes.SetCDSize(GetTaskPropValue(rTaskDefinition.GetConfiguration())); + rBufferSizes.SetLANSize(GetTaskPropValue(rTaskDefinition.GetConfiguration())); // log - const TBufferSizes& rbs1 = rBuffer.GetSizes(); - TString strFormat; - strFormat = _T("Changing buffer size from [Def:%defsize, One:%onesize, Two:%twosize, CD:%cdsize, LAN:%lansize] to [Def:%defsize2, One:%onesize2, Two:%twosize2, CD:%cdsize2, LAN:%lansize2]"); + strFormat = _T("Changing buffer size to [Def:%defsize2, One:%onesize2, Two:%twosize2, CD:%cdsize2, LAN:%lansize2]"); - strFormat.Replace(_T("%defsize"), boost::lexical_cast(rbs1.GetDefaultSize()).c_str()); - strFormat.Replace(_T("%onesize"), boost::lexical_cast(rbs1.GetOneDiskSize()).c_str()); - strFormat.Replace(_T("%twosize"), boost::lexical_cast(rbs1.GetTwoDisksSize()).c_str()); - strFormat.Replace(_T("%cdsize"), boost::lexical_cast(rbs1.GetCDSize()).c_str()); - strFormat.Replace(_T("%lansize"), boost::lexical_cast(rbs1.GetLANSize()).c_str()); - strFormat.Replace(_T("%defsize2"), boost::lexical_cast(bs.GetDefaultSize()).c_str()); - strFormat.Replace(_T("%onesize2"), boost::lexical_cast(bs.GetOneDiskSize()).c_str()); - strFormat.Replace(_T("%twosize2"), boost::lexical_cast(bs.GetTwoDisksSize()).c_str()); - strFormat.Replace(_T("%cdsize2"), boost::lexical_cast(bs.GetCDSize()).c_str()); - strFormat.Replace(_T("%lansize2"), boost::lexical_cast(bs.GetLANSize()).c_str()); + strFormat.Replace(_T("%defsize2"), boost::lexical_cast(rBufferSizes.GetDefaultSize()).c_str()); + strFormat.Replace(_T("%onesize2"), boost::lexical_cast(rBufferSizes.GetOneDiskSize()).c_str()); + strFormat.Replace(_T("%twosize2"), boost::lexical_cast(rBufferSizes.GetTwoDisksSize()).c_str()); + strFormat.Replace(_T("%cdsize2"), boost::lexical_cast(rBufferSizes.GetCDSize()).c_str()); + strFormat.Replace(_T("%lansize2"), boost::lexical_cast(rBufferSizes.GetLANSize()).c_str()); rLog.logi(strFormat); - rBuffer.Create(bs); + + if(!rBuffer.IsInitialized()) + { + size_t stMaxSize = rBufferSizes.GetMaxSize(); + size_t stPageSize = GetTaskPropValue(rTaskDefinition.GetConfiguration()); + size_t stChunkSize = GetTaskPropValue(rTaskDefinition.GetConfiguration()); + + chcore::TDataBufferManager::CheckBufferConfig(stMaxSize, stPageSize, stChunkSize); + + rBuffer.Initialize(stMaxSize, stPageSize, stChunkSize); + } + else + { + size_t stMaxSize = rBufferSizes.GetMaxSize(); + rBuffer.CheckResizeSize(stMaxSize); + + rBuffer.ChangeMaxMemorySize(stMaxSize); + } + + return true; } + else + return false; } TSubTaskBase::ESubOperationResult TSubTaskCopyMove::OpenSourceFileFB(TLocalFilesystemFile& fileSrc, const TSmartPath& spPathToOpen, bool bNoBuffering) @@ -944,7 +994,7 @@ return TSubTaskBase::eSubResult_Continue; } -TSubTaskBase::ESubOperationResult TSubTaskCopyMove::ReadFileFB(TLocalFilesystemFile& file, TDataBuffer& rBuffer, DWORD dwToRead, DWORD& rdwBytesRead, const TSmartPath& pathFile, bool& bSkip) +TSubTaskBase::ESubOperationResult TSubTaskCopyMove::ReadFileFB(TLocalFilesystemFile& file, chcore::TSimpleDataBuffer& rBuffer, DWORD dwToRead, DWORD& rdwBytesRead, const TSmartPath& pathFile, bool& bSkip) { IFeedbackHandler* piFeedbackHandler = GetContext().GetFeedbackHandler(); icpf::log_file& rLog = GetContext().GetLog(); @@ -995,7 +1045,7 @@ return TSubTaskBase::eSubResult_Continue; } -TSubTaskBase::ESubOperationResult TSubTaskCopyMove::WriteFileFB(TLocalFilesystemFile& file, TDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten, const TSmartPath& pathFile, bool& bSkip) +TSubTaskBase::ESubOperationResult TSubTaskCopyMove::WriteFileFB(TLocalFilesystemFile& file, chcore::TSimpleDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten, const TSmartPath& pathFile, bool& bSkip) { IFeedbackHandler* piFeedbackHandler = GetContext().GetFeedbackHandler(); icpf::log_file& rLog = GetContext().GetLog(); @@ -1047,7 +1097,7 @@ return TSubTaskBase::eSubResult_Continue; } -TSubTaskBase::ESubOperationResult TSubTaskCopyMove::WriteFileExFB(TLocalFilesystemFile& file, TDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten, const TSmartPath& pathFile, bool& bSkip, bool bNoBuffer) +TSubTaskBase::ESubOperationResult TSubTaskCopyMove::WriteFileExFB(TLocalFilesystemFile& file, chcore::TSimpleDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten, const TSmartPath& pathFile, bool& bSkip, bool bNoBuffer) { TString strFormat; TSubTaskBase::ESubOperationResult eResult = TSubTaskBase::eSubResult_Continue; Index: src/libchcore/TSubTaskCopyMove.h =================================================================== diff -u -N -rdc1988138aa8e37ce585fed1caa25294781578ac -rfdf4929dc7df1376ed439b7271765f1a4ca31de6 --- src/libchcore/TSubTaskCopyMove.h (.../TSubTaskCopyMove.h) (revision dc1988138aa8e37ce585fed1caa25294781578ac) +++ src/libchcore/TSubTaskCopyMove.h (.../TSubTaskCopyMove.h) (revision fdf4929dc7df1376ed439b7271765f1a4ca31de6) @@ -25,16 +25,20 @@ #include "libchcore.h" #include "TSubTaskBase.h" +#include "DataBuffer.h" BEGIN_CHCORE_NAMESPACE -class TDataBuffer; class TLocalFilesystemFile; typedef boost::shared_ptr TFileInfoPtr; struct CUSTOM_COPY_PARAMS; class TReadBinarySerializer; class TWriteBinarySerializer; +class TDataBufferManager; +class TSimpleDataBuffer; +class TBufferSizes; + namespace details { /////////////////////////////////////////////////////////////////////////// @@ -82,8 +86,8 @@ virtual void GetStatsSnapshot(TSubTaskStatsSnapshot& rStats) const; private: - int GetBufferIndex(const TFileInfoPtr& spFileInfo); - void RecreateBufferIfNeeded(TDataBuffer& rBuffer, bool bInitialCreate); + TBufferSizes::EBufferType GetBufferIndex(const TBufferSizes& rBufferSizes, const TFileInfoPtr& spFileInfo); + bool AdjustBufferIfNeeded(TDataBufferManager& rBuffer, TBufferSizes& rBufferSizes); ESubOperationResult CustomCopyFileFB(CUSTOM_COPY_PARAMS* pData); @@ -96,9 +100,9 @@ ESubOperationResult SetFilePointerFB(TLocalFilesystemFile& file, long long llDistance, const TSmartPath& pathFile, bool& bSkip); ESubOperationResult SetEndOfFileFB(TLocalFilesystemFile& file, const TSmartPath& pathFile, bool& bSkip); - ESubOperationResult ReadFileFB(TLocalFilesystemFile& file, TDataBuffer& rBuffer, DWORD dwToRead, DWORD& rdwBytesRead, const TSmartPath& pathFile, bool& bSkip); - ESubOperationResult WriteFileFB(TLocalFilesystemFile& file, TDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten, const TSmartPath& pathFile, bool& bSkip); - ESubOperationResult WriteFileExFB(TLocalFilesystemFile& file, TDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten, const TSmartPath& pathFile, bool& bSkip, bool bNoBuffer); + ESubOperationResult ReadFileFB(TLocalFilesystemFile& file, chcore::TSimpleDataBuffer& rBuffer, DWORD dwToRead, DWORD& rdwBytesRead, const TSmartPath& pathFile, bool& bSkip); + ESubOperationResult WriteFileFB(TLocalFilesystemFile& file, chcore::TSimpleDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten, const TSmartPath& pathFile, bool& bSkip); + ESubOperationResult WriteFileExFB(TLocalFilesystemFile& file, chcore::TSimpleDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten, const TSmartPath& pathFile, bool& bSkip, bool bNoBuffer); ESubOperationResult CreateDirectoryFB(const TSmartPath& pathDirectory); ESubOperationResult CheckForFreeSpaceFB(); Index: src/libchcore/TTaskConfigTracker.cpp =================================================================== diff -u -N -rfb4c4006dee5aaf815d08bc3e89312445b994307 -rfdf4929dc7df1376ed439b7271765f1a4ca31de6 --- src/libchcore/TTaskConfigTracker.cpp (.../TTaskConfigTracker.cpp) (revision fb4c4006dee5aaf815d08bc3e89312445b994307) +++ src/libchcore/TTaskConfigTracker.cpp (.../TTaskConfigTracker.cpp) (revision fdf4929dc7df1376ed439b7271765f1a4ca31de6) @@ -210,6 +210,10 @@ return eTO_DisableBuffering; else if(strOption == TaskPropData::GetPropertyName()) return eTO_DisableBufferingMinSize; + else if(strOption == TaskPropData::GetPropertyName()) + return eTO_BufferChunkSize; + else if(strOption == TaskPropData::GetPropertyName()) + return eTO_BufferPageSize; else if(strOption == TaskPropData::GetPropertyName()) return eTO_SetDestinationAttributes; Index: src/libchcore/TTaskConfiguration.h =================================================================== diff -u -N -rba802caea92ee56a154d1da3fe89a4b2f7875f0e -rfdf4929dc7df1376ed439b7271765f1a4ca31de6 --- src/libchcore/TTaskConfiguration.h (.../TTaskConfiguration.h) (revision ba802caea92ee56a154d1da3fe89a4b2f7875f0e) +++ src/libchcore/TTaskConfiguration.h (.../TTaskConfiguration.h) (revision fdf4929dc7df1376ed439b7271765f1a4ca31de6) @@ -39,6 +39,8 @@ eTO_LANBufferSize, eTO_DisableBuffering, eTO_DisableBufferingMinSize, + eTO_BufferChunkSize, + eTO_BufferPageSize, eTO_SetDestinationAttributes, eTO_SetDestinationDateTime, @@ -99,6 +101,8 @@ TASK_PROPERTY_MINMAX(eTO_TwoDisksBufferSize, unsigned int, _T("Buffer.TwoPhysicalDisksSize"), 524288, 1, 0xffffffff); TASK_PROPERTY_MINMAX(eTO_CDBufferSize, unsigned int, _T("Buffer.CDSize"), 262144, 1, 0xffffffff); TASK_PROPERTY_MINMAX(eTO_LANBufferSize, unsigned int, _T("Buffer.LANSize"), 131072, 1, 0xffffffff); +TASK_PROPERTY_MINMAX(eTO_BufferChunkSize, unsigned int, _T("Buffer.ChunkSize"), 65536, 1, 0xffffffff); +TASK_PROPERTY_MINMAX(eTO_BufferPageSize, unsigned int, _T("Buffer.PageSize"), 512384, 1, 0xffffffff); TASK_PROPERTY(eTO_DisableBuffering, bool, _T("Operation.Buffering.DisableBufferingForLargeFiles"), true); TASK_PROPERTY_MINMAX(eTO_DisableBufferingMinSize, int, _T("Operation.Buffering.MinSizeOfFileToDisableBuffering"), 2097152, 1, 0xffffffff); @@ -119,7 +123,7 @@ TASK_PROPERTY(eTO_CreateDirectoriesRelativeToRoot, bool, _T("Operation.CreateDirectoriesRelativeToRoot"), false); TASK_PROPERTY(eTO_IgnoreDirectories, bool, _T("Operation.IgnoreDirectories"), false); -TASK_PROPERTY(eTO_Filters, chcore::TFileFiltersArray, _T("Operation.Filtering"), chcore::TFileFiltersArray()); +TASK_PROPERTY(eTO_Filters, chcore::TFileFiltersArray, _T("Operation.Filtering"), TFileFiltersArray()); // Naming settings TASK_PROPERTY(eTO_AlternateFilenameFormatString_First, TString, _T("Naming.AlternateFilenameFormatFirst"), _T("Copy of %name"));