Index: ext/gmock/gmock.vcxproj =================================================================== diff -u -N -r122f5f17f602b989c626ab31bb7f32c4211f93bd -r6103ac74583f2136b821dc67515ed8469abd8155 --- ext/gmock/gmock.vcxproj (.../gmock.vcxproj) (revision 122f5f17f602b989c626ab31bb7f32c4211f93bd) +++ ext/gmock/gmock.vcxproj (.../gmock.vcxproj) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -119,61 +119,61 @@ $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ $(ProjectName)32d - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ $(ProjectName)32d - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ AllRules.ruleset false $(ProjectName)64d - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ AllRules.ruleset false $(ProjectName)64d - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ $(ProjectName)32 - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ $(ProjectName)32 - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ $(ProjectName)64 - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ $(ProjectName)64 - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) Index: src/ch/CfgProperties.h =================================================================== diff -u -N -r75318f0d3808d8d3c02dbc333c80b6d6e07fae13 -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/ch/CfgProperties.h (.../CfgProperties.h) (revision 75318f0d3808d8d3c02dbc333c80b6d6e07fae13) +++ src/ch/CfgProperties.h (.../CfgProperties.h) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -230,8 +230,6 @@ ADAPT_TASK_PROPERTY(PP_BFLAN, chcore::eTO_LANBufferSize); ADAPT_TASK_PROPERTY(PP_BFUSENOBUFFERING, chcore::eTO_DisableBuffering); ADAPT_TASK_PROPERTY(PP_BFBOUNDARYLIMIT, chcore::eTO_DisableBufferingMinSize); -ADAPT_TASK_PROPERTY(PP_BUFFERCHUNKSIZE, chcore::eTO_BufferChunkSize); -ADAPT_TASK_PROPERTY(PP_BUFFERPAGESIZE, chcore::eTO_BufferPageSize); ADAPT_TASK_PROPERTY(PP_CMSETDESTATTRIBUTES, chcore::eTO_SetDestinationAttributes); ADAPT_TASK_PROPERTY(PP_CMSETDESTDATE, chcore::eTO_SetDestinationDateTime); Index: src/ch/ch.vc120.vcxproj =================================================================== diff -u -N -r671f4b1792a20d98b186f4e0a9cc6a620dede019 -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/ch/ch.vc120.vcxproj (.../ch.vc120.vcxproj) (revision 671f4b1792a20d98b186f4e0a9cc6a620dede019) +++ src/ch/ch.vc120.vcxproj (.../ch.vc120.vcxproj) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -124,15 +124,15 @@ $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ true - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ true - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ @@ -141,8 +141,8 @@ $(ProjectName)64 false NativeRecommendedRules.ruleset - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ @@ -151,38 +151,38 @@ $(ProjectName)64 false NativeRecommendedRules.ruleset - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ false - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ false - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ false $(ProjectName)64 - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ false $(ProjectName)64 - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) Index: src/chext/chext.vc120.vcxproj =================================================================== diff -u -N -r122f5f17f602b989c626ab31bb7f32c4211f93bd -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/chext/chext.vc120.vcxproj (.../chext.vc120.vcxproj) (revision 122f5f17f602b989c626ab31bb7f32c4211f93bd) +++ src/chext/chext.vc120.vcxproj (.../chext.vc120.vcxproj) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -129,16 +129,16 @@ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ true true - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ true true - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ @@ -148,8 +148,8 @@ NativeRecommendedRules.ruleset false $(ProjectName)64 - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ @@ -159,42 +159,42 @@ NativeRecommendedRules.ruleset false $(ProjectName)64 - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ true false - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ true false - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ true false $(ProjectName)64 - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ true false $(ProjectName)64 - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) Index: src/libchcore/ErrorCodes.h =================================================================== diff -u -N -r2e384de25de613cb582a966df7d1cb9468f1c825 -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/libchcore/ErrorCodes.h (.../ErrorCodes.h) (revision 2e384de25de613cb582a966df7d1cb9468f1c825) +++ src/libchcore/ErrorCodes.h (.../ErrorCodes.h) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -76,6 +76,8 @@ eErr_FixedDriveWithoutDriveLetter = 3000, eErr_CannotGetFileInfo = 3001, eErr_CannotDeleteFile = 3002, + eErr_CannotReadFile = 3003, + eErr_CannotWriteFile = 3004, // Task handling errors (4000+) eErr_MissingTaskSerializationPath = 4000, Index: src/libchcore/IOverlappedDataBufferQueue.h =================================================================== diff -u -N --- src/libchcore/IOverlappedDataBufferQueue.h (revision 0) +++ src/libchcore/IOverlappedDataBufferQueue.h (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -0,0 +1,46 @@ +// ============================================================================ +// Copyright (C) 2001-2015 by Jozef Starosczyk +// ixen@copyhandler.com +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU Library General Public License +// (version 2) as published by the Free Software Foundation; +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this program; if not, write to the +// Free Software Foundation, Inc., +// 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// ============================================================================ +#ifndef __IOVERLAPPEDDATABUFFERQUEUE_H__ +#define __IOVERLAPPEDDATABUFFERQUEUE_H__ + +#include "libchcore.h" + +BEGIN_CHCORE_NAMESPACE + +class TOverlappedDataBuffer; + +class IOverlappedDataBufferQueue +{ +public: + virtual ~IOverlappedDataBufferQueue(); + + // buffer management + virtual void AddEmptyBuffer(TOverlappedDataBuffer* pBuffer) = 0; + virtual TOverlappedDataBuffer* GetEmptyBuffer() = 0; + + virtual void AddFullBuffer(TOverlappedDataBuffer* pBuffer) = 0; + virtual TOverlappedDataBuffer* GetFullBuffer() = 0; + + virtual void AddFinishedBuffer(TOverlappedDataBuffer* pBuffer) = 0; + virtual TOverlappedDataBuffer* GetFinishedBuffer() = 0; +}; + +END_CHCORE_NAMESPACE + +#endif Index: src/libchcore/RoundingFunctions.h =================================================================== diff -u -N --- src/libchcore/RoundingFunctions.h (revision 0) +++ src/libchcore/RoundingFunctions.h (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -0,0 +1,30 @@ +// ============================================================================ +// Copyright (C) 2001-2015 by Jozef Starosczyk +// ixen@copyhandler.com +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU Library General Public License +// (version 2) as published by the Free Software Foundation; +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this program; if not, write to the +// Free Software Foundation, Inc., +// 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// ============================================================================ +#ifndef __ROUNDINGFUNCTIONS_H__ +#define __ROUNDINGFUNCTIONS_H__ + +#include "libchcore.h" + +BEGIN_CHCORE_NAMESPACE + +template T RoundUp(T number, T roundValue) { return ((number + roundValue - 1) & ~(roundValue - 1)); } + +END_CHCORE_NAMESPACE + +#endif Index: src/libchcore/TDataBuffer.cpp =================================================================== diff -u -N --- src/libchcore/TDataBuffer.cpp (revision 548382442cbf7bed7f744b279ce3f66b54992724) +++ src/libchcore/TDataBuffer.cpp (revision 0) @@ -1,632 +0,0 @@ -// ============================================================================ -// Copyright (C) 2001-2012 by Jozef Starosczyk -// ixen@copyhandler.com -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU Library General Public License -// (version 2) as published by the Free Software Foundation; -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU Library General Public -// License along with this program; if not, write to the -// Free Software Foundation, Inc., -// 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -// ============================================================================ -/// @file TDataBuffer.cpp -/// @date 2012/03/04 -/// @brief Contains class representing buffer for data. -// ============================================================================ -#include "stdafx.h" -#include "TDataBuffer.h" -#include -#include "TCoreException.h" -#include "ErrorCodes.h" - -BEGIN_CHCORE_NAMESPACE - -namespace details -{ - TVirtualAllocMemoryBlock::TVirtualAllocMemoryBlock(size_t stSize, size_t stChunkSize) : - m_pMemory(NULL), - m_stMemorySize(0), - m_stChunkSize(0) - { - AllocBlock(stSize, stChunkSize); - } - - TVirtualAllocMemoryBlock::~TVirtualAllocMemoryBlock() - { - try - { - FreeBlock(); - } - catch(...) - { - } - } - - void TVirtualAllocMemoryBlock::GetFreeChunks(std::list& rListChunks) - { - rListChunks.insert(rListChunks.end(), m_setFreeChunks.begin(), m_setFreeChunks.end()); - m_setFreeChunks.clear(); - } - - void TVirtualAllocMemoryBlock::ReleaseChunks(std::list& rListChunks) - { - std::list::iterator iterList = rListChunks.begin(); - while(iterList != rListChunks.end()) - { - if(ReleaseChunk(*iterList)) - iterList = rListChunks.erase(iterList); - else - ++iterList; - } - } - - bool TVirtualAllocMemoryBlock::ReleaseChunk(LPVOID pChunk) - { - if(IsValidChunk(pChunk)) - { - m_setFreeChunks.insert(pChunk); - return true; - } - return false; - } - - size_t TVirtualAllocMemoryBlock::CountOwnChunks(const std::list& rListChunks) - { - std::set setChunks; - for(std::list::const_iterator iterList = rListChunks.begin(); iterList != rListChunks.end(); ++iterList) - { - if(IsValidChunk(*iterList)) - setChunks.insert(*iterList); - } - - // include chunks already owned - return setChunks.size() + m_setFreeChunks.size(); - } - - bool TVirtualAllocMemoryBlock::IsChunkOwner(LPVOID pChunk) const - { - LPVOID pMemoryEnd = (BYTE*)m_pMemory + m_stMemorySize; - return(pChunk >= m_pMemory && pChunk < pMemoryEnd); - } - - bool TVirtualAllocMemoryBlock::AreAllChunksFree() const - { - if(m_stChunkSize == 0) - return true; - return m_setFreeChunks.size() == m_stMemorySize / m_stChunkSize; - } - - bool TVirtualAllocMemoryBlock::HasFreeChunks() const - { - return !m_setFreeChunks.empty(); - } - - void TVirtualAllocMemoryBlock::AllocBlock(size_t stSize, size_t stChunkSize) - { - FreeBlock(); - - // allocate - LPVOID pBuffer = VirtualAlloc(NULL, stSize, MEM_COMMIT, PAGE_READWRITE); - if(!pBuffer) - THROW_CORE_EXCEPTION(eErr_CannotAllocateMemory); - - m_pMemory = pBuffer; - m_stMemorySize = stSize; - m_stChunkSize = stChunkSize; - - // slice the page to buffers - size_t stSliceCount = m_stMemorySize / m_stChunkSize; - for(size_t stIndex = 0; stIndex < stSliceCount; ++stIndex) - { - LPVOID pSimpleBuffer = (BYTE*)pBuffer + stIndex * stChunkSize; - m_setFreeChunks.insert(pSimpleBuffer); - } - } - - void TVirtualAllocMemoryBlock::FreeBlock() - { - if(m_pMemory) - { - VirtualFree(m_pMemory, 0, MEM_RELEASE); - m_stMemorySize = 0; - m_stChunkSize = 0; - } - } - - bool TVirtualAllocMemoryBlock::IsValidChunk(LPVOID pChunk) const - { - if(IsChunkOwner(pChunk)) - { - bool bValidPtr = (((BYTE*)pChunk - (BYTE*)m_pMemory) % m_stChunkSize) == 0; - _ASSERTE(bValidPtr); - return bValidPtr; - } - else - return false; - - } -} - -/////////////////////////////////////////////////////////////////////////////////// -// class TSimpleDataBuffer - -TSimpleDataBuffer::TSimpleDataBuffer() : - m_pBuffer(NULL), - m_pBufferManager(NULL), - m_stBufferSize(0), - m_stDataSize(0) -{ -} - -TSimpleDataBuffer::~TSimpleDataBuffer() -{ - ReleaseBuffer(); -} - -LPVOID TSimpleDataBuffer::GetBufferPtr() -{ - return m_pBuffer; -} - -void TSimpleDataBuffer::ReleaseBuffer() -{ - if(m_pBufferManager && m_pBuffer) - m_pBufferManager->ReleaseBuffer(*this); -} - -void TSimpleDataBuffer::Initialize(TDataBufferManager& rBufferManager, LPVOID pBuffer, size_t stBufferSize) -{ - ReleaseBuffer(); - - m_pBufferManager = &rBufferManager; - m_pBuffer = pBuffer; - m_stBufferSize = stBufferSize; -} - -void TSimpleDataBuffer::SetDataSize(size_t stDataSize) -{ - if(stDataSize != 0 && (stDataSize > m_stBufferSize || !m_pBuffer)) - THROW_CORE_EXCEPTION(eErr_InvalidArgument); - - m_stDataSize = stDataSize; -} - -void TSimpleDataBuffer::CutDataFromBuffer(size_t stCount) -{ - if(stCount >= m_stBufferSize || !m_pBuffer) - return; // nothing to do - - memmove(m_pBuffer, (BYTE*)m_pBuffer + stCount, m_stBufferSize - stCount); -} - -/////////////////////////////////////////////////////////////////////////////////// -// class TDataBufferManager - -TDataBufferManager::TDataBufferManager() : - m_stMaxMemory(0), - m_stPageSize(0), - m_stBufferSize(0), - m_stAllocBlocksToFree(0) -{ -} - -TDataBufferManager::~TDataBufferManager() -{ - try - { - FreeAllAllocBlocks(); - } - catch(...) - { - } -} - -bool TDataBufferManager::CheckBufferConfig(size_t& stMaxMemory, size_t& stPageSize, size_t& stBufferSize) -{ - bool bResult = true; - - // first the user-facing buffer size - if(stBufferSize == 0) - { - stBufferSize = DefaultBufferSize; - bResult = false; - } - else - { - size_t stNewSize = RoundUp(stBufferSize, DefaultAllocGranularity); - if(stBufferSize != stNewSize) - { - stBufferSize = stNewSize; - bResult = false; - } - } - - // now the page size - if(stPageSize == 0) - { - stPageSize = std::max(DefaultPageSize, RoundUp(DefaultPageSize, stBufferSize)); - bResult = false; - } - else - { - size_t stNewSize = RoundUp(stPageSize, stBufferSize); - if(stPageSize != stNewSize) - { - stPageSize = stNewSize; - bResult = false; - } - } - - if(stMaxMemory == 0) - { - stMaxMemory = std::max(DefaultMaxMemory, RoundUp(DefaultMaxMemory, stPageSize)); - bResult = false; - } - else - { - size_t stNewSize = RoundUp(stMaxMemory, stPageSize); - if(stMaxMemory != stNewSize) - { - stMaxMemory = stNewSize; - bResult = false; - } - } - - return bResult; -} - -bool TDataBufferManager::CheckBufferConfig(size_t& stMaxMemory) -{ - size_t stDefaultPageSize = DefaultPageSize; - size_t stDefaultBufferSize = DefaultBufferSize; - return CheckBufferConfig(stMaxMemory, stDefaultPageSize, stDefaultBufferSize); -} - -void TDataBufferManager::Initialize(size_t stMaxMemory) -{ - Initialize(stMaxMemory, DefaultPageSize, DefaultBufferSize); -} - -void TDataBufferManager::Initialize(size_t stMaxMemory, size_t stPageSize, size_t stBufferSize) -{ - FreeAllAllocBlocks(); - - // validate input (note that input parameters should already be checked by caller) - if(!CheckBufferConfig(stMaxMemory, stPageSize, stBufferSize)) - THROW_CORE_EXCEPTION(eErr_InvalidArgument); - - m_stMaxMemory = stMaxMemory; - m_stPageSize = stPageSize; - m_stBufferSize = stBufferSize; - - // allocate - if(!AllocNewPage()) - THROW_CORE_EXCEPTION(eErr_CannotAllocateMemory); -} - -bool TDataBufferManager::IsInitialized() const -{ - if(m_stPageSize == 0 || m_stMaxMemory == 0 || m_stBufferSize == 0) - return false; - return true; -} - -bool TDataBufferManager::CheckResizeSize(size_t& stNewMaxSize) -{ - if(!IsInitialized()) - { - stNewMaxSize = 0; - return false; - } - - size_t stPageSize = m_stPageSize; - size_t stBufferSize = m_stBufferSize; - - bool bRes = CheckBufferConfig(stNewMaxSize, stPageSize, stBufferSize); - // make sure the page size and buffer size are unchanged after the call - _ASSERTE(stPageSize == m_stPageSize && stBufferSize == m_stBufferSize); - if(stPageSize != m_stPageSize || stBufferSize != m_stBufferSize) - THROW_CORE_EXCEPTION(eErr_InternalProblem); - - return bRes; -} - -void TDataBufferManager::ChangeMaxMemorySize(size_t stNewMaxSize) -{ - if(!CheckResizeSize(stNewMaxSize)) - THROW_CORE_EXCEPTION(eErr_InvalidArgument); - - if(stNewMaxSize >= m_stMaxMemory) - m_stMaxMemory = stNewMaxSize; - else - { - size_t stCurrentMaxPages = m_stMaxMemory / m_stPageSize; - size_t stNewMaxPages = stNewMaxSize / m_stPageSize; - size_t stPagesToFree = stCurrentMaxPages - stNewMaxPages; - size_t stPagesStillUnallocated = stCurrentMaxPages - m_vVirtualAllocBlocks.size(); - - // first free the memory that has not been allocated yet - if(stPagesStillUnallocated != 0 && stPagesToFree != 0) - { - size_t stUnallocatedPagesToFree = std::min(stPagesStillUnallocated, stPagesToFree); - m_stMaxMemory -= stUnallocatedPagesToFree * m_stPageSize; - stPagesToFree -= stUnallocatedPagesToFree; - } - - // is there still too much memory that needs to be freed? - if(stPagesToFree != 0) - { - // free pages that are already allocated - FreeAllocatedPages(stPagesToFree); - } - } -} - -size_t TDataBufferManager::GetRealAllocatedMemorySize() const -{ - return m_stPageSize * m_vVirtualAllocBlocks.size(); -} - -bool TDataBufferManager::HasFreeBuffer() const -{ - return !m_listUnusedBuffers.empty() || CanAllocPage(); -} - -size_t TDataBufferManager::GetCountOfFreeBuffers() const -{ - if(!IsInitialized()) - return 0; - - size_t stActivePages = m_vVirtualAllocBlocks.size() - m_stAllocBlocksToFree; - - // count of unallocated pages - size_t stCurrentMaxPages = m_stMaxMemory / m_stPageSize; - size_t stPagesStillUnallocated = stCurrentMaxPages - stActivePages; - - return m_listUnusedBuffers.size() + stPagesStillUnallocated * m_stPageSize / m_stBufferSize; -} - -bool TDataBufferManager::HasFreeBufferNA() const -{ - return !m_listUnusedBuffers.empty(); -} - -size_t TDataBufferManager::GetCountOfFreeBuffersNA() const -{ - return m_listUnusedBuffers.size(); -} - -bool TDataBufferManager::CanAllocPage() const -{ - if(!IsInitialized()) - return false; - - size_t stActivePages = m_vVirtualAllocBlocks.size() - m_stAllocBlocksToFree; - size_t stMaxPages = m_stMaxMemory / m_stPageSize; - return stActivePages < stMaxPages; -} - -bool TDataBufferManager::AllocNewPage() -{ - if(!CanAllocPage()) - return false; - - // re-use the old block if possible - if(m_stAllocBlocksToFree != 0) - { - for(MemoryBlocksVector::iterator iterMem = m_vVirtualAllocBlocks.begin(); iterMem != m_vVirtualAllocBlocks.end(); ++iterMem) - { - if((*iterMem).second == eBlock_ToFree && (*iterMem).first->HasFreeChunks()) - { - (*iterMem).second = eBlock_Active; - --m_stAllocBlocksToFree; - (*iterMem).first->GetFreeChunks(m_listUnusedBuffers); - - return true; - } - } - } - - // alloc new block if can't re-use the old one - details::TVirtualAllocMemoryBlockPtr spAllocBlock(new details::TVirtualAllocMemoryBlock(m_stPageSize, m_stBufferSize)); - m_vVirtualAllocBlocks.push_back(std::make_pair(spAllocBlock, eBlock_Active)); - spAllocBlock->GetFreeChunks(m_listUnusedBuffers); - - return true; -} - -void TDataBufferManager::FreeAllocatedPages(size_t stPagesCount) -{ - if(stPagesCount == 0) - return; - - std::vector > vFreeBuffers; - for(MemoryBlocksVector::iterator iterAllocBlock = m_vVirtualAllocBlocks.begin(); - iterAllocBlock != m_vVirtualAllocBlocks.end() && (*iterAllocBlock).second == eBlock_Active; - ++iterAllocBlock) - { - vFreeBuffers.push_back(std::make_pair((*iterAllocBlock).first, (*iterAllocBlock).first->CountOwnChunks(m_listUnusedBuffers))); - } - - // sort by the count of free blocks - std::sort(vFreeBuffers.begin(), vFreeBuffers.end(), - boost::bind(&std::pair::second, _1) > boost::bind(&std::pair::second, _2)); - - // and free pages with the most free blocks inside - size_t stPagesToProcess = std::min(stPagesCount, vFreeBuffers.size()); - for(size_t stIndex = 0; stIndex < stPagesToProcess; ++stIndex) - { - FreePage(vFreeBuffers[stIndex].first); - m_stMaxMemory -= m_stPageSize; - } -} - -// function expects arrays to be sorted -bool TDataBufferManager::FreePage(const details::TVirtualAllocMemoryBlockPtr& spAllocBlock) -{ - // locate the entry - MemoryBlocksVector::iterator iterBlock = std::find_if(m_vVirtualAllocBlocks.begin(), m_vVirtualAllocBlocks.end(), - boost::bind(&std::pair::first, _1) == spAllocBlock); - - if(iterBlock == m_vVirtualAllocBlocks.end()) - THROW_CORE_EXCEPTION(eErr_InvalidArgument); - - // remove the entries from unused buffers - spAllocBlock->ReleaseChunks(m_listUnusedBuffers); - - // if some buffers are still in use - mark the whole block as additional cleaning needed - if(!spAllocBlock->AreAllChunksFree()) - { - if((*iterBlock).second == eBlock_Active) - { - (*iterBlock).second = eBlock_ToFree; - ++m_stAllocBlocksToFree; - } - } - else - { - if((*iterBlock).second == eBlock_ToFree) - --m_stAllocBlocksToFree; - m_vVirtualAllocBlocks.erase(iterBlock); - return true; // erased - } - - return false; // not erased -} - -void TDataBufferManager::ReclaimPage(const details::TVirtualAllocMemoryBlockPtr& spAllocBlock) -{ - // locate the entry - MemoryBlocksVector::iterator iterBlock = std::find_if(m_vVirtualAllocBlocks.begin(), m_vVirtualAllocBlocks.end(), - boost::bind(&std::pair::first, _1) == spAllocBlock); - - if(iterBlock == m_vVirtualAllocBlocks.end()) - THROW_CORE_EXCEPTION(eErr_InvalidArgument); - - // remove the entries from unused buffers - if((*iterBlock).second == eBlock_ToFree) - { - spAllocBlock->GetFreeChunks(m_listUnusedBuffers); - (*iterBlock).second = eBlock_Active; - --m_stAllocBlocksToFree; - } -} - -bool TDataBufferManager::GetFreeBuffer(TSimpleDataBuffer& rSimpleBuffer) -{ - if(m_listUnusedBuffers.empty()) - { - // try to alloc new page; we won't get one if max memory would be exceeded or allocation failed - // this one also populates the buffers list - if(CanAllocPage()) - { - if(!AllocNewPage()) - THROW_CORE_EXCEPTION(eErr_CannotAllocateMemory); - } - } - - if(!m_listUnusedBuffers.empty()) - { - LPVOID pBuffer = m_listUnusedBuffers.front(); - m_listUnusedBuffers.pop_front(); - rSimpleBuffer.Initialize(*this, pBuffer, m_stBufferSize); - return true; - } - - return false; -} - -void TDataBufferManager::ReleaseBuffer(TSimpleDataBuffer& rSimpleBuffer) -{ - if(rSimpleBuffer.m_pBuffer) - { - if(m_stAllocBlocksToFree != 0) - { - // return the buffer to the rightful owner in case we're trying to reduce memory footprint - for(MemoryBlocksVector::iterator iterBlock = m_vVirtualAllocBlocks.begin(); iterBlock != m_vVirtualAllocBlocks.end() && (*iterBlock).second == eBlock_ToFree; ++iterBlock) - { - const details::TVirtualAllocMemoryBlockPtr& spAllocBlock = (*iterBlock).first; - if(spAllocBlock->IsChunkOwner(rSimpleBuffer.m_pBuffer)) - { - spAllocBlock->ReleaseChunk(rSimpleBuffer.m_pBuffer); - if(spAllocBlock->AreAllChunksFree()) - { - --m_stAllocBlocksToFree; - m_vVirtualAllocBlocks.erase(iterBlock); - } - - return; - } - } - - // at this point we know that the buffer belongs to an active page - // and at the same time we have some pages still to be freed - m_listUnusedBuffers.push_back(rSimpleBuffer.m_pBuffer); - ReorganizePages(); - return; - } - - m_listUnusedBuffers.push_back(rSimpleBuffer.m_pBuffer); - } -} - -void TDataBufferManager::ReorganizePages() -{ - // prepare sorted pages - std::vector > vFreeBuffers; - for(MemoryBlocksVector::iterator iterAllocBlock = m_vVirtualAllocBlocks.begin(); - iterAllocBlock != m_vVirtualAllocBlocks.end(); - ++iterAllocBlock) - { - vFreeBuffers.push_back(std::make_pair((*iterAllocBlock).first, (*iterAllocBlock).first->CountOwnChunks(m_listUnusedBuffers))); - } - - // sort by the count of free blocks - std::sort(vFreeBuffers.begin(), vFreeBuffers.end(), - boost::bind(&std::pair::second, _1) > boost::bind(&std::pair::second, _2)); - - // and free pages with the most free blocks inside - size_t stPagesToFree = std::min(m_stAllocBlocksToFree, vFreeBuffers.size()); - - size_t stIndex = 0; - for(; stIndex < stPagesToFree; ++stIndex) - { - FreePage(vFreeBuffers[stIndex].first); - } - - // activate some other pages - size_t stPagesToActivate = std::min(stIndex + m_stAllocBlocksToFree, vFreeBuffers.size()); - for(; stIndex < stPagesToActivate; ++stIndex) - { - ReclaimPage(vFreeBuffers[stIndex].first); - } -} - -void TDataBufferManager::FreeAllAllocBlocks() -{ - for(MemoryBlocksVector::iterator iterAllocBlock = m_vVirtualAllocBlocks.begin(); iterAllocBlock != m_vVirtualAllocBlocks.end(); ++iterAllocBlock) - { - (*iterAllocBlock).first->ReleaseChunks(m_listUnusedBuffers); - _ASSERTE((*iterAllocBlock).first->AreAllChunksFree()); // without throwing on this condition, because there might be a situation that - // some hanged thread did not release the buffer - } - - _ASSERTE(m_listUnusedBuffers.empty()); // and all buffers should be returned to the pool by the caller - if(!m_listUnusedBuffers.empty()) - THROW_CORE_EXCEPTION(eErr_InternalProblem); - - m_vVirtualAllocBlocks.clear(); - - m_stBufferSize = 0; - m_stPageSize = 0; - m_stMaxMemory = 0; -} - -END_CHCORE_NAMESPACE Index: src/libchcore/TDataBuffer.h =================================================================== diff -u -N --- src/libchcore/TDataBuffer.h (revision 2cc200f4b40dedfdebafab18bf07c733be47da8a) +++ src/libchcore/TDataBuffer.h (revision 0) @@ -1,184 +0,0 @@ -// ============================================================================ -// Copyright (C) 2001-2012 by Jozef Starosczyk -// ixen@copyhandler.com -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU Library General Public License -// (version 2) as published by the Free Software Foundation; -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU Library General Public -// License along with this program; if not, write to the -// Free Software Foundation, Inc., -// 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -// ============================================================================ -/// @file TDataBuffer.h -/// @date 2012/03/04 -/// @brief Contains class representing buffer for data. -// ============================================================================ -#ifndef __TDATABUFFER_H__ -#define __TDATABUFFER_H__ - -#include "libchcore.h" - -BEGIN_CHCORE_NAMESPACE - -namespace details -{ - class TVirtualAllocMemoryBlock - { - public: - TVirtualAllocMemoryBlock(size_t stSize, size_t stChunkSize); - ~TVirtualAllocMemoryBlock(); - - void GetFreeChunks(std::list& rListChunks); - void ReleaseChunks(std::list& rListChunks); - bool ReleaseChunk(LPVOID pChunk); - - size_t CountOwnChunks(const std::list& rListChunks); - - bool IsChunkOwner(LPVOID pChunk) const; - bool AreAllChunksFree() const; - bool HasFreeChunks() const; - - private: - TVirtualAllocMemoryBlock(const TVirtualAllocMemoryBlock&); - TVirtualAllocMemoryBlock& operator=(const TVirtualAllocMemoryBlock&); - - void AllocBlock(size_t stSize, size_t stChunkSize); - void FreeBlock(); - - bool IsValidChunk(LPVOID pChunk) const; - - private: - LPVOID m_pMemory; - std::set m_setFreeChunks; - size_t m_stMemorySize; - size_t m_stChunkSize; - }; - - typedef boost::shared_ptr TVirtualAllocMemoryBlockPtr; -} - -template T RoundUp(T number, T roundValue) { return ((number + roundValue - 1) & ~(roundValue - 1)); } - -class TDataBufferManager; - -class LIBCHCORE_API TSimpleDataBuffer -{ -public: - TSimpleDataBuffer(); - ~TSimpleDataBuffer(); - - LPVOID GetBufferPtr(); - void ReleaseBuffer(); - - void SetDataSize(size_t stDataSize); - size_t GetDataSize() const { return m_stDataSize; } - size_t GetBufferSize() const { return m_stBufferSize; } - - void CutDataFromBuffer(size_t stCount); - -private: - TSimpleDataBuffer(const TSimpleDataBuffer&); - TSimpleDataBuffer& operator=(const TSimpleDataBuffer&); - - void Initialize(TDataBufferManager& rBufferManager, LPVOID pBuffer, size_t stBufferSize); - -private: - LPVOID m_pBuffer; - TDataBufferManager* m_pBufferManager; - size_t m_stBufferSize; - size_t m_stDataSize; - - friend class TDataBufferManager; -}; - -typedef boost::shared_ptr TSimpleDataBufferPtr; - -class LIBCHCORE_API TDataBufferManager -{ -public: - static const size_t DefaultAllocGranularity = 4096; - static const size_t DefaultBufferSize = 65536; - static const size_t DefaultPageSize = 1024*1024; - static const size_t DefaultMaxMemory = 1024*1024; - -public: - TDataBufferManager(); - ~TDataBufferManager(); - - // buffer size verification functions - should be called prior to initializing object - // to sanitize input - static bool CheckBufferConfig(size_t& stMaxMemory, size_t& stPageSize, size_t& stBufferSize); - static bool CheckBufferConfig(size_t& stMaxMemory); - - // initialization - void Initialize(size_t stMaxMemory); - void Initialize(size_t stMaxMemory, size_t stPageSize, size_t stBufferSize); - bool IsInitialized() const; - - // buffer resizing - bool CheckResizeSize(size_t& stNewMaxSize); - void ChangeMaxMemorySize(size_t stNewMaxSize); - - // current settings - size_t GetMaxMemorySize() const { return m_stMaxMemory; } - size_t GetPageSize() const { return m_stPageSize; } - size_t GetSimpleBufferSize() const { return m_stBufferSize; } - - size_t GetRealAllocatedMemorySize() const; - - // buffer info (regardless of the page allocation status) - bool HasFreeBuffer() const; - size_t GetCountOfFreeBuffers() const; - - // buffer info (without allocating additional pages) - bool HasFreeBufferNA() const; // checks if a buffer is available without allocating any new memory - size_t GetCountOfFreeBuffersNA() const; // how many free buffers are there that can be used without allocating additional memory - - // buffer retrieval - bool GetFreeBuffer(TSimpleDataBuffer& rSimpleBuffer); - void ReleaseBuffer(TSimpleDataBuffer& rSimpleBuffer); - -private: - void FreeAllAllocBlocks(); - - // page handling - bool CanAllocPage() const; // checks if a buffer can be returned after allocating new page of memory - bool AllocNewPage(); - - void FreeAllocatedPages(size_t stPagesCount); - bool FreePage(const details::TVirtualAllocMemoryBlockPtr& spAllocBlock); - void ReclaimPage(const details::TVirtualAllocMemoryBlockPtr& spAllocBlock); - void ReorganizePages(); - -private: - enum EBlockState - { - eBlock_Active, - eBlock_ToFree - }; - - typedef std::vector > MemoryBlocksVector; - -#pragma warning(push) -#pragma warning(disable: 4251) - MemoryBlocksVector m_vVirtualAllocBlocks; - std::list m_listUnusedBuffers; -#pragma warning(pop) - - size_t m_stAllocBlocksToFree; - - size_t m_stMaxMemory; // maximum amount of memory to use - size_t m_stPageSize; // size of a single page of real memory to be allocated (allocation granularity) - size_t m_stBufferSize; // size of a single chunk of memory retrievable by caller -}; - -END_CHCORE_NAMESPACE - -#endif Index: src/libchcore/TEvent.cpp =================================================================== diff -u -N --- src/libchcore/TEvent.cpp (revision 0) +++ src/libchcore/TEvent.cpp (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -0,0 +1,38 @@ +// ============================================================================ +// Copyright (C) 2001-2015 by Jozef Starosczyk +// ixen@copyhandler.com +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU Library General Public License +// (version 2) as published by the Free Software Foundation; +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this program; if not, write to the +// Free Software Foundation, Inc., +// 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// ============================================================================ +#include "stdafx.h" +#include "TEvent.h" +#include "TCoreException.h" +#include "ErrorCodes.h" + +BEGIN_CHCORE_NAMESPACE + +TEvent::TEvent(bool bManualReset, bool bInitialState) +{ + m_hEvent = CreateEvent(NULL, bManualReset, bInitialState, NULL); + if (m_hEvent == NULL) + THROW_CORE_EXCEPTION(eErr_CannotCreateEvent); +} + +TEvent::~TEvent() +{ + CloseHandle(m_hEvent); +} + +END_CHCORE_NAMESPACE Index: src/libchcore/TEvent.h =================================================================== diff -u -N --- src/libchcore/TEvent.h (revision 0) +++ src/libchcore/TEvent.h (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -0,0 +1,44 @@ +// ============================================================================ +// Copyright (C) 2001-2015 by Jozef Starosczyk +// ixen@copyhandler.com +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU Library General Public License +// (version 2) as published by the Free Software Foundation; +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this program; if not, write to the +// Free Software Foundation, Inc., +// 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// ============================================================================ +#ifndef __TEVENT_H__ +#define __TEVENT_H__ + +#include "libchcore.h" + +BEGIN_CHCORE_NAMESPACE + +class LIBCHCORE_API TEvent +{ +public: + TEvent(bool bManualReset, bool bInitialState); + virtual ~TEvent(); + + HANDLE Get() const { return m_hEvent; } + void SetEvent() { ::SetEvent(m_hEvent); } + void ResetEvent() { ::ResetEvent(m_hEvent); } + + HANDLE Handle() const { return m_hEvent; } + +private: + HANDLE m_hEvent; +}; + +END_CHCORE_NAMESPACE + +#endif Index: src/libchcore/TLocalFilesystem.cpp =================================================================== diff -u -N -r16df8fcf9d5b3317338aece64762771419beaf4a -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/libchcore/TLocalFilesystem.cpp (.../TLocalFilesystem.cpp) (revision 16df8fcf9d5b3317338aece64762771419beaf4a) +++ src/libchcore/TLocalFilesystem.cpp (.../TLocalFilesystem.cpp) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -23,6 +23,7 @@ #include "stdafx.h" #include "TLocalFilesystem.h" #include +#include #include "TAutoHandles.h" #include "TFileInfo.h" #include "DataBuffer.h" @@ -33,11 +34,12 @@ #pragma warning(disable: 4201) #include #pragma warning(pop) -#include "TDataBuffer.h" #include "TCoreException.h" #include "ErrorCodes.h" #include "TPathContainer.h" #include "TFileTime.h" +#include "TOverlappedDataBuffer.h" +#include "RoundingFunctions.h" BEGIN_CHCORE_NAMESPACE @@ -376,7 +378,8 @@ TLocalFilesystemFile::TLocalFilesystemFile() : m_hFile(INVALID_HANDLE_VALUE), - m_pathFile() + m_pathFile(), + m_bNoBuffering(false) { } @@ -389,29 +392,38 @@ { Close(); - m_hFile = ::CreateFile(TLocalFilesystem::PrependPathExtensionIfNeeded(pathFile).ToString(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN | (bNoBuffering ? FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH : 0), NULL); + m_pathFile = TLocalFilesystem::PrependPathExtensionIfNeeded(pathFile); + m_hFile = ::CreateFile(m_pathFile.ToString(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN | (bNoBuffering ? FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH : 0), NULL); if(m_hFile == INVALID_HANDLE_VALUE) return false; + + m_bNoBuffering = bNoBuffering; return true; } bool TLocalFilesystemFile::CreateNewForWriting(const TSmartPath& pathFile, bool bNoBuffering) { Close(); - m_hFile = ::CreateFile(TLocalFilesystem::PrependPathExtensionIfNeeded(pathFile).ToString(), GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_NEW, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN | (bNoBuffering ? FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH : 0), NULL); + m_pathFile = TLocalFilesystem::PrependPathExtensionIfNeeded(pathFile); + m_hFile = ::CreateFile(m_pathFile.ToString(), GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_NEW, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN | (bNoBuffering ? FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH : 0), NULL); if(m_hFile == INVALID_HANDLE_VALUE) return false; + + m_bNoBuffering = bNoBuffering; return true; } bool TLocalFilesystemFile::OpenExistingForWriting(const TSmartPath& pathFile, bool bNoBuffering) { Close(); - m_hFile = CreateFile(TLocalFilesystem::PrependPathExtensionIfNeeded(pathFile).ToString(), GENERIC_WRITE, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN | (bNoBuffering ? FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH : 0), NULL); + m_pathFile = TLocalFilesystem::PrependPathExtensionIfNeeded(pathFile); + m_hFile = CreateFile(m_pathFile.ToString(), GENERIC_WRITE, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN | (bNoBuffering ? FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH : 0), NULL); if(m_hFile == INVALID_HANDLE_VALUE) return false; + + m_bNoBuffering = bNoBuffering; return true; } @@ -436,20 +448,69 @@ return ::SetEndOfFile(m_hFile) != FALSE; } -bool TLocalFilesystemFile::ReadFile(TSimpleDataBuffer& rBuffer, DWORD dwToRead, DWORD& rdwBytesRead) +bool TLocalFilesystemFile::ReadFile(TOverlappedDataBuffer& rBuffer) { - if(!IsOpen()) - return false; + if (!IsOpen()) + THROW_CORE_EXCEPTION(eErr_InternalProblem); - return ::ReadFile(m_hFile, rBuffer.GetBufferPtr(), dwToRead, &rdwBytesRead, NULL) != FALSE; + if (!::ReadFileEx(m_hFile, rBuffer.GetBufferPtr(), rBuffer.GetRequestedDataSize(), &rBuffer, OverlappedReadCompleted)) + { + DWORD dwLastError = GetLastError(); + switch(dwLastError) + { + case ERROR_IO_PENDING: + return true; + + case ERROR_HANDLE_EOF: + { + rBuffer.SetBytesTransferred(0); + rBuffer.SetStatusCode(ERROR_SUCCESS); + rBuffer.SetLastPart(true); + + rBuffer.RequeueAsFull(); // basically the same as OverlappedReadCompleted + + return true; + } + } + + return false; + } + return true; } -bool TLocalFilesystemFile::WriteFile(TSimpleDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten) +bool TLocalFilesystemFile::WriteFile(TOverlappedDataBuffer& rBuffer) { - if(!IsOpen()) + if (!IsOpen()) + THROW_CORE_EXCEPTION(eErr_InternalProblem); + + DWORD dwToWrite = boost::numeric_cast(rBuffer.GetBytesTransferred()); + unsigned long long ullNewFileSize = 0; + + if (m_bNoBuffering && rBuffer.IsLastPart()) + { + dwToWrite = RoundUp(dwToWrite, MaxSectorSize); + if(dwToWrite != boost::numeric_cast(rBuffer.GetBytesTransferred())) + ullNewFileSize = rBuffer.GetFilePosition() + dwToWrite; // new size + } + + if (!::WriteFileEx(m_hFile, rBuffer.GetBufferPtr(), dwToWrite, &rBuffer, OverlappedWriteCompleted)) return false; - return ::WriteFile(m_hFile, rBuffer.GetBufferPtr(), dwToWrite, &rdwBytesWritten, NULL) != NULL && dwToWrite == rdwBytesWritten; + if(ullNewFileSize != 0) + { + if(!OpenExistingForWriting(m_pathFile, false)) + return false; + + //seek + if(!SetFilePointer(ullNewFileSize, FILE_BEGIN)) + return false; + + //set eof + if(!SetEndOfFile()) + return false; + } + + return true; } void TLocalFilesystemFile::Close() Index: src/libchcore/TLocalFilesystem.h =================================================================== diff -u -N -r16df8fcf9d5b3317338aece64762771419beaf4a -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/libchcore/TLocalFilesystem.h (.../TLocalFilesystem.h) (revision 16df8fcf9d5b3317338aece64762771419beaf4a) +++ src/libchcore/TLocalFilesystem.h (.../TLocalFilesystem.h) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -37,6 +37,7 @@ class TLocalFilesystemFile; class TSimpleDataBuffer; class TFileTime; +class TOverlappedDataBuffer; class LIBCHCORE_API TLocalFilesystem { @@ -108,6 +109,9 @@ class LIBCHCORE_API TLocalFilesystemFile { public: + static const unsigned int MaxSectorSize = 4096; + +public: ~TLocalFilesystemFile(); bool OpenExistingForReading(const TSmartPath& pathFile, bool bNoBuffering); @@ -117,8 +121,8 @@ bool SetFilePointer(long long llNewPos, DWORD dwMoveMethod); bool SetEndOfFile(); - bool ReadFile(TSimpleDataBuffer& rBuffer, DWORD dwToRead, DWORD& rdwBytesRead); - bool WriteFile(TSimpleDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten); + bool ReadFile(TOverlappedDataBuffer& rBuffer); + bool WriteFile(TOverlappedDataBuffer& rBuffer); bool IsOpen() const { return m_hFile != INVALID_HANDLE_VALUE; } unsigned long long GetFileSize() const; @@ -131,6 +135,7 @@ private: TSmartPath m_pathFile; HANDLE m_hFile; + bool m_bNoBuffering; friend class TLocalFilesystem; }; Index: src/libchcore/TOverlappedDataBuffer.cpp =================================================================== diff -u -N --- src/libchcore/TOverlappedDataBuffer.cpp (revision 0) +++ src/libchcore/TOverlappedDataBuffer.cpp (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -0,0 +1,121 @@ +// ============================================================================ +// Copyright (C) 2001-2015 by Jozef Starosczyk +// ixen@copyhandler.com +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU Library General Public License +// (version 2) as published by the Free Software Foundation; +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this program; if not, write to the +// Free Software Foundation, Inc., +// 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// ============================================================================ +/// @file TDataBuffer.cpp +/// @date 2012/03/04 +/// @brief Contains class representing buffer for data. +// ============================================================================ +#include "stdafx.h" +#include "TOverlappedDataBuffer.h" +#include +#include "TCoreException.h" +#include "ErrorCodes.h" +#include "IOverlappedDataBufferQueue.h" +#include "RoundingFunctions.h" + +BEGIN_CHCORE_NAMESPACE + +/////////////////////////////////////////////////////////////////////////////////// +// class TOverlappedDataBuffer +VOID CALLBACK OverlappedReadCompleted(DWORD dwErrorCode, DWORD /*dwNumberOfBytesTransfered*/, LPOVERLAPPED lpOverlapped) +{ + TOverlappedDataBuffer* pBuffer = (TOverlappedDataBuffer*) lpOverlapped; + bool bEof = (dwErrorCode == ERROR_HANDLE_EOF || + (dwErrorCode == ERROR_SUCCESS && pBuffer->GetBytesTransferred() != pBuffer->GetRequestedDataSize())); + + pBuffer->SetLastPart(bEof); + pBuffer->RequeueAsFull(); +} + +VOID CALLBACK OverlappedWriteCompleted(DWORD /*dwErrorCode*/, DWORD /*dwNumberOfBytesTransfered*/, LPOVERLAPPED lpOverlapped) +{ + TOverlappedDataBuffer* pBuffer = (TOverlappedDataBuffer*) lpOverlapped; + + pBuffer->RequeueAsFinished(); +} + +TOverlappedDataBuffer::TOverlappedDataBuffer(size_t stBufferSize, IOverlappedDataBufferQueue* pQueue) : + m_pBuffer(NULL), + m_stBufferSize(0), + m_bLastPart(false), + m_pQueue(pQueue), + m_dwRequestedDataSize(0) +{ + if (!m_pQueue) + THROW_CORE_EXCEPTION(eErr_InvalidPointer); + + // initialize OVERLAPPED members + Internal = 0; + InternalHigh = 0; + Offset = 0; + OffsetHigh = 0; + hEvent = NULL; + + // create buffer + ReinitializeBuffer(stBufferSize); +} + +TOverlappedDataBuffer::~TOverlappedDataBuffer() +{ + ReleaseBuffer(); +} + +void chcore::TOverlappedDataBuffer::ReinitializeBuffer(size_t stNewBufferSize) +{ + if (stNewBufferSize > m_stBufferSize) + { + ReleaseBuffer(); + + m_pBuffer = VirtualAlloc(NULL, stNewBufferSize, MEM_COMMIT, PAGE_READWRITE); + if (!m_pBuffer) + THROW_CORE_EXCEPTION(eErr_CannotAllocateMemory); + m_stBufferSize = stNewBufferSize; + } +} + +void TOverlappedDataBuffer::ReleaseBuffer() +{ + if (m_pBuffer) + { + VirtualFree(m_pBuffer, 0, MEM_RELEASE); + m_stBufferSize = 0; + m_pBuffer = nullptr; + } +} + +LPVOID TOverlappedDataBuffer::GetBufferPtr() +{ + return m_pBuffer; +} + +void chcore::TOverlappedDataBuffer::RequeueAsEmpty() +{ + m_pQueue->AddEmptyBuffer(this); +} + +void chcore::TOverlappedDataBuffer::RequeueAsFull() +{ + m_pQueue->AddFullBuffer(this); +} + +void chcore::TOverlappedDataBuffer::RequeueAsFinished() +{ + m_pQueue->AddFinishedBuffer(this); +} + +END_CHCORE_NAMESPACE Index: src/libchcore/TOverlappedDataBuffer.h =================================================================== diff -u -N --- src/libchcore/TOverlappedDataBuffer.h (revision 0) +++ src/libchcore/TOverlappedDataBuffer.h (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -0,0 +1,92 @@ +// ============================================================================ +// Copyright (C) 2001-2012 by Jozef Starosczyk +// ixen@copyhandler.com +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU Library General Public License +// (version 2) as published by the Free Software Foundation; +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this program; if not, write to the +// Free Software Foundation, Inc., +// 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// ============================================================================ +/// @file TDataBuffer.h +/// @date 2012/03/04 +/// @brief Contains class representing buffer for data. +// ============================================================================ +#ifndef __TDATABUFFER_H__ +#define __TDATABUFFER_H__ + +#include "libchcore.h" + +BEGIN_CHCORE_NAMESPACE + +class IOverlappedDataBufferQueue; + +VOID CALLBACK OverlappedReadCompleted(DWORD dwErrorCode, DWORD dwNumberOfBytesTransfered, LPOVERLAPPED lpOverlapped); +VOID CALLBACK OverlappedWriteCompleted(DWORD dwErrorCode, DWORD dwNumberOfBytesTransfered, LPOVERLAPPED lpOverlapped); + +class TOverlappedDataBuffer : public OVERLAPPED +{ +public: + // construction/destruction + TOverlappedDataBuffer(size_t stBufferSize, IOverlappedDataBufferQueue* pQueue); + TOverlappedDataBuffer(const TOverlappedDataBuffer&) = delete; + TOverlappedDataBuffer(TOverlappedDataBuffer&& rSrc) = delete; + + ~TOverlappedDataBuffer(); + + // operators + TOverlappedDataBuffer& operator=(const TOverlappedDataBuffer&) = delete; + TOverlappedDataBuffer& operator=(TOverlappedDataBuffer&& rSrc) = delete; + + // interface methods + void ReinitializeBuffer(size_t stNewBufferSize); + LPVOID GetBufferPtr(); + + size_t GetBufferSize() const { return m_stBufferSize; } + + DWORD GetRequestedDataSize() const { return m_dwRequestedDataSize; } + void SetRequestedDataSize(DWORD val) { m_dwRequestedDataSize = val; } + + void SetLastPart(bool bLastPart) { m_bLastPart = bLastPart; } + bool IsLastPart() const { return m_bLastPart; } + + void RequeueAsEmpty(); + void RequeueAsFull(); + void RequeueAsFinished(); + + // OVERLAPPED interface + ULONG_PTR GetStatusCode() const { return Internal; } + void SetStatusCode(ULONG_PTR ulStatusCode) { Internal = ulStatusCode; } + + void SetBytesTransferred(ULONG_PTR ulBytes) { InternalHigh = ulBytes; } + ULONG_PTR GetBytesTransferred() const { return InternalHigh; } + + unsigned long long GetFilePosition() const { return (unsigned long long)OffsetHigh << 32 | Offset; } + void SetFilePosition(unsigned long long ullPosition) { OffsetHigh = (DWORD) (ullPosition >> 32); Offset = (DWORD) ullPosition; } + + unsigned long long GetBufferOrder() const { return m_ullBufferOrder; } + void SetBufferOrder(unsigned long long ullOrder) { m_ullBufferOrder = ullOrder; } + +private: + void ReleaseBuffer(); + +private: + LPVOID m_pBuffer; // pointer to the allocated buffer + size_t m_stBufferSize; // total buffer size + DWORD m_dwRequestedDataSize; // part of the buffer that is to be used for data transfer (<= m_stBufferSize) + bool m_bLastPart; // marks the last part of the file + unsigned long long m_ullBufferOrder; // marks the order of this buffer + IOverlappedDataBufferQueue* m_pQueue; // pointer to the queue where this object resides +}; + +END_CHCORE_NAMESPACE + +#endif Index: src/libchcore/TOverlappedDataBufferQueue.cpp =================================================================== diff -u -N --- src/libchcore/TOverlappedDataBufferQueue.cpp (revision 0) +++ src/libchcore/TOverlappedDataBufferQueue.cpp (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -0,0 +1,219 @@ +// ============================================================================ +// Copyright (C) 2001-2015 by Jozef Starosczyk +// ixen@copyhandler.com +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU Library General Public License +// (version 2) as published by the Free Software Foundation; +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this program; if not, write to the +// Free Software Foundation, Inc., +// 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// ============================================================================ +#include "stdafx.h" +#include "TOverlappedDataBufferQueue.h" +#include "TOverlappedDataBuffer.h" +#include "TCoreException.h" +#include "ErrorCodes.h" + +BEGIN_CHCORE_NAMESPACE + +bool CompareBufferPositions::operator()(const TOverlappedDataBuffer* pBufferA, const TOverlappedDataBuffer* pBufferB) +{ + return pBufferA->GetBufferOrder() < pBufferB->GetBufferOrder(); +} + +chcore::TOverlappedDataBufferQueue::TOverlappedDataBufferQueue() : + m_eventReadPossible(true, true), + m_eventWritePossible(true, false), + m_eventWriteFinished(true, false), + m_stBufferSize(0), + m_ullNextExpectedWritePosition(0), + m_bDataSourceFinished(false), + m_ullNextReadBufferOrder(0), + m_ullNextWriteBufferOrder(0), + m_ullNextFinishedBufferOrder(0) +{ +} + +TOverlappedDataBufferQueue::TOverlappedDataBufferQueue(size_t stCount, size_t stBufferSize) : + m_eventReadPossible(true, true), + m_eventWritePossible(true, false), + m_eventWriteFinished(true, false), + m_stBufferSize(0), + m_ullNextExpectedWritePosition(0), + m_bDataSourceFinished(false), + m_ullNextReadBufferOrder(0), + m_ullNextWriteBufferOrder(0), + m_ullNextFinishedBufferOrder(0) +{ + ReinitializeBuffers(stCount, stBufferSize); +} + +TOverlappedDataBufferQueue::~TOverlappedDataBufferQueue() +{ +} + +TOverlappedDataBuffer* TOverlappedDataBufferQueue::GetEmptyBuffer() +{ + if (!m_listEmptyBuffers.empty()) + { + TOverlappedDataBuffer* pBuffer = m_listEmptyBuffers.front(); + m_listEmptyBuffers.pop_front(); + + pBuffer->SetBufferOrder(m_ullNextReadBufferOrder++); + if(m_listEmptyBuffers.empty()) + m_eventReadPossible.ResetEvent(); + + return pBuffer; + } + + return nullptr; +} + +void TOverlappedDataBufferQueue::AddEmptyBuffer(TOverlappedDataBuffer* pBuffer) +{ + m_listEmptyBuffers.push_back(pBuffer); + if (!m_bDataSourceFinished) + m_eventReadPossible.SetEvent(); + else + m_eventReadPossible.ResetEvent(); +} + +TOverlappedDataBuffer* TOverlappedDataBufferQueue::GetFullBuffer() +{ + if (!m_setFullBuffers.empty()) + { + TOverlappedDataBuffer* pBuffer = *m_setFullBuffers.begin(); + if (pBuffer->GetBufferOrder() != m_ullNextWriteBufferOrder) + return nullptr; + + m_setFullBuffers.erase(m_setFullBuffers.begin()); + m_ullNextExpectedWritePosition += pBuffer->GetBytesTransferred(); + + if(m_setFullBuffers.empty()) + m_eventWritePossible.ResetEvent(); + + ++m_ullNextWriteBufferOrder; + + return pBuffer; + } + + return nullptr; +} + +void TOverlappedDataBufferQueue::AddFullBuffer(TOverlappedDataBuffer* pBuffer) +{ + m_setFullBuffers.insert(pBuffer); + + if(pBuffer->IsLastPart()) + m_bDataSourceFinished = true; + + TOverlappedDataBuffer* pFirstBuffer = *m_setFullBuffers.begin(); + + if(pFirstBuffer->GetBufferOrder() == m_ullNextWriteBufferOrder) + m_eventWritePossible.SetEvent(); + else + m_eventWritePossible.ResetEvent(); +} + +TOverlappedDataBuffer* TOverlappedDataBufferQueue::GetFinishedBuffer() +{ + if(!m_setFinishedBuffers.empty()) + { + TOverlappedDataBuffer* pBuffer = *m_setFinishedBuffers.begin(); + if(pBuffer->GetBufferOrder() != m_ullNextFinishedBufferOrder) + return nullptr; + + m_setFinishedBuffers.erase(m_setFinishedBuffers.begin()); + + if(m_setFinishedBuffers.empty()) + m_eventWriteFinished.ResetEvent(); + + ++m_ullNextFinishedBufferOrder; + + return pBuffer; + } + + return nullptr; +} + +void TOverlappedDataBufferQueue::AddFinishedBuffer(TOverlappedDataBuffer* pBuffer) +{ + m_setFinishedBuffers.insert(pBuffer); + + TOverlappedDataBuffer* pFirstBuffer = *m_setFinishedBuffers.begin(); + + if(pFirstBuffer->GetBufferOrder() == m_ullNextFinishedBufferOrder) + m_eventWriteFinished.SetEvent(); + else + m_eventWriteFinished.ResetEvent(); +} + +void TOverlappedDataBufferQueue::ReinitializeBuffers(size_t stCount, size_t stBufferSize) +{ + // sanity check - if any of the buffers are still in use, we can't change the sizes + if (m_listAllBuffers.size() != m_listEmptyBuffers.size()) + THROW_CORE_EXCEPTION(eErr_InternalProblem); + + if (stBufferSize > m_stBufferSize) + { + // buffer sizes increased - clear current buffers and proceed with creating new ones + m_listAllBuffers.clear(); + m_listEmptyBuffers.clear(); + m_setFullBuffers.clear(); + } + else if (stCount == m_listAllBuffers.size()) + return; // nothing really changed + else if (stCount < m_listAllBuffers.size()) + stCount = m_listAllBuffers.size() - stCount; // allocate only the missing buffers + else if (stCount > m_listAllBuffers.size()) + { + // there are too many buffers - reduce + m_listEmptyBuffers.clear(); + m_setFullBuffers.clear(); + + size_t stCountToRemove = stCount - m_listAllBuffers.size(); + + m_listAllBuffers.erase(m_listAllBuffers.begin(), m_listAllBuffers.begin() + stCountToRemove); + for (const auto& upElement : m_listAllBuffers) + { + m_listEmptyBuffers.push_back(upElement.get()); + } + + return; + } + + // allocate buffers + while (stCount--) + { + auto upBuffer = std::make_unique(stBufferSize, this); + m_listEmptyBuffers.push_back(upBuffer.get()); + m_listAllBuffers.push_back(std::move(upBuffer)); + } + + m_stBufferSize = stCount; +} + +void TOverlappedDataBufferQueue::DataSourceChanged() +{ + if (m_listAllBuffers.size() != m_listEmptyBuffers.size()) + THROW_CORE_EXCEPTION(eErr_InternalProblem); + + m_bDataSourceFinished = false; + m_ullNextReadBufferOrder = 0; + m_ullNextWriteBufferOrder = 0; + m_ullNextFinishedBufferOrder = 0; + + m_eventReadPossible.SetEvent(); + m_eventWritePossible.ResetEvent(); + m_eventWriteFinished.ResetEvent(); +} + +END_CHCORE_NAMESPACE Index: src/libchcore/TOverlappedDataBufferQueue.h =================================================================== diff -u -N --- src/libchcore/TOverlappedDataBufferQueue.h (revision 0) +++ src/libchcore/TOverlappedDataBufferQueue.h (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -0,0 +1,84 @@ +// ============================================================================ +// Copyright (C) 2001-2014 by Jozef Starosczyk +// ixen@copyhandler.com +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU Library General Public License +// (version 2) as published by the Free Software Foundation; +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Library General Public +// License along with this program; if not, write to the +// Free Software Foundation, Inc., +// 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +// ============================================================================ +#ifndef __TOVERLAPPEDDATABUFFERQUEUE_H__ +#define __TOVERLAPPEDDATABUFFERQUEUE_H__ + +#include "libchcore.h" +#include +#include "TEvent.h" +#include "IOverlappedDataBufferQueue.h" + +BEGIN_CHCORE_NAMESPACE + +class TOverlappedDataBuffer; + +struct CompareBufferPositions +{ + bool operator()(const TOverlappedDataBuffer* rBufferA, const TOverlappedDataBuffer* rBufferB); +}; + +class TOverlappedDataBufferQueue : public IOverlappedDataBufferQueue +{ +public: + TOverlappedDataBufferQueue(); + TOverlappedDataBufferQueue(size_t stCount, size_t stBufferSize); + ~TOverlappedDataBufferQueue(); + + void ReinitializeBuffers(size_t stCount, size_t stBufferSize); + + // buffer management + virtual void AddEmptyBuffer(TOverlappedDataBuffer* pBuffer) override; + virtual TOverlappedDataBuffer* GetEmptyBuffer() override; + + virtual void AddFullBuffer(TOverlappedDataBuffer* pBuffer) override; + virtual TOverlappedDataBuffer* GetFullBuffer() override; + + virtual void AddFinishedBuffer(TOverlappedDataBuffer* pBuffer) override; + virtual TOverlappedDataBuffer* GetFinishedBuffer() override; + + // data source change + void DataSourceChanged(); + + // event access + HANDLE GetEventReadPossibleHandle() { return m_eventReadPossible.Handle(); } + HANDLE GetEventWritePossibleHandle() { return m_eventWritePossible.Handle(); } + HANDLE GetEventWriteFinishedHandle() { return m_eventWriteFinished.Handle(); } + +private: + std::deque> m_listAllBuffers; + size_t m_stBufferSize; + + std::list m_listEmptyBuffers; + std::set m_setFullBuffers; + std::set m_setFinishedBuffers; + + bool m_bDataSourceFinished; // input file was already read to the end + unsigned long long m_ullNextExpectedWritePosition; // current write file pointer + unsigned long long m_ullNextReadBufferOrder; // next order id for read buffers + unsigned long long m_ullNextWriteBufferOrder; // next order id to be processed when writing + unsigned long long m_ullNextFinishedBufferOrder; // next order id to be processed when finishing writing + + TEvent m_eventReadPossible; + TEvent m_eventWritePossible; + TEvent m_eventWriteFinished; +}; + +END_CHCORE_NAMESPACE + +#endif Index: src/libchcore/TSubTaskBase.h =================================================================== diff -u -N -r7d59ab9183c933f2fc2682a95fb5d26cf2f952d7 -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/libchcore/TSubTaskBase.h (.../TSubTaskBase.h) (revision 7d59ab9183c933f2fc2682a95fb5d26cf2f952d7) +++ src/libchcore/TSubTaskBase.h (.../TSubTaskBase.h) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -48,7 +48,8 @@ eSubResult_KillRequest, eSubResult_Error, eSubResult_CancelRequest, - eSubResult_PauseRequest + eSubResult_PauseRequest, + eSubResult_Retry, }; public: Index: src/libchcore/TSubTaskCopyMove.cpp =================================================================== diff -u -N -r671f4b1792a20d98b186f4e0a9cc6a620dede019 -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/libchcore/TSubTaskCopyMove.cpp (.../TSubTaskCopyMove.cpp) (revision 671f4b1792a20d98b186f4e0a9cc6a620dede019) +++ src/libchcore/TSubTaskCopyMove.cpp (.../TSubTaskCopyMove.cpp) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -36,25 +36,25 @@ #include #include "TFileInfo.h" #include "TFileInfoArray.h" -#include "TDataBuffer.h" #include "ErrorCodes.h" #include "TCoreException.h" #include "TPathContainer.h" #include "TScopedRunningTimeTracker.h" #include "TFeedbackHandlerWrapper.h" +#include "TOverlappedDataBufferQueue.h" +#include "TOverlappedDataBuffer.h" +#include "RoundingFunctions.h" +#include BEGIN_CHCORE_NAMESPACE -// assume max sectors of 4kB (for rounding) -#define MAXSECTORSIZE 4096 - struct CUSTOM_COPY_PARAMS { TFileInfoPtr spSrcFile; // CFileInfo - src file TSmartPath pathDstFile; // dest path with filename TBufferSizes tBufferSizes; - TDataBufferManager dbBuffer; // buffer handling + TOverlappedDataBufferQueue dbBuffer; // buffer handling bool bOnlyCreate; // flag from configuration - skips real copying - only create bool bProcessed; // has the element been processed ? (false if skipped) }; @@ -121,7 +121,7 @@ // remove changes in buffer sizes to avoid re-creation later rCfgTracker.RemoveModificationSet(TOptionsSet() % eTO_DefaultBufferSize % eTO_OneDiskBufferSize % eTO_TwoDisksBufferSize % eTO_CDBufferSize % eTO_LANBufferSize % eTO_UseOnlyDefaultBuffer); - AdjustBufferIfNeeded(ccp.dbBuffer, ccp.tBufferSizes); + AdjustBufferIfNeeded(ccp.dbBuffer, ccp.tBufferSizes, true); // log TString strFormat; @@ -269,7 +269,7 @@ TWorkerThreadController& rThreadController = GetContext().GetThreadController(); icpf::log_file& rLog = GetContext().GetLog(); const TConfig& rConfig = GetContext().GetConfig(); - + TLocalFilesystemFile fileSrc = TLocalFilesystem::CreateFileObject(); TLocalFilesystemFile fileDst = TLocalFilesystem::CreateFileObject(); @@ -291,121 +291,201 @@ return TSubTaskBase::eSubResult_Continue; // copying - std::list listDataBuffers; - std::list listEmptyBuffers; - - size_t stToRead = 0; - unsigned long ulRead = 0; - unsigned long ulWritten = 0; TBufferSizes::EBufferType eBufferIndex = TBufferSizes::eBuffer_Default; - bool bLastPart = false; - do - { - // kill flag checks - if(rThreadController.KillRequested()) - { - // log - strFormat = _T("Kill request while main copying file %srcpath -> %dstpath"); - strFormat.Replace(_T("%srcpath"), pData->spSrcFile->GetFullFilePath().ToString()); - strFormat.Replace(_T("%dstpath"), pData->pathDstFile.ToString()); - rLog.logi(strFormat.c_str()); - return TSubTaskBase::eSubResult_KillRequest; - } + // recreate buffer if needed + AdjustBufferIfNeeded(pData->dbBuffer, pData->tBufferSizes); + pData->dbBuffer.DataSourceChanged(); - // recreate buffer if needed - AdjustBufferIfNeeded(pData->dbBuffer, pData->tBufferSizes); + // establish count of data to read + eBufferIndex = GetBufferIndex(pData->tBufferSizes, pData->spSrcFile); + m_tSubTaskStats.SetCurrentBufferIndex(eBufferIndex); - // establish count of data to read - eBufferIndex = GetBufferIndex(pData->tBufferSizes, pData->spSrcFile); - m_tSubTaskStats.SetCurrentBufferIndex(eBufferIndex); + DWORD dwToRead = RoundUp(pData->tBufferSizes.GetSizeByType(eBufferIndex), TLocalFilesystemFile::MaxSectorSize); - stToRead = RoundUp((size_t)pData->tBufferSizes.GetSizeByType(eBufferIndex), pData->dbBuffer.GetSimpleBufferSize()); - size_t stBuffersToRead = stToRead / pData->dbBuffer.GetSimpleBufferSize(); + // read data from file to buffer + enum + { + eKillThread = 0, eWriteFinished, eWritePossible, eReadPossible, eHandleCount + }; + std::array arrHandles = { + rThreadController.GetKillThreadHandle(), + pData->dbBuffer.GetEventWriteFinishedHandle(), + pData->dbBuffer.GetEventWritePossibleHandle(), + pData->dbBuffer.GetEventReadPossibleHandle() + }; - // read data from file to buffer - for(size_t stIndex = 0; stIndex < stBuffersToRead; ++stIndex) + unsigned long long ullNextReadPos = 0; + bool bStopProcessing = false; + while(!bStopProcessing) + { + DWORD dwResult = WaitForMultipleObjectsEx(eHandleCount, arrHandles.data(), false, INFINITE, true); + switch(dwResult) { - // get new simple buffer - TSimpleDataBufferPtr spBuffer; - if(listEmptyBuffers.empty()) + case STATUS_USER_APC: + break; + + case WAIT_OBJECT_0 + eKillThread: { - spBuffer.reset(new TSimpleDataBuffer); - if(pData->dbBuffer.GetFreeBuffer(*spBuffer.get())) - listEmptyBuffers.push_back(spBuffer); - else - { - if(listDataBuffers.empty()) - THROW_CORE_EXCEPTION(eErr_InternalProblem); - break; - } + // log + strFormat = _T("Kill request while main copying file %srcpath -> %dstpath"); + strFormat.Replace(_T("%srcpath"), pData->spSrcFile->GetFullFilePath().ToString()); + strFormat.Replace(_T("%dstpath"), pData->pathDstFile.ToString()); + rLog.logi(strFormat.c_str()); + return TSubTaskBase::eSubResult_KillRequest; } - spBuffer = listEmptyBuffers.back(); - listEmptyBuffers.pop_back(); - - eResult = ReadFileFB(spFeedbackHandler, fileSrc, *spBuffer.get(), boost::numeric_cast(pData->dbBuffer.GetSimpleBufferSize()), ulRead, pData->spSrcFile->GetFullFilePath(), bSkip); - if(eResult != TSubTaskBase::eSubResult_Continue) - return eResult; - else if(bSkip) + case WAIT_OBJECT_0 + eReadPossible: { - // new stats - m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + ATLTRACE(_T("Read possible\n")); + TOverlappedDataBuffer* pBuffer = pData->dbBuffer.GetEmptyBuffer(); + if(!pBuffer) + THROW_CORE_EXCEPTION(eErr_InternalProblem); - pData->bProcessed = false; - return TSubTaskBase::eSubResult_Continue; + pBuffer->SetFilePosition(ullNextReadPos); + pBuffer->SetRequestedDataSize(dwToRead); + ullNextReadPos += dwToRead; + + eResult = ReadFileFB(spFeedbackHandler, fileSrc, *pBuffer, pData->spSrcFile->GetFullFilePath(), bSkip); + if(eResult != TSubTaskBase::eSubResult_Continue) + return eResult; + else if(bSkip) + { + // new stats + m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + + pData->bProcessed = false; + return TSubTaskBase::eSubResult_Continue; + } + break; } + case WAIT_OBJECT_0 + eWritePossible: + { + ATLTRACE(_T("Write possible\n")); + TOverlappedDataBuffer* pBuffer = pData->dbBuffer.GetFullBuffer(); + if(!pBuffer) + THROW_CORE_EXCEPTION(eErr_InternalProblem); - spBuffer->SetDataSize(ulRead); + // was there an error reported? + if(pBuffer->GetStatusCode() != ERROR_SUCCESS) + { + // read error encountered - handle it + eResult = HandleReadError(spFeedbackHandler, *pBuffer, pData->spSrcFile->GetFullFilePath(), bSkip); + if(eResult == TSubTaskBase::eSubResult_Retry) + { + // re-request read of the same data + eResult = ReadFileFB(spFeedbackHandler, fileSrc, *pBuffer, pData->spSrcFile->GetFullFilePath(), bSkip); + if(eResult != TSubTaskBase::eSubResult_Continue) + return eResult; + else if(bSkip) + { + // new stats + m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - if(ulRead > 0) - listDataBuffers.push_back(spBuffer); - else - listEmptyBuffers.push_back(spBuffer); + pData->bProcessed = false; + return TSubTaskBase::eSubResult_Continue; + } + } + else if(eResult != TSubTaskBase::eSubResult_Continue) + return eResult; + else if(bSkip) + { + // new stats + m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - bLastPart = (pData->dbBuffer.GetSimpleBufferSize() != ulRead); - if(bLastPart) - break; - } + pData->bProcessed = false; + return TSubTaskBase::eSubResult_Continue; + } + } + else + { + eResult = WriteFileFB(spFeedbackHandler, fileDst, *pBuffer, pData->pathDstFile, bSkip); + if(eResult != TSubTaskBase::eSubResult_Continue) + return eResult; + else if(bSkip) + { + // new stats + m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - while(!listDataBuffers.empty()) - { - TSimpleDataBufferPtr spBuffer = listDataBuffers.front(); - listDataBuffers.pop_front(); + pData->bProcessed = false; + return TSubTaskBase::eSubResult_Continue; + } + } - eResult = WriteFileExFB(spFeedbackHandler, fileDst, *spBuffer.get(), boost::numeric_cast(spBuffer->GetDataSize()), ulWritten, pData->pathDstFile, bSkip, bNoBuffer); - if(eResult != TSubTaskBase::eSubResult_Continue) - return eResult; - else if(bSkip) + break; + } + + case WAIT_OBJECT_0 + eWriteFinished: { - // new stats - m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + ATLTRACE(_T("Write finished\n")); + TOverlappedDataBuffer* pBuffer = pData->dbBuffer.GetFinishedBuffer(); + if(!pBuffer) + THROW_CORE_EXCEPTION(eErr_InternalProblem); - pData->bProcessed = false; - return TSubTaskBase::eSubResult_Continue; - } + if(pBuffer->GetStatusCode() != ERROR_SUCCESS) + { + eResult = HandleWriteError(spFeedbackHandler, *pBuffer, pData->spSrcFile->GetFullFilePath(), bSkip); + if(eResult == TSubTaskBase::eSubResult_Retry) + { + eResult = WriteFileFB(spFeedbackHandler, fileDst, *pBuffer, pData->pathDstFile, bSkip); + if(eResult != TSubTaskBase::eSubResult_Continue) + return eResult; + else if(bSkip) + { + // new stats + m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - listEmptyBuffers.push_back(spBuffer); + pData->bProcessed = false; + return TSubTaskBase::eSubResult_Continue; + } + } + else if(eResult != TSubTaskBase::eSubResult_Continue) + return eResult; + else if(bSkip) + { + // new stats + m_tSubTaskStats.IncreaseProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); + m_tSubTaskStats.IncreaseCurrentItemProcessedSize(pData->spSrcFile->GetLength64() - m_tSubTaskStats.GetCurrentItemProcessedSize()); - unsigned long long ullCITotalSize = m_tSubTaskStats.GetCurrentItemTotalSize(); - unsigned long long ullCIProcessedSize = m_tSubTaskStats.GetCurrentItemProcessedSize(); + pData->bProcessed = false; + return TSubTaskBase::eSubResult_Continue; + } + } + else + { + unsigned long long ullCITotalSize = m_tSubTaskStats.GetCurrentItemTotalSize(); + unsigned long long ullCIProcessedSize = m_tSubTaskStats.GetCurrentItemProcessedSize(); - if(ullCIProcessedSize + ulWritten > ullCITotalSize) - { - // total size changed - pData->spSrcFile->SetLength64(ullCIProcessedSize + ulWritten); - m_tSubTaskStats.IncreaseCurrentItemTotalSize(ullCIProcessedSize + ulWritten - ullCITotalSize); - m_tSubTaskStats.IncreaseTotalSize(ullCIProcessedSize + ulWritten - ullCITotalSize); + unsigned long long ullWritten = pBuffer->GetBytesTransferred(); + if(ullCIProcessedSize + ullWritten > ullCITotalSize) + { + // total size changed + pData->spSrcFile->SetLength64(ullCIProcessedSize + ullWritten); + m_tSubTaskStats.IncreaseCurrentItemTotalSize(ullCIProcessedSize + ullWritten - ullCITotalSize); + m_tSubTaskStats.IncreaseTotalSize(ullCIProcessedSize + ullWritten - ullCITotalSize); + } + + // new stats + m_tSubTaskStats.IncreaseProcessedSize(ullWritten); + m_tSubTaskStats.IncreaseCurrentItemProcessedSize(ullWritten); + + // stop iterating through file + bStopProcessing = pBuffer->IsLastPart(); + pBuffer->RequeueAsEmpty(); + } + + break; } - // new stats - m_tSubTaskStats.IncreaseProcessedSize(ulWritten); - m_tSubTaskStats.IncreaseCurrentItemProcessedSize(ulWritten); + default: + THROW_CORE_EXCEPTION(eErr_UnhandledCase); } } - while(!bLastPart); // fix the stats for files shorter than expected unsigned long long ullCITotalSize = m_tSubTaskStats.GetCurrentItemTotalSize(); @@ -533,7 +613,7 @@ if(ullSeekTo != 0) // src and dst files exists, requested resume at the specified index { // try to move file pointers to the end - ULONGLONG ullMove = (bNoBuffer ? ROUNDDOWN(ullSeekTo, MAXSECTORSIZE) : ullSeekTo); + ULONGLONG ullMove = (bNoBuffer ? ROUNDDOWN(ullSeekTo, TLocalFilesystemFile::MaxSectorSize) : ullSeekTo); eResult = SetFilePointerFB(spFeedbackHandler, fileSrc, ullMove, pData->spSrcFile->GetFullFilePath(), bSkip); if(eResult != TSubTaskBase::eSubResult_Continue) @@ -600,13 +680,13 @@ return eResult; } -bool TSubTaskCopyMove::AdjustBufferIfNeeded(chcore::TDataBufferManager& rBuffer, TBufferSizes& rBufferSizes) +bool TSubTaskCopyMove::AdjustBufferIfNeeded(TOverlappedDataBufferQueue& rBuffer, TBufferSizes& rBufferSizes, bool bForce) { const TConfig& rConfig = GetContext().GetConfig(); TTaskConfigTracker& rCfgTracker = GetContext().GetCfgTracker(); icpf::log_file& rLog = GetContext().GetLog(); - if(!rBuffer.IsInitialized() || (rCfgTracker.IsModified() && rCfgTracker.IsModified(TOptionsSet() % eTO_DefaultBufferSize % eTO_OneDiskBufferSize % eTO_TwoDisksBufferSize % eTO_CDBufferSize % eTO_LANBufferSize % eTO_UseOnlyDefaultBuffer, true))) + if(bForce || (rCfgTracker.IsModified() && rCfgTracker.IsModified(TOptionsSet() % eTO_DefaultBufferSize % eTO_OneDiskBufferSize % eTO_TwoDisksBufferSize % eTO_CDBufferSize % eTO_LANBufferSize % eTO_UseOnlyDefaultBuffer, true))) { rBufferSizes.SetOnlyDefault(GetTaskPropValue(rConfig)); rBufferSizes.SetDefaultSize(GetTaskPropValue(rConfig)); @@ -627,28 +707,12 @@ rLog.logi(strFormat.c_str()); - if(!rBuffer.IsInitialized()) - { - size_t stMaxSize = rBufferSizes.GetMaxSize(); - size_t stPageSize = GetTaskPropValue(rConfig); - size_t stChunkSize = GetTaskPropValue(rConfig); + rBuffer.ReinitializeBuffers(2, rBufferSizes.GetMaxSize()); - chcore::TDataBufferManager::CheckBufferConfig(stMaxSize, stPageSize, stChunkSize); - - rBuffer.Initialize(stMaxSize, stPageSize, stChunkSize); - } - else - { - size_t stMaxSize = rBufferSizes.GetMaxSize(); - rBuffer.CheckResizeSize(stMaxSize); - - rBuffer.ChangeMaxMemorySize(stMaxSize); - } - - return true; + return true; // buffer adjusted } - else - return false; + + return false; // buffer did not need adjusting } TSubTaskBase::ESubOperationResult TSubTaskCopyMove::OpenSourceFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& fileSrc, const TSmartPath& spPathToOpen, bool bNoBuffering) @@ -983,7 +1047,7 @@ return TSubTaskBase::eSubResult_Continue; } -TSubTaskBase::ESubOperationResult TSubTaskCopyMove::ReadFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& file, chcore::TSimpleDataBuffer& rBuffer, DWORD dwToRead, DWORD& rdwBytesRead, const TSmartPath& pathFile, bool& bSkip) +TSubTaskBase::ESubOperationResult TSubTaskCopyMove::ReadFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& file, TOverlappedDataBuffer& rBuffer, const TSmartPath& pathFile, bool& bSkip) { icpf::log_file& rLog = GetContext().GetLog(); @@ -993,18 +1057,15 @@ { bRetry = false; - if(!file.ReadFile(rBuffer, dwToRead, rdwBytesRead)) + if(!file.ReadFile(rBuffer)) { - // log - DWORD dwLastError = GetLastError(); - - TString strFormat = _T("Error %errno while trying to read %count bytes from source file %path (CustomCopyFileFB)"); - strFormat.Replace(_t("%errno"), boost::lexical_cast(dwLastError).c_str()); - strFormat.Replace(_t("%count"), boost::lexical_cast(dwToRead).c_str()); + TString strFormat = _T("Error %errno while requesting read of %count bytes from source file %path (CustomCopyFileFB)"); + strFormat.Replace(_t("%errno"), boost::lexical_cast(GetLastError()).c_str()); + strFormat.Replace(_t("%count"), boost::lexical_cast(rBuffer.GetRequestedDataSize()).c_str()); strFormat.Replace(_t("%path"), pathFile.ToString()); rLog.loge(strFormat.c_str()); - EFeedbackResult frResult = spFeedbackHandler->FileError(pathFile.ToWString(), TString(), EFileError::eReadError, dwLastError); + EFeedbackResult frResult = spFeedbackHandler->FileError(pathFile.ToWString(), TString(), EFileError::eReadError, GetLastError()); switch(frResult) { case EFeedbackResult::eResult_Cancel: @@ -1032,25 +1093,66 @@ return TSubTaskBase::eSubResult_Continue; } -TSubTaskBase::ESubOperationResult TSubTaskCopyMove::WriteFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& file, chcore::TSimpleDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten, const TSmartPath& pathFile, bool& bSkip) +TSubTaskBase::ESubOperationResult TSubTaskCopyMove::HandleReadError(const IFeedbackHandlerPtr& spFeedbackHandler, + TOverlappedDataBuffer& rBuffer, + const TSmartPath& pathFile, + bool& bSkip) { icpf::log_file& rLog = GetContext().GetLog(); + DWORD dwLastError = boost::numeric_cast(rBuffer.GetStatusCode()); bSkip = false; + // log + TString strFormat = _T("Error %errno while requesting read of %count bytes from source file %path (CustomCopyFileFB)"); + strFormat.Replace(_t("%errno"), boost::lexical_cast(dwLastError).c_str()); + strFormat.Replace(_t("%count"), boost::lexical_cast(rBuffer.GetRequestedDataSize()).c_str()); + strFormat.Replace(_t("%path"), pathFile.ToString()); + rLog.loge(strFormat.c_str()); + + EFeedbackResult frResult = spFeedbackHandler->FileError(pathFile.ToWString(), TString(), EFileError::eReadError, dwLastError); + switch(frResult) + { + case EFeedbackResult::eResult_Cancel: + return TSubTaskBase::eSubResult_CancelRequest; + + case EFeedbackResult::eResult_Retry: + return TSubTaskBase::eSubResult_Retry; + + case EFeedbackResult::eResult_Pause: + return TSubTaskBase::eSubResult_PauseRequest; + + case EFeedbackResult::eResult_Skip: + bSkip = true; + return TSubTaskBase::eSubResult_Continue; + + default: + BOOST_ASSERT(FALSE); // unknown result + THROW_CORE_EXCEPTION(eErr_UnhandledCase); + } +} + +TSubTaskBase::ESubOperationResult TSubTaskCopyMove::WriteFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& file, TOverlappedDataBuffer& rBuffer, const TSmartPath& pathFile, bool& bSkip) +{ + icpf::log_file& rLog = GetContext().GetLog(); + + bSkip = false; + bool bRetry = false; do { bRetry = false; - if(!file.WriteFile(rBuffer, dwToWrite, rdwBytesWritten)) + if(!file.WriteFile(rBuffer)) { // log DWORD dwLastError = GetLastError(); + if (dwLastError == ERROR_IO_PENDING) + break; TString strFormat = _T("Error %errno while trying to write %count bytes to destination file %path (CustomCopyFileFB)"); strFormat.Replace(_t("%errno"), boost::lexical_cast(dwLastError).c_str()); - strFormat.Replace(_t("%count"), boost::lexical_cast(dwToWrite).c_str()); + strFormat.Replace(_t("%count"), boost::lexical_cast(rBuffer.GetBytesTransferred()).c_str()); strFormat.Replace(_t("%path"), pathFile.ToString()); rLog.loge(strFormat.c_str()); @@ -1082,69 +1184,43 @@ return TSubTaskBase::eSubResult_Continue; } -TSubTaskBase::ESubOperationResult TSubTaskCopyMove::WriteFileExFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& file, chcore::TSimpleDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten, const TSmartPath& pathFile, bool& bSkip, bool bNoBuffer) +TSubTaskBase::ESubOperationResult TSubTaskCopyMove::HandleWriteError(const IFeedbackHandlerPtr& spFeedbackHandler, + TOverlappedDataBuffer& rBuffer, + const TSmartPath& pathFile, + bool& bSkip) { - TString strFormat; - TSubTaskBase::ESubOperationResult eResult = TSubTaskBase::eSubResult_Continue; + icpf::log_file& rLog = GetContext().GetLog(); + DWORD dwLastError = boost::numeric_cast(rBuffer.GetStatusCode()); - rdwBytesWritten = 0; + bSkip = false; - // copying - unsigned long ulWritten = 0; - bool bNonAlignedSize = (dwToWrite % MAXSECTORSIZE) != 0; + // log + TString strFormat = _T("Error %errno while trying to write %count bytes to destination file %path (CustomCopyFileFB)"); + strFormat.Replace(_t("%errno"), boost::lexical_cast(rBuffer.GetStatusCode()).c_str()); + strFormat.Replace(_t("%count"), boost::lexical_cast(rBuffer.GetBytesTransferred()).c_str()); + strFormat.Replace(_t("%path"), pathFile.ToString()); + rLog.loge(strFormat.c_str()); - // handle not aligned part at the end of file when no buffering is enabled - if(bNoBuffer && bNonAlignedSize) + EFeedbackResult frResult = spFeedbackHandler->FileError(pathFile.ToWString(), TString(), EFileError::eWriteError, dwLastError); + switch(frResult) { - // count of data read from the file is less than requested - we're at the end of source file - // and this is the operation with system buffering turned off + case EFeedbackResult::eResult_Cancel: + return TSubTaskBase::eSubResult_CancelRequest; - // write as much as possible to the destination file with no buffering - // NOTE: as an alternative, we could write more data to the destination file and then truncate the file - unsigned long ulDataToWrite = ROUNDDOWN(dwToWrite, MAXSECTORSIZE); - if(ulDataToWrite > 0) - { - eResult = WriteFileFB(spFeedbackHandler, file, rBuffer, ulDataToWrite, ulWritten, pathFile, bSkip); - if(eResult != TSubTaskBase::eSubResult_Continue || bSkip) - return eResult; + case EFeedbackResult::eResult_Retry: + return TSubTaskBase::eSubResult_Retry; - // calculate count of bytes left to be written - rdwBytesWritten = ulWritten; - dwToWrite -= ulWritten; + case EFeedbackResult::eResult_Pause: + return TSubTaskBase::eSubResult_PauseRequest; - // now remove part of data from buffer (ulWritten bytes) - rBuffer.CutDataFromBuffer(ulWritten); - } + case EFeedbackResult::eResult_Skip: + bSkip = true; + return TSubTaskBase::eSubResult_Continue; - // close and re-open the destination file with buffering option for append - file.Close(); - - // are there any more data to be written? - if(dwToWrite != 0) - { - // re-open the destination file, this time with standard buffering to allow writing not aligned part of file data - eResult = OpenExistingDestinationFileFB(spFeedbackHandler, file, pathFile, false); - if(eResult != TSubTaskBase::eSubResult_Continue || !file.IsOpen()) - return eResult; - - // move file pointer to the end of destination file - eResult = SetFilePointerFB(spFeedbackHandler, file, m_tSubTaskStats.GetCurrentItemProcessedSize() + rdwBytesWritten, pathFile, bSkip); - if(eResult != TSubTaskBase::eSubResult_Continue || bSkip) - return eResult; - } + default: + BOOST_ASSERT(FALSE); // unknown result + THROW_CORE_EXCEPTION(eErr_UnhandledCase); } - - // write - if(dwToWrite != 0) - { - eResult = WriteFileFB(spFeedbackHandler, file, rBuffer, dwToWrite, ulWritten, pathFile, bSkip); - if(eResult != TSubTaskBase::eSubResult_Continue || bSkip) - return eResult; - - rdwBytesWritten += ulWritten; - } - - return TSubTaskBase::eSubResult_Continue; } TSubTaskBase::ESubOperationResult TSubTaskCopyMove::CreateDirectoryFB(const IFeedbackHandlerPtr& spFeedbackHandler, const TSmartPath& pathDirectory) Index: src/libchcore/TSubTaskCopyMove.h =================================================================== diff -u -N -r7d59ab9183c933f2fc2682a95fb5d26cf2f952d7 -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/libchcore/TSubTaskCopyMove.h (.../TSubTaskCopyMove.h) (revision 7d59ab9183c933f2fc2682a95fb5d26cf2f952d7) +++ src/libchcore/TSubTaskCopyMove.h (.../TSubTaskCopyMove.h) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -37,6 +37,8 @@ class TDataBufferManager; class TSimpleDataBuffer; class TBufferSizes; +class TOverlappedDataBufferQueue; +class TOverlappedDataBuffer; class LIBCHCORE_API TSubTaskCopyMove : public TSubTaskBase { @@ -57,7 +59,7 @@ private: TBufferSizes::EBufferType GetBufferIndex(const TBufferSizes& rBufferSizes, const TFileInfoPtr& spFileInfo); - bool AdjustBufferIfNeeded(TDataBufferManager& rBuffer, TBufferSizes& rBufferSizes); + bool AdjustBufferIfNeeded(TOverlappedDataBufferQueue& rBuffer, TBufferSizes& rBufferSizes, bool bForce = false); ESubOperationResult CustomCopyFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, CUSTOM_COPY_PARAMS* pData); @@ -70,9 +72,11 @@ ESubOperationResult SetFilePointerFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& file, long long llDistance, const TSmartPath& pathFile, bool& bSkip); ESubOperationResult SetEndOfFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& file, const TSmartPath& pathFile, bool& bSkip); - ESubOperationResult ReadFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& file, chcore::TSimpleDataBuffer& rBuffer, DWORD dwToRead, DWORD& rdwBytesRead, const TSmartPath& pathFile, bool& bSkip); - ESubOperationResult WriteFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& file, chcore::TSimpleDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten, const TSmartPath& pathFile, bool& bSkip); - ESubOperationResult WriteFileExFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& file, chcore::TSimpleDataBuffer& rBuffer, DWORD dwToWrite, DWORD& rdwBytesWritten, const TSmartPath& pathFile, bool& bSkip, bool bNoBuffer); + ESubOperationResult ReadFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& file, TOverlappedDataBuffer& rBuffer, const TSmartPath& pathFile, bool& bSkip); + ESubOperationResult HandleReadError(const IFeedbackHandlerPtr& spFeedbackHandler, TOverlappedDataBuffer& rBuffer, const TSmartPath& pathFile, bool& bSkip); + + ESubOperationResult WriteFileFB(const IFeedbackHandlerPtr& spFeedbackHandler, TLocalFilesystemFile& file, TOverlappedDataBuffer& rBuffer, const TSmartPath& pathFile, bool& bSkip); + ESubOperationResult HandleWriteError(const IFeedbackHandlerPtr& spFeedbackHandler, TOverlappedDataBuffer& rBuffer, const TSmartPath& pathFile, bool& bSkip); ESubOperationResult CreateDirectoryFB(const IFeedbackHandlerPtr& spFeedbackHandler, const TSmartPath& pathDirectory); ESubOperationResult CheckForFreeSpaceFB(const IFeedbackHandlerPtr& spFeedbackHandler); Index: src/libchcore/TTaskConfigTracker.cpp =================================================================== diff -u -N -rb193a95402f2bf4c456fb9d65d111caaf6994823 -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/libchcore/TTaskConfigTracker.cpp (.../TTaskConfigTracker.cpp) (revision b193a95402f2bf4c456fb9d65d111caaf6994823) +++ src/libchcore/TTaskConfigTracker.cpp (.../TTaskConfigTracker.cpp) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -213,10 +213,6 @@ return eTO_DisableBuffering; else if(strOption == TaskPropData::GetPropertyName()) return eTO_DisableBufferingMinSize; - else if(strOption == TaskPropData::GetPropertyName()) - return eTO_BufferChunkSize; - else if(strOption == TaskPropData::GetPropertyName()) - return eTO_BufferPageSize; else if(strOption == TaskPropData::GetPropertyName()) return eTO_SetDestinationAttributes; Index: src/libchcore/TTaskConfiguration.h =================================================================== diff -u -N -rb193a95402f2bf4c456fb9d65d111caaf6994823 -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/libchcore/TTaskConfiguration.h (.../TTaskConfiguration.h) (revision b193a95402f2bf4c456fb9d65d111caaf6994823) +++ src/libchcore/TTaskConfiguration.h (.../TTaskConfiguration.h) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -39,8 +39,6 @@ eTO_LANBufferSize, eTO_DisableBuffering, eTO_DisableBufferingMinSize, - eTO_BufferChunkSize, - eTO_BufferPageSize, eTO_SetDestinationAttributes, eTO_SetDestinationDateTime, @@ -99,8 +97,6 @@ TASK_PROPERTY_MINMAX(eTO_TwoDisksBufferSize, unsigned int, _T("Buffer.TwoPhysicalDisksSize"), 524288, 1, 0xffffffff); TASK_PROPERTY_MINMAX(eTO_CDBufferSize, unsigned int, _T("Buffer.CDSize"), 262144, 1, 0xffffffff); TASK_PROPERTY_MINMAX(eTO_LANBufferSize, unsigned int, _T("Buffer.LANSize"), 131072, 1, 0xffffffff); -TASK_PROPERTY_MINMAX(eTO_BufferChunkSize, unsigned int, _T("Buffer.ChunkSize"), 65536, 1, 0xffffffff); -TASK_PROPERTY_MINMAX(eTO_BufferPageSize, unsigned int, _T("Buffer.PageSize"), 512384, 1, 0xffffffff); TASK_PROPERTY(eTO_DisableBuffering, bool, _T("Operation.Buffering.DisableBufferingForLargeFiles"), true); TASK_PROPERTY_MINMAX(eTO_DisableBufferingMinSize, int, _T("Operation.Buffering.MinSizeOfFileToDisableBuffering"), 2097152, 1, 0xffffffff); Index: src/libchcore/TWorkerThreadController.cpp =================================================================== diff -u -N -r548382442cbf7bed7f744b279ce3f66b54992724 -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/libchcore/TWorkerThreadController.cpp (.../TWorkerThreadController.cpp) (revision 548382442cbf7bed7f744b279ce3f66b54992724) +++ src/libchcore/TWorkerThreadController.cpp (.../TWorkerThreadController.cpp) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -164,6 +164,11 @@ return (m_hKillThread && WaitForSingleObject(m_hKillThread, dwWaitForSignal) == WAIT_OBJECT_0); } +HANDLE TWorkerThreadController::GetKillThreadHandle() const +{ + return m_hKillThread; +} + void TWorkerThreadController::RemoveZombieData(boost::upgrade_lock& rUpgradeLock) { // if thread is already stopped, then there is nothing to do Index: src/libchcore/TWorkerThreadController.h =================================================================== diff -u -N -rab32897e61cc637a1e28d9dc3f0489b8d16a429c -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/libchcore/TWorkerThreadController.h (.../TWorkerThreadController.h) (revision ab32897e61cc637a1e28d9dc3f0489b8d16a429c) +++ src/libchcore/TWorkerThreadController.h (.../TWorkerThreadController.h) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -47,6 +47,7 @@ // methods to be used only inside the thread being controlled bool KillRequested(DWORD dwWaitForSignal = 0); + HANDLE GetKillThreadHandle() const; protected: void RemoveZombieData(boost::upgrade_lock& rUpgradeLock); Index: src/libchcore/libchcore.vc120.vcxproj =================================================================== diff -u -N -rcdb4c898156398dd4f4bf8abd7c854eff42f6ae2 -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/libchcore/libchcore.vc120.vcxproj (.../libchcore.vc120.vcxproj) (revision cdb4c898156398dd4f4bf8abd7c854eff42f6ae2) +++ src/libchcore/libchcore.vc120.vcxproj (.../libchcore.vc120.vcxproj) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -121,16 +121,16 @@ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ true $(ProjectName)32ud - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ true $(ProjectName)32ud - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ @@ -139,8 +139,8 @@ $(ProjectName)64ud NativeRecommendedRules.ruleset false - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ @@ -149,40 +149,40 @@ $(ProjectName)64ud NativeRecommendedRules.ruleset false - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ false $(ProjectName)32u - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ false $(ProjectName)32u - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x32\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x32\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ false $(ProjectName)64u - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) $(SolutionDir)bin\$(Configuration)\ $(SolutionDir)intermediate\vc120\$(Platform)\$(ProjectName)_$(Configuration)\ false $(ProjectName)64u - C:\dev\boost_1_57_0;$(IncludePath) - C:\dev\boost_1_57_0\lib-12.0\x64\lib;$(LibraryPath) + C:\dev\boost_1_58_0;$(IncludePath) + C:\dev\boost_1_58_0\lib-12.0\x64\lib;$(LibraryPath) @@ -512,6 +512,7 @@ + @@ -521,17 +522,20 @@ + + + @@ -576,7 +580,7 @@ - + @@ -621,6 +625,7 @@ + @@ -686,12 +691,14 @@ true true + + @@ -733,7 +740,7 @@ - + Index: src/libchcore/libchcore.vc120.vcxproj.filters =================================================================== diff -u -N -rcdb4c898156398dd4f4bf8abd7c854eff42f6ae2 -r6103ac74583f2136b821dc67515ed8469abd8155 --- src/libchcore/libchcore.vc120.vcxproj.filters (.../libchcore.vc120.vcxproj.filters) (revision cdb4c898156398dd4f4bf8abd7c854eff42f6ae2) +++ src/libchcore/libchcore.vc120.vcxproj.filters (.../libchcore.vc120.vcxproj.filters) (revision 6103ac74583f2136b821dc67515ed8469abd8155) @@ -57,6 +57,12 @@ {9d121063-367c-4424-8009-12b3635e0fed} + + {30d60be9-a936-4191-a66d-64e127f3d258} + + + {202d13d3-126b-4811-8c1c-a14b4f0476b7} + @@ -116,9 +122,6 @@ Source Files\Tools - - Source Files\Tools - Source Files\Tools @@ -131,9 +134,6 @@ Source Files\Tools - - Source Files\Tools - Source Files\Library files @@ -344,6 +344,24 @@ Source Files\Serialization\Fake + + Source Files\Filesystems + + + Source Files\Tools + + + Source Files\Tools\OverlappedBuffer + + + Source Files\Tools\OverlappedBuffer + + + Source Files\Tools\OverlappedBuffer + + + Source Files\Tools + @@ -394,9 +412,6 @@ Source Files\Tools - - Source Files\Tools - Source Files\Tools @@ -409,9 +424,6 @@ Source Files\Tools - - Source Files\Tools - Source Files\Library files @@ -634,5 +646,20 @@ Source Files\Serialization\Fake + + Source Files\Filesystems + + + Source Files\Tools + + + Source Files\Tools\OverlappedBuffer + + + Source Files\Tools\OverlappedBuffer + + + Source Files\Tools\OverlappedBuffer + \ No newline at end of file