Fix code convention check

Makefile

@@ -24,7 +24,7 @@ unittest: build
 	build/cpp/testtaskqueue.bin
 
 check_convention:
-	pep8 py tests --max-line-length=109
+	pycodestyle py tests --max-line-length=109
 
 .PHONY: install_binary
 install_binary:

cpp/Osmosis/Client/CheckIn.cpp

@@ -13,10 +13,11 @@ CheckIn::CheckIn(        const boost::filesystem::path &  directory,
 		Chain::ObjectStoreInterface &    objectStore,
 		bool                             md5,
 		const boost::filesystem::path &  progressReport,
-		unsigned                         progressReportIntervalSeconds ) :
+		unsigned                         progressReportIntervalSeconds,
+		bool                             followSymlinks ) :
 	_label( label ),
 	_md5( md5 ),
-	_digestDirectory( directory, md5, _ignores ),
+	_digestDirectory( directory, md5, _ignores, followSymlinks ),
 	_putConnection( objectStore.connect() ),
 	_putQueue( CHECK_EXISTING_THREADS ),
 	_checkInProgress( progressReport, _digestDirectory, _putQueue, progressReportIntervalSeconds )

cpp/Osmosis/Client/CheckIn.h

@@ -18,7 +18,8 @@ class CheckIn
 			Chain::ObjectStoreInterface &    objectStore,
 			bool                             md5,
 			const boost::filesystem::path &  progressReport,
-			unsigned                         progressReportIntervalSeconds );
+			unsigned                         progressReportIntervalSeconds,
+			bool                             followSymlinks );
 
 	~CheckIn();
 

cpp/Osmosis/Client/CheckOut.cpp

@@ -28,7 +28,7 @@ CheckOut::CheckOut(       const boost::filesystem::path &  directory,
 	_removeUnknownFiles( removeUnknownFiles ),
 	_myUIDandGIDcheckout( myUIDandGIDcheckout ),
 	_ignores( ignores ),
-	_digestDirectory( directory, md5, ignores ),
+	_digestDirectory( directory, md5, ignores, false ),
 	_checkOutProgress( progressReport, _digestDirectory, progressReportIntervalSeconds ),
 	_chainTouch( chainTouch )
 {}

cpp/Osmosis/Client/DigestDirectory.h

@@ -16,11 +16,13 @@ class DigestDirectory
 public:
 	DigestDirectory(        const boost::filesystem::path &  directory,
 				bool                             md5,
-				const Ignores &                  ignores ) :
+				const Ignores &                  ignores,
+				bool                             followSymlinks ) :
 		_directory( directory ),
 		_ignores( ignores ),
 		_toDigestTaskQueue( 1 ),
-		_digestedQueue( digestionThreads() )
+		_digestedQueue( digestionThreads() ),
+		_followSymlinks( followSymlinks )
 	{
 		for ( unsigned i = 0; i < digestionThreads(); ++ i )
 			_threads.push_back( std::thread(
@@ -61,6 +63,7 @@ class DigestDirectory
 	PathTaskQueue                  _toDigestTaskQueue;
 	DigestedTaskQueue              _digestedQueue;
 	std::vector< std::thread >     _threads;
+	bool                           _followSymlinks;
 
 	void threadEntryPoint()
 	{
@@ -95,12 +98,15 @@ class DigestDirectory
 				i.no_push();
 				continue;
 			}
+			if ( status.isSymlink() and _followSymlinks ) {
+			    status = FileStatus( status.symlink() );
+			}
 			boost::filesystem::path relative = path.string().substr( prefixLength );
 			{
 				std::lock_guard< std::mutex > lock( _dirListMutex );
 				_dirList.add( relative, status );
 			}
-			if ( status.syncContent() )
+			if ( status.syncContent( _followSymlinks ) )
 				_toDigestTaskQueue.put( std::move( relative ) );
 		}
 		_toDigestTaskQueue.producerDone();

cpp/Osmosis/FileStatus.cpp

@@ -41,10 +41,10 @@ const boost::filesystem::path FileStatus::symlink() const
 	return _symlink;
 }
 
-bool FileStatus::syncContent() const
+bool FileStatus::syncContent( bool followSymlinks ) const
 {
 	if ( isDirectory() or isCharacter() or isBlock() or
-			isFIFO() or isSymlink() or isSocket() )
+	     isFIFO() or isSocket() or ( isSymlink() and not followSymlinks ) )
 		return false;
 	ASSERT( isRegular() );
 	return true;

cpp/Osmosis/FileStatus.h

@@ -23,7 +23,7 @@ class FileStatus
 
 	const boost::filesystem::path symlink() const;
 
-	bool syncContent() const;
+	bool syncContent( bool followSymlinks = false ) const;
 
 	bool isRegular() const;
 	bool isDirectory() const;

cpp/Osmosis/ObjectStore/Purge.cpp

@@ -11,10 +11,10 @@ Purge::Purge( Store & store, Labels & labels ):
 	_labels( labels )
 {}
 
-void Purge::purge( boost::filesystem::path & dirToPurge )
+void Purge::purge()
 {
 	BACKTRACE_BEGIN
-	startWithAllObjects( dirToPurge );
+	startWithAllObjects();
 	size_t before = _staleHashes.size();
 	TRACE_INFO( "Found " << before << " objects" );
 	takeOutAllLabels();
@@ -26,10 +26,10 @@ void Purge::purge( boost::filesystem::path & dirToPurge )
 	BACKTRACE_END
 }
 
-void Purge::startWithAllObjects( boost::filesystem::path & dirToPurge )
+void Purge::startWithAllObjects()
 {
 	BACKTRACE_BEGIN
-	for ( auto i = _store.list( dirToPurge ); not i.done(); i.next() )
+	for ( auto i = _store.list(); not i.done(); i.next() )
 		_staleHashes.emplace( * i );
 	BACKTRACE_END
 }

cpp/Osmosis/ObjectStore/Purge.h

@@ -2,7 +2,6 @@
 #define __OSMOSIS_OBJECT_STORE_PURGE_H__
 
 #include <unordered_set>
-#include <boost/filesystem.hpp>
 #include "Osmosis/ObjectStore/Labels.h"
 #include "Osmosis/ObjectStore/Store.h"
 
@@ -15,14 +14,14 @@ class Purge
 public:
 	Purge( Store & store, Labels & labels );
 
-	void purge( boost::filesystem::path & dirToPurge );
+	void purge();
 
 private:
 	Store &                     _store;
 	Labels &                    _labels;
 	std::unordered_set< Hash >  _staleHashes;
 
-	void startWithAllObjects( boost::filesystem::path & dirToPurge );
+	void startWithAllObjects();
 
 	void takeOutAllLabels();
 

cpp/Osmosis/ObjectStore/Store.cpp

@@ -68,12 +68,6 @@ ObjectsIterator Store::list() const
 	return std::move( iterator );
 }
 
-ObjectsIterator Store::list( boost::filesystem::path rootPath ) const
-{
-	ObjectsIterator iterator( rootPath );
-	return std::move( iterator );
-}
-
 boost::filesystem::path Store::absoluteFilename( const Hash & hash ) const
 {
 	return _rootPath / hash.relativeFilename();

cpp/Osmosis/ObjectStore/Store.h

@@ -25,8 +25,6 @@ class Store
 
 	ObjectsIterator list() const;
 
-	ObjectsIterator list( boost::filesystem::path rootDir ) const;
-
 private:
 	boost::filesystem::path _rootPath;
 

cpp/Osmosis/main.cpp

@@ -1,4 +1,3 @@
-#include <boost/range/iterator_range.hpp>
 #include <boost/program_options.hpp>
 #include "Osmosis/Server/Server.h"
 #include "Osmosis/Server/BroadcastServer.h"
@@ -11,7 +10,6 @@
 #include "Osmosis/ObjectStore/LeastRecentlyUsed.h"
 #include "Osmosis/ObjectStore/Purge.h"
 #include "Osmosis/FilesystemUtils.h"
-#include "Osmosis/Client/Typedefs.h"
 
 std::mutex globalTraceLock;
 
@@ -52,6 +50,7 @@ void checkIn( const boost::program_options::variables_map & options )
 	const unsigned int tcpTimeout = options[ "tcpTimeout" ].as< unsigned int >();
 	boost::filesystem::path workDir = stripTrailingSlash( options[ "arg1" ].as< std::string >() );
 	std::string label = options[ "arg2" ].as< std::string >();
+	bool followSymlinks = options.count( "followSymlinks" ) > 0;
 	Osmosis::Chain::Chain chain( options[ "objectStores" ].as< std::string >(), false, false, tcpTimeout );
 	if ( chain.count() > 1 )
 		THROW( Error, "--objectStores must contain one object store in a checkin operation" );
@@ -61,7 +60,8 @@ void checkIn( const boost::program_options::variables_map & options )
 	if ( boost::filesystem::exists( draftsPath ) )
 		THROW( Error, "workDir must not contain " << draftsPath );
 
-	Osmosis::Client::CheckIn instance( workDir, label, chain.single(), md5, reportFile, reportIntervalSeconds );
+	Osmosis::Client::CheckIn instance( workDir, label, chain.single(), md5, reportFile,
+                                       reportIntervalSeconds, followSymlinks );
 	instance.go();
 	BACKTRACE_END
 }
@@ -148,55 +148,13 @@ void eraseLabel( const boost::program_options::variables_map & options )
 	instance.eraseLabel( label );
 }
 
-void purgeDir( unsigned int threadID, boost::filesystem::path  rootPath, Osmosis::Client::PathTaskQueue & tasksQueue )
+void purge( const boost::program_options::variables_map & options )
 {
+	boost::filesystem::path rootPath( options[ "objectStoreRootPath" ].as< std::string >() );
 	Osmosis::ObjectStore::Store store( rootPath );
 	Osmosis::ObjectStore::Labels labels( rootPath, store );
 	Osmosis::ObjectStore::Purge purge( store, labels );
-	while ( tasksQueue.size() > 0 ) {
-		boost::filesystem::path dirToPurge = tasksQueue.get();
-		TRACE_INFO("Thread #" << threadID << " handling " << dirToPurge );
-		purge.purge( dirToPurge );
-	}
-	TRACE_INFO("Thread #" << threadID << " finished." );
-}
-
-void purge( const boost::program_options::variables_map & options )
-{
-	boost::filesystem::path rootPath( options[ "objectStoreRootPath" ].as< std::string >() );
-	unsigned nrThreads = options[ "nrPurgeThreads" ].as< unsigned >() ;
-
-	std::vector< std::thread > threads;
-	Osmosis::Client::PathTaskQueue purgeTasks( 1 );
-
-	// adding task per dir
-	for(auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator( rootPath ), {})) {
-		if ( entry.path().string().length() == 31) {
-			TRACE_INFO( "Adding a task for " << entry.path() );
-			boost::filesystem::path entryPath( entry.path() );
-			purgeTasks.put( std::move( entryPath ) );
-		}
-	}
-
-
-	// create threads
-	for ( unsigned i = 0; i < nrThreads; ++ i ) {
-		TRACE_INFO("Generating thread " << i << "...");
-
-		threads.push_back( std::thread(
-            purgeDir,
-			i,
-			std::ref( rootPath ),
-			std::ref( purgeTasks )
-			) );
-
-	}
-
-	TRACE_INFO("waiting for purge threads to finish...");
-	for ( auto & i : threads ) {
-		i.join();
-	}
-
+	purge.purge();
 }
 
 void renameLabel( const boost::program_options::variables_map & options )
@@ -341,10 +299,10 @@ int main( int argc, char * argv [] )
 		( "broadcastToLocalhost", "Use this to broadcast to 127.0.0.7" )
 		("timeout", boost::program_options::value< unsigned short >()->default_value( 1000 ),
 			"Timeout in seconds, for the 'whohaslabel' command")
-		("tcpTimeout", boost::program_options::value< unsigned int >()->default_value( 20000 ),
+		("tcpTimeout", boost::program_options::value< unsigned int >()->default_value( 7000 ),
 			"Timeout in milliseconds for actions on TCP sockets")
-		("nrPurgeThreads", boost::program_options::value< unsigned int >()->default_value( 1 ),
-			"Number of threads dedicated for purge");
+		( "followSymlinks", "Follow symlinks (use the poinetd file instead of the symlink file) in checkin");
+
 
 	boost::program_options::options_description positionalDescription( "positionals" );
 	positionalDescription.add_options()

py/osmosis/policy/cleanupleavelast.py

@@ -33,6 +33,7 @@ def go(self):
         if removed:
             objectStore.purge()
 
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.DEBUG)
     from osmosis import objectstore

py/osmosis/policy/disk.py

@@ -8,6 +8,6 @@ def dfPercent(location):
     try:
         line = " ".join(output.split("\n")[1:])
         return int(re.split(r"\s+", line)[4].strip("%"))
-    except:
+    except Exception:
         logging.exception("Unable to parse DF output:\n%(output)s", dict(output=output))
         raise

tests/fakeservers.py

@@ -40,7 +40,7 @@ def run(self):
             while True:
                 self._conn, peer = self._sock.accept()
                 self._serve()
-        except:
+        except Exception as ex:
             logging.exception("Fake Server")
 
     def readLog(self):

tests/httpserver.py

@@ -37,7 +37,7 @@ def _waitForTCPServer(self):
             try:
                 sock.connect((self.hostname(), self._port))
                 return
-            except:
+            except Exception:
                 pass
             finally:
                 sock.close()