# Logging mechanism

I have the following code:

#include <iostream>
#include <sstream>
#include <fstream>
#include <string>
#include <cassert>
#include <vector>
#include <cstdlib>
#define BOOST_FILESYSTEM_NO_DEPRECATED
#include <boost/filesystem.hpp>

using namespace std;
using namespace boost::filesystem;

static ofstream Log; // For information-level messages
static stringstream LogErrorStream;  // Buffer soft errors to output them separately after the informational messages in the log file.

// Convert any type to its string representation
template<typename T> static string ToString( const T& obj )
{
stringstream ss;
ss << obj;
return ss.str();
}

// Escape HTML special characters
static string EscapeHTMLSpecialChars( const path& PathName, const bool& href = false )
{
string in = PathName.string();
string ret;
for ( decltype( in.size() ) i = 0; i < in.size(); i++ )
{
if ( in[i] == '&' )
{
ret += "&amp;";
}
else if ( in[i] == '<' )
{
ret += "&lt;";
}
else if ( in[i] == '>' )
{
ret += "&gt;";
}
else if ( in[i] == '"' )
{
ret += "&quot;";
}
else if ( in[i] == '\'' )
{
ret += "&#39;";
}
else if ( href && in[i] == '#' ) // If the input is an href attribute
{
ret += "%23";
}
else if ( href && in[i] == '?' )
{
ret += "%3F";
}
else
{
ret += in[i];
}
}
return ret;
}

// Convert the input size ( in bytes ) to its nearest units in the ratio of 1024.
// ( Trying to do how Windows reports size of a file on right clicking and checking its properties )
static string RoundSize( const long long& size )
{
double ret = ( double )size;
static const vector<string> units {"bytes", "KiB", "MiB", "GiB", "TiB"};
const unsigned ratio = 1024;
unsigned i;
for ( i = 0; ret > ratio && i < units.size() - 1; i++ )
{
ret /= ratio;
}
}

// Iterate through a directory and store everything found ( regular files, directories or any other special files ) in the input container
static void DirectoryIterate( const path& dirPath, vector<path>& dirContents )
{
if ( is_directory( dirPath ) )
{
copy( directory_iterator( dirPath ), directory_iterator(), back_inserter( dirContents ) );
}
}

// Returns the difference in height in the filesystem tree, between the directory "parent" and the file/folder "descendant"
static int HeightDiff( const path parent, path descendant )
{
int diff = 0;
while ( descendant != parent )
{
descendant = descendant.parent_path();
if ( descendant.empty() )
{
diff = -1;  // "descendant" is in fact not a descendant of "parent"
break;
}
diff++;
}
return diff;
}

// Returns true if the file/folder "descendant" is a descendant of the directory "parent"
static bool IsDescendant( const path parent, path descendant )
{
return HeightDiff( parent, descendant ) >= 1;
}

// Create a set of HTML files containing information about source directory's contents and store it in the destination directory, in a directory structure similar to the source directory
// Returns the total size of the source directory
static long long Snapshot( const path& sourcePath, const path& destinationPath )
{
Log << sourcePath << endl;

// This should be executed only once during the whole program execution ( during the first invocation of Snapshot() )
static bool isDescendant = IsDescendant( sourcePath, destinationPath );
if ( isDescendant )
{
LogErrorStream << "The destination path cannot be a descendant of the source path!! Please provide an alternate destination path !!" << endl;
return -1;
}

boost::system::error_code ec;

long long sourcePathSize = 0; // Total size of the source directory

vector<path> dirContents, files, directories;
try
{
DirectoryIterate( sourcePath, dirContents );
}
catch ( const filesystem_error& ex )
{
LogErrorStream << ex.what() << endl;
return 0; // cannot iterate through the directory, so no point in going further
}

sort( dirContents.begin(), dirContents.end() ); // sort, since directory iteration is not ordered on some file systems
for ( const auto& item : dirContents )
{
ec.clear();
if ( is_directory( item, ec ) )
{
directories.push_back( item );
}
else if ( !ec )
{
files.push_back( item );
}
else
{
LogErrorStream << "Failed to determine if " << absolute( item ) << " is a directory or not : " << ec.message() << endl;
}
}

const path pwd = destinationPath / sourcePath.filename(); // Present working directory
ec.clear();
create_directory( pwd, ec );
if ( ec )
{
LogErrorStream << "Failed to create " << absolute( pwd ) << " : " << ec.message() << endl;
return 0;
}

// Create the output file.
const path outFilePath = ( pwd / sourcePath.filename() ).string() + ".html";
ofstream outFile( outFilePath.string() );
if ( !outFile )
{
LogErrorStream << "Failed to create " << absolute( outFilePath ) << " : " << strerror( errno ) << endl;
return 0;
}

// Write the HTML file header.
outFile << ""
"<!DOCTYPE html>\n"
"<html>\n"
"<meta charset=\"UTF-8\">\n"
"<title>" << EscapeHTMLSpecialChars( sourcePath.filename() ) << "</title>\n"
"<body>\n";

// Write information about the files contained in the source directory
outFile << ""
"<h1> Files </h1>\n"
"<table>\n";
for ( const auto& file : files )
{
outFile << ""
" <tr>\n"
"  <td>" << EscapeHTMLSpecialChars( file.filename() ) << "</td>\n"
"  <td>";

ec.clear();
auto size = file_size( file, ec );
if ( ec )
{
LogErrorStream << "Failed to read size of " << absolute( file ) << " : " << ec.message() << endl;
}
else
{
outFile << RoundSize( size );
sourcePathSize += size;
}
outFile << ""
"</td>\n"
" </tr>\n";
}
outFile << "</table>\n";

// Write information about the directories contained in the source directory
outFile << ""
"<h1> Directories </h1>\n"
"<table>\n";
for ( const auto& directory : directories )
{
long long size = Snapshot( sourcePath / directory.filename(), pwd );
sourcePathSize += size;
outFile << ""
" <tr>\n"
"  <td><a href=\"" << EscapeHTMLSpecialChars( ( directory.filename() / directory.filename() ).generic_string(), true ) << ".html\">" << EscapeHTMLSpecialChars( directory.filename() ) << "</a></td>\n" <<
"  <td>" << RoundSize( size ) << "</td>\n"
" </tr>\n";
}
outFile << "</table>\n";

// Write the footer
outFile << ""
"<br>\n"
"<h3>Total directory size = " << RoundSize( sourcePathSize ) << "</h3><br>\n"
"</body>\n"
"</html>\n";

return sourcePathSize;
}

int main( int argc, char** argv )
{
const path defaultLogFilePath = "DirectorySnapshotLog.txt";

if ( argc < 3 )
{
cout << "Usage : " << argv[0] << " <source_directory_path> <destination_directory_path> [log_file_path=" << defaultLogFilePath << "]\n";
return -1;
}

const path LogFilePath = ( ( argc >= 4 ) ? path( argv[3] ) : defaultLogFilePath );
Log.open( LogFilePath.string() );
if ( !Log )
{
cerr << "Error creating " << absolute( LogFilePath ) << " : " << strerror( errno ) << endl;
return -1;
}

try
{
Snapshot( canonical( argv[1] ), canonical( argv[2] ) );
}
catch ( const filesystem_error& ex )
{
LogErrorStream << ex.what() << endl;
}

if ( Log )
{
if ( LogErrorStream.str().empty() )
{
cout << "The program ran without any errors.\n";
}
else
{
Log << "\nERRORS -:\n\n" << LogErrorStream.str() << endl;
cout << "There were some errors during the execution of this program !\n\nCheck " << absolute( LogFilePath ) << " for details.\n";
return -1;
}
}
}


My current logging strategy is to output all the information-level messages as soon as they are encountered using Log, and to output the error-level messages only after all the information-level messages have been output. For implementing this, I store the error-level messages in a stringstream object, and output it at the end of the program.

The rationale for this strategy is that I want to see which directory the program was operating in currently by looking at the log file and refreshing it intermittently. I am not using stdout for information-level logging because I want to store it for debugging in case the program crashes.

The reason I am skeptical about this strategy is that in case the program crashes, the error-level messages might not get written to the file. So the only alternative I can think of this to log all the error messages as soon as I encounter them instead of buffering them first.

Which of the two options will better suit the program flow? Are there any better alternatives than these two?

I am using GCC 4.9.2 with the -std=c++14 option.

• Please Please Please. Do nto reinvent your own logging system. Yes its fun yes it is trivial to get something simple working. BUT. To get something correct working is non trivial and hard. Please use syslog. There are so many tools the work on top of syslog that loging any place else is bad idea. – Martin York Jun 19 '15 at 17:26

Which of the two options will better suit the program flow ?

Depends. Neither is fit for all situations, or optimal.

Are there any better alternatives than these two ?

Yes. Normally, logging is a module/part of the functionality of an application, orthogonal to the rest of the functionality (that is, you should be able to alter the app logic without changing the logging code implementation, and alter logging code without altering application logic).

Because of this, logging is usually implemented as a separate library, and should support:

• multiple sinks (write message to multiple logs at the same time)
• prioritized dispatching of messages (a warning could/should be processed differently than the error)
• configuration at runtime

As alternatives, have a look at some c++ logging frameworks (result of google search) and this question.

For an efficient logging mechanism, you want:

• it to be processed in a separate thread (logging service) fed by senders (log_msg, log_msgf ...) through a (threaded) queue.

• multiple sources and message levels

• the "send" method(s) to do as little as possible (cull messages without output, get the time, build/copy the (formatted) text and send to the service). Being able to quickly ignore a message if the source has no output for the level of the message is primordial. A pool for the messages you dispatch to the logging service is most likely a must.

• to be able to output on different medium using filters (ie: errors from that source to these outputs)

• the output to "summarize" the output with something like "the above line has been repeated X times", helping the IOs to stay low (this happens quite often)

• to reduce IOs, which implies buffered writing on the outputs. And thus you need to also able to flush one or all outputs in some cases (ie: critical message before aborting)

• to be able to issue the order to "reopen" the outputs (so you can "roll" the logs)

• You also want ask yourself if you allow yourself to lose a message (ie: what if the disk and the service queue are full? Block or wait?)

Logging is really a very interesting problem.

• And already solved. Everything mentioned above and much already done by syslog. – Martin York Jun 19 '15 at 17:27
• syslog may be an output but it's not on every architecture. Its performance let to be desired as it is not at a strong strategic place for culling (among other points). It is no contest with a specialized logger for one's application. Using syslog as one of the output can be an option of course. – Hurricane Jun 19 '15 at 23:09
• Not on every architecture. True. But on every *nix. I am not sure what you mean by culling. As there are tools that allow visualization of syslogs in many formats including veyr good filtering. It will beat any hand written logging implimentation (unless you happen to be a logging expert (all of which I count one hand)). – Martin York Jun 20 '15 at 1:06
• What I mean by culling: on the program side, you may have outputs for very low level debug (ie: debug7, say, for the inners of memory allocation, mutual exclusion, ...). You don't want to compute a format/printf of the message nor send it to the dispatch. What you want is to check if the output makes sense in the current logging setup so you can just "return" from the logging call without doing anything further. By delegating this to the logger daemon you just do a lot of work (and IOs) for no net result. – Hurricane Jun 21 '15 at 8:37
• About the visualization tools: nothing has been said about not following a standard line pattern. The subject here as far as I'm concerned is how to generate the logs with the smallest cost for the program and the highest possible throughput. – Hurricane Jun 21 '15 at 8:47