I'm not familiar with threads but I am not against them either. A bit of background the original program would have taken about 2 years to run. I had modified it and knocked down that time to around 40 days.
Can I make it even faster by using threads or updating my code?
The reason it is 40 days is because it has to navigate through a directory of about 6 million files. That amount of files slows this program down.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Data.SqlClient;
using System.IO;
using System.Data;
using System.Threading;
namespace LattitudeDocumentationOrganizer
{
class Program
{
static string connString = LattitudeDocumentationOrganizer.Properties.Settings.Default.ConnectionString;
static void Main(string[] args)
{
string fileName;
Guid documentID;
string location;
DateTime createdDate;
bool runLoop = true;
int x = 0;
string msg;
int fileCount = 0;
int errorCount = 0;
int number = 0;
while (runLoop)
{
Console.Clear();
// reset variables
fileName = "";
documentID = Guid.Empty;
location = "";
using (SqlConnection connection = new SqlConnection(connString))
{
connection.Open();
// Run through
string sql = @"select top 1 d.UID, d.CreatedDate, d.Location, m.number from master m
inner join documentation_attachments da
on m.number = da.accountid
inner join documentation d
on da.documentid = d.uid
where m.qlevel in (998,999)
and d.location is not null
and uid not in (select documentid from DocumentationIssues)";
using (SqlCommand command = new SqlCommand(sql, connection))
{
using (SqlDataReader reader = command.ExecuteReader())
{
// If no data is returned assume no data is left and report statistic and exit loop
if (!reader.HasRows)
{
runLoop = false;
Console.WriteLine("Processed {0} files successfully.", fileCount);
Console.WriteLine("Did not process {0} files successfully.", errorCount);
Console.WriteLine("No more files were found with the current query");
Console.ReadLine();
Console.WriteLine("Exiting program.");
break;
}
// Close the loop out as a specific day and time.
DateTime currentDate = DateTime.Now;
if (currentDate.DayOfWeek == DayOfWeek.Monday && currentDate.Hour >= 20)
{
runLoop = false;
Console.WriteLine("Processed {0} files successfully.", fileCount);
Console.WriteLine("Did not process {0} files successfully.", errorCount);
Console.WriteLine("The time set for this process to end has been reached. Program exited at {0}", DateTime.Now);
Console.ReadLine();
Console.WriteLine("Exiting program.");
break;
}
Console.WriteLine("Processing data...");
while (reader.Read())
{
// Row Values
// 0 = UID
// 1 = CreatedDate
// 2 = Location
documentID = reader.GetGuid(0);
fileName = reader.GetSqlValue(0).ToString() + ".zip";
location = reader.GetString(2);
createdDate = reader.GetDateTime(1);
number = reader.GetInt32(3);
Console.WriteLine("Current File #: {0}", fileCount);
Console.WriteLine("Working on document {0}", documentID);
FileInfo fileinfo = new FileInfo(Path.Combine(location, fileName));
if (!fileinfo.Exists)
{
// Log error to DocumentationIssues
msg = "This file does not exist";
LogError(documentID, location, null, msg, number);
Console.WriteLine("This file did not exist, logged it to the database and moving on to the next file.");
errorCount++;
}
else
{
// file exists begin process to create new folders
var fileYear = "DOCS" + createdDate.Year.ToString();
var fileMonth = createdDate.ToString("MMM");
var rootDir = @"\\server";
Console.WriteLine("File Exists, checking to make sure the directories needed exist.");
if (!Directory.Exists(Path.Combine(rootDir, fileYear)))
{
// Error, cannot create root level network share folder. Log to Database.
// Error Root Level Folder Missing
// Directory.CreateDirectory(Path.Combine(rootDir,fileYear));
// Log error to DocumentationIssues
msg = "Could not create root folder, log to skip this file";
LogError(documentID, location, Path.Combine(rootDir, fileYear), msg, number);
errorCount++;
}
else
{
if (!Directory.Exists(Path.Combine(rootDir, fileYear, fileMonth)))
{
// Create the month folder
Directory.CreateDirectory(Path.Combine(rootDir, fileYear, fileMonth));
Console.WriteLine("The month folder did not exist, created {0} folder", fileMonth);
}
// Call method to update location in database and move tile
UpdateDocument(documentID, Path.Combine(rootDir, fileYear, fileMonth));
fileinfo.MoveTo(Path.Combine(rootDir, fileYear, fileMonth, fileName));
//File.Move(Path.Combine(location, fileName), Path.Combine(rootDir, fileYear, fileMonth, fileName));
msg = "SUCCESS";
LogError(documentID, location, Path.Combine(rootDir, fileYear, fileMonth), msg, number);
//runLoop = false;
Console.WriteLine("Successfully moved and logged the file, checking for more files.");
fileCount++;
}
}
}
}
}
}
}
}
static void UpdateDocument(Guid id, string newLocation)
{
using (SqlConnection connection = new SqlConnection(connString))
{
connection.Open();
string sql = "update documentation set location = @newLocation where uid = @id";
using (SqlCommand command = new SqlCommand(sql, connection))
{
command.Parameters.Add(new SqlParameter("id", id));
command.Parameters.Add(new SqlParameter("newLocation", newLocation));
command.ExecuteNonQuery();
}
}
}
static void LogError(Guid id, string prevLocation, string newLocation, string msg, int number)
{
if (newLocation == null)
{
newLocation = "no new location";
}
using (SqlConnection connection = new SqlConnection(connString))
{
connection.Open();
string sql = @"insert into documentationissues (documentid, oldLocation, newLocation, errorMessage, dateAdded, number)
values (@id, @prevLocation, @newLocation, @msg, GETDATE(), @number)";
using (SqlCommand command = new SqlCommand(sql, connection))
{
command.Parameters.Add(new SqlParameter("id", id));
command.Parameters.Add(new SqlParameter("prevLocation", prevLocation));
command.Parameters.Add(new SqlParameter("newLocation", newLocation));
command.Parameters.Add(new SqlParameter("msg", msg));
command.Parameters.Add(new SqlParameter("number", number));
command.ExecuteNonQuery();
}
}
}
}
}
