I have looked online for transposing programs, but have found that they have extreme RAM use for large tables. The RAM use is so extreme I can't run most jobs on my laptop. I also have a Perl program, this uses less memory than online tools, but the RAM use is still extreme (~50 GB RAM for a 8 GB file) or extremely slow (days to run, this takes a few seconds depending on the input size).
I have written a program in GNU99 C which works like this:
./2tranpose -i in.tsv -o out.tsv
and the source code for 2transpose.c is as follows:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define GNU_SOURCE
void help () {
puts("This program is run thus:");
puts("./transpose -i infile -o outfile");
}
int main(int argc, char * argv[]) {
unsigned short int infile = 0, outfile = 0;
//I didn't like getopt's version of reading files, this is my own design
for (unsigned short int loop = 1; loop < argc; loop++) {
if (strcmp(argv[loop], "-i") == 0) {
if (loop == (argc-1)) {//prevents segfault on argv
printf("the '%s' option requires something to come after it.\n", argv[loop]);
printf("Failed at %s line %u\n", __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
infile = loop+1;
loop++;
} else if (strcmp(argv[loop], "-o") == 0) {
if (loop == (argc-1)) {//prevents segfault on argv
printf("the '%s' option requires something to come after it.\n", argv[loop]);
printf("Failed at %s line %u\n", __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
outfile = loop+1;
loop++;
} else {
printf("%s isn't a recognized option.\n", argv[loop]);
exit(EXIT_FAILURE);
}
}
if (infile == 0) {
puts("infile not defined.");
help();
exit(EXIT_FAILURE);
}
if (outfile == 0) {
puts("outfile not defined.");
help();
exit(EXIT_FAILURE);
}
FILE *restrict fh = fopen(argv[infile], "r");
if (fh == NULL) {
printf("failed to read %s\n", argv[infile]);
perror("");
exit(EXIT_FAILURE);
}
unsigned int nrow = 0, ncol = 0;
char *line = NULL;
size_t len = 0;
ssize_t read = 0;
size_t ALL_DATA_size = sizeof(char);
char ***restrict ALL_DATA = malloc(sizeof(char));
while ((read = getline ( &line, &len, fh)) != -1) {
char *restrict tmp_string = NULL;
char * tmp_pointer = NULL;
tmp_string = strtok_r(line, "\t ", &tmp_pointer);
size_t this_row_size = sizeof(char) * strlen(tmp_string) + sizeof(char*);
ALL_DATA_size += this_row_size;
ALL_DATA = realloc(ALL_DATA, ALL_DATA_size);
if (ALL_DATA == NULL) {
printf("ALL_DATA failed realloc at %s line %u at %s line %u\n", __FILE__, __LINE__, argv[infile], nrow+1);
perror("");
exit(EXIT_FAILURE);
}
ALL_DATA[nrow] = malloc(this_row_size);
if (ALL_DATA[nrow] == NULL) {
printf("ALL_DATA[%u] failed realloc at %s line %u at %s line %u\n", nrow, __FILE__, __LINE__, argv[infile], nrow+1);
perror("");
exit(EXIT_FAILURE);
}
ALL_DATA[nrow][0] = malloc(this_row_size);
if (ALL_DATA[nrow][0] == NULL) {
printf("failed to realloc ALL_DATA[%u][0] @ %s line %u\n", nrow, __FILE__, __LINE__);
perror("");
exit(EXIT_FAILURE);
}
strcpy(ALL_DATA[nrow][0], tmp_string);
ncol = 1;
while (tmp_string != NULL) {
tmp_string = strtok_r(NULL, "\t ", &tmp_pointer);
if (tmp_string == NULL) {
break;
}
const unsigned int STR_LENGTH = sizeof(char)*strlen(tmp_string) + sizeof(char*);
this_row_size += STR_LENGTH ;
ALL_DATA_size += STR_LENGTH ;
ALL_DATA = realloc(ALL_DATA, ALL_DATA_size);
if (ALL_DATA == NULL) {
printf("failed to realloc ALL_DATA for row %u @ %s line %u w/ string %s with length %u\n", nrow, __FILE__, __LINE__, tmp_string, STR_LENGTH);
perror("");
exit(EXIT_FAILURE);
}
ALL_DATA[nrow] = realloc(ALL_DATA[nrow], this_row_size);
if (ALL_DATA[nrow] == NULL) {
printf("failed to realloc ALL_DATA[%u] @ %s line %u\n", nrow,__FILE__, __LINE__);
perror("");
exit(EXIT_FAILURE);
}
ALL_DATA[nrow][ncol] = malloc(STR_LENGTH);
if (ALL_DATA[nrow][ncol] == NULL) {
printf("failed to realloc ALL_DATA[%u][%u] @ %s line %u\n", nrow, ncol, __FILE__, __LINE__);
perror("");
exit(EXIT_FAILURE);
}
strcpy(ALL_DATA[nrow][ncol], tmp_string);
ncol++;
}
const unsigned int LENGTH = strlen(ALL_DATA[nrow][ncol-1]);
if (ALL_DATA[nrow][ncol-1][LENGTH-1] == '\n') {
ALL_DATA[nrow][ncol-1][LENGTH-1] = '\0';
}
nrow++;
}
fclose(fh);
free(line); line = NULL;
FILE *restrict out = fopen(argv[outfile], "w");
if (out == NULL) {
printf("failed to write %s\n", argv[outfile]);
perror("");
exit(EXIT_FAILURE);
}
for (unsigned int row = 0; row < ncol; row++) {
fprintf(out, "%s", ALL_DATA[0][row]);
for (unsigned int col = 1; col < nrow; col++) {
fprintf(out, "\t%s", ALL_DATA[col][row]);
}
fprintf(out, "\n");
}
fclose(out);
for (unsigned int row = 0; row < nrow; row++) {
for (unsigned int col = 0; col < ncol; col++) {
free(ALL_DATA[row][col]); ALL_DATA[row][col] = NULL;
}
free(ALL_DATA[row]); ALL_DATA[row] = NULL;
}
free(ALL_DATA); ALL_DATA = NULL;
return 0;
}
from what I can tell, this program works, but I'd like to keep RAM use at a minimum (which was the goal of the program).
Overall, the program isn't complicated, it saves strings into a massive 2D array and then switches rows with columns. But, the devil is in the details.
Is this good coding style in C? perhaps this can be made more efficient?
ALL_DATA
, which is a pointer to pointers, withchar ***restrict ALL_DATA = malloc(sizeof(char));
? \$\endgroup\$const char *rowptr[]
combined withmmap
might be fast and memory efficient. \$\endgroup\$