You want to limit file size, measured in bytes.
You write records, one for each task
, terminated by line.separator
.
You are concerned about performance (with a limit of just 50 kB on file size - oh well).
- do not encode
char
s more than once (with 2n (guesstimated: 50) records per file, your average char
gets encoded n+1 (26) times with the second generateFile()
presented)
- using "old Java IO", have a
FilterOutputStream
tally bytes
- do not use a buffered
Writer
: it would obscure record boundaries
giving it a try:
/** Writes a limited amount of bytes to each of
* a succession of files numbered from 0 where necessary.
* Buffers liberally. */
static class SequentialFileOutputStream extends java.io.FilterOutputStream {
final static int MAXBUF = 1<<20;
final byte[]buffer;
int pos, index;
final long limit;
long left;
File file;
String path, name, suffix;
/** Creates an OutputStream that starts writing
* to <code>file</code>, creating numbered files as necessary. */
public
SequentialFileOutputStream(File file, boolean append, long limit)
throws IOException
{
super(null);
File parent = (this.file = file).getParentFile();
if (null == parent)
parent = new File(".");
// checks path, too
if (parent.getFreeSpace() < limit)
throw(new java.io.IOException("free < limit"));
this.limit = left = limit;
buffer = new byte[(int) Math.min(limit, MAXBUF)];
}
/** Closes the current part, if any. */
void closePart() throws IOException {
if (null != out) try {
flush();
} finally { try {
out.close();
} finally {
out = null;
}}
}
/** Finishes the current part, if any.
* Prepares a new one as far as necessary. */
void nextPart() throws IOException {
closePart();
}
/** Ensures <code>out</code> is open and buffer reset. */
void ensureOpen() throws FileNotFoundException {
if (null != out)
return;
left = limit;
pos = 0;
if (null == suffix) {
path = file.getPath();
name = file.getName();
int dot = name.lastIndexOf('.');
if (0 <= dot) {
suffix = name.substring(dot);
path = path.substring(0,
path.length() - suffix.length()) + '_';
} else
suffix = "";
out = new java.io.FileOutputStream(file);
} else {
if (null != file) {
File zero = new File(path + '0' + suffix);
zero.delete();
file.renameTo(zero);
file = null;
}
out = new java.io.FileOutputStream(path + ++index + suffix);
}
}
@Override
public void write(int b) throws IOException {
byte[]a = { (byte) b };
write(a, 0, a.length);
}
@Override
public void write(byte[] b, int off, int len)
throws IOException {
if (len <= 0 || null == b)
return;
ensureOpen();
if (left < len && 0 < pos) {
flush();
nextPart();
}
if (limit < len)
out.write(b, off, len);
else {
System.arraycopy(b, off, buffer, pos, len);
pos += len;
left -= len;
}
}
@Override // closePart()?
public void flush() throws IOException {
if (0 < pos) {
ensureOpen();
out.write(buffer, 0, pos);
pos = 0;
}
out.flush();
}
@Override
public void close() throws IOException { closePart(); }
}
private final String clientId;
private final Collection<Task> tasks;
public FileSplitter(String clientId, Collection<Task> tasks) {
this.clientId = clientId;
this.tasks = tasks;
}
/** Kludge to flush an OutputStreamWriter's encoder's buffer */
static class FlushWriter extends java.io.OutputStreamWriter {
final static Class[]NO_CLASSES = {};
static java.lang.reflect.Method flushEncoder;
// further constructors left as an exercise
public FlushWriter(OutputStream out) { super(out); }
/** Flushes the encoder's buffer */
java.io.Writer flushEncoder() {
try {
if (null == flushEncoder) {
final Class<?> osWriter = FlushWriter.class.getSuperclass();
flushEncoder = osWriter
.getDeclaredMethod("flushBuffer", NO_CLASSES);
flushEncoder.setAccessible(true);
}
flushEncoder.invoke(this, (Object[])null);
} catch (ReflectiveOperationException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return this;
}
}
public void generateFile() {
String fileName = "tasks_info_" + clientId + ".txt",
lineSeparator = System.getProperty("line.separator");
try (FlushWriter writer = new FlushWriter(
new SequentialFileOutputStream(
new File(new File("."), fileName), false, 9000))) {
for (Task task : tasks) {
writer.append(task.toString()).append(lineSeparator);
writer.flushEncoder();
}
} catch (IOException ex) {
ex.printStackTrace();
}
}
public static void main(String[] args) {
Task t = new Task() {@Override public String toString() {
return "I am the most useless task ever";
}};
Collection<Task>tasks = new java.util.ArrayList<>(999);
for (int i = 999 ; 0 < --i ; )
tasks.add(t);
new FileSplitter("broke", tasks).generateFile();
}
The non-buffered OutputStreamWriter
didn't suffice, in the end:
Sunsoft's encoder buffers. Trying to keep that from interfering messed things up.
The generateFile()
from the question seems to have a correctness issue with the way it guesstimates byte count: it extends sb
and accumulates "the length of byte[]
s gotten from String
representations of sb
". It should rather
- set the size to said length - or -
- accumulate the "byte length" of the
String
for each Task
which, in combination with instantiating sb
with an appropriate length, should lead to run time linear in the number of bytes in each file rather than quadratic.
But once you "have the byte[]
", you might as well collect those.
This seems to call for java.nio.ByteBuffer
&co:
public void generateFile() {
ByteBuffer bb = ByteBuffer.allocate(limit);
for (Task task : tasks) {
byte[]taskBytes = task.toString().getBytes(
java.nio.charset.StandardCharsets.UTF_8);
int taskLen = taskBytes.length+LS_LENGTH;
if (bb.remaining() < taskLen)
buffer2file(bb, index++);
if (limit < taskBytes.length+2*LS_LENGTH) {
ByteBuffer big = ByteBuffer.allocate(taskLen);
big.put(taskBytes).put(LINE_SEPARATOR_BYTES);
buffer2file(big, index++);
} else
bb.put(taskBytes).put(LINE_SEPARATOR_BYTES);
}
buffer2file(bb, index++);
}
/** Write a numbered client file if <code>bb</code> not empty. */
private void buffer2file(ByteBuffer bb, int i) {
bb.flip();
if (bb.hasRemaining()) {
String fileName = "tasks_info_" + clientId + "_" + i + ".txt";
try (java.nio.channels.FileChannel
fc = java.nio.channels.FileChannel.open(
java.nio.file.Paths.get(fileName, NO_STRINGS),
java.nio.file.StandardOpenOption.CREATE,
java.nio.file.StandardOpenOption.WRITE)) {
fc.write(bb);
} catch (IOException ex) {
ex.printStackTrace();
}
}
bb.flip();
}
static final String NO_STRINGS[] = {},
LINE_SEPARATOR = System.getProperty("line.separator");
static final byte[]
LINE_SEPARATOR_BYTES = LINE_SEPARATOR.getBytes();
static final int LS_LENGTH = LINE_SEPARATOR_BYTES.length;
UTF-8
, but write bytes to the file in the default encoding, which might be different.. \$\endgroup\$ – RobAu Feb 1 '18 at 11:53