I'm using this code in an app that has hundreds of daily users. The code works, and passes all of the test cases I have.
Occasionally some users report a problem that I don't get right to the bottom of, and there is a possibility it is related to this PermaDict. Can you see anything I may have missed in terms of robustness, or corner cases that may cause it to fail? (And of course, any other Code Review feedback is welcome).
import pickle, json, csv, os, shutil class PermaDict(dict): ''' Persistent dictionary with an API compatible with shelve and anydbm. The dict is kept in memory, so the dictionary operations run as fast as a regular dictionary. Write to disk is delayed until close or sync (similar to gdbm's fast mode). Input file format is automatically discovered. Output file format is selectable between pickle, json, and csv. All three serialization formats are backed by fast C implementations. ''' def __init__(self, filename, flag='c', mode=None, format='json', *args, **kwds): self.flag = flag # r=readonly, c=create, or n=new self.mode = mode # None or an octal triple like 0644 self.format = format # 'csv', 'json', or 'pickle' self.filename = filename if flag != 'n' and os.access(filename, os.R_OK): fileobj = open(filename, 'rb') # 'rb' if format=='pickle' else 'r') with fileobj: self.load(fileobj) dict.__init__(self, *args, **kwds) def sync(self): 'Write dict to disk' if self.flag == 'r': return filename = self.filename tempname = filename + '.tmp' try: with open(tempname, 'wb') as fileobj: # if self.format=='pickle' else 'w') as fileobj: self.dump(fileobj) except Exception: os.remove(tempname) raise shutil.move(tempname, self.filename) # atomic commit if self.mode is not None: os.chmod(self.filename, self.mode) def close(self): self.sync() def __enter__(self): return self def __exit__(self, *exc_info): self.close() def dump(self, fileobj): if self.format == 'csv': csv.writer(fileobj).writerows(self.items()) elif self.format == 'json': json.dump(self, fileobj, separators=(',', ':')) elif self.format == 'pickle': pickle.dump(dict(self), fileobj, 2) else: raise NotImplementedError('Unknown format: ' + repr(self.format)) def load(self, fileobj): # try formats from most restrictive to least restrictive for loader in (pickle.load, json.load, csv.reader): fileobj.seek(0) try: return self.update(loader(fileobj)) except Exception: pass raise ValueError('File not in a supported format')
Example usage of a "PermaDict" is:
Opening/initialising application state:
state_filename = os.path.join(app_pathname,"AppState.txt") try: state = permadict.PermaDict(state_filename) except: pub.sendMessage(b"ERROR_MESSAGE", msg = "I can't figure out how to read your saved information!") raise state['global']['root_dir_name'] = root_dir_name state['global']['app_dir_name'] = app_dir_name state.sync()
def _CallForRefresh(self): # we're going to do our stuff, processing files etc... # while we're doing that, we don't want to interrupt ourselves # due to making a file system change self._eventHandler.SetQueueMode() # Do the stuff... self._mainWindow.Refresh() self._appState.sync()
Down in _mainWindow.Refresh(), we have stuff like:
wx.Dialog.__init__(self, None, -1, _("Messaging"), style=wx.DEFAULT_FRAME_STYLE, pos=(app_state["Chat"]["pos_x"], app_state["Chat"]["pos_y"]), size=(app_state["Chat"]["width"], app_state["Chat"]["height"]) )
def OnResize(self, event): (self._appState['Chat']['width'], self._appState['Chat']['height']) = event.GetSize() event.Skip()
And of course:
# Here we handle incoming events from the GUI def OnAppEvent(self, event, value = None): # most important thing: if we have to die, then do it if event == AppEvents.APP_EXIT or event == AppEvents.APP_AUTO_EXIT: if event == AppEvents.APP_AUTO_EXIT: wx.Sound.PlaySound('autoExit.wav') wx.MilliSleep(500) self._appState.close() self._mainWindow.ShowStatusIcon(False) sys.exit()
One obvious way to screw up usage of this storage is to fail to sync() it and then bomb out without going through OnExit() ... I think I have that covered (and at least, it's something I'm aware of, as opposed to anything suspect in the PermaDict code that I might not be aware of!)