Source Code for Module pycosmicstar.filedict

  1  """filedict.py 
  2  a Persistent Dictionary in Python 
  3   
  4  Author: Erez Shinan 
  5  Date  : 31-May-2009 
  6  """ 
  7   
  8  from collections import UserDict 
  9  import pickle 
 10  import sqlite3 
 11  import hashlib 
 12   
 13 -class Solutions: 
 14      Sqlite3 = 0 
 15   
 16 -class FileDict(UserDict): 
 17      """A dictionary that stores its data persistantly in a file 
 18   
 19      Options: 
 20          filename - which file to use 
 21          connection - use an existing connection instead of a filename (overrides filename) 
 22          table - which table name to use for storing data (default: 'dict') 
 23   
 24      """ 
 25   
 26 -    def __init__(self, filename=None, solution=Solutions.Sqlite3, **options): 
 27          assert solution == Solutions.Sqlite3, "Only sqlite3 is supported right now" 
 28          try: 
 29              self.__conn = options.pop('connection') 
 30          except KeyError: 
 31              if not filename: 
 32                  raise ValueError("Must provide 'connection' or 'filename'") 
 33              self.__conn = sqlite3.connect(filename) 
 34   
 35          self.__tablename = options.pop('table', 'dict') 
 36   
 37          self._nocommit = False 
 38   
 39          assert not options, "Unrecognized options: %s" % options 
 40   
 41          self.__conn.execute('CREATE TABLE IF NOT EXISTS %s (id integer primary key, hash integer, key blob, value blob);'%self.__tablename) 
 42          self.__conn.execute('CREATE INDEX IF NOT EXISTS %s_index ON %s(hash);' % (self.__tablename, self.__tablename)) 
 43          self.__conn.commit() 
 44   
 45 -    def _commit(self): 
 46          if self._nocommit: 
 47              return 
 48   
 49          self.__conn.commit() 
 50   
 51 -    def __pack(self, value): 
 52          return pickle.dumps(value, 3) 
 53 -    def __unpack(self, value): 
 54          return pickle.loads(value) 
 55   
 56 -    def __hash(self, data): 
 57          binary_data = self.__pack(data) 
 58          hash = int(hashlib.md5(binary_data).hexdigest(),16) 
 59          # We need a 32bit hash: 
 60          return hash % 0x7FFFFFFF 
 61   
 62 -    def __get_id(self, key): 
 63          cursor = self.__conn.execute('SELECT key,id FROM %s WHERE hash=?;'%self.__tablename, (self.__hash(key),)) 
 64          for k,id in cursor: 
 65              if self.__unpack(k) == key: 
 66                  return id 
 67   
 68          raise KeyError(key) 
 69   
 70 -    def __getitem__(self, key): 
 71          cursor = self.__conn.execute('SELECT key,value FROM %s WHERE hash=?;'%self.__tablename, (self.__hash(key),)) 
 72          for k,v in cursor: 
 73              if self.__unpack(k) == key: 
 74                  return self.__unpack(v) 
 75   
 76          raise KeyError(key) 
 77   
 78 -    def __setitem(self, key, value): 
 79          value_pickle = self.__pack(value) 
 80   
 81          try: 
 82              id = self.__get_id(key) 
 83              cursor = self.__conn.execute('UPDATE %s SET value=? WHERE id=?;'%self.__tablename, (value_pickle, id) ) 
 84          except KeyError: 
 85              key_pickle = self.__pack(key) 
 86              cursor = self.__conn.execute('INSERT INTO %s (hash, key, value) values (?, ?, ?);' 
 87                      %self.__tablename, (self.__hash(key), key_pickle, value_pickle) ) 
 88   
 89          assert cursor.rowcount == 1 
 90   
 91 -    def __setitem__(self, key, value): 
 92          self.__setitem(key, value) 
 93          self._commit() 
 94   
 95 -    def __delitem__(self, key): 
 96          id = self.__get_id(key) 
 97          cursor = self.__conn.execute('DELETE FROM %s WHERE id=?;'%self.__tablename, (id,)) 
 98          if cursor.rowcount <= 0: 
 99              raise KeyError(key) 
100   
101          self._commit() 
102   
103   
104 -    def update(self, d): 
105          for k,v in d.items(): 
106              self.__setitem(k, v) 
107          self._commit() 
108   
109 -    def __iter__(self): 
110          return (self.__unpack(x[0]) for x in self.__conn.execute('SELECT key FROM %s;'%self.__tablename) ) 
111 -    def keys(self): 
112          return iter(self) 
113 -    def values(self): 
114          return (self.__unpack(x[0]) for x in self.__conn.execute('SELECT value FROM %s;'%self.__tablename) ) 
115 -    def items(self): 
116          return (list(map(self.__unpack, x)) for x in self.__conn.execute('SELECT key,value FROM %s;'%self.__tablename) ) 
117   
118 -    def __contains__(self, key): 
119          try: 
120              self.__get_id(key) 
121              return True 
122          except KeyError: 
123              return False 
124   
125 -    def __len__(self): 
126          return self.__conn.execute('SELECT COUNT(*) FROM %s;' % self.__tablename).fetchone()[0] 
127   
128 -    def __del__(self): 
129          try: 
130              self.__conn 
131          except AttributeError: 
132              pass 
133          else: 
134              self.__conn.commit() 
135   
136      @property 
137 -    def batch(self): 
138          return self._Batch(self) 
139   
140 -    class _Batch: 
141 -        def __init__(self, d): 
142              self.__d = d 
143   
144 -        def __enter__(self): 
145              self.__old_nocommit = self.__d._nocommit 
146              self.__d._nocommit = True 
147              return self.__d 
148   
149 -        def __exit__(self, type, value, traceback): 
150              self.__d._nocommit = self.__old_nocommit 
151              self.__d._commit() 
152              return True 
153