Significant speedups with minimal changes

shapefile.py

@@ -10,7 +10,7 @@
 
 __version__ = "1.2.3"
 
-from struct import pack, unpack, calcsize, error
+from struct import pack, unpack, calcsize, error, Struct
 import os
 import sys
 import time
@@ -346,7 +346,8 @@ def __shape(self):
             record.partTypes = _Array('i', unpack("<%si" % nParts, f.read(nParts * 4)))
         # Read points - produces a list of [x,y] values
         if nPoints:
-            record.points = [_Array('d', unpack("<2d", f.read(16))) for p in range(nPoints)]
+            flat = unpack("<%sd" % (2 * nPoints), f.read(16*nPoints))
+            record.points = list(izip(*(iter(flat),) * 2))
         # Read z extremes and values
         if shapeType in (13,15,18,31):
             (zmin, zmax) = unpack("<2d", f.read(16))
@@ -389,10 +390,12 @@ def __shapeIndex(self, i=None):
             numRecords = shxRecordLength // 8
             # Jump to the first record.
             shx.seek(100)
-            for r in range(numRecords):
-                # Offsets are 16-bit words just like the file length
-                self._offsets.append(unpack(">i", shx.read(4))[0] * 2)
-                shx.seek(shx.tell() + 4)
+            shxRecords = _Array('i')
+            # Each offset consists of two nrs, only the first one matters
+            shxRecords.fromfile(shx, 2 * numRecords)
+            if sys.byteorder != 'big':
+                 shxRecords.byteswap()
+            self._offsets = [2 * el for el in shxRecords[::2]]
         if not i == None:
             return self._offsets[i]
 
@@ -469,6 +472,8 @@ def __dbfHeader(self):
         if terminator != b("\r"):
             raise ShapefileException("Shapefile dbf header lacks expected terminator. (likely corrupt?)")
         self.fields.insert(0, ('DeletionFlag', 'C', 1, 0))
+        fmt,fmtSize = self.__recordFmt()
+        self.__recStruct = Struct(fmt)
 
     def __recordFmt(self):
         """Calculates the size of a .shp geometry record."""
@@ -481,8 +486,7 @@ def __recordFmt(self):
     def __record(self):
         """Reads and returns a dbf record row as a list of values."""
         f = self.__getFileObj(self.dbf)
-        recFmt = self.__recordFmt()
-        recordContents = unpack(recFmt[0], f.read(recFmt[1]))
+        recordContents = self.__recStruct.unpack(f.read(self.__recStruct.size))
         if recordContents[0] != b(' '):
             # deleted record
             return None
@@ -535,7 +539,7 @@ def record(self, i=0):
         if not self.numRecords:
             self.__dbfHeader()
         i = self.__restrictIndex(i)
-        recSize = self.__recordFmt()[1]
+        recSize = self.__recStruct.size
         f.seek(0)
         f.seek(self.__dbfHeaderLength() + (i * recSize))
         return self.__record()
@@ -544,13 +548,11 @@ def records(self):
         """Returns all records in a dbf file."""
         if not self.numRecords:
             self.__dbfHeader()
-        records = []
         f = self.__getFileObj(self.dbf)
         f.seek(self.__dbfHeaderLength())
-        for i in range(self.numRecords):
-            r = self.__record()
-            if r:
-                records.append(r)
+        flat = unpack(self.__recStruct.format * self.numRecords, f.read(self.__recStruct.size * self.numRecords))
+        rowlen = len(self.fields) - 1
+        records = list(izip(*(iter(flat),) * rowlen))
         return records
 
     def iterRecords(self):