From 40d47670cdb4deafd479f84e0ef02a64b6a5d346 Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:05:31 +0000 Subject: [PATCH 1/5] Change list.append("") in while loop to += [""]*N --- src/shapefile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shapefile.py b/src/shapefile.py index 4986b43..ff7b5a3 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -4403,8 +4403,8 @@ def record( fieldCount = sum(1 for field in self.fields if field[0] != "DeletionFlag") if recordList: record = list(recordList) - while len(record) < fieldCount: - record.append("") + record += [""] * (fieldCount - len(record)) + elif recordDict: record = [] for field in self.fields: From ce3ee6972d346ad7a38755d532d1d198c15c90aa Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:04:54 +0000 Subject: [PATCH 2/5] Get rid of DeletionFlag everwhere but .fields and _record_fmt Clarify comment where deletion flag is written to .dbf Delete test_record_subfields_delflag_notvalid --- src/shapefile.py | 86 ++++++++++++++++++----------------------- tests/test_shapefile.py | 9 ----- 2 files changed, 38 insertions(+), 57 deletions(-) diff --git a/src/shapefile.py b/src/shapefile.py index ff7b5a3..108f4ca 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -2982,7 +2982,7 @@ def __init__( self.encodingErrors = encodingErrors self.strict = strict - self.fields: list[Field] = [] + self.data_fields: list[Field] = [] self._fieldLookup: dict[str, int] = {} self._dbfHeader() @@ -3005,7 +3005,7 @@ def _dbfHeader(self) -> None: # read fields numFields = (self._dbfHdrLength - 33) // 32 for __ in range(numFields): - self.fields.append( + self.data_fields.append( Field.from_byte_stream( b_io=self.file, encoding=self.encoding, @@ -3019,40 +3019,37 @@ def _dbfHeader(self) -> None: "Dbf header lacks expected terminator. (likely corrupt?)" ) - # insert deletion field at start - self.fields.insert(0, Field("DeletionFlag", FieldType.C, 1, 0)) - # store all field positions for easy lookups # note: fieldLookup gives the index position of a field inside Reader.fields - self._fieldLookup = {f.name: i for i, f in enumerate(self.fields)} + self._fieldLookup = {f.name: i for i, f in enumerate(self.data_fields)} - # by default, read all fields except the deletion flag, hence "[1:]" # note: recLookup gives the index position of a field inside a _Record list - fieldnames = [f.name for f in self.fields[1:]] + fieldnames = [f.name for f in self.data_fields] __fieldTuples, recLookup, recStruct = self._record_fields(fieldnames) self._fullRecStruct = recStruct self._fullRecLookup = recLookup @property - def data_fields(self) -> list[Field]: - """All fields except the DeletionFlag.""" - return self.fields[1:] + def fields(self) -> list[Field]: + """To preserve legacy behaviour, preppend data fields with the DeletionFlag.""" + return [Field("DeletionFlag", FieldType.C, 1, 0), *self.data_fields] def _record_fmt(self, fields: Container[str] | None = None) -> str: - """Calculates the format and size of a .dbf record. Optional 'fields' arg + """Calculates the struct format string for of a .dbf record. Optional 'fields' arg specifies which fieldnames to unpack and which to ignore. Note that this - always includes the DeletionFlag at index 0, regardless of the 'fields' arg. + The DeletionFlag is always included at index 0 (to correctly unpack the dbf record). + The items in fields may appear in any order. """ - structcodes = [f"{fieldinfo.size}s" for fieldinfo in self.fields] - if fields is not None: - # only unpack specified fields, ignore others using padbytes (x) - structcodes = [ - code - if fieldinfo.name in fields - or fieldinfo.name == "DeletionFlag" # always unpack delflag - else f"{fieldinfo.size}x" - for fieldinfo, code in zip(self.fields, structcodes) - ] + structcodes = ["s"] # Always unpack DeletionFlag + for fieldinfo in self.data_fields: + if ( + fieldinfo.size >= 2 + ): # "1s" or "s", makes no difference to struct fmt strs + structcodes.append(str(fieldinfo.size)) + structcodes.append( + "s" if fields is None or fieldinfo.name in fields else "x" + ) + fmt = "".join(structcodes) fmt_size = calcsize(fmt) # total size of fields should add up to recordlength from the header @@ -3069,30 +3066,29 @@ def _record_fields( """Returns the necessary info required to unpack a record's fields, restricted to a subset of fieldnames 'fields' if specified. Returns a list of field info tuples, a name-index lookup dict, - and a Struct instance for unpacking these fields. Note that DeletionFlag - is not a valid field. + and a Struct instance for unpacking these fields. """ if fields is not None: # restrict info to the specified fields # first ignore repeated field names (order doesn't matter) - unique_fields = list(set(fields)) + unique_fields = set(fields) # get the struct fmt = self._record_fmt(fields=unique_fields) recStruct = Struct(fmt) # make sure the given fieldnames exist for name in unique_fields: - if name not in self._fieldLookup or name == "DeletionFlag": + if name not in self._fieldLookup: raise ValueError(f'"{name}" is not a valid field name') # fetch relevant field info tuples - fieldTuples = [] - for fieldinfo in self.fields[1:]: - if fieldinfo.name in unique_fields: - fieldTuples.append(fieldinfo) + fieldTuples = [ + fieldinfo + for fieldinfo in self.data_fields + if fieldinfo.name in unique_fields + ] # store the field positions recLookup = {f.name: i for i, f in enumerate(fieldTuples)} else: - # use all the dbf fields - fieldTuples = self.fields[1:] # sans deletion flag + fieldTuples = self.data_fields recStruct = self._fullRecStruct recLookup = self._fullRecLookup return fieldTuples, recLookup, recStruct @@ -3849,11 +3845,11 @@ def numRecords(self) -> int | None: @property def fields(self) -> list[Field]: + """Includes DeletionFlag to preserve legacy behaviour.""" return self.dbf_reader.fields @property def data_fields(self) -> list[Field]: - """All fields except the DeletionFlag.""" return self.dbf_reader.data_fields def record(self, i: int = 0, fields: list[str] | None = None) -> _Record | None: @@ -4347,21 +4343,19 @@ def _header(self) -> None: version = 3 year, month, day = time.localtime()[:3] year -= 1900 - # Get all fields, ignoring DeletionFlag if specified - fields = [field for field in self.fields if field.name != "DeletionFlag"] # Ensure has at least one field - if not fields: + if not self.fields: raise dbfFileException( "Shapefile dbf file must contain at least one field." ) numRecs = self.recNum - numFields = len(fields) + numFields = len(self.fields) headerLength = numFields * 32 + 33 if headerLength >= 65535: raise dbfFileException( "Shapefile dbf header length exceeds maximum length." ) - recordLength = sum(field.size for field in fields) + 1 + recordLength = sum(field.size for field in self.fields) + 1 header = pack( " None: f.write(header) # Field descriptors - for field in fields: + for field in self.fields: f.write( field.encode_field_descriptor( encoding=self.encoding, @@ -4400,7 +4394,7 @@ def record( field/value pairs only fields matching the already registered fields will be added.""" record: list[RecordValue] - fieldCount = sum(1 for field in self.fields if field[0] != "DeletionFlag") + fieldCount = len(self.fields) if recordList: record = list(recordList) record += [""] * (fieldCount - len(record)) @@ -4408,8 +4402,6 @@ def record( elif recordDict: record = [] for field in self.fields: - if field[0] == "DeletionFlag": - continue # ignore deletionflag field in case it was specified if field[0] in recordDict: val = recordDict[field[0]] if val is None: @@ -4434,14 +4426,12 @@ def _record(self, record: list[RecordValue]) -> None: # allowing us to write the dbf header # cannot change the fields after this point self._header() - # first byte of the record is deletion flag, always disabled + # first byte of the record is deletion flag + # (b" " for not deleted, b"*" for deleted). record_stream.write(b" ") # begin - fields = ( - field for field in self.fields if field[0] != "DeletionFlag" - ) # ignore deletionflag field in case it was specified - for (fieldName, fieldType, size, deci), value in zip(fields, record): + for (fieldName, fieldType, size, deci), value in zip(self.fields, record): # write # fieldName, fieldType, size and deci were already checked # when their Field instance was created and added to self.fields diff --git a/tests/test_shapefile.py b/tests/test_shapefile.py index aafbf23..33c7fb6 100644 --- a/tests/test_shapefile.py +++ b/tests/test_shapefile.py @@ -1029,15 +1029,6 @@ def test_record_subfields_unordered(): test_record_attributes(fields=fields) -def test_record_subfields_delflag_notvalid(): - """ - Assert that reader does not consider DeletionFlag as a valid field name. - """ - fields = ["DeletionFlag", "AREA", "POP1990", "MALES", "FEMALES", "MOBILEHOME"] - with pytest.raises(ValueError): - test_record_attributes(fields=fields) - - def test_record_subfields_duplicates(): """ Assert that reader correctly retrieves only a subset From 2b47f61c8117511ae6293fc890ec16df42352088 Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:35:10 +0000 Subject: [PATCH 3/5] Speed up iterShapeRecords with bbox --- src/shapefile.py | 56 +++++++++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/src/shapefile.py b/src/shapefile.py index 108f4ca..a86d8c5 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -3100,9 +3100,11 @@ def _record( recStruct: Struct, oid: int | None = None, ) -> _Record | None: - """Reads and returns a dbf record row as a list of values. Requires specifying - a list of field info Field namedtuples 'fieldTuples', a record name-index dict 'recLookup', - and a Struct instance 'recStruct' for unpacking these fields. + """Reads and returns a dbf record row as a list of values. Requires self.file + to already be pointing to the start of the record (e.g. from a previous call + to self.file.seek). Also requires specifying a list of field info Field + namedtuples 'fieldTuples', a record name-index dict 'recLookup', and a Struct + instance 'recStruct' for unpacking these fields. None is returned for 'deleted' records (those with their deletion flag marked). """ @@ -3203,9 +3205,7 @@ def record(self, i: int = 0, fields: list[str] | None = None) -> _Record | None: Returns None if record's deletion flag is marked. """ i = ensure_within_bounds(i, self.numRecords) - recSize = self._record_length - self.file.seek(0) - self.file.seek(self._dbfHdrLength + (i * recSize)) + self.seek_to_rec(index=i) fieldTuples, recLookup, recStruct = self._record_fields(fields) return self._record( oid=i, fieldTuples=fieldTuples, recLookup=recLookup, recStruct=recStruct @@ -3232,9 +3232,7 @@ def records( Set deleted_as_None=True to insert None for these, to preserve the indexing, as for DbfReader.record """ - return [ - record for record in self.iterRecords(fields, start, stop, deleted_as_None) - ] + return list(self.iterRecords(fields, start, stop, deleted_as_None)) def iterRecords( self, @@ -3276,8 +3274,7 @@ def iterRecords( ) elif stop < 0: stop = range(self.numRecords)[stop] - recSize = self._record_length - self.file.seek(self._dbfHdrLength + (start * recSize)) + self.seek_to_rec(index=start) fieldTuples, recLookup, recStruct = self._record_fields(fields) for i in range(start, stop): r = self._record( @@ -3286,6 +3283,10 @@ def iterRecords( if r is not None or deleted_as_None: yield r + def seek_to_rec(self, index: int) -> None: + recSize = self._record_length + self.file.seek(self._dbfHdrLength + (index * recSize)) + class ShxReader(_HasCheckedReadableFile): """Reads an shx file (a file of file positions @@ -4153,16 +4154,23 @@ def shapes(self, bbox: BBox | None = None) -> Shapes: shapes = Shapes(self.iterShapes(bbox=bbox)) return shapes - def iterShapes(self, bbox: BBox | None = None) -> Iterator[Shape | None]: + def iterShapes( + self, + bbox: BBox | None = None, + outside_bbox_as_None: bool = False, + ) -> Iterator[Shape | None]: """Returns a generator of shapes in a shapefile. Useful for handling large shapefiles. To only read shapes within a given spatial region, specify the 'bbox' - arg as a list or tuple of xmin,ymin,xmax,ymax. + arg as a list or tuple of xmin,ymin,xmax,ymax. To return None for + shapes outside bbox instead of skipping them, use outside_bbox_as_None=True. """ shape_lengths_B = self.shx_reader.shape_lengths_B if self._shx else None bbox = BBox(*bbox) if bbox is not None else None yield from self.shp_reader.iterShapes( - shape_lengths_B=shape_lengths_B, bbox=bbox + shape_lengths_B=shape_lengths_B, + bbox=bbox, + outside_bbox_as_None=outside_bbox_as_None, ) self.numShapes = len(self.shp_reader.headers_cache) @@ -4224,14 +4232,22 @@ def iterShapeRecords( yield ShapeRecord(shape=shape, record=record) else: # Only yield ShapeRecords whose shape.bbox overlaps with bbox. - # TODO: internal _record method should be faster but would have to - # make sure to seek to correct file location... - # fieldTuples,recLookup,recStruct = self._record_fields(fields) - for shape in self.iterShapes(bbox=bbox): + fieldTuples, recLookup, recStruct = self.dbf_reader._record_fields(fields) + last_oid: None | int = None + for shape in self.iterShapes(bbox=bbox, outside_bbox_as_None=True): if shape: - # record = self._record(oid=i, fieldTuples=fieldTuples, recLookup=recLookup, recStruct=recStruct) - record = self.dbf_reader.record(i=shape.oid, fields=fields) + if shape.oid - 1 != last_oid: + self.dbf_reader.seek_to_rec(index=shape.oid) + + record = self.dbf_reader._record( + fieldTuples=fieldTuples, + recLookup=recLookup, + recStruct=recStruct, + oid=shape.oid, + ) + last_oid = shape.oid + yield ShapeRecord(shape=shape, record=record) From 16cc637ee88208dc6ead15f21b7d80fd272bf4c5 Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:02:12 +0000 Subject: [PATCH 4/5] Make Doctests refer to Reader.data_fields instead of Reader.fields[1:] --- README.md | 31 ++++++++++++++----------------- src/shapefile.py | 5 +++-- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 060aad0..392421b 100644 --- a/README.md +++ b/README.md @@ -423,22 +423,20 @@ A new field can be created as follows: Using this method the conversion from string to enum is done automatically. -To see the fields for the Reader object above (sf) call the "fields" +To see the fields for the Reader object above (sf) call the "data_fields" attribute: + >>> sf.data_fields + [Field(name="AREA", field_type=FieldType.N, size=18, decimal=5), Field(name="BKG_KEY", field_type=FieldType.C, size=12, decimal=0), Field(name="POP1990", field_type=FieldType.N, size=9, decimal=0), Field(name="POP90_SQMI", field_type=FieldType.N, size=10, decimal=1), Field(name="HOUSEHOLDS", field_type=FieldType.N, size=9, decimal=0), Field(name="MALES", field_type=FieldType.N, size=9, decimal=0), Field(name="FEMALES", field_type=FieldType.N, size=9, decimal=0), Field(name="WHITE", field_type=FieldType.N, size=9, decimal=0), Field(name="BLACK", field_type=FieldType.N, size=8, decimal=0), Field(name="AMERI_ES", field_type=FieldType.N, size=7, decimal=0), Field(name="ASIAN_PI", field_type=FieldType.N, size=8, decimal=0), Field(name="OTHER", field_type=FieldType.N, size=8, decimal=0), Field(name="HISPANIC", field_type=FieldType.N, size=8, decimal=0), Field(name="AGE_UNDER5", field_type=FieldType.N, size=8, decimal=0), Field(name="AGE_5_17", field_type=FieldType.N, size=8, decimal=0), Field(name="AGE_18_29", field_type=FieldType.N, size=8, decimal=0), Field(name="AGE_30_49", field_type=FieldType.N, size=8, decimal=0), Field(name="AGE_50_64", field_type=FieldType.N, size=8, decimal=0), Field(name="AGE_65_UP", field_type=FieldType.N, size=8, decimal=0), Field(name="NEVERMARRY", field_type=FieldType.N, size=8, decimal=0), Field(name="MARRIED", field_type=FieldType.N, size=9, decimal=0), Field(name="SEPARATED", field_type=FieldType.N, size=7, decimal=0), Field(name="WIDOWED", field_type=FieldType.N, size=8, decimal=0), Field(name="DIVORCED", field_type=FieldType.N, size=8, decimal=0), Field(name="HSEHLD_1_M", field_type=FieldType.N, size=8, decimal=0), Field(name="HSEHLD_1_F", field_type=FieldType.N, size=8, decimal=0), Field(name="MARHH_CHD", field_type=FieldType.N, size=8, decimal=0), Field(name="MARHH_NO_C", field_type=FieldType.N, size=8, decimal=0), Field(name="MHH_CHILD", field_type=FieldType.N, size=7, decimal=0), Field(name="FHH_CHILD", field_type=FieldType.N, size=7, decimal=0), Field(name="HSE_UNITS", field_type=FieldType.N, size=9, decimal=0), Field(name="VACANT", field_type=FieldType.N, size=7, decimal=0), Field(name="OWNER_OCC", field_type=FieldType.N, size=8, decimal=0), Field(name="RENTER_OCC", field_type=FieldType.N, size=8, decimal=0), Field(name="MEDIAN_VAL", field_type=FieldType.N, size=7, decimal=0), Field(name="MEDIANRENT", field_type=FieldType.N, size=4, decimal=0), Field(name="UNITS_1DET", field_type=FieldType.N, size=8, decimal=0), Field(name="UNITS_1ATT", field_type=FieldType.N, size=7, decimal=0), Field(name="UNITS2", field_type=FieldType.N, size=7, decimal=0), Field(name="UNITS3_9", field_type=FieldType.N, size=8, decimal=0), Field(name="UNITS10_49", field_type=FieldType.N, size=8, decimal=0), Field(name="UNITS50_UP", field_type=FieldType.N, size=8, decimal=0), Field(name="MOBILEHOME", field_type=FieldType.N, size=7, decimal=0)] - >>> sf.fields - [Field(name="DeletionFlag", field_type=FieldType.C, size=1, decimal=0), Field(name="AREA", field_type=FieldType.N, size=18, decimal=5), Field(name="BKG_KEY", field_type=FieldType.C, size=12, decimal=0), Field(name="POP1990", field_type=FieldType.N, size=9, decimal=0), Field(name="POP90_SQMI", field_type=FieldType.N, size=10, decimal=1), Field(name="HOUSEHOLDS", field_type=FieldType.N, size=9, decimal=0), Field(name="MALES", field_type=FieldType.N, size=9, decimal=0), Field(name="FEMALES", field_type=FieldType.N, size=9, decimal=0), Field(name="WHITE", field_type=FieldType.N, size=9, decimal=0), Field(name="BLACK", field_type=FieldType.N, size=8, decimal=0), Field(name="AMERI_ES", field_type=FieldType.N, size=7, decimal=0), Field(name="ASIAN_PI", field_type=FieldType.N, size=8, decimal=0), Field(name="OTHER", field_type=FieldType.N, size=8, decimal=0), Field(name="HISPANIC", field_type=FieldType.N, size=8, decimal=0), Field(name="AGE_UNDER5", field_type=FieldType.N, size=8, decimal=0), Field(name="AGE_5_17", field_type=FieldType.N, size=8, decimal=0), Field(name="AGE_18_29", field_type=FieldType.N, size=8, decimal=0), Field(name="AGE_30_49", field_type=FieldType.N, size=8, decimal=0), Field(name="AGE_50_64", field_type=FieldType.N, size=8, decimal=0), Field(name="AGE_65_UP", field_type=FieldType.N, size=8, decimal=0), Field(name="NEVERMARRY", field_type=FieldType.N, size=8, decimal=0), Field(name="MARRIED", field_type=FieldType.N, size=9, decimal=0), Field(name="SEPARATED", field_type=FieldType.N, size=7, decimal=0), Field(name="WIDOWED", field_type=FieldType.N, size=8, decimal=0), Field(name="DIVORCED", field_type=FieldType.N, size=8, decimal=0), Field(name="HSEHLD_1_M", field_type=FieldType.N, size=8, decimal=0), Field(name="HSEHLD_1_F", field_type=FieldType.N, size=8, decimal=0), Field(name="MARHH_CHD", field_type=FieldType.N, size=8, decimal=0), Field(name="MARHH_NO_C", field_type=FieldType.N, size=8, decimal=0), Field(name="MHH_CHILD", field_type=FieldType.N, size=7, decimal=0), Field(name="FHH_CHILD", field_type=FieldType.N, size=7, decimal=0), Field(name="HSE_UNITS", field_type=FieldType.N, size=9, decimal=0), Field(name="VACANT", field_type=FieldType.N, size=7, decimal=0), Field(name="OWNER_OCC", field_type=FieldType.N, size=8, decimal=0), Field(name="RENTER_OCC", field_type=FieldType.N, size=8, decimal=0), Field(name="MEDIAN_VAL", field_type=FieldType.N, size=7, decimal=0), Field(name="MEDIANRENT", field_type=FieldType.N, size=4, decimal=0), Field(name="UNITS_1DET", field_type=FieldType.N, size=8, decimal=0), Field(name="UNITS_1ATT", field_type=FieldType.N, size=7, decimal=0), Field(name="UNITS2", field_type=FieldType.N, size=7, decimal=0), Field(name="UNITS3_9", field_type=FieldType.N, size=8, decimal=0), Field(name="UNITS10_49", field_type=FieldType.N, size=8, decimal=0), Field(name="UNITS50_UP", field_type=FieldType.N, size=8, decimal=0), Field(name="MOBILEHOME", field_type=FieldType.N, size=7, decimal=0)] +From this, e.g. to get a list of all field names: -The first field of a dbf file is always a 1-byte field called "DeletionFlag", -which indicates records that have been deleted but not removed. However, -since this flag is very rarely used, PyShp currently will return all records -regardless of their deletion flag, and the flag is also not included in the list of -record values. In other words, the DeletionFlag field has no real purpose, and -should in most cases be ignored. For instance, to get a list of all fieldnames: + >>> fieldnames = [f.name for f in sf.data_fields] - - >>> fieldnames = [f[0] for f in sf.fields[1:]] +The first byte of each dbf file record is a "DeletionFlag". To preserve compatibility with older +versions of PyShp, a list of fields starting with this pseudo-field is returned by sf.fields. However, +the DeletionFlag field is very rarely used, has no real purpose for shapefiles (as records and shapes must +be balanced), and so should in most cases be ignored. You can get a list of the shapefile's records by calling the records() method: @@ -448,8 +446,7 @@ You can get a list of the shapefile's records by calling the records() method: >>> len(records) 663 -To read a single record call the record() method with the record's index: - +To read a single record, call the record() method with the record's index: >>> rec = sf.record(3) @@ -868,7 +865,7 @@ This can be particularly useful for copying from one file to another: >>> r = shapefile.Reader('tests/shapefiles/test/polygon') >>> w = shapefile.Writer('tests/shapefiles/test/copy') - >>> w.fields = r.fields[1:] # skip first deletion field + >>> w.fields = r.data_fields >>> # adding existing Shape objects >>> for shaperec in r.iterShapeRecords(): @@ -1003,7 +1000,7 @@ should give you the same unicode string you started with. >>> w = shapefile.Writer("tests/shapefiles/test/latin_as_utf8.shp", encoding="utf8") - >>> w.fields = r.fields[1:] + >>> w.fields = r.data_fields >>> w.record(*r.record(0)) >>> w.null() >>> r.close() @@ -1252,7 +1249,7 @@ memory. The following example copies the contents of a shapefile to a new file 1 >>> # copy over fields from the reader >>> r = shapefile.Reader("tests/shapefiles/blockgroups") - >>> for field in r.fields[1:]: + >>> for field in r.data_fields: ... w.field(*field) >>> # copy the shapefile to writer 10 times @@ -1285,7 +1282,7 @@ file one record at a time, modify or filter the contents, and write it back out. >>> # copy over the relevant fields from the reader >>> r = shapefile.Reader("tests/shapefiles/blockgroups") - >>> for field in r.fields[1:]: + >>> for field in r.data_fields: ... if field[0] in keep_fields: ... w.field(*field) diff --git a/src/shapefile.py b/src/shapefile.py index a86d8c5..bc2c6cd 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -4371,6 +4371,7 @@ def _header(self) -> None: raise dbfFileException( "Shapefile dbf header length exceeds maximum length." ) + # + 1 for deletion flag. recordLength = sum(field.size for field in self.fields) + 1 header = pack( " tuple[int, int]: - """Appends s to the file. Returns shape's offset and length in B""" + """Appends s to the file. Returns shape's offset and length in Bytes""" if not isinstance(s, Shape): if isinstance(s, HasGeoInterface): shape_dict = s.__geo_interface__ @@ -4774,7 +4775,7 @@ def shape( return self._shp_record(s) def _shp_record(self, s: Shape) -> tuple[int, int]: - """Appends s to the file. Returns shape's offset and length in B""" + """Appends s to the file. Returns shape's offset and length in Bytes""" offset = self.file.tell() self.shpNum += 1 From d7780785c6edd5d9b57970bffc2111c95539f1d8 Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:21:12 +0000 Subject: [PATCH 5/5] Update changelog + Readme --- README.md | 17 ++++++++++++----- changelog.txt | 4 ++++ src/shapefile.py | 2 +- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 392421b..49e35b3 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The Python Shapefile Library (PyShp) reads and writes ESRI Shapefiles in pure Py - **Author**: [Joel Lawhead](https://github.com/GeospatialPython) - **Maintainers**: [James Parrott](https://github.com/JamesParrott) & [Karim Bahgat](https://github.com/karimbahgat) -- **Version**: 3.1.8.dev +- **Version**: 3.1.8 - **Date**: 9th Nov 2026 - **License**: [MIT](https://github.com/GeospatialPython/pyshp/blob/master/LICENSE.TXT) @@ -93,14 +93,17 @@ part of your geospatial project. # Recent version Changes See also the Older version Changes section below + +## 3.1.8 +### Internal code improvements + - Avoid unnecessary seeks in Reader.iterShapeRecords with a bbox. + - Removed lots of deletion field checks (prefer dbfReader.data_fields). + - Replace two while loops with simpler alternatives. + ## 3.1.7 ### Bug fix - Non-positive .shp record lengths (@lunnuyenhi-dev) and mismatched shape types now raise CorruptShpShapeRecord instead of hanging forever, raising a KeyError, or passing silently. -## 3.1.6 -### Feature - - Encodings can now be read from .cpg files (and optionally written to them). - # The Basics @@ -1511,6 +1514,10 @@ Zac Miller # Older version changes +## 3.1.6 +### Feature + - Encodings can now be read from .cpg files (and optionally written to them). + ## 3.1.5 ### Bug fix - Fixed another bug causing dates before the year 1000 to be encoded as less than 8 chars under "%Y%m%d (found by [Thomas Beierlein](https://github.com/GeospatialPython/pyshp/issues/435)) diff --git a/changelog.txt b/changelog.txt index 9416a46..9a8b996 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,7 @@ +VERSION 3.1.8 +2026-11-07 + * Avoid unnecessary seeks in Reader.iterShapeRecords with a bbox. Removed lots of deletion field checks (prefer dbfReader.data_fields). Replace two while loops with simpler alternatives. + VERSION 3.1.7 2026-11-07 * Improved handling of corrupt .shp record headers [@lunnuyenhi-dev] (see [issue 453](https://github.com/GeospatialPython/pyshp/issues/453)) diff --git a/src/shapefile.py b/src/shapefile.py index bc2c6cd..bccf7db 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -8,7 +8,7 @@ from __future__ import annotations -__version__ = "3.1.8.dev" +__version__ = "3.1.8" import abc import array