From 283ea2c906ae08f4bb8b6b621f9a69e65218309d Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:58:19 +0000 Subject: [PATCH 1/3] Warn or raise if dbd record length doesn't match that in the header. --- README.md | 2 +- src/shapefile.py | 44 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 49e35b3..04e1c88 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The Python Shapefile Library (PyShp) reads and writes ESRI Shapefiles in pure Py - **Author**: [Joel Lawhead](https://github.com/GeospatialPython) - **Maintainers**: [James Parrott](https://github.com/JamesParrott) & [Karim Bahgat](https://github.com/karimbahgat) -- **Version**: 3.1.8 +- **Version**: 3.1.9.dev - **Date**: 9th Nov 2026 - **License**: [MIT](https://github.com/GeospatialPython/pyshp/blob/master/LICENSE.TXT) diff --git a/src/shapefile.py b/src/shapefile.py index bccf7db..fda632b 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -8,7 +8,7 @@ from __future__ import annotations -__version__ = "3.1.8" +__version__ = "3.1.9.dev" import abc import array @@ -239,6 +239,10 @@ class PossibleDataLoss(Warning): pass +class CorruptDbfRecord(Warning): + pass + + class DbfStringDataLoss(ValueError): pass @@ -3054,8 +3058,23 @@ def _record_fmt(self, fields: Container[str] | None = None) -> str: fmt_size = calcsize(fmt) # total size of fields should add up to recordlength from the header # so calculate the difference, and concat a field of that many pad bytes. - if fmt_size < self._record_length: - num_pad_bytes = self._record_length - fmt_size + + num_pad_bytes = self._record_length - fmt_size + if num_pad_bytes != 0: + msg = ( + f"The (fixed) record length declared in the file header: {self._record_length} " + f"did not match the record size: {fmt_size} " + f"calculated from the struct format string: {fmt}" + ) + + if fmt_size > self._record_length or self.strict: + raise dbfFileException(msg) + else: + warnings.warn( + f"{msg}. Struct fmt string will be padded with '{num_pad_bytes}x' (any such bytes in the record will be ignored). ", + category=CorruptDbfRecord, + ) + fmt = f"{fmt}{num_pad_bytes}x" return fmt @@ -3111,15 +3130,22 @@ def _record( # The only format chars from self._record_fmt, in recStruct from _record_fields, # are s and x (ascii encoded str and pad byte) so everything in recordContents is bytes # https://docs.python.org/3/library/struct.html#format-characters - recordContents = recStruct.unpack(self.file.read(recStruct.size)) + deletionFlag, *recordContents = recStruct.unpack(self.file.read(recStruct.size)) - # deletion flag field is always unpacked as first value (see _record_fmt) - if recordContents[0] != b" ": - # deleted record + if deletionFlag == b"*": + # "The byte's value is a space (0x20), if the record is active, + # or an asterisk (0x2A), if the record is deleted. " + # https://en.wikipedia.org/wiki/.dbf#Database_records return None - # drop deletion flag from values - recordContents = recordContents[1:] + if deletionFlag != b" ": + warnings.warn( + ( + f"On record: {oid}, deletion flag not marked for deletion (not b'*') " + f"but also not an ascii space( not b' '). Got: {deletionFlag!r}" + ), + category=CorruptDbfRecord, + ) # check that values match fields if len(fieldTuples) != len(recordContents): From b0514245573b8d579d4fe480bdf7cb6ff71e08b2 Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:15:43 +0000 Subject: [PATCH 2/3] Support ignoreing deletion flags .dbf files. Warn or raise if deletion flag invalid. --- src/shapefile.py | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/src/shapefile.py b/src/shapefile.py index fda632b..9f4e223 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -2483,6 +2483,7 @@ def __init__( field_positions: dict[str, int], values: Iterable[RecordValue], oid: int | None = None, + deletion_flag: bytes = b" ", ): """ A Record should be created by the Reader class @@ -2496,6 +2497,7 @@ def __init__( self._oid = oid else: self._oid = -1 + self.deletion_flag = deletion_flag list.__init__(self, values) def __getattr__(self, item: str) -> RecordValue: @@ -3118,6 +3120,7 @@ def _record( recLookup: dict[str, int], recStruct: Struct, oid: int | None = None, + ignore_deletion_flag: bool = False, ) -> _Record | None: """Reads and returns a dbf record row as a list of values. Requires self.file to already be pointing to the start of the record (e.g. from a previous call @@ -3130,22 +3133,24 @@ def _record( # The only format chars from self._record_fmt, in recStruct from _record_fields, # are s and x (ascii encoded str and pad byte) so everything in recordContents is bytes # https://docs.python.org/3/library/struct.html#format-characters - deletionFlag, *recordContents = recStruct.unpack(self.file.read(recStruct.size)) + deletion_flag, *recordContents = recStruct.unpack( + self.file.read(recStruct.size) + ) - if deletionFlag == b"*": + if deletion_flag == b"*": # "The byte's value is a space (0x20), if the record is active, # or an asterisk (0x2A), if the record is deleted. " # https://en.wikipedia.org/wiki/.dbf#Database_records - return None - - if deletionFlag != b" ": - warnings.warn( - ( - f"On record: {oid}, deletion flag not marked for deletion (not b'*') " - f"but also not an ascii space( not b' '). Got: {deletionFlag!r}" - ), - category=CorruptDbfRecord, + if not ignore_deletion_flag: + return None + elif deletion_flag != b" " and not ignore_deletion_flag: + msg = ( + f"On record: {oid}, deletion flag not marked for deletion (not b'*') " + f"but also not an ascii space( not b' '). Got: {deletion_flag!r}" ) + if self.strict: + raise dbfFileException(msg) + warnings.warn(msg, category=CorruptDbfRecord) # check that values match fields if len(fieldTuples) != len(recordContents): @@ -3220,9 +3225,14 @@ def _record( ) record.append(value) - return _Record(recLookup, record, oid) + return _Record(recLookup, record, oid=oid, deletion_flag=deletion_flag) - def record(self, i: int = 0, fields: list[str] | None = None) -> _Record | None: + def record( + self, + i: int = 0, + fields: list[str] | None = None, + ignore_deletion_flag: bool = False, + ) -> _Record | None: """Returns a specific dbf record based on the supplied index. To only read some of the fields, specify the 'fields' arg as a list of one or more fieldnames. @@ -3234,7 +3244,11 @@ def record(self, i: int = 0, fields: list[str] | None = None) -> _Record | None: self.seek_to_rec(index=i) fieldTuples, recLookup, recStruct = self._record_fields(fields) return self._record( - oid=i, fieldTuples=fieldTuples, recLookup=recLookup, recStruct=recStruct + oid=i, + fieldTuples=fieldTuples, + recLookup=recLookup, + recStruct=recStruct, + ignore_deletion_flag=ignore_deletion_flag, ) def records( From d12bced32e56e2bc8e0da785282ec425156030e3 Mon Sep 17 00:00:00 2001 From: James Parrott <80779630+JamesParrott@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:20:30 +0000 Subject: [PATCH 3/3] Use _attr on _Record --- src/shapefile.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/shapefile.py b/src/shapefile.py index 9f4e223..89583be 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -2497,9 +2497,13 @@ def __init__( self._oid = oid else: self._oid = -1 - self.deletion_flag = deletion_flag + self._deletion_flag = deletion_flag list.__init__(self, values) + @property + def deletion_flag(self) -> bytes: + return self._deletion_flag + def __getattr__(self, item: str) -> RecordValue: """ __getattr__ is called if an attribute is used that does