BUG: Patch read_csv NA values behaviour

Patches the following behaviour when `na_values` is passed in as a
dictionary:    1. Prevent aliasing in case `na_values` was defined in
a broader scope.  2. Respect column indices as keys when doing NA
conversions.    Closes #14203.

Author: gfyoung <gfyoung17@gmail.com>

Closes #14751 from gfyoung/csv-na-values-patching and squashes the following commits:

cac422c [gfyoung] BUG: Respect column indices for dict-like na_values
1439c27 [gfyoung] BUG: Prevent aliasing of dict na_values

(cherry picked from commit dd8cba2767)
This commit is contained in:
gfyoung 2016-12-16 18:30:26 -05:00 committed by Joris Van den Bossche
parent c520b25944
commit c9e5bf41f7
4 changed files with 62 additions and 13 deletions

View File

@ -40,6 +40,8 @@ Bug Fixes
- Compat with ``dateutil==2.6.0``; segfault reported in the testing suite (:issue:`14621`)
- Allow ``nanoseconds`` in ``Timestamp.replace`` as a kwarg (:issue:`14621`)
- Bug in ``pd.read_csv`` in which aliasing was being done for ``na_values`` when passed in as a dictionary (:issue:`14203`)
- Bug in ``pd.read_csv`` in which column indices for a dict-like ``na_values`` were not being respected (:issue:`14203`)
- Bug in ``pd.read_csv`` where reading files fails, if the number of headers is equal to the number of lines in the file (:issue:`14515`)
- Bug in ``pd.read_csv`` for the Python engine in which an unhelpful error message was being raised when multi-char delimiters were not being respected with quotes (:issue:`14582`)
- Fix bugs (:issue:`14734`, :issue:`13654`) in ``pd.read_sas`` and ``pandas.io.sas.sas7bdat.SAS7BDATReader`` that caused problems when reading a SAS file incrementally.

View File

@ -2040,8 +2040,27 @@ class PythonParser(ParserBase):
col = self.orig_names[col]
clean_conv[col] = f
return self._convert_to_ndarrays(data, self.na_values, self.na_fvalues,
self.verbose, clean_conv)
# Apply NA values.
clean_na_values = {}
clean_na_fvalues = {}
if isinstance(self.na_values, dict):
for col in self.na_values:
na_value = self.na_values[col]
na_fvalue = self.na_fvalues[col]
if isinstance(col, int) and col not in self.orig_names:
col = self.orig_names[col]
clean_na_values[col] = na_value
clean_na_fvalues[col] = na_fvalue
clean_na_values = self.na_values
clean_na_fvalues = self.na_fvalues
return self._convert_to_ndarrays(data, clean_na_values,
clean_na_fvalues, self.verbose,
def _to_recarray(self, data, columns):
dtypes = []
@ -2749,6 +2768,7 @@ def _clean_na_values(na_values, keep_default_na=True):
na_values = []
na_fvalues = set()
elif isinstance(na_values, dict):
na_values = na_values.copy() # Prevent aliasing.
if keep_default_na:
for k, v in compat.iteritems(na_values):
if not is_list_like(v):

View File

@ -266,3 +266,26 @@ nan,B
out = self.read_csv(StringIO(data), names=names,
na_values={'a': 2, 'b': 1})
tm.assert_frame_equal(out, expected)
def test_na_values_dict_aliasing(self):
na_values = {'a': 2, 'b': 1}
na_values_copy = na_values.copy()
names = ['a', 'b']
data = '1,2\n2,1'
expected = DataFrame([[1.0, 2.0], [np.nan, np.nan]], columns=names)
out = self.read_csv(StringIO(data), names=names, na_values=na_values)
tm.assert_frame_equal(out, expected)
tm.assert_dict_equal(na_values, na_values_copy)
def test_na_values_dict_col_index(self):
# see gh-14203
data = 'a\nfoo\n1'
na_values = {0: 'foo'}
out = self.read_csv(StringIO(data), na_values=na_values)
expected = DataFrame({'a': [np.nan, 1]})
tm.assert_frame_equal(out, expected)

View File

@ -1243,19 +1243,23 @@ cdef class TextReader:
return None, set()
if isinstance(self.na_values, dict):
key = None
values = None
if name is not None and name in self.na_values:
values = self.na_values[name]
if values is not None and not isinstance(values, list):
values = list(values)
fvalues = self.na_fvalues[name]
if fvalues is not None and not isinstance(fvalues, set):
fvalues = set(fvalues)
if i in self.na_values:
return self.na_values[i], self.na_fvalues[i]
return _NA_VALUES, set()
key = name
elif i in self.na_values:
key = i
else: # No na_values provided for this column.
return _NA_VALUES, set()
values = self.na_values[key]
if values is not None and not isinstance(values, list):
values = list(values)
fvalues = self.na_fvalues[key]
if fvalues is not None and not isinstance(fvalues, set):
fvalues = set(fvalues)
return _ensure_encoded(values), fvalues