diff --git a/csv-core/README.md b/csv-core/README.md index 07afd997..044b9964 100644 --- a/csv-core/README.md +++ b/csv-core/README.md @@ -52,7 +52,8 @@ loop { match result { ReadFieldResult::InputEmpty => {}, ReadFieldResult::OutputFull => panic!("field too large"), - ReadFieldResult::Field { record_end } => { + ReadFieldResult::Field { record_end } + | ReadFieldResult::Invalid { record_end } => { count_fields += 1; if record_end { count_records += 1; diff --git a/csv-core/benches/bench.rs b/csv-core/benches/bench.rs index 2aa24a92..57201fa8 100644 --- a/csv-core/benches/bench.rs +++ b/csv-core/benches/bench.rs @@ -65,7 +65,7 @@ fn count_fields(rdr: &mut Reader, mut data: &[u8]) -> u64 { match res { InputEmpty => {} OutputFull => panic!("field too large"), - Field { .. } => { + Field { .. } | Invalid { .. } => { count += 1; } End => break, @@ -86,7 +86,7 @@ fn count_records(rdr: &mut Reader, mut data: &[u8]) -> u64 { match res { InputEmpty => {} OutputFull | OutputEndsFull => panic!("field too large"), - Record => count += 1, + Record | Invalid => count += 1, End => break, } } diff --git a/csv-core/src/lib.rs b/csv-core/src/lib.rs index 1e41cb5d..77a593bd 100644 --- a/csv-core/src/lib.rs +++ b/csv-core/src/lib.rs @@ -36,7 +36,8 @@ loop { match result { ReadFieldResult::InputEmpty => {}, ReadFieldResult::OutputFull => panic!("field too large"), - ReadFieldResult::Field { record_end } => { + ReadFieldResult::Field { record_end } + | ReadFieldResult::Invalid { record_end } => { count_fields += 1; if record_end { count_records += 1; diff --git a/csv-core/src/reader.rs b/csv-core/src/reader.rs index 5bcbdfaa..2acd122f 100644 --- a/csv-core/src/reader.rs +++ b/csv-core/src/reader.rs @@ -92,6 +92,9 @@ use crate::Terminator; /// default. /// * Records are permitted to be of varying length. /// * Empty lines (that do not include other whitespace) are ignored. +/// +/// A reader built with [`ReaderBuilder::strict`] still finds the same parse, +/// but reports fields and records containing invalid quoting. #[derive(Clone, Debug)] pub struct Reader { /// A table-based DFA for parsing CSV. @@ -115,6 +118,12 @@ pub struct Reader { /// If enabled (the default), then quotes are respected. When disabled, /// quotes are not treated specially. quoting: bool, + /// Whether to report fields and records with invalid quoting. + strict: bool, + /// Whether the field being parsed has invalid quoting. + invalid_field: bool, + /// Whether the record being parsed has invalid quoting. + invalid_record: bool, /// Whether to use the NFA for parsing. /// /// Generally this is for debugging. There's otherwise no good reason @@ -141,6 +150,9 @@ impl Default for Reader { double_quote: true, comment: None, quoting: true, + strict: false, + invalid_field: false, + invalid_record: false, use_nfa: false, line: 1, has_read: false, @@ -238,6 +250,29 @@ impl ReaderBuilder { self } + /// Enable or disable strict parsing. + /// + /// When enabled, a field with invalid quoting is reported as + /// `ReadFieldResult::Invalid` by `read_field`, and a record containing + /// such a field is reported as `ReadRecordResult::Invalid` by + /// `read_record`. The parse itself is unchanged: the field data and + /// field ends written to the output buffers are the same as without + /// strict parsing. + /// + /// Quoting is invalid when a quote appears in a field that does not start + /// with a quote, when the closing quote of a quoted field is followed by + /// anything other than a delimiter, a record terminator or the end of the + /// data, or when the data ends inside a quoted field. A quote that is part + /// of a doubled quote or that follows the escape byte inside a quoted field + /// is not a closing quote. Strict parsing has no effect when quoting is + /// disabled. + /// + /// This is disabled by default. + pub fn strict(&mut self, yes: bool) -> &mut ReaderBuilder { + self.rdr.strict = yes; + self + } + /// A convenience method for specifying a configuration to read ASCII /// delimited text. /// @@ -280,6 +315,15 @@ pub enum ReadFieldResult { /// This state can only be returned when an empty input buffer is provided /// by the caller. End, + /// The end of a field with invalid quoting was found. + /// + /// This is only returned when strict parsing is enabled. It is returned + /// in place of `Field`, and the field data is the same as it would be for + /// `Field`. + Invalid { + /// Whether this was the last field in a record or not. + record_end: bool, + }, } impl ReadFieldResult { @@ -349,11 +393,18 @@ pub enum ReadRecordResult { /// This state can only be returned when an empty input buffer is provided /// by the caller. End, + /// The end of a record with invalid quoting in at least one of its fields + /// was found. + /// + /// This is only returned when strict parsing is enabled. It is returned + /// in place of `Record`, and the record data and field ends are the same + /// as they would be for `Record`. + Invalid, } impl ReadRecordResult { fn is_record(&self) -> bool { - *self == ReadRecordResult::Record + matches!(*self, ReadRecordResult::Record | ReadRecordResult::Invalid) } fn from_nfa( @@ -485,6 +536,8 @@ impl Reader { self.line = 1; self.has_read = false; self.output_pos = 0; + self.invalid_field = false; + self.invalid_record = false; } /// Return the current line number as measured by the number of occurrences @@ -531,7 +584,8 @@ impl Reader { /// # Errors /// /// This CSV reader can never return an error. Instead, it prefers *a* - /// parse over *no* parse. + /// parse over *no* parse. When strict parsing is enabled, a field with + /// invalid quoting is reported with `ReadFieldResult::Invalid`. pub fn read_field( &mut self, input: &[u8], @@ -587,7 +641,8 @@ impl Reader { /// # Errors /// /// This CSV reader can never return an error. Instead, it prefers *a* - /// parse over *no* parse. + /// parse over *no* parse. When strict parsing is enabled, a record with + /// invalid quoting is reported with `ReadRecordResult::Invalid`. pub fn read_record( &mut self, input: &[u8], @@ -626,6 +681,9 @@ impl Reader { ends: &mut [usize], ) -> (ReadRecordResult, usize, usize, usize) { if input.is_empty() { + if self.is_invalid_end_dfa(self.dfa_state) { + self.set_invalid(); + } let s = self.transition_final_dfa(self.dfa_state); let res = self.dfa.new_read_record_result(s, true, false, false, false); @@ -644,7 +702,7 @@ impl Reader { self.dfa_state = s; ends[0] = self.output_pos; self.output_pos = 0; - (res, 0, 0, 1) + (self.finish_record(res), 0, 0, 1) } _ => { self.dfa_state = s; @@ -661,7 +719,10 @@ impl Reader { let (mut nin, mut nout, mut nend) = (0, 0, 0); let mut state = self.dfa_state; while nin < input.len() && nout < output.len() && nend < ends.len() { - let (s, has_out) = self.dfa.get_output(state, input[nin]); + let (s, has_out, invalid) = self.dfa.get_output(state, input[nin]); + if invalid { + self.set_invalid(); + } self.line += (input[nin] == b'\n') as u64; state = s; if has_out { @@ -689,6 +750,7 @@ impl Reader { nout >= output.len(), nend >= ends.len(), ); + let res = self.finish_record(res); self.dfa_state = state; if res.is_record() { self.output_pos = 0; @@ -705,6 +767,9 @@ impl Reader { output: &mut [u8], ) -> (ReadFieldResult, usize, usize) { if input.is_empty() { + if self.is_invalid_end_dfa(self.dfa_state) { + self.set_invalid(); + } self.dfa_state = self.transition_final_dfa(self.dfa_state); let res = self.dfa.new_read_field_result( self.dfa_state, @@ -712,7 +777,7 @@ impl Reader { false, false, ); - return (res, 0, 0); + return (self.finish_field(res), 0, 0); } if output.is_empty() { return (ReadFieldResult::OutputFull, 0, 0); @@ -722,7 +787,10 @@ impl Reader { while nin < input.len() && nout < output.len() { let b = input[nin]; self.line += (b == b'\n') as u64; - let (s, has_out) = self.dfa.get_output(state, b); + let (s, has_out, invalid) = self.dfa.get_output(state, b); + if invalid { + self.set_invalid(); + } state = s; if has_out { output[nout] = b; @@ -740,7 +808,7 @@ impl Reader { nout >= output.len(), ); self.dfa_state = state; - (res, nin, nout) + (self.finish_field(res), nin, nout) } /// Perform the final state transition, i.e., when the caller indicates @@ -819,13 +887,21 @@ impl Reader { // possible combinations of state and input byte. for &state in NFA_STATES { for c in (0..256).map(|c| c as u8) { + let mut nfa_from = state; let mut nfa_result = (state, NfaInputAction::Epsilon); // Consume NFA states until we hit a non-epsilon transition. while nfa_result.0 != NfaState::End && nfa_result.1 == NfaInputAction::Epsilon { + nfa_from = nfa_result.0; nfa_result = self.transition_nfa(nfa_result.0, c); } + let invalid = self.is_invalid_transition( + nfa_from, + c, + nfa_result.0, + &nfa_result.1, + ); let from = self.dfa.new_state(state); let to = self.dfa.new_state(nfa_result.0); self.dfa.set( @@ -833,6 +909,7 @@ impl Reader { c, to, nfa_result.1 == NfaInputAction::CopyToOutput, + invalid, ); } } @@ -853,6 +930,9 @@ impl Reader { ends: &mut [usize], ) -> (ReadRecordResult, usize, usize, usize) { if input.is_empty() { + if self.is_invalid_end(self.nfa_state) { + self.set_invalid(); + } let s = self.transition_final_nfa(self.nfa_state); let res = ReadRecordResult::from_nfa(s, false, false, false); return match res { @@ -863,7 +943,7 @@ impl Reader { self.nfa_state = s; ends[0] = self.output_pos; self.output_pos = 0; - (res, 0, 0, 1) + (self.finish_record(res), 0, 0, 1) } _ => { self.nfa_state = s; @@ -877,10 +957,13 @@ impl Reader { if ends.is_empty() { return (ReadRecordResult::OutputEndsFull, 0, 0, 0); } - let (mut nin, mut nout, mut nend) = (0, self.output_pos, 0); + let (mut nin, mut nout, mut nend) = (0, 0, 0); let mut state = self.nfa_state; while nin < input.len() && nout < output.len() && nend < ends.len() { let (s, io) = self.transition_nfa(state, input[nin]); + if self.is_invalid_transition(state, input[nin], s, &io) { + self.set_invalid(); + } match io { NfaInputAction::CopyToOutput => { output[nout] = input[nin]; @@ -894,7 +977,7 @@ impl Reader { } state = s; if state.is_field_final() { - ends[nend] = nout; + ends[nend] = self.output_pos + nout; nend += 1; if state != NfaState::EndFieldDelim { break; @@ -907,8 +990,13 @@ impl Reader { nout >= output.len(), nend >= ends.len(), ); + let res = self.finish_record(res); self.nfa_state = state; - self.output_pos = if res.is_record() { 0 } else { nout }; + if res.is_record() { + self.output_pos = 0; + } else { + self.output_pos += nout; + } (res, nin, nout, nend) } @@ -919,9 +1007,12 @@ impl Reader { output: &mut [u8], ) -> (ReadFieldResult, usize, usize) { if input.is_empty() { + if self.is_invalid_end(self.nfa_state) { + self.set_invalid(); + } self.nfa_state = self.transition_final_nfa(self.nfa_state); let res = ReadFieldResult::from_nfa(self.nfa_state, false, false); - return (res, 0, 0); + return (self.finish_field(res), 0, 0); } if output.is_empty() { // If the output buffer is empty, then we can never make progress, @@ -932,6 +1023,9 @@ impl Reader { let mut state = self.nfa_state; while nin < input.len() && nout < output.len() { let (s, io) = self.transition_nfa(state, input[nin]); + if self.is_invalid_transition(state, input[nin], s, &io) { + self.set_invalid(); + } match io { NfaInputAction::CopyToOutput => { output[nout] = input[nin]; @@ -954,7 +1048,95 @@ impl Reader { nout >= output.len(), ); self.nfa_state = state; - (res, nin, nout) + (self.finish_field(res), nin, nout) + } + + /// Returns true if strict parsing rejects the given NFA transition on the + /// input byte `c`. + /// + /// Only a transition that copies a byte into an unquoted field can be + /// invalid: either a quote inside a field that did not start with a quote, + /// or a byte following the closing quote of a quoted field that is + /// neither a delimiter, a record terminator nor a doubled quote. + fn is_invalid_transition( + &self, + from: NfaState, + c: u8, + to: NfaState, + action: &NfaInputAction, + ) -> bool { + if !self.strict + || !self.quoting + || to != NfaState::InField + || *action != NfaInputAction::CopyToOutput + { + return false; + } + match from { + NfaState::InField => c == self.quote, + NfaState::InDoubleEscapedQuote => true, + _ => false, + } + } + + /// Returns true if strict parsing rejects the end of the data in the + /// given NFA state, which is the case inside a quoted field. + fn is_invalid_end(&self, state: NfaState) -> bool { + self.strict + && matches!( + state, + NfaState::InQuotedField | NfaState::InEscapedQuote + ) + } + + /// Like `is_invalid_end`, but for a DFA state. + fn is_invalid_end_dfa(&self, state: DfaState) -> bool { + self.strict + && (state == self.dfa.in_quoted || state == self.dfa.in_escaped) + } + + /// Marks the current field and record as having invalid quoting. + fn set_invalid(&mut self) { + self.invalid_field = true; + self.invalid_record = true; + } + + /// Reports the end of a field as invalid if strict parsing rejected any + /// part of it. + fn finish_field(&mut self, res: ReadFieldResult) -> ReadFieldResult { + match res { + ReadFieldResult::Field { record_end } => { + let invalid = self.invalid_field; + self.invalid_field = false; + if record_end { + self.invalid_record = false; + } + if invalid { + ReadFieldResult::Invalid { record_end } + } else { + res + } + } + res => res, + } + } + + /// Reports the end of a record as invalid if strict parsing rejected any + /// part of it. + fn finish_record(&mut self, res: ReadRecordResult) -> ReadRecordResult { + match res { + ReadRecordResult::Record => { + let invalid = self.invalid_record; + self.invalid_field = false; + self.invalid_record = false; + if invalid { + ReadRecordResult::Invalid + } else { + res + } + } + res => res, + } } /// Compute the final NFA transition after all caller-provided input has @@ -1105,6 +1287,10 @@ struct Dfa { /// whether a particular `(state, equivalence class)` pair should emit an /// output byte. has_output: [bool; TRANS_SIZE], + /// A table with the same layout as `trans`, except its values indicate + /// whether a particular `(state, equivalence class)` pair is rejected by + /// strict parsing. + invalid: [bool; TRANS_SIZE], /// A map from input byte to equivalence class. /// /// This is responsible for reducing the effective alphabet size from @@ -1114,6 +1300,9 @@ struct Dfa { in_field: DfaState, /// The DFA state corresponding to being inside an quoted field. in_quoted: DfaState, + /// The DFA state corresponding to having seen an escape byte inside a + /// quoted field. + in_escaped: DfaState, /// The minimum DFA state that indicates a field has been parsed. All DFA /// states greater than this are also final-field states. final_field: DfaState, @@ -1127,9 +1316,11 @@ impl Dfa { Dfa { trans: [DfaState(0); TRANS_SIZE], has_output: [false; TRANS_SIZE], + invalid: [false; TRANS_SIZE], classes: DfaClasses::new(), in_field: DfaState(0), in_quoted: DfaState(0), + in_escaped: DfaState(0), final_field: DfaState(0), final_record: DfaState(0), } @@ -1149,22 +1340,31 @@ impl Dfa { self.new_state(NfaState::EndRecord) } - fn get_output(&self, state: DfaState, c: u8) -> (DfaState, bool) { + fn get_output(&self, state: DfaState, c: u8) -> (DfaState, bool, bool) { let cls = self.classes.classes[c as usize]; let idx = state.0 as usize + cls as usize; - (self.trans[idx], self.has_output[idx]) + (self.trans[idx], self.has_output[idx], self.invalid[idx]) } - fn set(&mut self, from: DfaState, c: u8, to: DfaState, output: bool) { + fn set( + &mut self, + from: DfaState, + c: u8, + to: DfaState, + output: bool, + invalid: bool, + ) { let cls = self.classes.classes[c as usize]; let idx = from.0 as usize + cls as usize; self.trans[idx] = to; self.has_output[idx] = output; + self.invalid[idx] = invalid; } fn finish(&mut self) { self.in_field = self.new_state(NfaState::InField); self.in_quoted = self.new_state(NfaState::InQuotedField); + self.in_escaped = self.new_state(NfaState::InEscapedQuote); self.final_field = self.new_state(NfaState::EndFieldDelim); self.final_record = self.new_state(NfaState::EndRecord); } @@ -1413,7 +1613,8 @@ mod tests { } } ReadFieldResult::OutputFull => panic!("field too large"), - ReadFieldResult::Field { record_end } => { + ReadFieldResult::Field { record_end } + | ReadFieldResult::Invalid { record_end } => { let s = str::from_utf8(&field[..outpos]).unwrap(); row.push(Field::from(s).unwrap()); outpos = 0; @@ -1456,7 +1657,7 @@ mod tests { } OutputFull => panic!("record too large (out buffer)"), OutputEndsFull => panic!("record too large (end buffer)"), - Record => { + Record | Invalid => { let s = str::from_utf8(&record[..outpos]).unwrap(); let mut start = 0; let mut row = Row::new(); diff --git a/csv-core/tests/strict.rs b/csv-core/tests/strict.rs new file mode 100644 index 00000000..4806ea2a --- /dev/null +++ b/csv-core/tests/strict.rs @@ -0,0 +1,984 @@ +use csv_core::{ + ReadFieldResult, ReadRecordResult, Reader, ReaderBuilder, Terminator, +}; + +/// One parsed field: its unescaped bytes, whether it ended its record and +/// whether it was reported as `Invalid`. +#[derive(Clone, Debug, PartialEq)] +struct Field { + data: Vec, + record_end: bool, + invalid: bool, +} + +/// One parsed record: its fields and whether it was reported as `Invalid`. +#[derive(Clone, Debug, PartialEq)] +struct Record { + fields: Vec>, + invalid: bool, +} + +/// A parser configuration applied to a `ReaderBuilder`. +#[derive(Clone, Copy, Debug)] +struct Config { + delimiter: u8, + quote: u8, + escape: Option, + double_quote: bool, + quoting: bool, + comment: Option, + term: Option, +} + +impl Default for Config { + fn default() -> Config { + Config { + delimiter: b',', + quote: b'"', + escape: None, + double_quote: true, + quoting: true, + comment: None, + term: None, + } + } +} + +fn build(cfg: &Config, strict: bool, nfa: bool) -> Reader { + let mut b = ReaderBuilder::new(); + b.delimiter(cfg.delimiter) + .quote(cfg.quote) + .escape(cfg.escape) + .double_quote(cfg.double_quote) + .quoting(cfg.quoting) + .comment(cfg.comment) + .nfa(nfa) + .strict(strict); + if let Some(t) = cfg.term { + b.terminator(Terminator::Any(t)); + } + b.build() +} + +/// Reads all fields, feeding `chunk` input bytes and offering `outcap` +/// output bytes per call. Also returns the `(nin, nout)` of every call with +/// the result kind, `Invalid` folded into `Field`. +fn fields_with( + rdr: &mut Reader, + data: &[u8], + chunk: usize, + outcap: usize, +) -> (Vec, Vec<(String, usize, usize)>) { + let mut out = vec![0u8; data.len() + 16]; + let mut outlen = 0; + let mut pos = 0; + let mut limit = chunk.min(data.len()); + let mut fields = vec![]; + let mut trace = vec![]; + for _ in 0..100_000 { + if pos == limit && limit < data.len() { + limit = (limit + chunk).min(data.len()); + } + let end = (outlen + outcap).min(out.len()); + let (res, nin, nout) = + rdr.read_field(&data[pos..limit], &mut out[outlen..end]); + pos += nin; + outlen += nout; + let kind = match res { + ReadFieldResult::Invalid { record_end } => { + format!("Field {{ record_end: {} }}", record_end) + } + ref other => format!("{:?}", other), + }; + trace.push((kind, nin, nout)); + match res { + ReadFieldResult::InputEmpty | ReadFieldResult::OutputFull => {} + ReadFieldResult::Field { record_end } => { + fields.push(Field { + data: out[..outlen].to_vec(), + record_end, + invalid: false, + }); + outlen = 0; + } + ReadFieldResult::Invalid { record_end } => { + fields.push(Field { + data: out[..outlen].to_vec(), + record_end, + invalid: true, + }); + outlen = 0; + } + ReadFieldResult::End => return (fields, trace), + } + } + panic!( + "the reader never returned End for {:?}", + String::from_utf8_lossy(data) + ); +} + +/// Reads all records, feeding `chunk` input bytes, offering `outcap` output +/// bytes and `endscap` field ends per call. Also returns the +/// `(nin, nout, nend)` of every call with the result kind, `Invalid` folded +/// into `Record`. +fn records_with( + rdr: &mut Reader, + data: &[u8], + chunk: usize, + outcap: usize, + endscap: usize, +) -> (Vec, Vec<(String, usize, usize, usize)>) { + let mut out = vec![0u8; data.len() + 16]; + let mut ends = vec![0usize; data.len() + 16]; + let (mut outlen, mut endlen) = (0, 0); + let mut pos = 0; + let mut limit = chunk.min(data.len()); + let mut records = vec![]; + let mut trace = vec![]; + for _ in 0..100_000 { + if pos == limit && limit < data.len() { + limit = (limit + chunk).min(data.len()); + } + let oend = (outlen + outcap).min(out.len()); + let eend = (endlen + endscap).min(ends.len()); + let (res, nin, nout, nend) = rdr.read_record( + &data[pos..limit], + &mut out[outlen..oend], + &mut ends[endlen..eend], + ); + pos += nin; + outlen += nout; + endlen += nend; + let kind = match res { + ReadRecordResult::Invalid => "Record".to_string(), + ref other => format!("{:?}", other), + }; + trace.push((kind, nin, nout, nend)); + let invalid = match res { + ReadRecordResult::InputEmpty + | ReadRecordResult::OutputFull + | ReadRecordResult::OutputEndsFull => continue, + ReadRecordResult::End => return (records, trace), + ReadRecordResult::Record => false, + ReadRecordResult::Invalid => true, + }; + let mut fields = vec![]; + let mut start = 0; + for &e in &ends[..endlen] { + fields.push(out[start..e].to_vec()); + start = e; + } + records.push(Record { fields, invalid }); + outlen = 0; + endlen = 0; + } + panic!( + "the reader never returned End for {:?}", + String::from_utf8_lossy(data) + ); +} + +fn fields(cfg: &Config, strict: bool, nfa: bool, data: &[u8]) -> Vec { + fields_with(&mut build(cfg, strict, nfa), data, data.len().max(1), 1 << 20) + .0 +} + +fn records(cfg: &Config, strict: bool, nfa: bool, data: &[u8]) -> Vec { + records_with( + &mut build(cfg, strict, nfa), + data, + data.len().max(1), + 1 << 20, + 1 << 20, + ) + .0 +} + +fn f(data: &str, record_end: bool, invalid: bool) -> Field { + Field { data: data.as_bytes().to_vec(), record_end, invalid } +} + +fn r(fields: &[&str], invalid: bool) -> Record { + Record { + fields: fields.iter().map(|s| s.as_bytes().to_vec()).collect(), + invalid, + } +} + +/// Checks that the strict reader reports exactly the expected fields and +/// records, with both parsers and with every field ending up as the +/// non-strict reader parses it. +fn check( + cfg: &Config, + data: &str, + want_fields: &[Field], + want_records: &[Record], +) { + let data = data.as_bytes(); + for &nfa in &[false, true] { + let engine = if nfa { "nfa" } else { "dfa" }; + assert_eq!( + fields(cfg, true, nfa, data), + want_fields, + "{} read_field of {:?}", + engine, + String::from_utf8_lossy(data) + ); + assert_eq!( + records(cfg, true, nfa, data), + want_records, + "{} read_record of {:?}", + engine, + String::from_utf8_lossy(data) + ); + let lenient: Vec = want_fields + .iter() + .map(|x| Field { invalid: false, ..x.clone() }) + .collect(); + assert_eq!( + fields(cfg, false, nfa, data), + lenient, + "{} non-strict read_field", + engine + ); + } +} + +#[test] +fn quote_inside_unquoted_field_is_invalid() { + check( + &Config::default(), + "a\"b,c\n", + &[f("a\"b", false, true), f("c", true, false)], + &[r(&["a\"b", "c"], true)], + ); +} + +#[test] +fn quote_after_leading_space_is_invalid() { + check( + &Config::default(), + "x, \"y\"\n", + &[f("x", false, false), f(" \"y\"", true, true)], + &[r(&["x", " \"y\""], true)], + ); +} + +#[test] +fn byte_after_closing_quote_is_invalid() { + check( + &Config::default(), + "\"a\"b,c\n\"d\" ,e\n", + &[ + f("ab", false, true), + f("c", true, false), + f("d ", false, true), + f("e", true, false), + ], + &[r(&["ab", "c"], true), r(&["d ", "e"], true)], + ); +} + +#[test] +fn closing_quote_before_delimiter_terminator_or_end_is_valid() { + check( + &Config::default(), + "\"a\",\"b\"\n\"c\",\"d\"\r\n\"e\",\"f\"\r\"g\",\"h\"", + &[ + f("a", false, false), + f("b", true, false), + f("c", false, false), + f("d", true, false), + f("e", false, false), + f("f", true, false), + f("g", false, false), + f("h", true, false), + ], + &[ + r(&["a", "b"], false), + r(&["c", "d"], false), + r(&["e", "f"], false), + r(&["g", "h"], false), + ], + ); +} + +#[test] +fn doubled_quotes_are_valid() { + check( + &Config::default(), + "\"a\"\"b\",\"\"\"\"\n\"\",x\n", + &[ + f("a\"b", false, false), + f("\"", true, false), + f("", false, false), + f("x", true, false), + ], + &[r(&["a\"b", "\""], false), r(&["", "x"], false)], + ); +} + +#[test] +fn doubled_quote_is_invalid_without_double_quote() { + let cfg = Config { double_quote: false, ..Config::default() }; + check( + &cfg, + "\"a\"\"b\",c\n", + &[f("a\"b\"", false, true), f("c", true, false)], + &[r(&["a\"b\"", "c"], true)], + ); +} + +#[test] +fn escaped_quote_is_valid() { + let cfg = Config { + double_quote: false, + escape: Some(b'\\'), + ..Config::default() + }; + check( + &cfg, + "\"a\\\"b\",c\n\"x\"\\y,z\n", + &[ + f("a\"b", false, false), + f("c", true, false), + f("x\\y", false, true), + f("z", true, false), + ], + &[r(&["a\"b", "c"], false), r(&["x\\y", "z"], true)], + ); +} + +#[test] +fn delimiter_and_newline_inside_quotes_are_valid() { + check( + &Config::default(), + "\"a,b\nc\",d\n", + &[f("a,b\nc", false, false), f("d", true, false)], + &[r(&["a,b\nc", "d"], false)], + ); +} + +#[test] +fn end_of_data_inside_quotes_is_invalid() { + check( + &Config::default(), + "a,b\nc,\"d,e\nf", + &[ + f("a", false, false), + f("b", true, false), + f("c", false, false), + f("d,e\nf", true, true), + ], + &[r(&["a", "b"], false), r(&["c", "d,e\nf"], true)], + ); +} + +#[test] +fn end_of_data_after_escape_is_invalid() { + let cfg = Config { + double_quote: false, + escape: Some(b'\\'), + ..Config::default() + }; + check(&cfg, "\"ab\\", &[f("ab", true, true)], &[r(&["ab"], true)]); +} + +#[test] +fn lone_quote_field_is_invalid() { + check( + &Config::default(), + "a,\"", + &[f("a", false, false), f("", true, true)], + &[r(&["a", ""], true)], + ); + check( + &Config::default(), + "\"\"\"\n", + &[f("\"\n", true, true)], + &[r(&["\"\n"], true)], + ); +} + +#[test] +fn several_errors_in_one_field_report_it_once() { + check( + &Config::default(), + "\"a\"b\"c\"d,e\n", + &[f("ab\"c\"d", false, true), f("e", true, false)], + &[r(&["ab\"c\"d", "e"], true)], + ); +} + +#[test] +fn only_the_bad_record_is_invalid() { + check( + &Config::default(), + "a,b\nc\"d,e\nf,g\n", + &[ + f("a", false, false), + f("b", true, false), + f("c\"d", false, true), + f("e", true, false), + f("f", false, false), + f("g", true, false), + ], + &[ + r(&["a", "b"], false), + r(&["c\"d", "e"], true), + r(&["f", "g"], false), + ], + ); +} + +#[test] +fn comment_lines_are_not_checked() { + let cfg = Config { comment: Some(b'#'), ..Config::default() }; + check( + &cfg, + "#\"x\"y a\"b\na,b\n#\"unterminated\nc,d", + &[ + f("a", false, false), + f("b", true, false), + f("c", false, false), + f("d", true, false), + ], + &[r(&["a", "b"], false), r(&["c", "d"], false)], + ); +} + +#[test] +fn custom_quote_delimiter_and_terminator() { + let cfg = Config { + delimiter: b';', + quote: b'\'', + term: Some(b'|'), + ..Config::default() + }; + check( + &cfg, + "'a';b\"c|'d'e;f|x';y", + &[ + f("a", false, false), + f("b\"c", true, false), + f("de", false, true), + f("f", true, false), + f("x'", false, true), + f("y", true, false), + ], + &[ + r(&["a", "b\"c"], false), + r(&["de", "f"], true), + r(&["x'", "y"], true), + ], + ); +} + +#[test] +fn newline_after_closing_quote_is_invalid_with_custom_terminator() { + let cfg = Config { term: Some(b'|'), ..Config::default() }; + check( + &cfg, + "\"a\"\n|\"b\"|", + &[f("a\n", true, true), f("b", true, false)], + &[r(&["a\n"], true), r(&["b"], false)], + ); +} + +#[test] +fn quoting_disabled_never_reports_invalid() { + let cfg = Config { quoting: false, ..Config::default() }; + check( + &cfg, + "a\"b,\"c\"d\n\"e", + &[ + f("a\"b", false, false), + f("\"c\"d", true, false), + f("\"e", true, false), + ], + &[r(&["a\"b", "\"c\"d"], false), r(&["\"e"], false)], + ); +} + +#[test] +fn strict_is_off_by_default() { + let data = b"a\"b,\"c\"d\n\"e"; + for &nfa in &[false, true] { + let mut b = ReaderBuilder::new(); + b.nfa(nfa); + let got = fields_with(&mut b.build(), data, data.len(), 1 << 20).0; + assert!(got.iter().all(|x| !x.invalid), "{:?}", got); + let got = + records_with(&mut b.build(), data, data.len(), 1 << 20, 1 << 20).0; + assert!(got.iter().all(|x| !x.invalid), "{:?}", got); + let got = records_with( + &mut Reader::new(), + data, + data.len(), + 1 << 20, + 1 << 20, + ) + .0; + assert!(got.iter().all(|x| !x.invalid), "{:?}", got); + } +} + +#[test] +fn strict_can_be_switched_off_again() { + let mut b = ReaderBuilder::new(); + b.strict(true).strict(false); + let data = b"a\"b,c\n"; + let got = + records_with(&mut b.build(), data, data.len(), 1 << 20, 1 << 20).0; + assert_eq!(got, vec![r(&["a\"b", "c"], false)]); +} + +#[test] +fn invalid_byte_in_a_later_call() { + for &nfa in &[false, true] { + let mut rdr = build(&Config::default(), true, nfa); + let mut out = [0u8; 64]; + let (res, nin, nout) = rdr.read_field(b"\"ab\"", &mut out); + assert_eq!((res, nin, nout), (ReadFieldResult::InputEmpty, 4, 2)); + let (res, nin, nout) = rdr.read_field(b"c,d\n", &mut out[2..]); + assert_eq!( + (res, nin, nout), + (ReadFieldResult::Invalid { record_end: false }, 2, 1) + ); + assert_eq!(&out[..3], b"abc"); + let (res, _, _) = rdr.read_field(b"d\n", &mut out); + assert_eq!(res, ReadFieldResult::Field { record_end: true }); + let (res, _, _) = rdr.read_field(b"", &mut out); + assert_eq!(res, ReadFieldResult::End); + } + // A record split over two calls. + for &nfa in &[false, true] { + let mut out = [0u8; 64]; + let mut rdr = build(&Config::default(), true, nfa); + let mut ends = [0usize; 8]; + let (res, nin, nout, nend) = + rdr.read_record(b"x,\"ab\"", &mut out, &mut ends); + assert_eq!( + (res, nin, nout, nend), + (ReadRecordResult::InputEmpty, 6, 3, 1) + ); + let (res, nin, nout, nend) = + rdr.read_record(b"!\ny\n", &mut out[3..], &mut ends[1..]); + assert_eq!( + (res, nin, nout, nend), + (ReadRecordResult::Invalid, 2, 1, 1) + ); + assert_eq!((&out[..4], &ends[..2]), (&b"xab!"[..], &[1usize, 4][..])); + let (res, _, _, _) = rdr.read_record(b"y\n", &mut out, &mut ends); + assert_eq!(res, ReadRecordResult::Record); + + // Split right before the quote of an unquoted field. + let mut rdr = build(&Config::default(), true, nfa); + let (res, nin, nout, nend) = + rdr.read_record(b"a", &mut out, &mut ends); + assert_eq!( + (res, nin, nout, nend), + (ReadRecordResult::InputEmpty, 1, 1, 0) + ); + let (res, nin, nout, nend) = + rdr.read_record(b"\"b,c\n", &mut out[1..], &mut ends); + assert_eq!( + (res, nin, nout, nend), + (ReadRecordResult::Invalid, 5, 3, 2) + ); + assert_eq!((&out[..4], &ends[..2]), (&b"a\"bc"[..], &[3usize, 4][..])); + } +} + +#[test] +fn unterminated_quote_reported_at_end_of_input() { + for &nfa in &[false, true] { + let mut rdr = build(&Config::default(), true, nfa); + let mut out = [0u8; 64]; + let mut ends = [0usize; 8]; + let (res, _, _, _) = rdr.read_record(b"a,\"bc", &mut out, &mut ends); + assert_eq!(res, ReadRecordResult::InputEmpty); + // No room for the final field end: nothing is reported yet. + let (res, nin, nout, nend) = + rdr.read_record(b"", &mut out[3..], &mut []); + assert_eq!( + (res, nin, nout, nend), + (ReadRecordResult::OutputEndsFull, 0, 0, 0) + ); + let (res, nin, nout, nend) = + rdr.read_record(b"", &mut out[3..], &mut ends[1..]); + assert_eq!( + (res, nin, nout, nend), + (ReadRecordResult::Invalid, 0, 0, 1) + ); + assert_eq!(&ends[..2], &[1usize, 3][..]); + let (res, _, _, _) = rdr.read_record(b"", &mut out, &mut ends); + assert_eq!(res, ReadRecordResult::End); + } +} + +#[test] +fn reset_forgets_a_partly_read_invalid_field() { + for &nfa in &[false, true] { + let mut rdr = build(&Config::default(), true, nfa); + let mut out = [0u8; 64]; + let mut ends = [0usize; 8]; + let (res, _, _) = rdr.read_field(b"\"a\"b", &mut out); + assert_eq!(res, ReadFieldResult::InputEmpty); + rdr.reset(); + let (res, _, _) = rdr.read_field(b"c,", &mut out); + assert_eq!(res, ReadFieldResult::Field { record_end: false }); + + let mut rdr = build(&Config::default(), true, nfa); + let (res, _, _, _) = rdr.read_record(b"a\"b,c", &mut out, &mut ends); + assert_eq!(res, ReadRecordResult::InputEmpty); + rdr.reset(); + let (res, _, _, _) = rdr.read_record(b"d,e\n", &mut out, &mut ends); + assert_eq!(res, ReadRecordResult::Record); + } +} + +#[test] +fn field_reads_then_record_reads_keep_their_own_verdicts() { + for &nfa in &[false, true] { + let mut rdr = build(&Config::default(), true, nfa); + let mut out = [0u8; 64]; + let mut ends = [0usize; 8]; + let data = b"a\"b,c\nd,e\n"; + let (res, nin, _) = rdr.read_field(data, &mut out); + assert_eq!(res, ReadFieldResult::Invalid { record_end: false }); + let (res, nin2, _) = rdr.read_field(&data[nin..], &mut out); + assert_eq!(res, ReadFieldResult::Field { record_end: true }); + let (res, _, _, _) = + rdr.read_record(&data[nin + nin2..], &mut out, &mut ends); + assert_eq!(res, ReadRecordResult::Record); + } +} + +// Deterministic generator of CSV documents whose invalid fields are known by +// construction. + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self + .0 + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + self.0 >> 33 + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + fn pick(&mut self, xs: &[T]) -> T { + xs[self.below(xs.len() as u64) as usize] + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +enum LineEnd { + Lf, + CrLf, + Cr, + Any(u8), +} + +fn plain(rng: &mut Rng, cfg: &Config, min: u64) -> Vec { + let mut alphabet = b"abcxyz019 ".to_vec(); + if let Some(e) = cfg.escape { + alphabet.push(e); + } + let n = min + rng.below(4); + (0..n).map(|_| rng.pick(&alphabet)).collect() +} + +fn quoted_body(rng: &mut Rng, cfg: &Config, eol: LineEnd) -> Vec { + let mut v = vec![]; + for _ in 0..rng.below(6) { + match rng.below(7) { + 0 => v.push(cfg.delimiter), + 1 => v.push(b'\n'), + 2 => v.push(b'\r'), + 3 => { + if let LineEnd::Any(t) = eol { + v.push(t); + } + } + 4 => { + if cfg.double_quote { + v.push(cfg.quote); + v.push(cfg.quote); + } else if let Some(e) = cfg.escape { + v.push(e); + v.push(cfg.quote); + } + } + _ => v.push(rng.pick(b"abcxyz ")), + } + } + v +} + +fn quoted(rng: &mut Rng, cfg: &Config, eol: LineEnd) -> Vec { + let mut v = vec![cfg.quote]; + v.extend(quoted_body(rng, cfg, eol)); + v.push(cfg.quote); + v +} + +/// Returns a field and whether it has invalid quoting. +fn gen_field( + rng: &mut Rng, + cfg: &Config, + eol: LineEnd, + first: bool, +) -> (Vec, bool) { + // A record's first field is never empty, so no record is a blank line, + // and never starts with the comment byte. + let min = if first { 1 } else { 0 }; + match rng.below(6) { + 0 | 1 => (plain(rng, cfg, min), false), + 2 | 3 => (quoted(rng, cfg, eol), false), + 4 => { + // A quote inside an unquoted field. + let mut v = plain(rng, cfg, 1); + v.push(cfg.quote); + v.extend(plain(rng, cfg, 0)); + if rng.below(2) == 0 { + v.push(cfg.quote); + } + (v, true) + } + _ => { + // Something other than a delimiter or terminator after the + // closing quote. + let mut v = quoted(rng, cfg, eol); + if !cfg.double_quote && rng.below(3) == 0 { + v.push(cfg.quote); + } else { + v.push(rng.pick(b"abz ")); + } + v.extend(plain(rng, cfg, 0)); + if rng.below(3) == 0 { + v.push(cfg.quote); + } + (v, true) + } + } +} + +struct Doc { + data: Vec, + // Validity of every field, per record. + records: Vec>, +} + +fn gen_doc(rng: &mut Rng, cfg: &Config, eol: LineEnd) -> Doc { + let term: &[u8] = match eol { + LineEnd::Lf => b"\n", + LineEnd::CrLf => b"\r\n", + LineEnd::Cr => b"\r", + LineEnd::Any(_) => b"", + }; + let term: Vec = match eol { + LineEnd::Any(t) => vec![t], + _ => term.to_vec(), + }; + let mut data = vec![]; + let mut records = vec![]; + let nrec = 1 + rng.below(5); + for i in 0..nrec { + let newline_ends = eol == LineEnd::Lf || eol == LineEnd::CrLf; + if cfg.comment.is_some() && newline_ends && rng.below(4) == 0 { + data.push(cfg.comment.unwrap()); + data.extend(plain(rng, cfg, 0)); + data.push(cfg.quote); + data.extend(plain(rng, cfg, 0)); + data.push(b'\n'); + } + if rng.below(5) == 0 { + data.extend(&term); + } + let nfields = 1 + rng.below(4); + let mut valid = vec![]; + for j in 0..nfields { + if j > 0 { + data.push(cfg.delimiter); + } + let (field, invalid) = gen_field(rng, cfg, eol, j == 0); + data.extend(field); + valid.push(invalid); + } + let last = i + 1 == nrec; + if last && rng.below(3) == 0 { + // The data ends inside a quoted field of the last record. + data.push(cfg.delimiter); + data.push(cfg.quote); + data.extend(quoted_body(rng, cfg, eol)); + if cfg.escape.is_some() && rng.below(2) == 0 { + data.push(cfg.escape.unwrap()); + } + valid.push(true); + } else if !last || rng.below(2) == 0 { + data.extend(&term); + } + records.push(valid); + } + Doc { data, records } +} + +fn configs() -> Vec<(Config, LineEnd)> { + let mut v = vec![]; + for &(delimiter, quote) in + &[(b',', b'"'), (b';', b'\''), (b'\t', b'"'), (b'|', b'\'')] + { + for &(double_quote, escape) in &[ + (true, None), + (false, Some(b'\\')), + (false, None), + (true, Some(b'\\')), + ] { + for &eol in + &[LineEnd::Lf, LineEnd::CrLf, LineEnd::Cr, LineEnd::Any(b'~')] + { + let comments: &[Option] = match eol { + LineEnd::Lf | LineEnd::CrLf => &[None, Some(b'#')], + _ => &[None], + }; + for &comment in comments { + let cfg = Config { + delimiter, + quote, + escape, + double_quote, + quoting: true, + comment, + term: match eol { + LineEnd::Any(t) => Some(t), + _ => None, + }, + }; + v.push((cfg, eol)); + } + } + } + } + v +} + +#[test] +fn generated_documents_by_record() { + let mut rng = Rng(0x5eed_a781); + for (cfg, eol) in configs() { + for _ in 0..12 { + let doc = gen_doc(&mut rng, &cfg, eol); + let shown = String::from_utf8_lossy(&doc.data).into_owned(); + let want: Vec = + doc.records.iter().map(|fs| fs.iter().any(|&x| x)).collect(); + for &nfa in &[false, true] { + for &(chunk, outcap, endscap) in &[ + (usize::MAX, 1 << 20, 1 << 20), + (1, 1 << 20, 1 << 20), + (3, 2, 1), + (7, 5, 2), + ] { + let chunk = chunk.min(doc.data.len().max(1)); + let (lenient, lenient_trace) = records_with( + &mut build(&cfg, false, nfa), + &doc.data, + chunk, + outcap, + endscap, + ); + let (strict, strict_trace) = records_with( + &mut build(&cfg, true, nfa), + &doc.data, + chunk, + outcap, + endscap, + ); + let ctx = format!( + "{:?} nfa={} chunk={} outcap={} endscap={} data={:?}", + cfg, nfa, chunk, outcap, endscap, shown + ); + assert_eq!( + strict_trace, lenient_trace, + "call results differ from the non-strict reader: {}", + ctx + ); + let data: Vec<_> = + strict.iter().map(|x| x.fields.clone()).collect(); + let lenient_data: Vec<_> = + lenient.iter().map(|x| x.fields.clone()).collect(); + assert_eq!( + data, lenient_data, + "record data differs from the non-strict reader: {}", + ctx + ); + let got: Vec = + strict.iter().map(|x| x.invalid).collect(); + assert_eq!(got, want, "invalid records: {}", ctx); + } + } + } + } +} + +#[test] +fn generated_documents_by_field() { + let mut rng = Rng(0xf1e1_d5); + for (cfg, eol) in configs() { + for _ in 0..12 { + let doc = gen_doc(&mut rng, &cfg, eol); + let shown = String::from_utf8_lossy(&doc.data).into_owned(); + let want: Vec = + doc.records.iter().flatten().copied().collect(); + for &nfa in &[false, true] { + for &(chunk, outcap) in + &[(usize::MAX, 1 << 20), (1, 1 << 20), (2, 1), (5, 3)] + { + let chunk = chunk.min(doc.data.len().max(1)); + let (lenient, lenient_trace) = fields_with( + &mut build(&cfg, false, nfa), + &doc.data, + chunk, + outcap, + ); + let (strict, strict_trace) = fields_with( + &mut build(&cfg, true, nfa), + &doc.data, + chunk, + outcap, + ); + let ctx = format!( + "{:?} nfa={} chunk={} outcap={} data={:?}", + cfg, nfa, chunk, outcap, shown + ); + assert_eq!( + strict_trace, lenient_trace, + "call results differ from the non-strict reader: {}", + ctx + ); + let data: Vec<_> = strict + .iter() + .map(|x| (x.data.clone(), x.record_end)) + .collect(); + let lenient_data: Vec<_> = lenient + .iter() + .map(|x| (x.data.clone(), x.record_end)) + .collect(); + assert_eq!( + data, lenient_data, + "field data differs from the non-strict reader: {}", + ctx + ); + let got: Vec = + strict.iter().map(|x| x.invalid).collect(); + assert_eq!(got, want, "invalid fields: {}", ctx); + } + } + } + } +} diff --git a/examples/tutorial-perf-core-01.rs b/examples/tutorial-perf-core-01.rs index 220918c3..bcec07c5 100644 --- a/examples/tutorial-perf-core-01.rs +++ b/examples/tutorial-perf-core-01.rs @@ -36,7 +36,9 @@ fn run(mut data: &[u8]) -> Option { } // This case happens when we've successfully read a field. If the // field is the last field in a record, then `record_end` is true. - ReadFieldResult::Field { record_end } => { + // (`Invalid` is only reported when strict parsing is enabled.) + ReadFieldResult::Field { record_end } + | ReadFieldResult::Invalid { record_end } => { if fieldidx == 0 && field == b"us" { inus = true; } else if inus && fieldidx == 3 && field == b"MA" { diff --git a/src/error.rs b/src/error.rs index 8ffc181b..43850639 100644 --- a/src/error.rs +++ b/src/error.rs @@ -14,8 +14,8 @@ pub type Result = result::Result; /// /// There are some important scenarios where an error is impossible to occur. /// For example, if a CSV reader is used on an in-memory buffer with the -/// `flexible` option enabled and one is reading records as raw byte strings, -/// then no error can occur. +/// `flexible` option enabled and the `strict` option disabled, and one is +/// reading records as raw byte strings, then no error can occur. #[derive(Debug)] pub struct Error(Box); @@ -95,6 +95,12 @@ pub enum ErrorKind { /// The deserialization error. err: DeserializeError, }, + /// This error occurs when a record has invalid quoting. This error only + /// occurs when the `strict` option in a CSV reader is enabled. + InvalidQuoting { + /// The position of the record with invalid quoting, if available. + pos: Option, + }, } impl ErrorKind { @@ -107,6 +113,7 @@ impl ErrorKind { ErrorKind::Utf8 { ref pos, .. } => pos.as_ref(), ErrorKind::UnequalLengths { ref pos, .. } => pos.as_ref(), ErrorKind::Deserialize { ref pos, .. } => pos.as_ref(), + ErrorKind::InvalidQuoting { ref pos } => pos.as_ref(), _ => None, } } @@ -188,6 +195,17 @@ impl fmt::Display for Error { pos.byte(), err ), + ErrorKind::InvalidQuoting { pos: None } => { + write!(f, "CSV parse error: found record with invalid quoting") + } + ErrorKind::InvalidQuoting { pos: Some(ref pos) } => write!( + f, + "CSV parse error: record {} (line: {}, byte: {}): \ + found record with invalid quoting", + pos.record(), + pos.line(), + pos.byte() + ), } } } diff --git a/src/reader.rs b/src/reader.rs index a34c9d5a..b81dd123 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -542,6 +542,28 @@ impl ReaderBuilder { self } + /// Enable or disable strict parsing of quotes. + /// + /// When enabled, reading a record with invalid quoting returns an error + /// of kind `ErrorKind::InvalidQuoting` whose position is the position of + /// that record. Quoting is invalid when a quote appears in a field that + /// does not start with a quote, when the closing quote of a quoted field + /// is followed by anything other than a delimiter, a record terminator or + /// the end of the data, or when the data ends inside a quoted field. + /// + /// A record with invalid quoting is consumed by the read that reports it, + /// so reading continues with the next record. It counts towards the + /// record numbers of the positions of later records, but it is never used + /// as the header row, and it is not compared with other records when the + /// `flexible` option is disabled. + /// + /// Strict parsing has no effect when quoting is disabled. This is + /// disabled by default. + pub fn strict(&mut self, yes: bool) -> &mut ReaderBuilder { + self.builder.strict(yes); + self + } + /// The comment character to use when parsing CSV. /// /// If the start of a record begins with the byte given here, then that @@ -1665,6 +1687,13 @@ impl Reader { self.state.add_record(record)?; return Ok(true); } + Invalid => { + record.set_len(endlen); + self.state.add_invalid_record(); + return Err(Error::new(ErrorKind::InvalidQuoting { + pos: record.position().cloned(), + })); + } End => { self.state.eof = ReaderEofState::Eof; return Ok(false); @@ -1898,6 +1927,13 @@ impl ReaderState { } Ok(()) } + + /// Counts a record with invalid quoting, which is otherwise ignored. + #[inline(always)] + fn add_invalid_record(&mut self) { + let i = self.cur_pos.record(); + self.cur_pos.set_record(i.checked_add(1).unwrap()); + } } /// An owned iterator over deserialized records. @@ -1908,20 +1944,20 @@ pub struct DeserializeRecordsIntoIter { rdr: Reader, rec: StringRecord, headers: Option, + headers_err: Option, + headers_pending: bool, _priv: PhantomData, } impl DeserializeRecordsIntoIter { fn new(mut rdr: Reader) -> DeserializeRecordsIntoIter { - let headers = if !rdr.state.has_headers { - None - } else { - rdr.headers().ok().cloned() - }; + let (headers, headers_err) = read_deserialize_headers(&mut rdr); DeserializeRecordsIntoIter { rdr, rec: StringRecord::new(), headers, + headers_err, + headers_pending: false, _priv: PhantomData, } } @@ -1948,6 +1984,14 @@ impl Iterator type Item = Result; fn next(&mut self) -> Option> { + if let Some(err) = next_headers_error( + &mut self.rdr, + &mut self.headers, + &mut self.headers_err, + &mut self.headers_pending, + ) { + return Some(Err(err)); + } match self.rdr.read_record(&mut self.rec) { Err(err) => Some(Err(err)), Ok(false) => None, @@ -1956,6 +2000,49 @@ impl Iterator } } +/// Reads the header row used by a deserializing iterator. +/// +/// A header row with invalid quoting is returned as an error, so that the +/// iterator can report it before taking its headers from the next row. +fn read_deserialize_headers( + rdr: &mut Reader, +) -> (Option, Option) { + if !rdr.state.has_headers { + return (None, None); + } + match rdr.headers() { + Ok(headers) => (Some(headers.clone()), None), + Err(err) => match *err.kind() { + ErrorKind::InvalidQuoting { .. } => (None, Some(err)), + _ => (None, None), + }, + } +} + +/// Returns the next header row error a deserializing iterator must report, +/// reading the header row again once a previous error has been reported. +fn next_headers_error( + rdr: &mut Reader, + headers: &mut Option, + headers_err: &mut Option, + headers_pending: &mut bool, +) -> Option { + if let Some(err) = headers_err.take() { + *headers_pending = true; + return Some(err); + } + if *headers_pending { + *headers_pending = false; + let (new_headers, err) = read_deserialize_headers(rdr); + *headers = new_headers; + if err.is_some() { + *headers_pending = true; + } + return err; + } + None +} + /// A borrowed iterator over deserialized records. /// /// The lifetime parameter `'r` refers to the lifetime of the underlying @@ -1966,20 +2053,20 @@ pub struct DeserializeRecordsIter<'r, R: 'r, D> { rdr: &'r mut Reader, rec: StringRecord, headers: Option, + headers_err: Option, + headers_pending: bool, _priv: PhantomData, } impl<'r, R: io::Read, D: DeserializeOwned> DeserializeRecordsIter<'r, R, D> { fn new(rdr: &'r mut Reader) -> DeserializeRecordsIter<'r, R, D> { - let headers = if !rdr.state.has_headers { - None - } else { - rdr.headers().ok().cloned() - }; + let (headers, headers_err) = read_deserialize_headers(rdr); DeserializeRecordsIter { rdr, rec: StringRecord::new(), headers, + headers_err, + headers_pending: false, _priv: PhantomData, } } @@ -2001,6 +2088,14 @@ impl<'r, R: io::Read, D: DeserializeOwned> Iterator type Item = Result; fn next(&mut self) -> Option> { + if let Some(err) = next_headers_error( + self.rdr, + &mut self.headers, + &mut self.headers_err, + &mut self.headers_pending, + ) { + return Some(Err(err)); + } match self.rdr.read_record(&mut self.rec) { Err(err) => Some(Err(err)), Ok(false) => None, diff --git a/src/tutorial.rs b/src/tutorial.rs index 46820534..312749d1 100644 --- a/src/tutorial.rs +++ b/src/tutorial.rs @@ -2362,7 +2362,9 @@ fn run(mut data: &[u8]) -> Option { } // This case happens when we've successfully read a field. If the // field is the last field in a record, then `record_end` is true. - ReadFieldResult::Field { record_end } => { + // (`Invalid` is only reported when strict parsing is enabled.) + ReadFieldResult::Field { record_end } + | ReadFieldResult::Invalid { record_end } => { if fieldidx == 0 && field == b"us" { inus = true; } else if inus && fieldidx == 3 && field == b"MA" { diff --git a/tests/strict.rs b/tests/strict.rs new file mode 100644 index 00000000..2dfeeb46 --- /dev/null +++ b/tests/strict.rs @@ -0,0 +1,682 @@ +use csv::{ + ByteRecord, Error, ErrorKind, ReaderBuilder, StringRecord, Terminator, +}; +use serde::Deserialize; + +type Pos = (u64, u64, u64); + +/// Returns `(byte, line, record)` of an `InvalidQuoting` error, and checks +/// that both position accessors agree with it. +fn invalid_pos(err: &Error) -> Pos { + let pos = match *err.kind() { + ErrorKind::InvalidQuoting { ref pos } => { + pos.clone().expect("InvalidQuoting error without a position") + } + ref other => { + panic!("expected an InvalidQuoting error, got {:?}", other) + } + }; + let via_error = err.position().expect("Error::position is None"); + let via_kind = err.kind().position().expect("ErrorKind::position is None"); + assert_eq!( + (via_error.byte(), via_error.line(), via_error.record()), + (pos.byte(), pos.line(), pos.record()) + ); + assert_eq!( + (via_kind.byte(), via_kind.line(), via_kind.record()), + (pos.byte(), pos.line(), pos.record()) + ); + (pos.byte(), pos.line(), pos.record()) +} + +#[derive(Debug, PartialEq)] +enum Item { + Rec(Vec), + Bad(Pos), +} + +fn rec(fields: &[&str]) -> Item { + Item::Rec(fields.iter().map(|s| s.to_string()).collect()) +} + +fn item(res: Result) -> Item { + match res { + Ok(r) => Item::Rec(r.iter().map(|s| s.to_string()).collect()), + Err(err) => Item::Bad(invalid_pos(&err)), + } +} + +fn byte_item(res: Result) -> Item { + match res { + Ok(r) => Item::Rec( + r.iter().map(|s| String::from_utf8(s.to_vec()).unwrap()).collect(), + ), + Err(err) => Item::Bad(invalid_pos(&err)), + } +} + +fn strict() -> ReaderBuilder { + let mut b = ReaderBuilder::new(); + b.strict(true); + b +} + +/// Reads `data` through every record reading API of a reader built by +/// `builder` and checks that each gives `want`. +fn check_all_apis(builder: &ReaderBuilder, data: &str, want: &[Item]) { + let got: Vec = + builder.from_reader(data.as_bytes()).records().map(item).collect(); + assert_eq!(got, want, "records() of {:?}", data); + let got: Vec = builder + .from_reader(data.as_bytes()) + .byte_records() + .map(byte_item) + .collect(); + assert_eq!(got, want, "byte_records() of {:?}", data); + let got: Vec = builder + .from_reader(data.as_bytes()) + .into_records() + .map(item) + .collect(); + assert_eq!(got, want, "into_records() of {:?}", data); + let got: Vec = builder + .from_reader(data.as_bytes()) + .into_byte_records() + .map(byte_item) + .collect(); + assert_eq!(got, want, "into_byte_records() of {:?}", data); + + let mut rdr = builder.from_reader(data.as_bytes()); + let mut got = vec![]; + let mut record = StringRecord::new(); + for _ in 0..want.len() + 5 { + match rdr.read_record(&mut record) { + Ok(true) => got.push(item(Ok(record.clone()))), + Ok(false) => break, + Err(err) => got.push(item(Err(err))), + } + } + assert_eq!(got, want, "read_record of {:?}", data); + assert!(rdr.is_done()); + + let mut rdr = builder.from_reader(data.as_bytes()); + let mut got = vec![]; + let mut record = ByteRecord::new(); + for _ in 0..want.len() + 5 { + match rdr.read_byte_record(&mut record) { + Ok(true) => got.push(byte_item(Ok(record.clone()))), + Ok(false) => break, + Err(err) => got.push(byte_item(Err(err))), + } + } + assert_eq!(got, want, "read_byte_record of {:?}", data); +} + +#[test] +fn invalid_record_is_reported_and_skipped() { + let data = "h1,h2\na,b\nc\"d,e\nf,g\n"; + check_all_apis( + &strict(), + data, + &[rec(&["a", "b"]), Item::Bad((10, 3, 2)), rec(&["f", "g"])], + ); +} + +#[test] +fn junk_after_closing_quote_is_reported() { + let mut b = strict(); + b.has_headers(false); + check_all_apis( + &b, + "\"a\"b,c\n\"d\" ,e\n\"f\",\"g\"\n", + &[Item::Bad((0, 1, 0)), Item::Bad((7, 2, 1)), rec(&["f", "g"])], + ); +} + +#[test] +fn valid_quoting_is_accepted() { + let mut b = strict(); + b.has_headers(false).flexible(true); + check_all_apis( + &b, + "\"a\"\"b\",\"c,d\ne\"\r\n\"\",x\r\"y\"", + &[rec(&["a\"b", "c,d\ne"]), rec(&["", "x"]), rec(&["y"])], + ); +} + +#[test] +fn unterminated_quote_is_reported_last() { + let mut b = strict(); + b.has_headers(false); + check_all_apis( + &b, + "a,b\nc,\"d\ne,f\n", + &[rec(&["a", "b"]), Item::Bad((4, 2, 1))], + ); +} + +#[test] +fn escape_and_double_quote_settings_are_respected() { + let mut b = strict(); + b.has_headers(false).double_quote(false).escape(Some(b'\\')); + check_all_apis( + &b, + "\"a\\\"b\",c\n\"d\"\"e\",f\n\"g\\", + &[rec(&["a\"b", "c"]), Item::Bad((9, 2, 1)), Item::Bad((18, 3, 2))], + ); +} + +#[test] +fn custom_quote_delimiter_terminator_and_comment() { + let mut b = strict(); + b.has_headers(false) + .delimiter(b';') + .quote(b'\'') + .terminator(Terminator::Any(b'\n')) + .comment(Some(b'#')); + check_all_apis( + &b, + "#'x'y\n'a';b\"c\nd'e;f\n'g'h;i\n", + &[rec(&["a", "b\"c"]), Item::Bad((14, 3, 1)), Item::Bad((20, 4, 2))], + ); +} + +#[test] +fn strict_is_off_by_default() { + let data = "a\"b,c\n\"d\"e,f\n\"g"; + let mut b = ReaderBuilder::new(); + b.has_headers(false).flexible(true); + check_all_apis( + &b, + data, + &[rec(&["a\"b", "c"]), rec(&["de", "f"]), rec(&["g"])], + ); + b.strict(true).strict(false); + check_all_apis( + &b, + data, + &[rec(&["a\"b", "c"]), rec(&["de", "f"]), rec(&["g"])], + ); +} + +#[test] +fn strict_does_nothing_without_quoting() { + let mut b = strict(); + b.has_headers(false).flexible(true).quoting(false); + check_all_apis( + &b, + "a\"b,c\n\"d\"e,f\n\"g", + &[rec(&["a\"b", "c"]), rec(&["\"d\"e", "f"]), rec(&["\"g"])], + ); +} + +#[test] +fn error_position_is_the_record_position() { + let data = "x,y\n\n\"a\nb\",c\n\"p\"q,r\ns,t\n\"u"; + let mut lenient = ReaderBuilder::new(); + lenient.has_headers(false).flexible(true); + let positions: Vec = lenient + .from_reader(data.as_bytes()) + .byte_records() + .map(|r| { + let r = r.unwrap(); + let p = r.position().unwrap(); + (p.byte(), p.line(), p.record()) + }) + .collect(); + assert_eq!(positions.len(), 5); + + let mut b = strict(); + b.has_headers(false).flexible(true); + let mut rdr = b.from_reader(data.as_bytes()); + let mut got = vec![]; + let mut record = ByteRecord::new(); + loop { + match rdr.read_byte_record(&mut record) { + Ok(true) => { + let p = record.position().unwrap(); + got.push((p.byte(), p.line(), p.record())); + } + Ok(false) => break, + Err(err) => got.push(invalid_pos(&err)), + } + } + assert_eq!(got, positions); +} + +#[test] +fn invalid_header_row_is_reported_by_headers() { + let data = "a\"b,c\nh1,h2\nx,y\n"; + let mut rdr = strict().from_reader(data.as_bytes()); + let err = rdr.headers().unwrap_err(); + assert_eq!(invalid_pos(&err), (0, 1, 0)); + assert_eq!(rdr.headers().unwrap(), vec!["h1", "h2"]); + let got: Vec = rdr.records().map(item).collect(); + assert_eq!(got, vec![rec(&["x", "y"])]); + assert_eq!(rdr.headers().unwrap(), vec!["h1", "h2"]); + + let mut rdr = strict().from_reader(data.as_bytes()); + let err = rdr.byte_headers().unwrap_err(); + assert_eq!(invalid_pos(&err), (0, 1, 0)); + assert_eq!(rdr.byte_headers().unwrap(), vec!["h1", "h2"]); +} + +#[test] +fn invalid_header_row_is_reported_by_first_read() { + let data = "a\"b,c\n\"h\"1,h2\nh1,h2\nx,y\n"; + check_all_apis( + &strict(), + data, + &[Item::Bad((0, 1, 0)), Item::Bad((6, 2, 1)), rec(&["x", "y"])], + ); + let mut rdr = strict().from_reader(data.as_bytes()); + let got: Vec = rdr.byte_records().map(byte_item).collect(); + assert_eq!(got.len(), 3); + assert_eq!(rdr.headers().unwrap(), vec!["h1", "h2"]); +} + +#[test] +fn record_numbers_count_invalid_records() { + let mut rdr = strict().from_reader("\"h\"x\nh\na\n\"b\"c\nd\n".as_bytes()); + let mut got = vec![]; + let mut record = StringRecord::new(); + loop { + match rdr.read_record(&mut record) { + Ok(true) => got.push(( + record.get(0).unwrap().to_string(), + record.position().unwrap().record(), + )), + Ok(false) => break, + Err(err) => got.push(("error".to_string(), invalid_pos(&err).2)), + } + } + assert_eq!( + got, + vec![ + ("error".to_string(), 0), + ("a".to_string(), 2), + ("error".to_string(), 3), + ("d".to_string(), 4) + ] + ); + assert_eq!(rdr.position().record(), 5); +} + +#[test] +fn invalid_records_are_not_compared_for_length() { + let mut b = strict(); + b.has_headers(false).flexible(false); + check_all_apis( + &b, + "a,b\n\"x\"y\nc,d\n\"p\"q,r,s\ne,f\n", + &[ + rec(&["a", "b"]), + Item::Bad((4, 2, 1)), + rec(&["c", "d"]), + Item::Bad((13, 4, 3)), + rec(&["e", "f"]), + ], + ); + + let mut rdr = b.from_reader("\"x\"y,1,2\na,b\nc,d,e\n".as_bytes()); + let mut it = rdr.records(); + assert_eq!(invalid_pos(&it.next().unwrap().unwrap_err()), (0, 1, 0)); + assert_eq!(it.next().unwrap().unwrap(), vec!["a", "b"]); + match *it.next().unwrap().unwrap_err().kind() { + ErrorKind::UnequalLengths { expected_len, len, .. } => { + assert_eq!((expected_len, len), (2, 3)); + } + ref other => panic!("expected UnequalLengths, got {:?}", other), + } + + let mut rdr = strict() + .flexible(false) + .from_reader("h1,h2\n\"a\"b\nc,d\n".as_bytes()); + let got: Vec = rdr.records().map(item).collect(); + assert_eq!(got, vec![Item::Bad((6, 2, 1)), rec(&["c", "d"])]); +} + +#[derive(Debug, Deserialize, PartialEq)] +struct Row { + name: String, + n: u32, +} + +fn row(name: &str, n: u32) -> Row { + Row { name: name.to_string(), n } +} + +#[test] +fn deserialize_reports_invalid_header_rows() { + let data = "\"n\"x,name\n\"n\"ame,n\nn,name\n1,bob\n\"2\"x,al\n3,cat\n"; + let want = vec![ + Err((0, 1, 0)), + Err((10, 2, 1)), + Ok(row("bob", 1)), + Err((32, 5, 4)), + Ok(row("cat", 3)), + ]; + + let mut rdr = strict().from_reader(data.as_bytes()); + let got: Vec> = + rdr.deserialize().map(|r| r.map_err(|e| invalid_pos(&e))).collect(); + assert_eq!(got, want, "deserialize()"); + assert_eq!(rdr.headers().unwrap(), vec!["n", "name"]); + + let got: Vec> = strict() + .from_reader(data.as_bytes()) + .into_deserialize() + .map(|r| r.map_err(|e| invalid_pos(&e))) + .collect(); + assert_eq!(got, want, "into_deserialize()"); +} + +#[test] +fn deserialize_without_headers_reports_invalid_rows() { + let mut b = strict(); + b.has_headers(false); + let data = "bob,1\n\"al\"x,2\ncat,3\n"; + let got: Vec> = b + .from_reader(data.as_bytes()) + .deserialize() + .map(|r| r.map_err(|e| invalid_pos(&e))) + .collect(); + assert_eq!( + got, + vec![Ok(row("bob", 1)), Err((6, 2, 1)), Ok(row("cat", 3))] + ); +} + +#[test] +fn invalid_quoting_error_is_displayable() { + let mut rdr = strict().has_headers(false).from_reader("a\"b\n".as_bytes()); + let err = rdr.records().next().unwrap().unwrap_err(); + assert!(!err.to_string().is_empty()); + assert!(!err.is_io_error()); +} + +// Deterministic generator of CSV documents whose invalid records are known +// by construction. + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self + .0 + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + self.0 >> 33 + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + fn pick(&mut self, xs: &[T]) -> T { + xs[self.below(xs.len() as u64) as usize] + } +} + +#[derive(Clone, Copy, Debug)] +struct Config { + delimiter: u8, + quote: u8, + escape: Option, + double_quote: bool, + comment: Option, + term: &'static [u8], +} + +impl Config { + fn builder(&self) -> ReaderBuilder { + let mut b = ReaderBuilder::new(); + b.delimiter(self.delimiter) + .quote(self.quote) + .escape(self.escape) + .double_quote(self.double_quote) + .comment(self.comment) + .has_headers(false) + .flexible(true); + if self.term == b"~" { + b.terminator(Terminator::Any(b'~')); + } + b + } +} + +fn plain(rng: &mut Rng, min: u64) -> Vec { + let n = min + rng.below(4); + (0..n).map(|_| rng.pick(b"abcxyz019 ")).collect() +} + +fn quoted(rng: &mut Rng, cfg: &Config) -> Vec { + let mut v = vec![cfg.quote]; + for _ in 0..rng.below(6) { + match rng.below(6) { + 0 => v.push(cfg.delimiter), + 1 => v.push(b'\n'), + 2 => v.extend(cfg.term), + 3 => { + if cfg.double_quote { + v.extend([cfg.quote, cfg.quote]); + } else if let Some(e) = cfg.escape { + v.extend([e, cfg.quote]); + } + } + _ => v.push(rng.pick(b"abcxyz ")), + } + } + v.push(cfg.quote); + v +} + +/// Returns a document and, per record, whether it has invalid quoting. +fn gen_doc(rng: &mut Rng, cfg: &Config) -> (Vec, Vec) { + let mut data = vec![]; + let mut invalid = vec![]; + let nrec = 1 + rng.below(6); + for i in 0..nrec { + if cfg.comment.is_some() && rng.below(4) == 0 { + data.push(b'#'); + data.extend(plain(rng, 0)); + data.push(cfg.quote); + data.push(b'\n'); + } + let mut bad = false; + for j in 0..1 + rng.below(4) { + if j > 0 { + data.push(cfg.delimiter); + } + match rng.below(5) { + 0 => data.extend(plain(rng, if j == 0 { 1 } else { 0 })), + 1 => data.extend(quoted(rng, cfg)), + 2 => { + data.extend(plain(rng, 1)); + data.push(cfg.quote); + data.extend(plain(rng, 0)); + bad = true; + } + 3 => { + data.extend(quoted(rng, cfg)); + data.push(rng.pick(b"xz ")); + bad = true; + } + _ => data.extend(quoted(rng, cfg)), + } + } + let last = i + 1 == nrec; + if last && rng.below(3) == 0 { + data.push(cfg.delimiter); + let q = quoted(rng, cfg); + data.extend(&q[..q.len() - 1]); + bad = true; + } else if !last || rng.below(2) == 0 { + data.extend(cfg.term); + } + invalid.push(bad); + } + (data, invalid) +} + +fn configs() -> Vec { + let mut v = vec![]; + for &(delimiter, quote) in &[(b',', b'"'), (b';', b'\''), (b'\t', b'"')] { + for &(double_quote, escape) in + &[(true, None), (false, Some(b'\\')), (false, None)] + { + for &term in &[&b"\n"[..], &b"\r\n"[..], &b"~"[..]] { + for &comment in &[None, Some(b'#')] { + if comment.is_some() && term == b"~" { + continue; + } + v.push(Config { + delimiter, + quote, + escape, + double_quote, + comment, + term, + }); + } + } + } + } + v +} + +#[derive(Debug, PartialEq)] +enum Out { + Rec(Vec>, Pos), + Bad(Pos), + Unequal(u64, u64, Pos), +} + +fn read_all(builder: &ReaderBuilder, data: &[u8]) -> Vec { + let mut rdr = builder.from_reader(data); + let mut out = vec![]; + let mut record = ByteRecord::new(); + for _ in 0..1000 { + match rdr.read_byte_record(&mut record) { + Ok(true) => { + let p = record.position().unwrap(); + out.push(Out::Rec( + record.iter().map(|f| f.to_vec()).collect(), + (p.byte(), p.line(), p.record()), + )); + } + Ok(false) => return out, + Err(err) => match *err.kind() { + ErrorKind::UnequalLengths { ref pos, expected_len, len } => { + let p = pos.clone().unwrap(); + out.push(Out::Unequal( + expected_len, + len, + (p.byte(), p.line(), p.record()), + )); + } + _ => out.push(Out::Bad(invalid_pos(&err))), + }, + } + } + panic!("reader did not finish"); +} + +#[test] +fn generated_documents() { + let mut rng = Rng(0xc5f_a781); + for cfg in configs() { + for _ in 0..25 { + let (data, invalid) = gen_doc(&mut rng, &cfg); + let shown = String::from_utf8_lossy(&data).into_owned(); + let lenient = read_all(&cfg.builder(), &data); + assert_eq!( + lenient.len(), + invalid.len(), + "generator mismatch for {:?} {:?}", + cfg, + shown + ); + + // flexible: every invalid record becomes an error at its position. + let want: Vec = lenient + .iter() + .zip(&invalid) + .map(|(o, &bad)| match *o { + Out::Rec(_, p) if bad => Out::Bad(p), + Out::Rec(ref f, p) => Out::Rec(f.clone(), p), + _ => unreachable!(), + }) + .collect(); + let mut b = cfg.builder(); + b.strict(true); + assert_eq!(read_all(&b, &data), want, "{:?} {:?}", cfg, shown); + + // not flexible: invalid records neither set nor break the length. + let mut expected_len = None; + let want: Vec = lenient + .iter() + .zip(&invalid) + .map(|(o, &bad)| match *o { + Out::Rec(_, p) if bad => Out::Bad(p), + Out::Rec(ref f, p) => match expected_len { + None => { + expected_len = Some(f.len() as u64); + Out::Rec(f.clone(), p) + } + Some(n) if n != f.len() as u64 => { + Out::Unequal(n, f.len() as u64, p) + } + Some(_) => Out::Rec(f.clone(), p), + }, + _ => unreachable!(), + }) + .collect(); + b.flexible(false); + assert_eq!( + read_all(&b, &data), + want, + "not flexible: {:?} {:?}", + cfg, + shown + ); + + // headers: the first record that is not invalid. + let mut b = cfg.builder(); + b.strict(true).has_headers(true); + let mut rdr = b.from_reader(&data[..]); + let first_good = invalid.iter().position(|&x| !x); + let mut errors = 0; + let headers = loop { + match rdr.byte_headers() { + Ok(h) => break h.clone(), + Err(err) => { + invalid_pos(&err); + errors += 1; + assert!( + errors <= invalid.len(), + "too many header errors: {:?}", + shown + ); + } + } + }; + match first_good { + Some(i) => { + assert_eq!(errors, i, "{:?}", shown); + match lenient[i] { + Out::Rec(ref f, _) => assert_eq!( + headers, + ByteRecord::from(f.clone()), + "{:?}", + shown + ), + _ => unreachable!(), + } + } + None => { + assert_eq!(errors, invalid.len(), "{:?}", shown); + assert!(headers.is_empty(), "{:?}", shown); + } + } + } + } +}