feat(bigquery): add PreserveAsciiControlCharacters support for CSV (#6448)
* feat(bigquery): add PreserveAsciiControlCharacters support for CSV
diff --git a/bigquery/external.go b/bigquery/external.go
index c5337a0..3d3442b 100644
--- a/bigquery/external.go
+++ b/bigquery/external.go
@@ -230,17 +230,22 @@
// An optional custom string that will represent a NULL
// value in CSV import data.
NullMarker string
+
+ // Preserves the embedded ASCII control characters (the first 32 characters in the ASCII-table,
+ // from '\\x00' to '\\x1F') when loading from CSV. Only applicable to CSV, ignored for other formats.
+ PreserveASCIIControlCharacters bool
}
func (o *CSVOptions) populateExternalDataConfig(c *bq.ExternalDataConfiguration) {
c.CsvOptions = &bq.CsvOptions{
- AllowJaggedRows: o.AllowJaggedRows,
- AllowQuotedNewlines: o.AllowQuotedNewlines,
- Encoding: string(o.Encoding),
- FieldDelimiter: o.FieldDelimiter,
- Quote: o.quote(),
- SkipLeadingRows: o.SkipLeadingRows,
- NullMarker: o.NullMarker,
+ AllowJaggedRows: o.AllowJaggedRows,
+ AllowQuotedNewlines: o.AllowQuotedNewlines,
+ Encoding: string(o.Encoding),
+ FieldDelimiter: o.FieldDelimiter,
+ Quote: o.quote(),
+ SkipLeadingRows: o.SkipLeadingRows,
+ NullMarker: o.NullMarker,
+ PreserveAsciiControlCharacters: o.PreserveASCIIControlCharacters,
}
}
@@ -267,12 +272,13 @@
func bqToCSVOptions(q *bq.CsvOptions) *CSVOptions {
o := &CSVOptions{
- AllowJaggedRows: q.AllowJaggedRows,
- AllowQuotedNewlines: q.AllowQuotedNewlines,
- Encoding: Encoding(q.Encoding),
- FieldDelimiter: q.FieldDelimiter,
- SkipLeadingRows: q.SkipLeadingRows,
- NullMarker: q.NullMarker,
+ AllowJaggedRows: q.AllowJaggedRows,
+ AllowQuotedNewlines: q.AllowQuotedNewlines,
+ Encoding: Encoding(q.Encoding),
+ FieldDelimiter: q.FieldDelimiter,
+ SkipLeadingRows: q.SkipLeadingRows,
+ NullMarker: q.NullMarker,
+ PreserveASCIIControlCharacters: q.PreserveAsciiControlCharacters,
}
o.setQuote(q.Quote)
return o
diff --git a/bigquery/file.go b/bigquery/file.go
index 44eab5d..9afdbb5 100644
--- a/bigquery/file.go
+++ b/bigquery/file.go
@@ -93,6 +93,7 @@
conf.IgnoreUnknownValues = fc.IgnoreUnknownValues
conf.MaxBadRecords = fc.MaxBadRecords
conf.NullMarker = fc.NullMarker
+ conf.PreserveAsciiControlCharacters = fc.PreserveASCIIControlCharacters
if fc.Schema != nil {
conf.Schema = fc.Schema.toBQ()
}
@@ -120,6 +121,7 @@
fc.Encoding = Encoding(conf.Encoding)
fc.FieldDelimiter = conf.FieldDelimiter
fc.CSVOptions.NullMarker = conf.NullMarker
+ fc.CSVOptions.PreserveASCIIControlCharacters = conf.PreserveAsciiControlCharacters
fc.CSVOptions.setQuote(conf.Quote)
}
diff --git a/bigquery/file_test.go b/bigquery/file_test.go
index e57633f..77ba8b2 100644
--- a/bigquery/file_test.go
+++ b/bigquery/file_test.go
@@ -33,13 +33,14 @@
nestedFieldSchema(),
},
CSVOptions: CSVOptions{
- Quote: hyphen,
- FieldDelimiter: "\t",
- SkipLeadingRows: 8,
- AllowJaggedRows: true,
- AllowQuotedNewlines: true,
- Encoding: UTF_8,
- NullMarker: "marker",
+ Quote: hyphen,
+ FieldDelimiter: "\t",
+ SkipLeadingRows: 8,
+ AllowJaggedRows: true,
+ AllowQuotedNewlines: true,
+ Encoding: UTF_8,
+ NullMarker: "marker",
+ PreserveASCIIControlCharacters: true,
},
}
)
@@ -63,16 +64,17 @@
description: "csv",
fileConfig: &fc,
want: &bq.JobConfigurationLoad{
- SourceFormat: "CSV",
- FieldDelimiter: "\t",
- SkipLeadingRows: 8,
- AllowJaggedRows: true,
- AllowQuotedNewlines: true,
- Autodetect: true,
- Encoding: "UTF-8",
- MaxBadRecords: 7,
- IgnoreUnknownValues: true,
- NullMarker: "marker",
+ SourceFormat: "CSV",
+ FieldDelimiter: "\t",
+ SkipLeadingRows: 8,
+ AllowJaggedRows: true,
+ AllowQuotedNewlines: true,
+ Autodetect: true,
+ Encoding: "UTF-8",
+ MaxBadRecords: 7,
+ IgnoreUnknownValues: true,
+ NullMarker: "marker",
+ PreserveAsciiControlCharacters: true,
Schema: &bq.TableSchema{
Fields: []*bq.TableFieldSchema{
bqStringFieldSchema(),
@@ -150,13 +152,14 @@
bqNestedFieldSchema(),
}},
CsvOptions: &bq.CsvOptions{
- AllowJaggedRows: true,
- AllowQuotedNewlines: true,
- Encoding: "UTF-8",
- FieldDelimiter: "\t",
- Quote: &hyphen,
- SkipLeadingRows: 8,
- NullMarker: "marker",
+ AllowJaggedRows: true,
+ AllowQuotedNewlines: true,
+ Encoding: "UTF-8",
+ FieldDelimiter: "\t",
+ Quote: &hyphen,
+ SkipLeadingRows: 8,
+ NullMarker: "marker",
+ PreserveAsciiControlCharacters: true,
},
},
},
