Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,10 @@ static void AssertTypeAndValue(
byte[]? expectedBytes = ctv.ExpectedValue as byte[];
Assert.True(expectedBytes != null && actualBytes.SequenceEqual(expectedBytes), Utils.FormatMessage($"byte[] values do not match expected values for {ctv.Name} for query [{query}]", environmentName));
}
else if (ctv.IsCalculatedResult)
{
Assert.True(ctv.IsValid(value), Utils.FormatMessage($"Actual value [{value}] for {ctv.Name} does not return true for the IsValid function for query [{query}]", environmentName));
}
else if (ctv.ExpectedValue is null)
{
Assert.True(value is null, Utils.FormatMessage($"Expected value [{ctv.ExpectedValue}] does not match actual value [{value}] for {ctv.Name} for query [{query}]", environmentName));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,15 @@ namespace Apache.Arrow.Adbc.Tests
/// </summary>
public class ColumnNetTypeArrowTypeValue
{
private Func<object?, bool>? _isValid;

/// <summary>
/// Instantiates <see cref="ColumnNetTypeArrowTypeValue"/>.
/// </summary>
/// <param name="name">The column name</param>
/// <param name="expectedNetType">The expected .NET type</param>
/// <param name="expectedArrowArrayType">The expected Arrow type</param>
/// <param name="expectedValue">The expected value</param>
public ColumnNetTypeArrowTypeValue(string name, Type expectedNetType, Type expectedArrowArrayType, object? expectedValue)
{
this.Name = name;
Expand All @@ -32,12 +41,38 @@ public ColumnNetTypeArrowTypeValue(string name, Type expectedNetType, Type expec
this.ExpectedValue = expectedValue;
}

/// <summary>
/// Instantiates <see cref="ColumnNetTypeArrowTypeValue"/>.
/// </summary>
/// <param name="name">The column name</param>
/// <param name="expectedNetType">The expected .NET type</param>
/// <param name="expectedArrowArrayType">The expected Arrow type</param>
/// <param name="expectedValue">The expected value</param>
/// <param name="isValid">A function that can be run to compare values</param>
/// <exception cref="ArgumentNullException"></exception>
public ColumnNetTypeArrowTypeValue(string name, Type expectedNetType, Type expectedArrowArrayType, bool expectedValue, Func<object?, bool> isValid)
{
this.Name = name;
this.ExpectedNetType = expectedNetType;
this.ExpectedArrowArrayType = expectedArrowArrayType;
this.ExpectedValue = expectedValue;

_isValid = isValid ?? throw new ArgumentNullException(nameof(isValid));
}

public string Name { get; set; }

public Type ExpectedNetType { get; set; }

public Type ExpectedArrowArrayType { get; set; }

public object? ExpectedValue { get; set; }

public bool IsCalculatedResult => _isValid != null;

public bool IsValid(object? value)
{
return _isValid == null ? false : Convert.ToBoolean(this.ExpectedValue) == _isValid(value);
}
}
}
23 changes: 11 additions & 12 deletions csharp/test/Drivers/Interop/Snowflake/CastTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -310,64 +310,64 @@ private static async Task ValidateCast(object? value, ArrowTypeId expectedType,
for (int i = 0; i < doubleArray.Length; i++)
{
Assert.Equal(Convert.ToDouble(value), doubleArray.GetValue(i));
};
}
break;
case ArrowTypeId.Int32:
var int32Array = (Int32Array)valueArray;
for (int i = 0; i < int32Array.Length; i++)
{
Assert.Equal(value, int32Array.GetValue(i));
};
}
break;
case ArrowTypeId.Int64:
var int64Array = (Int64Array)valueArray;
for (int i = 0; i < int64Array.Length; i++)
{
Assert.Equal(value, int64Array.GetValue(i));
};
}
break;
case ArrowTypeId.String:
var stringArray = (StringArray)valueArray;
for (int i = 0; i < stringArray.Length; i++)
{
Assert.Equal(value, stringArray.GetString(i));
};
}
break;
case ArrowTypeId.Boolean:
var booleanArray = (BooleanArray)valueArray;
for (int i = 0; i < booleanArray.Length; i++)
{
Assert.Equal(value, booleanArray.GetValue(i));
};
}
break;
case ArrowTypeId.Date64:
var date64Array = (Date64Array)valueArray;
for (int i = 0; i < date64Array.Length; i++)
{
Assert.Equal(value, date64Array.GetValue(i));
};
}
break;
case ArrowTypeId.Decimal128:
var decimal128Array = (Decimal128Array)valueArray;
for (int i = 0; i < decimal128Array.Length; i++)
{

Assert.Equal(value == null ? null : Convert.ToDecimal(value), decimal128Array.GetValue(i));
};
}
break;
case ArrowTypeId.Decimal256:
var decimal256Array = (Decimal256Array)valueArray;
for (int i = 0; i < decimal256Array.Length; i++)
{
Assert.Equal(value, decimal256Array.GetValue(i));
};
}
break;
case ArrowTypeId.Timestamp:
var timestampArray = (TimestampArray)valueArray;
for (int i = 0; i < timestampArray.Length; i++)
{
Assert.Equal(value == null ? null : new DateTimeOffset(Convert.ToDateTime(value).ToUniversalTime()), timestampArray.GetTimestamp(i));
};
}
break;
case ArrowTypeId.Time64:
var time64Array = (Time64Array)valueArray;
Expand All @@ -378,16 +378,15 @@ private static async Task ValidateCast(object? value, ArrowTypeId expectedType,
#else
Assert.Equal(Convert.ToInt64(value), time64Array.GetMicroSeconds(i));
#endif
};
}
break;
case ArrowTypeId.Date32:
var date32Array = (Date32Array)valueArray;
for (int i = 0; i < date32Array.Length; i++)
{
Assert.Equal(Convert.ToDateTime(value), date32Array.GetDateTimeOffset(i));
};
}
break;

default:
throw new ArgumentException(string.Format("Unexpected ArrowTypeId: {0}({1})", field.DataType.TypeId.ToString(), field.DataType.TypeId));

Expand Down
63 changes: 63 additions & 0 deletions csharp/test/Drivers/Interop/Snowflake/ClientTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,69 @@ public void VerifyTimestampPrecision()
Assert.Throws<Exception>(() => ValidateTimestampPrecision(SnowflakeConstants.OptionValueNanosecondsNoOverflow, query, expectedNanoseconddValues));
}

[SkippableFact, Order(6)]
public void VerifyTimestampPrecisionJson()
{
SnowflakeTestConfiguration testConfiguration = Utils.LoadTestConfiguration<SnowflakeTestConfiguration>(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE);
testConfiguration.MaxTimestampPrecision = "microseconds";

string tempTable = "pk_" + Guid.NewGuid().ToString().Replace("-", "");

using (Adbc.Client.AdbcConnection adbcConnection = GetSnowflakeAdbcConnectionUsingConnectionString(testConfiguration))
{
SampleDataBuilder sampleDataBuilder = new SampleDataBuilder();
sampleDataBuilder.Samples.Add(
new SampleData()
{
// create the table
PreQueryCommands = new List<string> {
@$"CREATE OR REPLACE TABLE {testConfiguration.Metadata.Catalog}.{testConfiguration.Metadata.Schema}.{tempTable}(
id INT PRIMARY KEY,
name STRING
);"
},
// run the SHOW PRIMARY KEYS command, which returns data as json
Query = $"SHOW PRIMARY KEYS IN TABLE {testConfiguration.Metadata.Catalog}.{testConfiguration.Metadata.Schema}.{tempTable}",
ExpectedValues = new List<ColumnNetTypeArrowTypeValue>()
{
new ColumnNetTypeArrowTypeValue("created_on", typeof(DateTimeOffset), typeof(TimestampType), true,
v =>
{
if (v is DateTimeOffset createdOn)
{
// the key will have just been created, so just compare that the two times
// are only a few minutes apart
DateTimeOffset now = DateTimeOffset.UtcNow;
TimeSpan difference = now - createdOn;
return difference.Duration() < TimeSpan.FromMinutes(2);
}
else
{
return false;
}
}),
new ColumnNetTypeArrowTypeValue("database_name", typeof(string), typeof(StringType), testConfiguration.Metadata.Catalog),
new ColumnNetTypeArrowTypeValue("schema_name", typeof(string), typeof(StringType), testConfiguration.Metadata.Schema),
new ColumnNetTypeArrowTypeValue("table_name", typeof(string), typeof(StringType), tempTable.ToUpper()),
new ColumnNetTypeArrowTypeValue("column_name", typeof(string), typeof(StringType), "ID"),
new ColumnNetTypeArrowTypeValue("key_sequence", typeof(SqlDecimal), typeof(Decimal128Type), new SqlDecimal(1m)),

// we dont control these, but also don't care about the value
new ColumnNetTypeArrowTypeValue("constraint_name", typeof(string), typeof(StringType), true, v => { return true; }),
new ColumnNetTypeArrowTypeValue("rely", typeof(string), typeof(StringType), true, v => { return true; }),
new ColumnNetTypeArrowTypeValue("comment", typeof(string), typeof(StringType), null)
},
// drop the table
PostQueryCommands = new List<string>() {
$"DROP TABLE {testConfiguration.Metadata.Catalog}.{testConfiguration.Metadata.Schema}.{tempTable}"
}
});

Tests.ClientTests.VerifyTypesAndValues(adbcConnection, sampleDataBuilder);
}
}


private void ValidateTimestampPrecision(string precision, string query, List<ColumnNetTypeArrowTypeValue> expectedValues)
{
SnowflakeTestConfiguration testConfiguration = Utils.LoadTestConfiguration<SnowflakeTestConfiguration>(SnowflakeTestingUtils.SNOWFLAKE_TEST_CONFIG_VARIABLE);
Expand Down
94 changes: 94 additions & 0 deletions go/adbc/driver/snowflake/driver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1724,6 +1724,100 @@ func (suite *SnowflakeTests) TestBooleanType() {
}
}

func (suite *SnowflakeTests) TestTimestampPrecisionJson() {
opts := suite.Quirks.DatabaseOptions()
opts[driver.OptionMaxTimestampPrecision] = "microseconds"

db, err := suite.driver.NewDatabase(opts)
suite.NoError(err)
defer validation.CheckedClose(suite.T(), db)
cnxn, err := db.Open(suite.ctx)
suite.NoError(err)
defer validation.CheckedClose(suite.T(), cnxn)
stmt, _ := cnxn.NewStatement()

id := uuid.New()
tempTable := "pk_" + strings.ReplaceAll(id.String(), "-", "")

query := fmt.Sprintf(`CREATE OR REPLACE TABLE %s.%s.%s (
id INT PRIMARY KEY,
name STRING);`, suite.Quirks.catalogName, suite.Quirks.schemaName, tempTable)

suite.Require().NoError(stmt.SetSqlQuery(query))
_, err = stmt.ExecuteUpdate(suite.ctx)
suite.Require().NoError(err)

query = fmt.Sprintf("SHOW PRIMARY KEYS IN TABLE %s.%s.%s", suite.Quirks.catalogName, suite.Quirks.schemaName, tempTable)
stmt, _ = cnxn.NewStatement()
suite.Require().NoError(stmt.SetSqlQuery(query))
rdr, _, err := stmt.ExecuteQuery(suite.ctx)
defer rdr.Release()
suite.Require().NoError(err)

suite.True(rdr.Next())
rec := rdr.Record()

suite.True(rec.NumRows() == 1)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suite.Equal?


// Get column indexes
getColIdx := func(name string) int {
for i := 0; i < int(rec.NumCols()); i++ {
if rec.ColumnName(i) == name {
return i
}
}
return -1
}

// Expected column names
dbIdx := getColIdx("database_name")
schemaIdx := getColIdx("schema_name")
tableIdx := getColIdx("table_name")
colIdx := getColIdx("column_name")
seqIdx := getColIdx("key_sequence")
createdIdx := getColIdx("created_on")

if dbIdx == -1 || schemaIdx == -1 || tableIdx == -1 || colIdx == -1 || seqIdx == -1 || createdIdx == -1 {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suite.Equal instead of panic?

Or just panic inside of getColIndex with the column name.

panic("Missing expected columns")
}

i := 0
dbName := rec.Column(dbIdx).(*array.String).Value(i)
schema := rec.Column(schemaIdx).(*array.String).Value(i)
tbl := rec.Column(tableIdx).(*array.String).Value(i)
column := rec.Column(colIdx).(*array.String).Value(i)
keySeq := rec.Column(seqIdx).(*array.Int64).Value(i)

// Created_on should be a timestamp array
createdCol := rec.Column(createdIdx).(*array.Timestamp)
created := time.Unix(0, int64(createdCol.Value(i))*int64(time.Microsecond)).UTC()

// Perform checks
if strings.EqualFold(dbName, suite.Quirks.catalogName) &&
strings.EqualFold(schema, suite.Quirks.schemaName) &&
strings.EqualFold(tbl, tempTable) &&
strings.EqualFold(column, "id") &&
keySeq == 1 {

now := time.Now().UTC()
diff := now.Sub(created)
if diff < 0 {
diff = -diff
}
// since this was just created, make sure the times are within a short difference
suite.Assert().True(diff <= 2*time.Minute)
} else {
panic("Invalid values")
}

query = fmt.Sprintf("DROP TABLE %s.%s.%s", suite.Quirks.catalogName, suite.Quirks.schemaName, tempTable)
stmt, _ = cnxn.NewStatement()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This isn't being cleaned up, though I suppose it doesn't seem to matter

suite.Require().NoError(stmt.SetSqlQuery(query))
_, err = stmt.ExecuteUpdate(suite.ctx)
defer rdr.Release()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't this a double release? And why is it all the way down here?

suite.Require().NoError(err)
}

func (suite *SnowflakeTests) TestTimestampPrecision() {

query_ntz := "select TO_TIMESTAMP_NTZ('0001-01-01 00:00:00.000000000') as Jan01_0001, TO_TIMESTAMP_NTZ('2025-06-02 10:37:56.123456789') as June02_2025, TO_TIMESTAMP_NTZ('9999-12-31 23:59:59.999999999') As December31_9999"
Expand Down
12 changes: 10 additions & 2 deletions go/adbc/driver/snowflake/record_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -479,8 +479,16 @@ func jsonDataToArrow(_ context.Context, bldr *array.RecordBuilder, rawData [][]*
return nil, err
}

fb.Append(arrow.Timestamp(sec*1e9 + nsec))

if maxTimestampPrecision == Microseconds {
tt := time.Unix(sec, nsec)
ts, err := getArrowTimestampFromTime(tt, arrow.Microsecond, arrow.Nanosecond, maxTimestampPrecision)
if err != nil {
return nil, err
}
fb.Append(ts)
} else {
fb.Append(arrow.Timestamp(sec*1e9 + nsec))
}
case *array.BinaryBuilder:
b, err := hex.DecodeString(*col)
if err != nil {
Expand Down
Loading