ARROW-33: [C++] Implement zero-copy array slicing, integrate with IPC code paths

cpp/src/arrow/CMakeLists.txt

@@ -53,6 +53,7 @@ ADD_ARROW_TEST(array-list-test)
 ADD_ARROW_TEST(array-primitive-test)
 ADD_ARROW_TEST(array-string-test)
 ADD_ARROW_TEST(array-struct-test)
+ADD_ARROW_TEST(array-union-test)
 ADD_ARROW_TEST(buffer-test)
 ADD_ARROW_TEST(column-test)
 ADD_ARROW_TEST(memory_pool-test)

cpp/src/arrow/array-dictionary-test.cc

@@ -34,7 +34,7 @@ namespace arrow {
 TEST(TestDictionary, Basics) {
   std::vector<int32_t> values = {100, 1000, 10000, 100000};
   std::shared_ptr<Array> dict;
-  ArrayFromVector<Int32Type, int32_t>(int32(), values, &dict);
+  ArrayFromVector<Int32Type, int32_t>(values, &dict);
 
   std::shared_ptr<DictionaryType> type1 =
       std::dynamic_pointer_cast<DictionaryType>(dictionary(int16(), dict));
@@ -54,66 +54,88 @@ TEST(TestDictionary, Equals) {
 
   std::shared_ptr<Array> dict;
   std::vector<std::string> dict_values = {"foo", "bar", "baz"};
-  ArrayFromVector<StringType, std::string>(utf8(), dict_values, &dict);
+  ArrayFromVector<StringType, std::string>(dict_values, &dict);
   std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
 
   std::shared_ptr<Array> dict2;
   std::vector<std::string> dict2_values = {"foo", "bar", "baz", "qux"};
-  ArrayFromVector<StringType, std::string>(utf8(), dict2_values, &dict2);
+  ArrayFromVector<StringType, std::string>(dict2_values, &dict2);
   std::shared_ptr<DataType> dict2_type = dictionary(int16(), dict2);
 
   std::shared_ptr<Array> indices;
   std::vector<int16_t> indices_values = {1, 2, -1, 0, 2, 0};
-  ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, indices_values, &indices);
+  ArrayFromVector<Int16Type, int16_t>(is_valid, indices_values, &indices);
 
   std::shared_ptr<Array> indices2;
   std::vector<int16_t> indices2_values = {1, 2, 0, 0, 2, 0};
-  ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, indices2_values, &indices2);
+  ArrayFromVector<Int16Type, int16_t>(is_valid, indices2_values, &indices2);
 
   std::shared_ptr<Array> indices3;
   std::vector<int16_t> indices3_values = {1, 1, 0, 0, 2, 0};
-  ArrayFromVector<Int16Type, int16_t>(int16(), is_valid, indices3_values, &indices3);
+  ArrayFromVector<Int16Type, int16_t>(is_valid, indices3_values, &indices3);
 
-  auto arr = std::make_shared<DictionaryArray>(dict_type, indices);
-  auto arr2 = std::make_shared<DictionaryArray>(dict_type, indices2);
-  auto arr3 = std::make_shared<DictionaryArray>(dict2_type, indices);
-  auto arr4 = std::make_shared<DictionaryArray>(dict_type, indices3);
+  auto array = std::make_shared<DictionaryArray>(dict_type, indices);
+  auto array2 = std::make_shared<DictionaryArray>(dict_type, indices2);
+  auto array3 = std::make_shared<DictionaryArray>(dict2_type, indices);
+  auto array4 = std::make_shared<DictionaryArray>(dict_type, indices3);
 
-  ASSERT_TRUE(arr->Equals(arr));
+  ASSERT_TRUE(array->Equals(array));
 
   // Equal, because the unequal index is masked by null
-  ASSERT_TRUE(arr->Equals(arr2));
+  ASSERT_TRUE(array->Equals(array2));
 
   // Unequal dictionaries
-  ASSERT_FALSE(arr->Equals(arr3));
+  ASSERT_FALSE(array->Equals(array3));
 
   // Unequal indices
-  ASSERT_FALSE(arr->Equals(arr4));
+  ASSERT_FALSE(array->Equals(array4));
 
   // RangeEquals
-  ASSERT_TRUE(arr->RangeEquals(3, 6, 3, arr4));
-  ASSERT_FALSE(arr->RangeEquals(1, 3, 1, arr4));
+  ASSERT_TRUE(array->RangeEquals(3, 6, 3, array4));
+  ASSERT_FALSE(array->RangeEquals(1, 3, 1, array4));
+
+  // ARROW-33 Test slices
+  const int size = array->length();
+
+  std::shared_ptr<Array> slice, slice2;
+  slice = array->Array::Slice(2);
+  slice2 = array->Array::Slice(2);
+  ASSERT_EQ(size - 2, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(2, array->length(), 0, slice));
+
+  // Chained slices
+  slice2 = array->Array::Slice(1)->Array::Slice(1);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(1, 3);
+  slice2 = array->Slice(1, 3);
+  ASSERT_EQ(3, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(1, 4, 0, slice));
 }
 
 TEST(TestDictionary, Validate) {
   std::vector<bool> is_valid = {true, true, false, true, true, true};
 
   std::shared_ptr<Array> dict;
   std::vector<std::string> dict_values = {"foo", "bar", "baz"};
-  ArrayFromVector<StringType, std::string>(utf8(), dict_values, &dict);
+  ArrayFromVector<StringType, std::string>(dict_values, &dict);
   std::shared_ptr<DataType> dict_type = dictionary(int16(), dict);
 
   std::shared_ptr<Array> indices;
   std::vector<uint8_t> indices_values = {1, 2, 0, 0, 2, 0};
-  ArrayFromVector<UInt8Type, uint8_t>(uint8(), is_valid, indices_values, &indices);
+  ArrayFromVector<UInt8Type, uint8_t>(is_valid, indices_values, &indices);
 
   std::shared_ptr<Array> indices2;
   std::vector<float> indices2_values = {1., 2., 0., 0., 2., 0.};
-  ArrayFromVector<FloatType, float>(float32(), is_valid, indices2_values, &indices2);
+  ArrayFromVector<FloatType, float>(is_valid, indices2_values, &indices2);
 
   std::shared_ptr<Array> indices3;
   std::vector<int64_t> indices3_values = {1, 2, 0, 0, 2, 0};
-  ArrayFromVector<Int64Type, int64_t>(int64(), is_valid, indices3_values, &indices3);
+  ArrayFromVector<Int64Type, int64_t>(is_valid, indices3_values, &indices3);
 
   std::shared_ptr<Array> arr = std::make_shared<DictionaryArray>(dict_type, indices);
   std::shared_ptr<Array> arr2 = std::make_shared<DictionaryArray>(dict_type, indices2);

cpp/src/arrow/array-list-test.cc

@@ -90,9 +90,9 @@ TEST_F(TestListBuilder, Equality) {
   Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
 
   std::shared_ptr<Array> array, equal_array, unequal_array;
-  vector<int32_t> equal_offsets = {0, 1, 2, 5};
-  vector<int32_t> equal_values = {1, 2, 3, 4, 5, 2, 2, 2};
-  vector<int32_t> unequal_offsets = {0, 1, 4};
+  vector<int32_t> equal_offsets = {0, 1, 2, 5, 6, 7, 8, 10};
+  vector<int32_t> equal_values = {1, 2, 3, 4, 5, 2, 2, 2, 5, 6};
+  vector<int32_t> unequal_offsets = {0, 1, 4, 7};
   vector<int32_t> unequal_values = {1, 2, 2, 2, 3, 4, 5};
 
   // setup two equal arrays
@@ -122,7 +122,27 @@ TEST_F(TestListBuilder, Equality) {
   EXPECT_FALSE(array->RangeEquals(0, 2, 0, unequal_array));
   EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_array));
   EXPECT_TRUE(array->RangeEquals(2, 3, 2, unequal_array));
-  EXPECT_TRUE(array->RangeEquals(3, 4, 1, unequal_array));
+
+  // Check with slices, ARROW-33
+  std::shared_ptr<Array> slice, slice2;
+
+  slice = array->Slice(2);
+  slice2 = array->Slice(2);
+  ASSERT_EQ(array->length() - 2, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(2, slice->length(), 0, slice));
+
+  // Chained slices
+  slice2 = array->Slice(1)->Slice(1);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(1, 4);
+  slice2 = array->Slice(1, 4);
+  ASSERT_EQ(4, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(1, 5, 0, slice));
 }
 
 TEST_F(TestListBuilder, TestResize) {}
@@ -137,9 +157,9 @@ TEST_F(TestListBuilder, TestAppendNull) {
   ASSERT_TRUE(result_->IsNull(0));
   ASSERT_TRUE(result_->IsNull(1));
 
-  ASSERT_EQ(0, result_->raw_offsets()[0]);
-  ASSERT_EQ(0, result_->offset(1));
-  ASSERT_EQ(0, result_->offset(2));
+  ASSERT_EQ(0, result_->raw_value_offsets()[0]);
+  ASSERT_EQ(0, result_->value_offset(1));
+  ASSERT_EQ(0, result_->value_offset(2));
 
   Int32Array* values = static_cast<Int32Array*>(result_->values().get());
   ASSERT_EQ(0, values->length());
@@ -154,7 +174,7 @@ void ValidateBasicListArray(const ListArray* result, const vector<int32_t>& valu
   ASSERT_EQ(3, result->length());
   vector<int32_t> ex_offsets = {0, 3, 3, 7};
   for (size_t i = 0; i < ex_offsets.size(); ++i) {
-    ASSERT_EQ(ex_offsets[i], result->offset(i));
+    ASSERT_EQ(ex_offsets[i], result->value_offset(i));
   }
 
   for (int i = 0; i < result->length(); ++i) {

cpp/src/arrow/array-primitive-test.cc

@@ -121,7 +121,7 @@ class TestPrimitiveBuilder : public TestBuilder {
     }
 
     auto expected =
-        std::make_shared<ArrayType>(size, ex_data, ex_null_count, ex_null_bitmap);
+        std::make_shared<ArrayType>(size, ex_data, ex_null_bitmap, ex_null_count);
 
     std::shared_ptr<Array> out;
     ASSERT_OK(builder->Finish(&out));
@@ -217,7 +217,7 @@ void TestPrimitiveBuilder<PBoolean>::Check(
   }
 
   auto expected =
-      std::make_shared<BooleanArray>(size, ex_data, ex_null_count, ex_null_bitmap);
+      std::make_shared<BooleanArray>(size, ex_data, ex_null_bitmap, ex_null_count);
 
   std::shared_ptr<Array> out;
   ASSERT_OK(builder->Finish(&out));
@@ -235,15 +235,14 @@ void TestPrimitiveBuilder<PBoolean>::Check(
 
   for (int i = 0; i < result->length(); ++i) {
     if (nullable) { ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i; }
-    bool actual = BitUtil::GetBit(result->raw_data(), i);
+    bool actual = BitUtil::GetBit(result->data()->data(), i);
     ASSERT_EQ(static_cast<bool>(draws_[i]), actual) << i;
   }
   ASSERT_TRUE(result->Equals(*expected));
 }
 
 typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
-    PInt32, PInt64, PFloat, PDouble>
-    Primitives;
+    PInt32, PInt64, PFloat, PDouble> Primitives;
 
 TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
 
@@ -347,6 +346,39 @@ TYPED_TEST(TestPrimitiveBuilder, Equality) {
       array->RangeEquals(first_valid_idx + 1, size, first_valid_idx + 1, unequal_array));
 }
 
+TYPED_TEST(TestPrimitiveBuilder, SliceEquality) {
+  DECL_T();
+
+  const int size = 1000;
+  this->RandomData(size);
+  vector<T>& draws = this->draws_;
+  vector<uint8_t>& valid_bytes = this->valid_bytes_;
+  auto builder = this->builder_.get();
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &array));
+
+  std::shared_ptr<Array> slice, slice2;
+
+  slice = array->Slice(5);
+  slice2 = array->Slice(5);
+  ASSERT_EQ(size - 5, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(5, array->length(), 0, slice));
+
+  // Chained slices
+  slice2 = array->Slice(2)->Slice(3);
+  ASSERT_TRUE(slice->Equals(slice2));
+
+  slice = array->Slice(5, 10);
+  slice2 = array->Slice(5, 10);
+  ASSERT_EQ(10, slice->length());
+
+  ASSERT_TRUE(slice->Equals(slice2));
+  ASSERT_TRUE(array->RangeEquals(5, 15, 0, slice));
+}
+
 TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
   DECL_T();
 
@@ -473,4 +505,40 @@ TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
   ASSERT_EQ(BitUtil::NextPower2(kMinBuilderCapacity + 100), this->builder_->capacity());
 }
 
+template <typename TYPE>
+void CheckSliceApproxEquals() {
+  using T = typename TYPE::c_type;
+
+  const int kSize = 50;
+  std::vector<T> draws1;
+  std::vector<T> draws2;
+
+  const uint32_t kSeed = 0;
+  test::random_real<T>(kSize, kSeed, 0, 100, &draws1);
+  test::random_real<T>(kSize, kSeed + 1, 0, 100, &draws2);
+
+  // Make the draws equal in the sliced segment, but unequal elsewhere (to
+  // catch not using the slice offset)
+  for (int i = 10; i < 30; ++i) {
+    draws2[i] = draws1[i];
+  }
+
+  std::vector<bool> is_valid;
+  test::random_is_valid(kSize, 0.1, &is_valid);
+
+  std::shared_ptr<Array> array1, array2;
+  ArrayFromVector<TYPE, T>(is_valid, draws1, &array1);
+  ArrayFromVector<TYPE, T>(is_valid, draws2, &array2);
+
+  std::shared_ptr<Array> slice1 = array1->Slice(10, 20);
+  std::shared_ptr<Array> slice2 = array2->Slice(10, 20);
+
+  ASSERT_TRUE(slice1->ApproxEquals(slice2));
+}
+
+TEST(TestPrimitiveAdHoc, FloatingSliceApproxEquals) {
+  CheckSliceApproxEquals<FloatType>();
+  CheckSliceApproxEquals<DoubleType>();
+}
+
 }  // namespace arrow