if (type.HasDictionary()) {

return ArrowArrayPhysicalType::DICTIONARY_ENCODED;

}

if (type.RunEndEncoded()) {

return ArrowArrayPhysicalType::RUN_END_ENCODED;

}

return ArrowArrayPhysicalType::DEFAULT;

int carry = 0;

while (shift--) {

for (int i = size - 1; i >= 0; --i) {

int next = (ar[i] & 1) ? 0x80 : 0;

ar[i] = UnsafeNumericCast<unsigned char>(carry | (ar[i] >> 1));

carry = next;

}

}

int64_t nested_offset = -1) {

if (nested_offset != -1) {

// The parent of this array is a list

// We just ignore the parent offset, it's already applied to the list

return UnsafeNumericCast<idx_t>(array.offset + nested_offset);

}

// Parent offset is set in the case of a struct, it applies to all child arrays

// 'chunk_offset' is how much of the chunk we've already scanned, in case the chunk size exceeds

// STANDARD_VECTOR_SIZE

return UnsafeNumericCast<idx_t>(array.offset + parent_offset) + state.chunk_offset;

int64_t parent_offset, int64_t nested_offset = -1, bool add_null = false) {

// In certains we don't need to or cannot copy arrow's validity mask to duckdb.

//

// The conditions where we do want to copy arrow's mask to duckdb are:

// 1. nulls exist

// 2. n_buffers > 0, meaning the array's arrow type is not `null`

// 3. the validity buffer (the first buffer) is not a nullptr

if (array.null_count != 0 && array.n_buffers > 0 && array.buffers[0]) {

auto bit_offset = GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);

mask.EnsureWritable();

#if STANDARD_VECTOR_SIZE > 64

auto n_bitmask_bytes = (size + 8 - 1) / 8;

if (bit_offset % 8 == 0) {

//! just memcpy nullmask

memcpy((void *)mask.GetData(), ArrowBufferData<uint8_t>(array, 0) + bit_offset / 8, n_bitmask_bytes);

} else {

//! need to re-align nullmask

vector<uint8_t> temp_nullmask(n_bitmask_bytes + 1);

memcpy(temp_nullmask.data(), ArrowBufferData<uint8_t>(array, 0) + bit_offset / 8, n_bitmask_bytes + 1);

ShiftRight(temp_nullmask.data(), NumericCast<int>(n_bitmask_bytes + 1),

NumericCast<int>(bit_offset % 8ull)); //! why this has to be a right shift is a mystery to me

memcpy((void *)mask.GetData(), data_ptr_cast(temp_nullmask.data()), n_bitmask_bytes);

}

#else

auto byte_offset = bit_offset / 8;

auto source_data = ArrowBufferData<uint8_t>(array, 0);

bit_offset %= 8;

for (idx_t i = 0; i < size; i++) {

mask.Set(i, source_data[byte_offset] & (1 << bit_offset));

bit_offset++;

if (bit_offset == 8) {

bit_offset = 0;

byte_offset++;

}

}

#endif

}

if (add_null) {

//! We are setting a validity mask of the data part of dictionary vector

//! For some reason, Nulls are allowed to be indexes, hence we need to set the last element here to be null

//! We might have to resize the mask

mask.Resize(size + 1);

mask.SetInvalid(size);

}

int64_t parent_offset, int64_t nested_offset, bool add_null = false) {

D_ASSERT(vector.GetVectorType() == VectorType::FLAT_VECTOR);

auto &mask = FlatVector::Validity(vector);

GetValidityMask(mask, array, scan_state, size, parent_offset, nested_offset, add_null);

idx_t effective_offset) {

ArrowListOffsetData result;

auto &start_offset = result.start_offset;

auto &list_size = result.list_size;

if (size == 0) {

start_offset = 0;

list_size = 0;

return result;

}

idx_t cur_offset = 0;

auto offsets = ArrowBufferData<BUFFER_TYPE>(array, 1) + effective_offset;

start_offset = offsets[0];

auto list_data = FlatVector::GetData<list_entry_t>(vector);

for (idx_t i = 0; i < size; i++) {

auto &le = list_data[i];

le.offset = cur_offset;

le.length = offsets[i + 1] - offsets[i];

cur_offset += le.length;

}

list_size = offsets[size];

list_size -= start_offset;

return result;

idx_t effective_offset) {

ArrowListOffsetData result;

auto &start_offset = result.start_offset;

auto &list_size = result.list_size;

list_size = 0;

auto offsets = ArrowBufferData<BUFFER_TYPE>(array, 1) + effective_offset;

auto sizes = ArrowBufferData<BUFFER_TYPE>(array, 2) + effective_offset;

// In ListArrays the offsets have to be sequential

// ListViewArrays do not have this same constraint

// for that reason we need to keep track of the lowest offset, so we can skip all the data that comes before it

// when we scan the child data

auto lowest_offset = size ? offsets[0] : 0;

auto list_data = FlatVector::GetData<list_entry_t>(vector);

for (idx_t i = 0; i < size; i++) {

auto &le = list_data[i];

le.offset = offsets[i];

le.length = sizes[i];

list_size += le.length;

if (sizes[i] != 0) {

lowest_offset = MinValue(lowest_offset, offsets[i]);

}

}

start_offset = lowest_offset;

if (start_offset) {

// We start scanning the child data at the 'start_offset' so we need to fix up the created list entries

for (idx_t i = 0; i < size; i++) {

auto &le = list_data[i];

le.offset = le.offset <= start_offset ? 0 : le.offset - start_offset;

}

}

return result;

const ArrowType &arrow_type, idx_t effective_offset) {

auto &list_info = arrow_type.GetTypeInfo<ArrowListInfo>();

auto size_type = list_info.GetSizeType();

if (list_info.IsView()) {

if (size_type == ArrowVariableSizeType::NORMAL) {

return ConvertArrowListViewOffsetsTemplated<uint32_t>(vector, array, size, effective_offset);

} else {

D_ASSERT(size_type == ArrowVariableSizeType::SUPER_SIZE);

return ConvertArrowListViewOffsetsTemplated<uint64_t>(vector, array, size, effective_offset);

}

} else {

if (size_type == ArrowVariableSizeType::NORMAL) {

return ConvertArrowListOffsetsTemplated<uint32_t>(vector, array, size, effective_offset);

} else {

D_ASSERT(size_type == ArrowVariableSizeType::SUPER_SIZE);

return ConvertArrowListOffsetsTemplated<uint64_t>(vector, array, size, effective_offset);

}

}

const ArrowType &arrow_type, int64_t nested_offset, const ValidityMask *parent_mask,

int64_t parent_offset) {

auto &scan_state = array_state.state;

auto &list_info = arrow_type.GetTypeInfo<ArrowListInfo>();

SetValidityMask(vector, array, scan_state, size, parent_offset, nested_offset);

auto effective_offset = GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);

auto list_data = ConvertArrowListOffsets(vector, array, size, arrow_type, effective_offset);

auto &start_offset = list_data.start_offset;

auto &list_size = list_data.list_size;

ListVector::Reserve(vector, list_size);

ListVector::SetListSize(vector, list_size);

auto &child_vector = ListVector::GetEntry(vector);

SetValidityMask(child_vector, *array.children[0], scan_state, list_size, array.offset,

NumericCast<int64_t>(start_offset));

auto &list_mask = FlatVector::Validity(vector);

if (parent_mask) {

//! Since this List is owned by a struct we must guarantee their validity map matches on Null

if (!parent_mask->AllValid()) {

for (idx_t i = 0; i < size; i++) {

if (!parent_mask->RowIsValid(i)) {

list_mask.SetInvalid(i);

}

}

}

}

auto &child_state = array_state.GetChild(0);

auto &child_array = *array.children[0];

auto &child_type = list_info.GetChild();

if (list_size == 0 && start_offset == 0) {

D_ASSERT(!child_array.dictionary);

ColumnArrowToDuckDB(child_vector, child_array, child_state, list_size, child_type, -1);

return;

}

auto array_physical_type = GetArrowArrayPhysicalType(child_type);

switch (array_physical_type) {

case ArrowArrayPhysicalType::DICTIONARY_ENCODED:

// TODO: add support for offsets

ColumnArrowToDuckDBDictionary(child_vector, child_array, child_state, list_size, child_type,

NumericCast<int64_t>(start_offset));

break;

case ArrowArrayPhysicalType::RUN_END_ENCODED:

ColumnArrowToDuckDBRunEndEncoded(child_vector, child_array, child_state, list_size, child_type,

NumericCast<int64_t>(start_offset));

break;

case ArrowArrayPhysicalType::DEFAULT:

ColumnArrowToDuckDB(child_vector, child_array, child_state, list_size, child_type,

NumericCast<int64_t>(start_offset));

break;

default:

throw NotImplementedException("ArrowArrayPhysicalType not recognized");

}

const ArrowType &arrow_type, int64_t nested_offset, const ValidityMask *parent_mask,

int64_t parent_offset) {

auto &array_info = arrow_type.GetTypeInfo<ArrowArrayInfo>();

auto &scan_state = array_state.state;

auto array_size = array_info.FixedSize();

auto child_count = array_size * size;

auto child_offset = GetEffectiveOffset(array, parent_offset, scan_state, nested_offset) * array_size;

SetValidityMask(vector, array, scan_state, size, parent_offset, nested_offset);

auto &child_vector = ArrayVector::GetEntry(vector);

SetValidityMask(child_vector, *array.children[0], scan_state, child_count, array.offset,

NumericCast<int64_t>(child_offset));

auto &array_mask = FlatVector::Validity(vector);

if (parent_mask) {

//! Since this List is owned by a struct we must guarantee their validity map matches on Null

if (!parent_mask->AllValid()) {

for (idx_t i = 0; i < size; i++) {

if (!parent_mask->RowIsValid(i)) {

array_mask.SetInvalid(i);

}

}

}

}

// Broadcast the validity mask to the child vector

if (!array_mask.AllValid()) {

auto &child_validity_mask = FlatVector::Validity(child_vector);

for (idx_t i = 0; i < size; i++) {

if (!array_mask.RowIsValid(i)) {

for (idx_t j = 0; j < array_size; j++) {

child_validity_mask.SetInvalid(i * array_size + j);

}

}

}

}

auto &child_state = array_state.GetChild(0);

auto &child_array = *array.children[0];

auto &child_type = array_info.GetChild();

if (child_count == 0 && child_offset == 0) {

D_ASSERT(!child_array.dictionary);

ColumnArrowToDuckDB(child_vector, child_array, child_state, child_count, child_type, -1);

} else {

if (child_array.dictionary) {

ColumnArrowToDuckDBDictionary(child_vector, child_array, child_state, child_count, child_type,

NumericCast<int64_t>(child_offset));

} else {

ColumnArrowToDuckDB(child_vector, child_array, child_state, child_count, child_type,

NumericCast<int64_t>(child_offset));

}

}

const ArrowType &arrow_type, int64_t nested_offset, int64_t parent_offset) {

SetValidityMask(vector, array, scan_state, size, parent_offset, nested_offset);

auto &string_info = arrow_type.GetTypeInfo<ArrowStringInfo>();

auto size_type = string_info.GetSizeType();

if (size_type == ArrowVariableSizeType::FIXED_SIZE) {

auto fixed_size = string_info.FixedSize();

//! Have to check validity mask before setting this up

idx_t offset = GetEffectiveOffset(array, parent_offset, scan_state, nested_offset) * fixed_size;

auto cdata = ArrowBufferData<char>(array, 1);

auto blob_len = fixed_size;

auto result = FlatVector::GetData<string_t>(vector);

for (idx_t row_idx = 0; row_idx < size; row_idx++) {

if (FlatVector::IsNull(vector, row_idx)) {

offset += blob_len;

continue;

}

auto bptr = cdata + offset;

result[row_idx] = StringVector::AddStringOrBlob(vector, bptr, blob_len);

offset += blob_len;

}

} else if (size_type == ArrowVariableSizeType::NORMAL) {

auto offsets =

ArrowBufferData<uint32_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);

auto cdata = ArrowBufferData<char>(array, 2);

auto result = FlatVector::GetData<string_t>(vector);

for (idx_t row_idx = 0; row_idx < size; row_idx++) {

if (FlatVector::IsNull(vector, row_idx)) {

continue;

}

auto bptr = cdata + offsets[row_idx];

auto blob_len = offsets[row_idx + 1] - offsets[row_idx];

result[row_idx] = StringVector::AddStringOrBlob(vector, bptr, blob_len);

}

} else {

//! Check if last offset is higher than max uint32

if (ArrowBufferData<uint64_t>(array, 1)[array.length] > NumericLimits<uint32_t>::Maximum()) { // LCOV_EXCL_START

throw ConversionException("DuckDB does not support Blobs over 4GB");

} // LCOV_EXCL_STOP

auto offsets =

ArrowBufferData<uint64_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);

auto cdata = ArrowBufferData<char>(array, 2);

auto result = FlatVector::GetData<string_t>(vector);

for (idx_t row_idx = 0; row_idx < size; row_idx++) {

if (FlatVector::IsNull(vector, row_idx)) {

continue;

}

auto bptr = cdata + offsets[row_idx];

auto blob_len = offsets[row_idx + 1] - offsets[row_idx];

result[row_idx] = StringVector::AddStringOrBlob(vector, bptr, blob_len);

}

}

auto valid_check = MapVector::CheckMapValidity(vector, count);

switch (valid_check) {

case MapInvalidReason::VALID:

break;

case MapInvalidReason::DUPLICATE_KEY: {

throw InvalidInputException("Arrow map contains duplicate key, which isn't supported by DuckDB map type");

}

case MapInvalidReason::NULL_KEY: {

throw InvalidInputException("Arrow map contains NULL as map key, which isn't supported by DuckDB map type");

}

default: {

throw InternalException("MapInvalidReason not implemented");

}

}

auto strings = FlatVector::GetData<string_t>(vector);

for (idx_t row_idx = 0; row_idx < size; row_idx++) {

if (FlatVector::IsNull(vector, row_idx)) {

continue;

}

auto cptr = cdata + offsets[row_idx];

auto str_len = offsets[row_idx + 1] - offsets[row_idx];

if (str_len > NumericLimits<uint32_t>::Maximum()) { // LCOV_EXCL_START

throw ConversionException("DuckDB does not support Strings over 4GB");

} // LCOV_EXCL_STOP

strings[row_idx] = string_t(cptr, UnsafeNumericCast<uint32_t>(str_len));

}

auto strings = FlatVector::GetData<string_t>(vector);

auto arrow_string = ArrowBufferData<arrow_string_view_t>(array, 1) + current_pos;

for (idx_t row_idx = 0; row_idx < size; row_idx++) {

if (FlatVector::IsNull(vector, row_idx)) {

continue;

}

auto length = UnsafeNumericCast<uint32_t>(arrow_string[row_idx].Length());

if (arrow_string[row_idx].IsInline()) {

// This string is inlined

// | Bytes 0-3 | Bytes 4-15 |

// |------------|---------------------------------------|

// | length | data (padded with 0) |

strings[row_idx] = string_t(arrow_string[row_idx].GetInlineData(), length);

} else {

// This string is not inlined, we have to check a different buffer and offsets

// | Bytes 0-3 | Bytes 4-7 | Bytes 8-11 | Bytes 12-15 |

// |------------|------------|------------|-------------|

// | length | prefix | buf. index | offset |

auto buffer_index = UnsafeNumericCast<uint32_t>(arrow_string[row_idx].GetBufferIndex());

int32_t offset = arrow_string[row_idx].GetOffset();

D_ASSERT(array.n_buffers > 2 + buffer_index);

auto c_data = ArrowBufferData<char>(array, 2 + buffer_index);

strings[row_idx] = string_t(&c_data[offset], length);

}

}

int64_t nested_offset, uint64_t parent_offset) {

auto internal_type = GetTypeIdSize(vector.GetType().InternalType());

auto data_ptr =

ArrowBufferData<data_t>(array, 1) +

internal_type * GetEffectiveOffset(array, NumericCast<int64_t>(parent_offset), scan_state, nested_offset);

FlatVector::SetData(vector, data_ptr);

int64_t nested_offset, int64_t parent_offset, idx_t size, int64_t conversion) {

auto tgt_ptr = FlatVector::GetData<dtime_t>(vector);

auto &validity_mask = FlatVector::Validity(vector);

auto src_ptr =

static_cast<const T *>(array.buffers[1]) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);

for (idx_t row = 0; row < size; row++) {

if (!validity_mask.RowIsValid(row)) {

continue;

}

if (!TryMultiplyOperator::Operation(static_cast<int64_t>(src_ptr[row]), conversion, tgt_ptr[row].micros)) {

throw ConversionException("Could not convert Time to Microsecond");

}

}

int64_t nested_offset, int64_t parent_offset, idx_t size) {

auto tgt_ptr = FlatVector::GetData<hugeint_t>(vector);

auto &validity_mask = FlatVector::Validity(vector);

auto src_ptr = static_cast<const hugeint_t *>(array.buffers[1]) +

GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);

for (idx_t row = 0; row < size; row++) {

if (!validity_mask.RowIsValid(row)) {

continue;

}

tgt_ptr[row].lower = static_cast<uint64_t>(BSwap(src_ptr[row].upper));

// flip Upper MSD

tgt_ptr[row].upper =

static_cast<int64_t>(static_cast<uint64_t>(BSwap(src_ptr[row].lower)) ^ (static_cast<uint64_t>(1) << 63));

}

int64_t nested_offset, int64_t parent_offset, idx_t size, int64_t conversion) {

auto tgt_ptr = FlatVector::GetData<timestamp_t>(vector);

auto &validity_mask = FlatVector::Validity(vector);

auto src_ptr =

ArrowBufferData<int64_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);

for (idx_t row = 0; row < size; row++) {

if (!validity_mask.RowIsValid(row)) {

continue;

}

if (!TryMultiplyOperator::Operation(src_ptr[row], conversion, tgt_ptr[row].value)) {

throw ConversionException("Could not convert TimestampTZ to Microsecond");

}

}

int64_t nested_offset, int64_t parent_offset, idx_t size, int64_t conversion) {

auto tgt_ptr = FlatVector::GetData<interval_t>(vector);

auto src_ptr =

ArrowBufferData<int64_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);

for (idx_t row = 0; row < size; row++) {

tgt_ptr[row].days = 0;

tgt_ptr[row].months = 0;

if (!TryMultiplyOperator::Operation(src_ptr[row], conversion, tgt_ptr[row].micros)) {

throw ConversionException("Could not convert Interval to Microsecond");

}

}

int64_t nested_offset, int64_t parent_offset, idx_t size) {

auto tgt_ptr = FlatVector::GetData<interval_t>(vector);

auto src_ptr =

ArrowBufferData<int32_t>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);

for (idx_t row = 0; row < size; row++) {

tgt_ptr[row].days = 0;

tgt_ptr[row].micros = 0;

tgt_ptr[row].months = src_ptr[row];

}

int64_t nested_offset, int64_t parent_offset, idx_t size) {

auto tgt_ptr = FlatVector::GetData<interval_t>(vector);

auto src_ptr =

ArrowBufferData<ArrowInterval>(array, 1) + GetEffectiveOffset(array, parent_offset, scan_state, nested_offset);

for (idx_t row = 0; row < size; row++) {

tgt_ptr[row].days = src_ptr[row].days;

tgt_ptr[row].micros = src_ptr[row].nanoseconds / Interval::NANOS_PER_MICRO;

tgt_ptr[row].months = src_ptr[row].months;

}

// Binary-search within the [0, count) range. For example:

// [0, 0, 0, 1, 1, 2] encoded as

// run_ends: [3, 5, 6]:

// 0, 1, 2 -> 0

// 3, 4 -> 1

// 5 -> 2

// 6, 7 .. -> 3 (3 == count [not found])

idx_t begin = 0;

idx_t end = count;

while (begin < end) {

idx_t middle = (begin + end) / 2;

// begin < end implies middle < end

if (offset >= static_cast<idx_t>(run_ends[middle])) {

// keep searching in [middle + 1, end)

begin = middle + 1;

} else {

// offset < run_ends[middle], so keep searching in [begin, middle)

end = middle;

}

}

return begin;

idx_t scan_offset, idx_t count) {

auto &runs = *run_end_encoding.run_ends;

auto &values = *run_end_encoding.values;

UnifiedVectorFormat run_end_format;

UnifiedVectorFormat value_format;

runs.ToUnifiedFormat(compressed_size, run_end_format);

values.ToUnifiedFormat(compressed_size, value_format);

auto run_ends_data = run_end_format.GetData<RUN_END_TYPE>(run_end_format);

auto values_data = value_format.GetData<VALUE_TYPE>(value_format);

auto result_data = FlatVector::GetData<VALUE_TYPE>(result);

auto &validity = FlatVector::Validity(result);

// According to the arrow spec, the 'run_ends' array is always valid

// so we will assume this is true and not check the validity map

// Now construct the result vector from the run_ends and the values

auto run = FindRunIndex(run_ends_data, compressed_size, scan_offset);

idx_t logical_index = scan_offset;

idx_t index = 0;

if (value_format.validity.AllValid()) {

// None of the compressed values are NULL

for (; run < compressed_size; ++run) {

auto run_end_index = run_end_format.sel->get_index(run);

auto value_index = value_format.sel->get_index(run);

auto &value = values_data[value_index];

auto run_end = static_cast<idx_t>(run_ends_data[run_end_index]);

D_ASSERT(run_end > (logical_index + index));

auto to_scan = run_end - (logical_index + index);

// Cap the amount to scan so we don't go over size

to_scan = MinValue<idx_t>(to_scan, (count - index));

for (idx_t i = 0; i < to_scan; i++) {

result_data[index + i] = value;

}

index += to_scan;

if (index >= count) {

if (logical_index + index >= run_end) {

// The last run was completed, forward the run index

++run;

}

break;

}

}

} else {

for (; run < compressed_size; ++run) {

auto run_end_index = run_end_format.sel->get_index(run);

auto value_index = value_format.sel->get_index(run);

auto run_end = static_cast<idx_t>(run_ends_data[run_end_index]);

D_ASSERT(run_end > (logical_index + index));

auto to_scan = run_end - (logical_index + index);

// Cap the amount to scan so we don't go over size

to_scan = MinValue<idx_t>(to_scan, (count - index));

if (value_format.validity.RowIsValidUnsafe(value_index)) {

auto &value = values_data[value_index];

for (idx_t i = 0; i < to_scan; i++) {

result_data[index + i] = value;

validity.SetValid(index + i);

}

} else {

for (idx_t i = 0; i < to_scan; i++) {

validity.SetInvalid(index + i);

}

}

index += to_scan;

if (index >= count) {

if (logical_index + index >= run_end) {

// The last run was completed, forward the run index

++run;

}

break;

}

}

}

idx_t scan_offset, idx_t size) {

auto &values = *run_end_encoding.values;

auto physical_type = values.GetType().InternalType();

switch (physical_type) {

case PhysicalType::INT8:

FlattenRunEnds<RUN_END_TYPE, int8_t>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::INT16:

FlattenRunEnds<RUN_END_TYPE, int16_t>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::INT32:

FlattenRunEnds<RUN_END_TYPE, int32_t>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::INT64:

FlattenRunEnds<RUN_END_TYPE, int64_t>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::INT128:

FlattenRunEnds<RUN_END_TYPE, hugeint_t>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::UINT8:

FlattenRunEnds<RUN_END_TYPE, uint8_t>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::UINT16:

FlattenRunEnds<RUN_END_TYPE, uint16_t>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::UINT32:

FlattenRunEnds<RUN_END_TYPE, uint32_t>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::UINT64:

FlattenRunEnds<RUN_END_TYPE, uint64_t>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::BOOL:

FlattenRunEnds<RUN_END_TYPE, bool>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::FLOAT:

FlattenRunEnds<RUN_END_TYPE, float>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::DOUBLE:

FlattenRunEnds<RUN_END_TYPE, double>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::INTERVAL:

FlattenRunEnds<RUN_END_TYPE, interval_t>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::VARCHAR: {

// Share the string heap, we don't need to allocate new strings, we just reference the existing ones

result.SetAuxiliary(values.GetAuxiliary());

FlattenRunEnds<RUN_END_TYPE, string_t>(result, run_end_encoding, compressed_size, scan_offset, size);

break;

}

default:

throw NotImplementedException("RunEndEncoded value type '%s' not supported yet", TypeIdToString(physical_type));

}

idx_t size, const ArrowType &arrow_type, int64_t nested_offset,

ValidityMask *parent_mask, uint64_t parent_offset) {

// Scan the 'run_ends' array

D_ASSERT(array.n_children == 2);

auto &run_ends_array = *array.children[0];

auto &values_array = *array.children[1];

auto &struct_info = arrow_type.GetTypeInfo<ArrowStructInfo>();

auto &run_ends_type = struct_info.GetChild(0);

auto &values_type = struct_info.GetChild(1);

D_ASSERT(vector.GetType() == values_type.GetDuckType());

auto &scan_state = array_state.state;

if (vector.GetBuffer()) {

vector.GetBuffer()->SetAuxiliaryData(make_uniq<ArrowAuxiliaryData>(array_state.owned_data));

}

D_ASSERT(run_ends_array.length == values_array.length);

auto compressed_size = NumericCast<idx_t>(run_ends_array.length);

// Create a vector for the run ends and the values

auto &run_end_encoding = array_state.RunEndEncoding();

if (!run_end_encoding.run_ends) {

// The run ends and values have not been scanned yet for this array

D_ASSERT(!run_end_encoding.values);

run_end_encoding.run_ends = make_uniq<Vector>(run_ends_type.GetDuckType(), compressed_size);

run_end_encoding.values = make_uniq<Vector>(values_type.GetDuckType(), compressed_size);

ColumnArrowToDuckDB(*run_end_encoding.run_ends, run_ends_array, array_state, compressed_size, run_ends_type);

auto &values = *run_end_encoding.values;

SetValidityMask(values, values_array, scan_state, compressed_size, NumericCast<int64_t>(parent_offset),

nested_offset);

ColumnArrowToDuckDB(values, values_array, array_state, compressed_size, values_type);

}

idx_t scan_offset = GetEffectiveOffset(array, NumericCast<int64_t>(parent_offset), scan_state, nested_offset);

auto physical_type = run_ends_type.GetDuckType().InternalType();

switch (physical_type) {

case PhysicalType::INT16:

FlattenRunEndsSwitch<int16_t>(vector, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::INT32:

FlattenRunEndsSwitch<int32_t>(vector, run_end_encoding, compressed_size, scan_offset, size);

break;

case PhysicalType::INT64:

FlattenRunEndsSwitch<int32_t>(vector, run_end_encoding, compressed_size, scan_offset, size);

break;

default:

throw NotImplementedException("Type '%s' not implemented for RunEndEncoding", TypeIdToString(physical_type));

}

uint64_t parent_offset, ArrowScanLocalState &scan_state, ValidityMask &val_mask,

DecimalBitWidth arrow_bit_width) {

switch (vector.GetType().InternalType()) {

case PhysicalType::INT16: {

auto tgt_ptr = FlatVector::GetData<int16_t>(vector);

for (idx_t row = 0; row < size; row++) {

if (val_mask.RowIsValid(row)) {

auto result = TryCast::Operation(src_ptr[row], tgt_ptr[row]);

D_ASSERT(result);

(void)result;

}

}

break;

}

case PhysicalType::INT32: {

if (arrow_bit_width == DecimalBitWidth::DECIMAL_32) {

FlatVector::SetData(vector, ArrowBufferData<data_t>(array, 1) +

GetTypeIdSize(vector.GetType().InternalType()) *

GetEffectiveOffset(array, NumericCast<int64_t>(parent_offset),

scan_state, nested_offset));

} else {

auto tgt_ptr = FlatVector::GetData<int32_t>(vector);

for (idx_t row = 0; row < size; row++) {

if (val_mask.RowIsValid(row)) {

auto result = TryCast::Operation(src_ptr[row], tgt_ptr[row]);

D_ASSERT(result);

(void)result;

}

}

}

break;

}

case PhysicalType::INT64: {

if (arrow_bit_width == DecimalBitWidth::DECIMAL_64) {

FlatVector::SetData(vector, ArrowBufferData<data_t>(array, 1) +

GetTypeIdSize(vector.GetType().InternalType()) *

GetEffectiveOffset(array, NumericCast<int64_t>(parent_offset),

scan_state, nested_offset));

} else {

auto tgt_ptr = FlatVector::GetData<int64_t>(vector);

for (idx_t row = 0; row < size; row++) {

if (val_mask.RowIsValid(row)) {

auto result = TryCast::Operation(src_ptr[row], tgt_ptr[row]);

D_ASSERT(result);

(void)result;

}

}

}

break;

}

case PhysicalType::INT128: {

if (arrow_bit_width == DecimalBitWidth::DECIMAL_128) {

FlatVector::SetData(vector, ArrowBufferData<data_t>(array, 1) +

GetTypeIdSize(vector.GetType().InternalType()) *

GetEffectiveOffset(array, NumericCast<int64_t>(parent_offset),

scan_state, nested_offset));

} else {

auto tgt_ptr = FlatVector::GetData<hugeint_t>(vector);

for (idx_t row = 0; row < size; row++) {

if (val_mask.RowIsValid(row)) {

auto result = TryCast::Operation(src_ptr[row], tgt_ptr[row]);

D_ASSERT(result);

(void)result;

}

}

}

break;

}

default:

throw NotImplementedException("Unsupported physical type for Decimal: %s",

TypeIdToString(vector.GetType().InternalType()));

}

const ArrowType &arrow_type, int64_t nested_offset, ValidityMask *parent_mask,

uint64_t parent_offset, bool ignore_extensions) {

auto &scan_state = array_state.state;

D_ASSERT(!array.dictionary);

if (!ignore_extensions && arrow_type.HasExtension()) {

if (arrow_type.extension_data->arrow_to_duckdb) {

// Convert the storage and then call the cast function

Vector input_data(arrow_type.extension_data->GetInternalType());

ColumnArrowToDuckDB(input_data, array, array_state, size, arrow_type, nested_offset, parent_mask,

parent_offset, /*ignore_extensions*/ true);

arrow_type.extension_data->arrow_to_duckdb(array_state.context, input_data, vector, size);

return;

}

}

if (vector.GetBuffer()) {

vector.GetBuffer()->SetAuxiliaryData(make_uniq<ArrowAuxiliaryData>(array_state.owned_data));

}

switch (vector.GetType().id()) {

case LogicalTypeId::SQLNULL:

vector.Reference(Value());

break;

case LogicalTypeId::BOOLEAN: {

//! Arrow bit-packs boolean values

//! Lets first figure out where we are in the source array

auto effective_offset =

GetEffectiveOffset(array, NumericCast<int64_t>(parent_offset), scan_state, nested_offset);

auto src_ptr = ArrowBufferData<uint8_t>(array, 1) + effective_offset / 8;

auto tgt_ptr = (uint8_t *)FlatVector::GetData(vector);

int src_pos = 0;

idx_t cur_bit = effective_offset % 8;

for (idx_t row = 0; row < size; row++) {

if ((src_ptr[src_pos] & (1 << cur_bit)) == 0) {

tgt_ptr[row] = 0;

} else {

tgt_ptr[row] = 1;

}

cur_bit++;

if (cur_bit == 8) {

src_pos++;

cur_bit = 0;

}

}

break;

}

case LogicalTypeId::TINYINT:

case LogicalTypeId::SMALLINT:

case LogicalTypeId::INTEGER:

case LogicalTypeId::FLOAT:

case LogicalTypeId::DOUBLE:

case LogicalTypeId::UTINYINT:

case LogicalTypeId::USMALLINT:

case LogicalTypeId::UINTEGER:

case LogicalTypeId::UBIGINT:

case LogicalTypeId::BIGINT:

case LogicalTypeId::HUGEINT:

case LogicalTypeId::UHUGEINT:

case LogicalTypeId::TIMESTAMP:

case LogicalTypeId::TIMESTAMP_SEC:

case LogicalTypeId::TIMESTAMP_MS:

case LogicalTypeId::TIMESTAMP_NS:

case LogicalTypeId::TIME_TZ: {

DirectConversion(vector, array, scan_state, nested_offset, parent_offset);

break;

}

case LogicalTypeId::UUID:

UUIDConversion(vector, array, scan_state, nested_offset, NumericCast<int64_t>(parent_offset), size);

break;

case LogicalTypeId::VARCHAR: {

auto &string_info = arrow_type.GetTypeInfo<ArrowStringInfo>();

auto size_type = string_info.GetSizeType();

switch (size_type) {

case ArrowVariableSizeType::SUPER_SIZE: {

auto cdata = ArrowBufferData<char>(array, 2);

auto offsets = ArrowBufferData<uint64_t>(array, 1) +

GetEffectiveOffset(array, NumericCast<int64_t>(parent_offset), scan_state, nested_offset);

SetVectorString(vector, size, cdata, offsets);

break;

}

case ArrowVariableSizeType::NORMAL:

case ArrowVariableSizeType::FIXED_SIZE: {

auto cdata = ArrowBufferData<char>(array, 2);

auto offsets = ArrowBufferData<uint32_t>(array, 1) +

GetEffectiveOffset(array, NumericCast<int64_t>(parent_offset), scan_state, nested_offset);

SetVectorString(vector, size, cdata, offsets);

break;

}

case ArrowVariableSizeType::VIEW: {

SetVectorStringView(

vector, size, array,

GetEffectiveOffset(array, NumericCast<int64_t>(parent_offset), scan_state, nested_offset));

break;

}

}

break;

}

case LogicalTypeId::DATE: {

auto &datetime_info = arrow_type.GetTypeInfo<ArrowDateTimeInfo>();

auto precision = datetime_info.GetDateTimeType();

switch (precision) {

case ArrowDateTimeType::DAYS: {

DirectConversion(vector, array, scan_state, nested_offset, parent_offset);

break;

}

case ArrowDateTimeType::MILLISECONDS: {

//! convert date from nanoseconds to days

auto src_ptr = ArrowBufferData<uint64_t>(array, 1) +

GetEffectiveOffset(array, NumericCast<int64_t>(parent_offset), scan_state, nested_offset);

auto tgt_ptr = FlatVector::GetData<date_t>(vector);

for (idx_t row = 0; row < size; row++) {

tgt_ptr[row] = date_t(UnsafeNumericCast<int32_t>(static_cast<int64_t>(src_ptr[row]) /

static_cast<int64_t>(1000 * 60 * 60 * 24)));

}

break;

}

default:

throw NotImplementedException("Unsupported precision for Date Type ");

}

break;

}

case LogicalTypeId::TIME: {

auto &datetime_info = arrow_type.GetTypeInfo<ArrowDateTimeInfo>();

auto precision = datetime_info.GetDateTimeType();

switch (precision) {

case ArrowDateTimeType::SECONDS: {

TimeConversion<int32_t>(vector, array, scan_state, nested_offset, NumericCast<int64_t>(parent_offset), size,

1000000);

break;

}

case ArrowDateTimeType::MILLISECONDS: {

TimeConversion<int32_t>(vector, array, scan_state, nested_offset, NumericCast<int64_t>(parent_offset), size,

1000);

break;

}

case ArrowDateTimeType::MICROSECONDS: {

TimeConversion<int64_t>(vector, array, scan_state, nested_offset, NumericCast<int64_t>(parent_offset), size,

1);

break;

}

case ArrowDateTimeType::NANOSECONDS: {

auto tgt_ptr = FlatVector::GetData<dtime_t>(vector);

auto src_ptr = ArrowBufferData<int64_t>(array, 1) +

GetEffectiveOffset(array, NumericCast<int64_t>(parent_offset), scan_state, nested_offset);

for (idx_t row = 0; row < size; row++) {

tgt_ptr[row].micros = src_ptr[row] / 1000;

}

break;

}

default:

throw NotImplementedException("Unsupported precision for Time Type ");